ratch 0.1

data/lib/ratch/options.rb

@@ -0,0 +1,57 @@
+
+module Ratch
+
+  # General Options
+
+  module GeneralOptions
+
+    #
+
+    def argv
+      @argv ||= ARGV.dup
+    end
+
+    # Convert command line argv to args.
+    #
+    # TODO Is this implmented as expected?
+
+    def command_parameters
+      argv.to_params
+    end
+
+    # Debug mode.
+
+    def debug?
+      @debug
+    end
+
+    #
+
+    def trace?
+      @trace ||= %w{--trace}.any?{|a| argv.delete(a)}
+    end
+
+    #
+
+    def trace!
+      @trace = true
+    end
+
+    #
+
+    def noharm?
+      @noharm ||= %w{--dryrun --dry-run --noharm}.any?{|a| argv.delete(a)}
+    end
+
+    alias_method :dryrun?, :noharm? ; module_function :dryrun?
+
+    #
+
+    def noharm!
+      @noharm = true
+    end
+    alias_method :dryrun!, :noharm! ; module_function :dryrun!
+
+  end
+
+end

data/lib/ratch/runnable.rb

@@ -0,0 +1,117 @@
+module Ratch
+
+  #
+
+  module Runnable
+
+    # TODO How to handle arguments?
+
+    def run(batchfile, arguments=nil)
+      BatchFile.new(batchfile).call
+#       # TODO probably should raise error instead
+#       abort "missing batch file -- #{batchfile}" unless File.file?(batchfile)
+#       script = File.read($0 = batchfile)
+#       #instance_eval(script)
+#       eval(script, binding, $0)
+    end
+
+    # Batch run, ie. run and cache.
+    # Usually this can be take care of by method_missing.
+    # But, in some cases, built in method names block task
+    # calls, so you have to use #batch to invoke those.
+
+    def batch(batchfile, arguments=nil)
+      task_cache[task] ||= run(batchfile, arguments)
+    end
+
+    # Task cache, which prevents batch runs from re-executing.
+
+    def task_cache
+      @@task_cache ||= {}
+    end
+
+    # Is a task complete or in the process of being completed?
+
+    def done?(task)
+      task == $0 or task_cache.key?(task)
+    end
+
+    # Shell runner.
+
+    def sh(cmd)
+      if noharm?
+        puts cmd
+        true
+      else
+        puts "--> system call: #{cmd}" if trace?
+        system(cmd)
+      end
+    end
+
+    # Abort running.
+
+    def abort(msg=nil)
+      puts msg if msg
+      exit 0
+    end
+
+  end
+
+  #
+  module OpenRunnable
+
+    # If method is missing try to run an external task
+    # or binary by that name. If it is a binary, arguments
+    # translate into commandline parameters. For example:
+    #
+    #   tar 'foo/', :x=>true, :v=>true, :z=>true, :f=>'foo.tar.gz'
+    #
+    # or
+    #
+    #   tar '-xvzf', "foo.tar.gz", "foo/"
+    #
+    # becomes
+    #
+    #   tar -x -v -z -f foo.tar.gz foo/
+    #
+    # If it is a task, it will be cached. Tasks only ever run once.
+    # To run them more than once you can manually execute them with #run.
+    # Likewise you can manually run and cache by calling #batch.
+    # This is good to know, b/c in some cases built in method names
+    # block task calls, so you have to #batch to invoke them.
+
+    def method_missing(sym,*args)
+      puts "method_missing: #{sym}" if debug?
+
+      name = sym.to_s
+
+      task  = task?(name)
+      done  = task && done?(task)
+      cache = task && !done && name[1,-1] != '!'
+      bin   = bin?(name) if (!task || done)
+      none  = task && done && !bin
+      #task  = name if bin
+
+      return super unless task || bin
+
+      return if none  # nothing to do
+
+      params = args.to_params
+
+      if bin
+        cmd = "#{File.basename(bin)} #{params}"
+        res = sh(cmd)
+      elsif task
+        cmd = "./#{task} #{params}"
+        puts "--> #{cache ? '' : 'not-'}cached execution: #{cmd}" if trace?
+        res = run(task, args)
+        if cache
+          #@task_cache[task] ||= (system(cmd); true)
+          task_cache[task] ||= res
+        end
+      end
+
+      return res
+    end
+  end
+end

data/lib/ratch/taskable.rb

@@ -0,0 +1,105 @@
+module Ratch
+
+  module Taskable
+
+    #def main
+    #  @main
+    #end
+
+    def tasks
+      @tasks ||= {}
+    end
+
+    # Define a main task.
+
+    def main(name, &block)
+      @main = Task.register(name, &block)
+      tasks[@main.name] = @main
+    end
+
+    # Define a task.
+
+    def task(name, &block)
+      tsk = Task.register(name, &block)
+      tasks[tsk.name] = tsk
+    end
+
+    # Define a file target.
+
+    def file(name, &block)
+      tasks[name.to_s] = FileTask.register(name, &block)
+    end
+
+  end
+
+  #
+  #
+  #
+
+  class Task
+
+    def self.tasks
+      @@tasks ||= {}
+    end
+
+    def self.register(name_deps, &block)
+      if Hash===name_deps
+        name = name_deps.keys[0]
+        deps = name_deps.values[0]
+      else
+        name = name_deps
+        deps = []
+      end
+      tasks[name.to_s] = new(name, *deps, &block)
+    end
+
+    attr :name
+    attr :preqs
+    attr :action
+
+    #
+
+    def initialize(name, *preqs, &action)
+      @name          = name.to_s
+      @preqs         = preqs
+      @action        = action
+    end
+
+    def call
+  #p plan
+      plan.each{ |name| @@tasks[name].action_call }
+      #action_call
+    end
+
+    #
+
+    def action_call
+      @action.call if @action
+    end
+
+    # TODO Check for circular dependencies.
+
+    def plan(list=[])
+      if list.include?(name)
+        raise "Circular dependency #{name}."
+      end
+
+      @preqs.each do |need|
+        need = need.to_s
+        next if list.include?(need)
+        @@tasks[need].plan(list)
+      end
+      list << name
+      return list
+    end
+
+  end
+
+  #
+  #
+  #
+
+  class FileTask < Task
+  end
+
+end

data/lib/ratch/taskutils.rb

@@ -0,0 +1,44 @@
+# TITLE:
+#
+#   TaskUtils
+#
+# SUMMARY:
+#
+#   All the task methods used by BatchRunner.
+#
+# COPYRIGHT:
+#
+#   Copyright (c) 2007 7rans
+
+require 'yaml'
+
+#require 'facets/hash/merge'  # for reverse_merge
+
+require 'ratch/options'
+require 'ratch/consoleutils'
+require 'ratch/configutils'
+require 'ratch/emailutils'
+require 'ratch/fileutils'
+require 'ratch/taskable'
+require 'ratch/runnable'
+
+
+
+module Ratch
+
+  # = TaskUtils
+
+  module TaskUtils
+    include GeneralOptions
+
+    include ConsoleUtils
+    include FileUtils
+    include ConfigUtils
+    include EmailUtils
+
+    include Taskable
+    include Runnable
+    include OpenRunnable
+  end
+
+end

data/lib/ratch/uploadutils.rb

@@ -0,0 +1,413 @@
+# = uploadutils.rb
+#
+# CREDIT Thomas Sawyer
+#
+# TODO Incorporate password into scp and ftp ?
+# TODO rsync needs --delete option
+
+require 'openssl'
+require 'shellwords'
+require 'tmpdir'
+
+require 'facets/openobject'
+
+module Ratch
+
+  # Upload files to host. These means of uploading are current
+  # supported: ftp, sftp, scp and rsync.
+  #
+  #     user       Username for host.
+  #     host       Host server's domain name.
+  #     root       Document root path on host.
+  #     copy       Directory of files to publish, or
+  #                Files to publish using from and to.
+  #
+  #     dryrun     If true only pretend to upload.
+  #     quiet      Supress all output.
+  #     verbose    Provide extra details.
+  #
+  # The copy parameter allows you to simply specify a file
+  # or directory which will be published to host's document
+  # root location.
+  #
+  # If you need more control over which files to publish
+  # where, you can use the copy parameter instead. Provide
+  # an array of pattern strings in the form of "{from} {to}".
+  # If the desitination is the host's document root you do
+  # not need to specify the {to} part. For example:
+  #
+  #     copy = [ 'web/*', 'doc/api/* doc/api' ]
+  #
+  # The first copies the files under your project's web directory
+  # to the host's document root. The second copies your projects
+  # doc/api files to the doc/api location on the host.
+  #
+  # The internal template used for the outbound destination
+  # is 'username@host:root/'.
+
+  module UploadUtils
+
+    module_function
+
+    #
+    # Upload via given protocol.
+    #
+
+    def upload( protocol, opts )
+      send(protocol.to_s.downcase,opts)
+    end
+
+    #
+    # Use ftp to upload files.
+    #
+
+    def ftp( keys )
+      keys = upload_parameters(keys)
+
+      # set transfer rules
+      if keys.stage
+        trans = stage_transfer(keys.stage)
+      else
+        files(keys.dir, keys.copy).each do |from|
+          trans << [from,from]
+        end
+      end
+
+      # append location of publication dir to from
+      dir = keys.dir
+      trans.collect!{ |from,to| [File.join(dir,from), to] }
+
+      if keys.dryrun
+        puts "ftp open #{keys.user}@#{keys.host}:#{keys.root}/"
+        keys.trans.each do |f, t|
+          puts "ftp put #{f} #{t}"
+        end
+      else
+        require 'net/ftp'
+        Net::FTP.open(keys.host) do |ftp|
+          ftp.login(keys.user) #password?
+          ftp.chdir(keys.root)
+          keys.trans.each do |f, t|
+            puts "ftp #{f} #{t}" unless keys.quiet
+            ftp.putbinaryfile( f, t, 1024 )
+          end
+        end
+      end
+    end
+
+    #
+    # Use sftp to upload files.
+    #
+
+    def sftp( keys )
+      keys = upload_parameters(keys)
+
+      # set transfer rules
+      if keys.stage
+        trans = stage_transfer(keys.stage)
+      else
+        files(keys.dir, keys.copy).each do |from|
+          trans << [from,from]
+        end
+      end
+
+      # append location of publication dir to from
+      dir = keys.dir
+      trans.collect!{ |from,to| [File.join(dir,from), to] }
+
+      if keys.dryrun
+        puts "sftp open #{keys.user}@#{keys.host}:#{keys.root}/"
+        keys.trans.each do |f,t|
+          puts "sftp put #{f} #{t}"
+        end
+      else
+        require 'net/sftp'
+        Net::SFTP.start(keys.host, keys.user, keys.pass) do |sftp|
+          #sftp.login( user )
+          sftp.chdir(keys.root)
+          keys.trans.each do |f,t|
+            puts "sftp #{f} #{t}" unless keys.quiet
+            sftp.put_file(f,t) #, 1024 )
+          end
+        end
+      end
+    end
+
+    #
+    # Use rsync to upload files.
+    #
+
+    def rsync( keys )
+      keys = upload_parameters(keys)
+
+      flags = []
+      flags << "-n" if keys.dryrun
+      flags << "-q" if keys.quiet
+      flags << "-v" if keys.verbose
+      flags << "--progress" unless keys.quiet
+      flags = flags.join(' ').strip
+      flags = ' ' + flags unless flags.empty?
+
+      manfile = 'Publish.txt'
+
+      if keys.stage
+        dir = stage_linkdir(keys.dir, keys.stage)
+        Dir.chdir(dir) do
+          cpy = files(keys.copy)
+        end
+        manifest = File.join(dir,manfile)
+        cmd = %{rsync#{flags} -L -arz --files-from='#{manifest}' #{dir} #{keys.user}@#{keys.host}:/#{keys.root}}
+      else
+        dir = keys.dir
+        cpy = files(dir, keys.copy)
+        manifest = File.join(dir,manfile)
+        cmd = %{rsync#{flags} -arz --files-from='#{manifest}' #{dir} #{keys.user}@#{keys.host}:/#{keys.root}}
+      end
+
+      #Dir.chdir(keys.dir) do
+        begin
+          File.open(manifest, 'w'){ |f| f << cpy.join("\n") }
+          ENV['RSYNC_PASSWORD'] = keys.pass if keys.pass
+          puts cmd unless keys.quiet
+          system cmd
+        ensure
+          ENV.delete('RSYNC_PASSWORD') if keys.pass
+        end
+      #end
+
+    end
+
+    # private (can't do b/c of module_function)
+
+    # parse publishing options.
+
+    def upload_parameters( keys )
+      keys = OpenObject.new(keys)
+
+      keys.copy = keys.copy || '**/*'
+      keys.host = keys.host || keys.domain
+      keys.user = keys.user || keys.username
+      keys.root = keys.root || '/'
+      #keys.pass = keys.pass || keys.password
+
+      # validate
+      raise ArgumentError, "missing publish parameter -- dir" unless keys.dir
+      raise ArgumentError, "missing publish parameter -- host" unless keys.host
+      raise ArgumentError, "missing publish parameter -- user" unless keys.user
+      #raise ArgumentError, "missing publish parameter -- copy" unless keys.copy
+      #raise ArgumentError, "missing publish parameter -- root" unless keys.root
+
+      keys.root = '' if keys.root.nil?
+      keys.root.sub!(/^\//,'')
+
+      if String===keys.copy and File.directory?(keys.copy)
+        copy = File.join(keys.copy, '*')
+      end
+      keys.copy = [keys.copy].flatten.compact
+
+  #     trans = []
+  #     keys.copy.each do |from|
+  #       #from, to = *Shellwords.shellwords(c)
+  #       #to = from if to.nil?
+  #       #to = to[1..-1] if to[0,1] == '/'
+  #       from.sub('*','**/*') unless from =~ /\*\*/
+  #       files = Dir.glob(from)
+  #       files.each do |f|
+  #         #t = File.join(to,File.basename(f))
+  #         #t = t[1..-1] if t[0,1] == '/'
+  #         trans << [f,f]
+  #       end
+  #     end
+  #     keys.trans = trans
+
+      return keys
+    end
+
+    # Put together the list of files to copy.
+
+    def files( dir, copy )
+      Dir.chdir(dir) do
+        del, add = copy.partition{ |f| /^[-]/ =~ f }
+
+        # remove - and + prefixes
+        del.collect!{ |f| f.sub(/^[-]/,'') }
+        add.collect!{ |f| f.sub(/^[+]/,'') }
+
+        #del.concat(must_exclude)
+
+        files = []
+        add.each{ |g| files += Dir.multiglob(g) }
+        del.each{ |g| files -= Dir.multiglob(g) }
+
+        files.collect!{ |f| f.sub(/^\//,'') }
+
+        return files
+      end
+    end
+
+    # Combine three part stage list into a two part from->to list.
+    #
+    # Using the stage list of three space separated fields.
+    #
+    #   fromdir file todir
+    #
+    # This is used to generate a from -> to list of the form:
+    #
+    #  fromdir/file todir/file
+    #
+
+    def stage_transfer( list )
+      trans = []
+      list.each do |line|
+        trans << Shellwords.shellwords(line)
+      end
+
+      trans.collect! do |from, base, to|
+        file = File.join(from,base)
+        to = File.join(to,base)
+        [from, to]
+      end
+    end
+
+    # When using stage options this will create temporary linked location.
+
+    def stage_linkdir( dir, list )
+      folder = File.join(Dir.tmpdir, 'ratchets', 'project', object_id.abs.to_s)
+      FileUtils.mkdir_p(folder)
+
+      Dir.chdir(dir) do
+        stage_transfer(list).each do |file, to|
+          link = File.join(folder,to)
+          FileUtils.ln_s(link,file)
+        end
+      end
+
+      return folder
+    end
+
+  end
+
+end
+
+
+=begin
+
+
+  #--
+  # SHELLS OUT! Need net/scp library to fix.
+  #++
+
+  # Use scp to upload files.
+
+  def scp( keys )
+    keys = upload_parameters(keys)
+
+    flags = []
+    flags << "-v" if keys.verbose
+    flags << "-q" if keys.quiet
+    flags = flags.join(' ').strip
+    flags = ' ' + flags unless flags.empty?
+
+    upload_stage(keys) do #|tmpdir|
+      cmd = "scp -r#{flags} * #{keys.user}@#{keys.host}:/#{keys.root}"
+      puts cmd unless keys.quiet
+      system cmd unless keys.dryrun
+    end
+  end
+
+  # Use rsync to upload files.
+
+  def rsync( keys )
+    keys = upload_parameters(keys)
+
+    flags = []
+    flags << "-n" if keys.dryrun
+    flags << "-v" if keys.verbose
+    flags << "-q" if keys.quiet
+    flags = flags.join(' ').strip
+    flags = ' ' + flags unless flags.empty?
+
+    upload_stage(keys) do #|tmpdir|
+      begin
+        ENV['RSYNC_PASSWORD'] = keys.pass if keys.pass
+        cmd = "rsync -R#{flags} -arz * #{keys.user}@#{keys.host} /#{keys.root}"
+      ensure
+        ENV.delete('RSYNC_PASSWORD') if keys.pass
+      end
+    end
+  end
+
+  # Use ftp to upload files.
+
+  def ftp( keys )
+    keys = upload_parameters(keys)
+
+    if keys.dryrun
+      puts "ftp open #{keys.user}@#{keys.host}:#{keys.root}/"
+      keys.trans.each do |f, t|
+        puts "ftp put #{f} #{t}"
+      end
+    else
+      require 'net/ftp'
+      Net::FTP.open(keys.host) do |ftp|
+        ftp.login(keys.user) #password?
+        ftp.chdir(keys.root)
+        keys.trans.each do |f, t|
+          puts "ftp #{f} #{t}" unless keys.quiet
+          ftp.putbinaryfile( f, t, 1024 )
+        end
+      end
+    end
+  end
+
+  # Use sftp to upload files.
+
+  def sftp( keys )
+    keys = upload_parameters(keys)
+
+    if keys.dryrun
+      puts "sftp open #{keys.user}@#{keys.host}:#{keys.root}/"
+      keys.trans.each do |f,t|
+        puts "sftp put #{f} #{t}"
+      end
+    else
+      require 'net/sftp'
+      Net::SFTP.start(keys.host, keys.user, keys.pass) do |sftp|
+        #sftp.login( user )
+        sftp.chdir(keys.root)
+        keys.trans.each do |f,t|
+          puts "sftp #{f} #{t}" unless keys.quiet
+          sftp.put_file(f,t) #, 1024 )
+        end
+      end
+    end
+  end
+
+
+  # Creates a stage from which the whole directory can be uploaded.
+  # This is needed for scp and rsync which have to shelled out,
+  # and can't conveniently copy one file at a time.
+
+  def upload_stage(keys) #:yield:
+    tmp = "scp_#{object_id.abs}_#{ Time.now.strftime("%Y%m%d%H%M%S")}"
+    tmpdir = File.join(Dir.tmpdir,tmp)
+
+    puts "mkdir -p #{tmpdir}" unless keys.quiet
+    FileUtils.mkdir_p(tmpdir)  # go ahead and do this even if dryrun
+
+    fu = keys.dryrun ? FileUtils::DryRun : FileUtils
+    keys.trans.each do |f, t|
+      to = File.join(tmpdir, t)
+      fu.mv(f,to)
+    end
+
+    puts "cd #{tmpdir}" unless keys.quiet
+    Dir.chdir(tmpdir) do
+      yield #(tmpdir)
+    end
+
+    puts "rm -r #{tmpdir}" unless keys.quiet
+    FileUtils.rm_r(tmpdir)  # now remove the temp dir
+  end
+
+=end