tap 0.7.9 → 0.8.0

data/lib/tap/file_task.rb

@@ -1,22 +1,111 @@
 module Tap
   
-  # FileTask provides methods for making files 
-  
-  # FileTask process adds error handling code to restore any files marked
-  # for backup or files that were made (using make or mkdir) if an unhandled
-  # error occurs during processing.  Set +restore_on_error+ to false to 
-  # turn off automatic restore.  In addtion, backed-up files can be removed
-  # when process completes by setting +remove_backed_up_files+ to true.
+  # FileTask provides methods for creating/modifying files such that you can
+  # rollback changes if an error occurs.  In addition, FileTask provides a 
+  # method to infer filepaths within the standard Tap directory structure.  
+  #
+  # == Creating Files/Rolling Back Changes
+  #
+  # FileTask tracks which files to roll back using the added_files array
+  # and the backed_up_files hash.  On an execute error, all added files are
+  # removed and then all backed up files (backed_up_files keys) are restored  
+  # using the corrsponding backup files (backed_up_files values). 
+  # 
+  # For consistency, all filepaths in added_files and backed_up_files should 
+  # be expanded using File.expand_path.  The easiest way to ensure files are
+  # properly set up for rollback is to use prepare before working with files
+  # and to create directories with mkdir.
+  #
+  #   # this file will be backed up and restored
+  #   File.open("file.txt", "w") {|f| f << "original content"}
+  #  
+  #   t = FileTask.new do |task, inputs|
+  #     task.mkdir("some/dir")                         # marked for rollback
+  #     task.prepare("file.txt", "path/to/file.txt")   # marked for rollback
+  #
+  #     File.open("file.txt", "w") {|f| f << "new content"}
+  #     File.touch("path/to/file.txt")
+  #
+  #     # raise an error to start rollback
+  #     raise "error!"
+  #   end
+  #
+  #   begin
+  #     File.exists?("some/dir")              # => false
+  #     File.exists?("path/to/file.txt")      # => false
+  #     t.execute(nil)
+  #   rescue
+  #     $!.message                            # => "error!"
+  #     File.exists?("some/dir")              # => false
+  #     File.exists?("path/to/file.txt")      # => false
+  #     File.read("file.txt")                 # => "original content"
+  #   end
+  #
+  #--
+  # TODO - make cleanup into remove_backup_files and simply remove this 
+  # configuration. People will figure it out themselves if they want that to happen.
+  #
+  # The FileTask configurations modify the default backup and restore behavior.
+  # 
+  # backup_dir:: The app directory alias for backups (default :backup)
+  # backup_timestamp::  A strftime format string used as a timestamp for backup
+  #                     files (default "%Y%m%d_%H%M%S")
+  # rollback_on_error:: Controls whether or not added and backed up files are
+  #                     rolled back on error (default true)
+  # cleanup_after_execute:: If true, backed up files will be removed after a 
+  #                          successful execution (default false)
+  #++
   class FileTask < Task
-    set_default_config({
-      :backup_dir => :backup,
-      :backup_timestamp => "%Y%m%d_%H%M%S",
-      :restore_on_error => true,
-      :remove_backed_up_files => false},
-      :make_accessors => true)
+    autoload(:FileUtils, "fileutils")
+    
+    class << self
       
-    attr_reader :inference_block, :backed_up_files, :made_files
-    attr_accessor :dirname
+      # A batch File.open method.  If a block is given, each file in the list will be 
+      # opened the open files passed to the block.  Files are automatically closed when 
+      # the block returns.  If no block is given, the open files are returned.
+      #
+      #   FileTask.open(["one.txt", "two.txt"], "w") do |one, two|
+      #     one << "one"
+      #     two << "two"
+      #   end
+      #
+      #   File.read("one.txt")                 # => "one"
+      #   File.read("two.txt")                 # => "two"
+      #
+      # Note that open normally takes and passes a list (ie an Array).  If you provide
+      # a single argument, it will be translated into an Array, and passed AS AN ARRAY
+      # to the block.
+      #
+      #   FileTask.open("file.txt", "w") do |array|
+      #     array.first << "content"
+      #   end
+      #
+      #   File.read("file.txt")                # => "content"
+      def open(list, mode="rb")
+        open_files = []
+        begin
+          [list].flatten.map {|path| path.to_str }.each do |filepath| 
+            open_files << File.open(filepath, mode)
+          end
+
+          block_given? ? yield(open_files) : open_files
+        ensure
+          open_files.each {|file| file.close } if block_given?
+        end
+      end
+    end
+      
+    write_inheritable_attribute(:backup_dir, :backup)  
+    class_inheritable_accessor(:backup_dir)
+    
+    write_inheritable_attribute(:backup_timestamp, "%Y%m%d_%H%M%S")  
+    class_inheritable_accessor(:backup_timestamp)
+    
+    write_inheritable_attribute(:rollback_on_error, true)  
+    class_inheritable_accessor(:rollback_on_error)
+    
+    attr_reader :inference_block, :backed_up_files, :added_files
+    attr_accessor :dirname, :backup_dir, :backup_timestamp, :rollback_on_error
     
     def initialize(*args)
       super
@@ -24,7 +113,10 @@ module Tap
       batch.each do |task|
         task.dirname = task.default_dirname
         task.backed_up_files = {}
-        task.made_files = []
+        task.added_files = []
+        task.backup_dir = self.class.backup_dir
+        task.backup_timestamp = self.class.backup_timestamp
+        task.rollback_on_error = self.class.rollback_on_error
       end
     end
     
@@ -43,32 +135,36 @@ module Tap
     #   t = FileTask.new 
     #   t.app[:data]                        # => "/data"
     #   t.dirname                           # => "tap/file_task"
-    #   t.infer_filepath(:data, "result.txt")   # => "/data/tap/file_task/result.txt"
+    #   t.filepath(:data, "result.txt")     # => "/data/tap/file_task/result.txt"
     #
     #   t.inference do |root, dir, path|
     #     File.join(root, dir, path.chomp(".txt") + ".yml")
     #   end
     #
-    #   t.infer_filepath(:data, "result.txt")   # => "/data/tap/file_task/result.yml"
+    #   t.filepath(:data, "result.txt")     # => "/data/tap/file_task/result.yml"
     #
-    def infer_filepath(dir, *paths) # :yields: app[dir], dirname, *paths
+    def filepath(dir, *paths) 
       inference_block ? 
         inference_block.call(app[dir], dirname, *paths) : 
         app.filepath(dir, dirname, *paths)
     end
     
-    # Makes a backup filepath for input filepath by doing the following:
-    # - insert a timestamp between the basename and extname of the filepath
-    # - identify a backup path as the part of the filepath relative to 
-    #   dirname, or the basename if the filepath is not relative to dirname
-    # - infer_filepath using the backup_dir and this backup path
+    # Makes a backup filepath relative to backup_dir by translating the input
+    # filepath and inserting a timestamp formatted using backup_timestamp.
+    # The filepath used during translation will be the filepath relative
+    # to dirname (if the input filepath is relative to dirname) or just 
+    # the basename of the filepath.  
     #
     #   t = FileTask.new("dir/name", :backup_dir => :backup, :backup_timestamp => "%Y%m%d")
-    #   t.app[:backup]                                 # => "/backup"
-    #   Date.today.to_s                                # => "2007-08-08"
+    #   t.dirname                                        # => "dir/name"
+    #   t.app[:backup]                                   # => "/backup"
+    #   Date.today.to_s                                  # => "2007-08-08"
     #
-    #   t.backup_filepath("path/to/folder/file.txt")   # => "/backup/file_20070808.txt"
-    #   t.backup_filepath("dir/name/folder/file.txt")  # => "/backup/folder/file_20070808.txt"
+    #   # uses path relative to dirname, if possible
+    #   t.backup_filepath("dir/name/folder/file.txt")    # => "/backup/folder/file_20070808.txt"
+    #
+    #   # otherwise uses basename
+    #   t.backup_filepath("path/to/folder/file.txt")     # => "/backup/file_20070808.txt"
     #   
     def backup_filepath(filepath)
       extname = File.extname(filepath)
@@ -79,33 +175,18 @@ module Tap
         backup_path[(split_index + dirname.length + 1)..-1] : 
         File.basename(backup_path)
 
-      infer_filepath(backup_dir, backup_path)
-    end
-    
-    # A batch File.open method.  If a block is given, each file in the list will be 
-    # opened the open files passed to the block.  Files are automatically closed when 
-    # the block returns.  If no block is given, the open files are returned.
-    def open(list, mode="rb")
-      open_files = []
-      begin
-        fu_list(list).each do |filepath| 
-          open_files << File.open(filepath, mode)
-        end
-        
-        block_given? ? yield(open_files) : open_files
-      ensure
-        open_files.each {|file| file.close } if block_given?
-      end
+      filepath(backup_dir, backup_path)
     end
 
     # Returns true if all of the targets are up to date relative to all of the sources
-    # AND the task config_file (if it exists). Single values or arrays can be provided 
-    # for both targets and sources.  
+    # AND the task config_file, if it exists. Single values or arrays can be provided 
+    # for both targets and sources.  Used to check if any work needs to be done for
+    # a given set of sources and configurations.
     #
-    # Returns false if +force?+ is true.
+    # Returns false (ie 'not up to date') if +force?+ is true.
     def uptodate?(targets, sources=[])
       if app.options.force
-        log_filepaths(:force, *targets)
+        log_basename(:force, *targets)
         false
       else
         targets = [targets] unless targets.kind_of?(Array)
@@ -119,12 +200,31 @@ module Tap
     end
     
     # Makes a backup of each file in list to backup_filepath(file) and registers 
-    # the files so that they can be restored using restore.  If copy is true, the 
-    # files will be copied to backup_filepath, otherwise the file is moved to 
-    # backup_filepath.  Raises an error if the file is already backed up.
+    # the files so that they can be restored using restore.  If backup_using_copy 
+    # is true, the files will be copied to backup_filepath, otherwise the file is 
+    # moved to backup_filepath.  Raises an error if the file is already listed
+    # in backed_up_files.
     #
     # Returns a list of the backup_filepaths.
-    def backup(list, copy=true)
+    #
+    #   file = "file.txt"
+    #   File.open(file, "w") {|f| f << "file content"}
+    #
+    #   t = FileTask.new
+    #   backed_up_file = t.backup(file).first   
+    #       
+    #   File.exists?(file)                       # => false
+    #   File.exists?(backed_up_file)             # => true
+    #   File.read(backed_up_file)                # => "file content"
+    #
+    #   File.open(file, "w") {|f| f << "new content"}
+    #   t.restore(file)
+    #
+    #   File.exists?(file)                       # => true
+    #   File.exists?(backed_up_file)             # => false
+    #   File.read(file)                          # => "file content"
+    #
+    def backup(list, backup_using_copy=false)
       fu_list(list).collect do |filepath|
         next unless File.exists?(filepath)
         
@@ -137,7 +237,7 @@ module Tap
         dir = File.dirname(target)
         mkdir(dir)
 
-        if copy
+        if backup_using_copy
           log :cp, "#{filepath} to #{target}", Logger::DEBUG
           FileUtils.cp(filepath, target)
         else
@@ -151,13 +251,33 @@ module Tap
       end
     end
     
-    # Restores a backed-up file to its original location and removes the
-    # directory of the backup if it is empty.  Returns the restored files.
+    # Restores each file in the input list using the backup file from
+    # backed_up_files.  The backup directory is removed if it is empty.
+    #  
+    # Returns a list of the restored files.
+    #
+    #   file = "file.txt"
+    #   File.open(file, "w") {|f| f << "file content"}
+    #
+    #   t = FileTask.new
+    #   backed_up_file = t.backup(file).first   
+    #       
+    #   File.exists?(file)                       # => true
+    #   File.exists?(backed_up_file)             # => true
+    #   File.read(backed_up_file)                # => "file content"
+    #
+    #   File.open(file, "w") {|f| f << "new content"}
+    #   t.restore(file)
+    #
+    #   File.exists?(file)                       # => true
+    #   File.exists?(backed_up_file)             # => false
+    #   File.read(file)                          # => "file content"
+    #
     def restore(list)
       fu_list(list).collect do |filepath|
         filepath = File.expand_path(filepath)
-        raise "No backed up file for: #{filepath}" unless backed_up_files.has_key?(filepath)
-        
+        next unless backed_up_files.has_key?(filepath)
+
         target = backed_up_files.delete(filepath)
       
         dir = File.dirname(filepath)
@@ -170,13 +290,36 @@ module Tap
         rmdir(dir)
         
         filepath
-      end
+      end.compact
     end
     
     # Creates the directories in list if they do not exist and adds
-    # them to made_files so they can be removed using rmdir.
+    # them to added_files so they can be removed using rmdir.  Creating
+    # directories in this way causes them to be rolled back upon an
+    # execution error.
     #
     # Returns the made directories.
+    #
+    #   t = FileTask.new do |task, inputs|
+    #     File.exists?("path")                  # => false
+    #
+    #     task.mkdir("path/to/dir")             # will be rolled back
+    #     File.exists?("path/to/dir")           # => true
+    #
+    #     FileUtils.mkdir("path/to/another")    # will not be rolled back
+    #     File.exists?("path/to/another")       # => true
+    #
+    #     raise "error!"
+    #   end
+    #
+    #   begin
+    #     t.execute(nil)
+    #   rescue
+    #     $!.message                            # => "error!"
+    #     File.exists?("path/to/dir")           # => false
+    #     File.exists?("path/to/another")       # => true
+    #   end
+    #    
     def mkdir(list)
       fu_list(list).each do |dir|
         dir = File.expand_path(dir)
@@ -190,17 +333,28 @@ module Tap
         make_paths.reverse_each do |dir|
           log :mkdir, dir, Logger::DEBUG 
           FileUtils.mkdir(dir) 
-          made_files << dir 
+          added_files << dir 
         end
       end
     end
     
     # Removes each directory in the input list, provided the directory is in
-    # made_files and the directory is empty.  When checking if the directory
+    # added_files and the directory is empty.  When checking if the directory
     # is empty, rmdir checks for regular files and hidden files.  Removed
-    # directories are removed from made_files.
+    # directories are removed from added_files.
     #
     # Returns a list of the removed directories.
+    #
+    #   t = FileTask.new
+    #   File.exists?("path")                  # => false  
+    #   FileUtils.mkdir("path")               # will not be removed
+    #
+    #   t.mkdir("path/to/dir")           
+    #   File.exists?("path/to/dir")           # => true
+    #
+    #   t.rmdir("path/to/dir")                
+    #   File.exists?("path")                  # => true  
+    #   File.exists?("path/to")               # => false  
     def rmdir(list) 
       removed = []
       fu_list(list).each do |dir|  
@@ -208,7 +362,7 @@ module Tap
       
         # remove directories and parents until the
         # directory was not made by the task 
-        while made_files.include?(dir)
+        while added_files.include?(dir)
           break unless Dir.entries(dir).delete_if {|d| d == "." || d == ".."}.empty?
           
           if File.exists?(dir)
@@ -216,20 +370,45 @@ module Tap
             FileUtils.rmdir(dir) 
           end
         
-          removed << made_files.delete(dir)
+          removed << added_files.delete(dir)
           dir = File.dirname(dir)
         end
       end
       removed
     end
     
-    # Make prepares the input list of files by backing them up (if they exist),
-    # ensuring that the parent directory for the file exists, and adds each file
-    # to made_files.  As a result the files can be removed using rm, and 
-    # the original files restored using restore.  
+    # Prepares the input list of files by backing them up (if they exist),
+    # ensuring that the parent directory for the file exists, and adding
+    # each file to added_files.  As a result the files can be removed 
+    # using rm, restored using restore, and will be rolled back upon an 
+    # execution error.
+    #
+    # Returns the prepared files.
+    #
+    #   File.open("file.txt", "w") {|f| f << "original content"}
     #
-    # Returns the made files.
-    def make(list) # :yields: list
+    #   t = FileTask.new do |task, inputs|
+    #     File.exists?("path")                  # => false
+    #
+    #     # backup... make parent dirs... prepare for restore     
+    #     task.prepare(["file.txt", "path/to/file.txt"])
+    #
+    #     File.open("file.txt", "w") {|f| f << "new content"}
+    #     File.touch("path/to/file.txt")
+    #
+    #     raise "error!"
+    #   end
+    #
+    #   begin
+    #     t.execute(nil)
+    #   rescue
+    #     $!.message                            # => "error!"
+    #     File.exists?("file.txt")              # => true
+    #     File.read("file.txt")                 # => "original content"
+    #     File.exists?("path")                  # => false
+    #   end
+    #
+    def prepare(list, backup_using_copy=false) 
       list = fu_list(list)
       existing_files, non_existant_files = list.partition do |filepath| 
         File.exists?(filepath)
@@ -237,8 +416,7 @@ module Tap
       
       # backup existing files
       existing_files.each do |filepath|
-        backup(filepath, false)
-        made_files << File.expand_path(filepath)
+        backup(filepath, backup_using_copy)
       end
       
       # ensure the parent directory exists  
@@ -246,24 +424,37 @@ module Tap
       non_existant_files.each do |filepath| 
         dir = File.dirname(filepath)
         mkdir(dir)
-        made_files << File.expand_path(filepath)
       end
-   
-      yield list if block_given?
       
+      list.each do |filepath|
+        added_files << File.expand_path(filepath)
+      end
+   
       list
     end
     
-    # Removes each file in the input list, provided the file is in made_files.  
+    # Removes each file in the input list, provided the file is in added_files.  
     # The parent directory of each file is removed using rmdir. Removed files 
-    # are removed from made_files.
+    # are removed from added_files.
     #
     # Returns the removed files and directories.
+    #
+    #   t = FileTask.new
+    #   File.exists?("path")                  # => false  
+    #   FileUtils.mkdir("path")               # will not be removed
+    #
+    #   t.make("path/to/file.txt")          
+    #   FileUtils.touch("path/to/file.txt") 
+    #   File.exists?("path/to/file.txt")      # => true
+    #
+    #   t.rm("path/to/file.txt")               
+    #   File.exists?("path")                  # => true  
+    #   File.exists?("path/to")               # => false
     def rm(list) 
       removed = []
       fu_list(list).each do |filepath|  
         filepath = File.expand_path(filepath)
-        next unless made_files.include?(filepath)
+        next unless added_files.include?(filepath)
         
         # if the file exists, remove it
         if File.exists?(filepath)
@@ -271,21 +462,87 @@ module Tap
           FileUtils.rm(filepath, :force => true) 
         end
 
-        removed << made_files.delete(filepath)
+        removed << added_files.delete(filepath)
         removed.concat rmdir(File.dirname(filepath))
       end
       removed
     end
+
+    def rollback
+      added_files.dup.each do |filepath| 
+        begin
+          case
+          when File.file?(filepath)
+            rm(filepath)
+          when File.directory?(filepath)
+            rmdir(filepath)
+          else
+            # assures non-existant files are cleared from added_files
+            # this is automatically done by rm and rmdir for existing files
+            added_files.delete(filepath)
+          end
+        rescue
+          yield $!
+        end
+      end
+   
+      backed_up_files.keys.each do |filepath|
+        begin
+          restore(filepath)
+        rescue
+          yield $!
+        end
+      end
+    end
+    
+    def cleanup(pattern=nil)
+      backed_up_files.each do |filepath, target|
+        next unless pattern == nil || target =~ pattern
+        
+        # the filepath needs to be added to added_files
+        # before it can be removed by rm
+        added_files << target
+        rm(target)
+        backed_up_files.delete(filepath)
+      end 
+    end
+    
+    # Run the system command +cmd+. If multiple arguments are given the command
+    # is not run with the shell (same semantics as Kernel::exec and Kernel::system).
+    #
+    # Example:
+    #   sh %{ls -ltr}
+    #
+    #   sh 'ls', 'file with spaces'
+    #
+    #   # check exit status after command runs
+    #   sh %{grep pattern file} do |ok, res|
+    #     if ! ok
+    #       puts "pattern not found (status = #{res.exitstatus})"
+    #     end
+    #   end
+    #
+    def sh(*cmd, &block)
+      # based on sh from Rake
+      unless block_given?
+        block = lambda { |ok, status|
+          ok or raise "Command failed with status (#{status.exitstatus}): [#{cmd.join(" ")}]"
+        }
+      end
+      log :sh, cmd.join(" ")
+      res = system(*cmd)
+      block.call(res, $?)
+    end
     
     # Logs the given action, with the basenames of the input filepaths.  
-    def log_filepaths(action, *filepaths)
+    def log_basename(action, filepaths, level=Logger::INFO)
       msg = filepaths.collect {|filepath| File.basename(filepath) }.join(',')
-      log(action , msg)
+      log(action, msg, level)
     end
      
     protected
 
-    attr_writer :inference_block, :backed_up_files, :made_files
+    attr_writer :inference_block, :backed_up_files, :added_files
   
     # The default_dirname is based on the name of the task, and the
     # index of the task in batch (if the task is batched):
@@ -300,51 +557,33 @@ module Tap
       batched? ? "#{name}_#{batch_index}" : name
     end
     
-    def after_execute
-      backed_up_files.values.each do |filepath|
-        # the filepath needs to be added to made_files
-        # before it can be removed by rm
-        made_files << filepath
-        rm(filepath)
-      end if remove_backed_up_files
+    # Clears added_files and backed_up_files so that  
+    # a failure will not affect previous executions
+    def before_execute
+      added_files.clear 
+      backed_up_files.clear
     end
     
+    # Removes made files/dirs and restores backed-up files upon 
+    # an execute error.  Collects any errors raised along the way
+    # and raises them in a Tap::Support::RunError.
     def on_execute_error(original_error)
-      restore_errors = []
-      
-      if restore_on_error
-        made_files.dup.each do |filepath| 
-          begin
-            case
-            when File.file?(filepath)
-              rm(filepath)
-            when File.directory?(filepath)
-              rmdir(filepath)
-            end
-          rescue
-            restore_errors << $!
-          end
-        end
-        
-        backed_up_files.keys.each do |filepath|
-          begin
-            restore(filepath)
-          rescue
-            restore_errors << $!
-          end
-        end
+      rollback_errors = []
+      if rollback_on_error
+        rollback {|error| rollback_errors << error}
       end
 
-      # Re-raise an error unless the original error was due
-      # to task termination and no restore errors occured
-      if restore_errors.empty?
+      # Re-raise the error if no rollback errors occured,
+      # otherwise, raise a RunError tracking the restore
+      # errors.
+      if rollback_errors.empty?
         raise original_error
       else
-        raise Support::RunError.new(original_error, restore_errors)
+        raise Support::RunError.new(original_error, rollback_errors)
       end
     end
     
-    private
+    protected
     
     # Lifted from FileUtils
     def fu_list(arg)