imw 0.1.1 → 0.2.0

data/lib/imw/runner.rb

@@ -18,7 +18,7 @@ module IMW
     def initialize *args
       @args    = args
       @options = DEFAULT_OPTIONS.dup
-      parser.parse!(args)       # will trim options from args
+      parser.parse!(args)
     end
 
     def parser
@@ -67,11 +67,12 @@ EOF
     end
 
     def handles
+      require 'set'
       matched_handles = Set.new
       if options[:selectors].blank?
-        matched_handles += IMW::REPOSITORY.keys
+        matched_handles += IMW.repository.keys
       else
-        keys = IMW::REPOSITORY.keys
+        keys = IMW.repository.keys
         unless keys.empty?
           options[:selectors].each do |selector|
             matched_handles += keys.find_all { |key| key =~ Regexp.new(selector) }
@@ -82,7 +83,7 @@ EOF
     end
 
     def datasets
-      handles.map { |handle| IMW::REPOSITORY[handle] }
+      handles.map { |handle| IMW.repository[handle] }
     end
 
     def list!

data/lib/imw/transforms/archiver.rb

@@ -0,0 +1,215 @@
+require 'imw/resource'
+
+module IMW
+  module Transforms
+    
+    # Packages an Array of input files into a single output archive.
+    # When the archive is extracted, all the input files given will be
+    # in a single directory with a chosen name.  The path to the output
+    # archive determines both the name of the archive and its type (tar,
+    # tar.bz2, zip, &c.).
+    # 
+    # If any of the input files are themselves archives, they will first
+    # be extracted, with only their contents winding up in the final
+    # directory (the file hierarchy of the archive will be preserved).
+    # If any of the input files are compressed, they will first be
+    # uncompressed before being added to the directory.
+    #
+    # Both local and remote files can be archived. An exmaple:
+    # 
+    #   archiver = IMW::Transforms::Archiver.new 'my_archive', '/path/to/my/regular_file.tsv', '/path/to/an/archive.tar.bz2', '/path/to/my_compressed_file.gz', 'http://mywebsite.com/index.html'
+    #   archiver.package! '/path/to/my_archive.zip'
+    #
+    # This will create a ZIP archive at
+    # <tt>/path/to/my_archive.zip</tt>.  When the ZIP archive is
+    # extracted its contents will look like
+    #
+    #   my_archive
+    #   |-- regular_file.tsv
+    #   |-- archive_file1
+    #   |-- archive_dir
+    #   |   |-- archive_file2
+    #   |   `-- archive_file3
+    #   |-- archive_file3
+    #   |-- my_compressed_file
+    #   `-- index.html
+    #
+    # Notice that
+    #
+    # - the name of the extracted directory is given by the first
+    #   argument to the Archiver when it was instantiated.
+    #
+    # - all files wind up in the top-level of this extracted directory
+    #   when possible (<tt>regular_file.tsv</tt>, <tt>index.html</tt>)
+    #
+    # - /path/to/archive.tar.bz2 was not directly included, but its
+    #   contents (<tt>archive_file1</tt>,
+    #   <tt>archive_dir/archive_file2</tt>,
+    #   <tt>archive_dir/archive_file3</tt>) were included instead.
+    #
+    # - /path/to/my_compressed_file.gz was first uncompressed before
+    #   being added to the archive.
+    #
+    # - the remote file <tt>http://mywebsite.com/index.html</tt> was
+    #   downloaded and included
+    #
+    # This process can take a while when the constituent files are
+    # large because there is quite a lot of preparation done to the
+    # files to make this nice output structure in the final archive.
+    # Further calls to <tt>package!</tt> on the same instance of
+    # Archiver will skip the preparation step (the intermediate
+    # results of which are sitting in IMW's temporary directory) and
+    # directly create the package, saving time when attempting to
+    # create multiple package formats from the same input data.
+    class Archiver
+
+      attr_accessor :name, :local_inputs, :remote_inputs
+
+      def initialize name, raw_inputs
+        @name   = name
+        self.inputs = raw_inputs
+      end
+
+      # Set the inputs for this archiver.
+      #
+      # @param [String, IMW::Resource] new_inputs the inputs to archive, local or remote
+      def inputs= new_inputs
+        @local_inputs, @remote_inputs = [], []
+        new_inputs.each do |obj|
+          input = obj.is_a?(IMW::Resource) ? obj : IMW.open(obj)  # take either paths/URIs or IMW::Resource objects
+          if input.is_local?
+            @local_inputs << (input.directory? ? input.resources : input)  # recurse through directories
+          else
+            @remote_inputs << input
+          end
+        end
+        @local_inputs.flatten!
+      end
+
+      # Return a list of error messages for this archiver.
+      #
+      # @return [Array] the error messages
+      def errors
+        @errors ||= []      
+      end
+
+      # Was this archiver successful (did it not have any errors)?
+      #
+      # @return [true, false]
+      def success?
+        errors.empty?
+      end
+
+      # A temporary directory to work in.  Its contents will
+      # ultimately consist of a directory named for the package
+      # containing all the input files.
+      #
+      # @return [String]
+      def tmp_dir
+        @tmp_dir ||= File.join(IMW.path_to(:tmp_root, 'packager'), (Time.now.to_i.to_s + "-" + $$.to_s)) # guaranteed unique on a node
+      end
+
+      # A directory which will contain all the content being packaged,
+      # including the contents of any archives that were included in
+      # the list of files to process.
+      #
+      # @return [String]
+      def dir
+        @dir ||= File.join(tmp_dir, name.to_s)
+      end
+
+      # Remove the +tmp_dir+ entirely, getting rid of all temporary
+      # files.
+      def clean!
+        FileUtils.rm_rf(tmp_dir)
+      end
+
+      # Copy, decompress, or extract the input paths to the temporary
+      # directory, readying them for packaging.
+      def prepare!
+        FileUtils.mkdir_p dir unless File.exist?(dir)
+        
+        local_inputs.each do |existing_file|
+          new_path      = File.join(dir, existing_file.basename)
+          case
+          when existing_file.is_archive?
+            FileUtils.cd(dir) do
+              existing_file.extract
+            end
+          when existing_file.is_compressed?
+            existing_file.cp(new_path).decompress!
+          else
+            existing_file.cp(new_path)
+          end
+        end
+        
+        remote_inputs.each do |remote_input|
+          remote_input.cp(File.join(dir, remote_input.effective_basename))
+        end
+      end        
+      
+      # Checks to see if all expected files exist in the temporary
+      # directory for this packager.
+      #
+      # @return [true, false]
+      def prepared?
+        local_inputs.each do |existing_file|
+          case
+          when existing_file.is_archive?
+            existing_file.contents.each do |archived_file_path|
+              return false unless File.exist?(File.join(dir, archived_file_path))
+            end
+          when existing_file.is_compressed?
+            return false unless File.exist?(File.join(dir, existing_file.decompressed_basename))
+          else
+            return false unless File.exist?(File.join(dir, existing_file.basename))
+          end
+        end
+
+        remote_inputs.each do |remote_input|
+          return false unless File.exist?(File.join(dir, remote_input.effective_basename))
+        end
+        
+        true
+      end
+      
+      # Package the contents of the temporary directory to an archive
+      # at +output+ but return exceptions instead of raising them.
+      #
+      # @param [String, IMW::Resource] output the path to the output package
+      # @param [Hash] options
+      # @return [RuntimeError, IMW::Resource] either the completed package or the error which was raised
+      def package output, options={}
+        begin
+          package! output, options={}
+        rescue => e
+          return e
+        end
+      end
+
+      # Package the contents of the temporary directory to an archive
+      # at +output+.  The extension of +output+ determines the kind of
+      # archive.
+      #
+      # @param [String, IMW::Resource] output the path to the output package
+      # @param [Hash] options
+      # @return [IMW::Resource] the completed package
+      def package! output, options={}
+        prepare!                          unless prepared?
+        output = IMW.open(output)
+        FileUtils.mkdir_p(output.dirname) unless File.exist?(output.dirname)        
+        output.rm!                        if output.exist?
+        FileUtils.cd(tmp_dir) { IMW.open(output.basename).create(*Dir["#{name}/**/*"]).mv(output.path) }
+        add_processing_error "Archiver: couldn't create archive #{output.path}" unless output.exists?
+        output
+      end
+
+      protected
+      def add_processing_error error # :nodoc:
+        IMW.logger.warn error      
+        errors << error
+      end
+      
+    end
+  end
+end

data/lib/imw/transforms/transferer.rb

@@ -0,0 +1,103 @@
+module IMW
+  module Transforms
+    class Transferer
+
+      attr_accessor :action, :source, :destination
+      
+      def initialize action, source, destination
+        @action      = normalize_action(action)
+        @source      = IMW.open(source)
+        @destination = IMW.open(destination)
+        raise IMW::PathError.new("Source and destination have the same URI: #{@source.uri}") if @source.uri.to_s == @destination.uri.to_s
+      end
+
+      def transfer!
+        if source.is_local? 
+          source.should_exist!("Cannot copy") # don't bother checking for remote resources
+          source_scheme = 'file'              # make sure it isn't blank
+        else
+          source_scheme = source.scheme
+        end
+        destination_scheme = destination.is_local? ? 'file' : destination.scheme
+        method             = "#{source_scheme}_to_#{destination_scheme}"
+        if respond_to?(method)
+          send(method)
+        else
+          raise IMW::NoMethodError.new("Do not know how to #{action} #{source.uri} => #{destination.uri} (#{source_scheme.inspect} => #{destination_scheme.inspect})")
+        end
+        destination.reopen
+      end
+
+      protected
+
+      def normalize_action action # :nodoc:
+        case action.to_sym
+        when :cp, :copy then :cp
+        when :mv, :move, :mv! then :mv
+        else raise IMW::ArgumentError.new("action (#{action}) must be one of `cp' (or `copy') or `mv' (or `move' or `mv!'")
+        end
+      end
+
+      #
+      # Purely local file
+      #
+
+      def file_to_file
+        FileUtils.send(action, source.path, destination.path)
+      end
+
+      #
+      # HTTP
+      #
+
+      def http_to_file
+        File.open(destination.path, 'w') { |f| f.write(source.read) }
+      end
+
+      #
+      # S3
+      #
+
+      def file_to_s3
+        IMW::Resources::Schemes::S3.put(source, destination)
+      end
+
+      def http_to_s3
+        IMW::Resources::Schemes::S3.put(source, destination)
+      end
+
+      def s3_to_file
+        IMW::Resources::Schemes::S3.get(source, destination)
+      end
+
+      def s3_to_s3
+        IMW::Resources::Schemes::S3.copy(source, destination)
+      end
+
+      #
+      # HDFS
+      #
+
+      def hdfs_to_hdfs
+        IMW::Resources::Schemes::HDFS.fs(action, source.path, destination.path)
+      end
+
+      def file_to_hdfs
+        IMW::Resources::Schemes::HDFS.fs(:put, source.path, destination.path)
+      end
+
+      def hdfs_to_file
+        IMW::Resources::Schemes::HDFS.fs(:get, source.path, destination.path)
+      end
+
+      def s3_to_hdfs
+        IMW::Resources::Schemes::HDFS.fs(action, source.s3n_url, destination.path)
+      end
+
+      def hdfs_to_s3
+        IMW::Resources::Schemes::HDFS.fs(action, source.path, destination.s3n_url)
+      end
+
+    end
+  end
+end

data/lib/imw/transforms.rb

@@ -0,0 +1,8 @@
+module IMW
+  module Transforms
+    autoload :Archiver,   'imw/transforms/archiver'
+    autoload :Transferer, 'imw/transforms/transferer'
+  end
+end
+
+

data/lib/imw/utils/error.rb

@@ -1,54 +1,50 @@
-#
-# h2. lib/imw/utils/error -- errors
-#
-# == About
-#
-# Error objects for IMW.
-#
-# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
-# Copyright:: Copyright (c) 2008 infochimps.org
-# License::   GPL 3.0
-# Website::   http://infinitemonkeywrench.org/
-#
-
 module IMW
 
-  # A generic error class.
-  class Error < StandardError
-  end
+  # Base error class which all IMW errors subclass.
+  Error = Class.new(StandardError)
 
-  class TypeError < TypeError
-  end
+  # Method undefined.
+  NoMethodError = Class.new(Error)
 
-  class ArgumentError < ArgumentError
-  end
+  # Type error.
+  TypeError = Class.new(Error)
 
-  class NotImplementedError < NotImplementedError
-  end
+  # Not implemented (typically because user needs to define a method
+  # when subclassing a base class).
+  NotImplementedError = Class.new(Error)
 
-  class ParseError < Error
-  end
+  # Error during parsing.
+  ParseError = Class.new(Error)
+
+  # Error with a non-existing, invalid, or inaccessible path.
+  PathError = Class.new(Error)
+
+  # Error communicating with a remote entity.
+  NetworkError = Class.new(Error)
+
+  # Error communicating with a remote entity.
+  ArgumentError = Class.new(Error)
 
   # An error meant to be used when a system call goes awry.  It will
   # report exit status and the process id of the offending call.
   class SystemCallError < IMW::Error
 
-    def initialize(message)
+    attr_reader :status, :message
+
+    def initialize(status, message)
+      @status  = status
       @message = message
     end
 
     def display
-      "(error code: #{$?.exitstatus}, pid: #{$?.pid}) #{@message}"
+      "(error code: #{status.exitstatus}, pid: #{status.pid}) #{message}"
     end
 
     def to_s
-      "(error code: #{$?.exitstatus}, pid: #{$?.pid}) #{@message}"
+      "(error code: #{status.exitstatus}, pid: #{status.pid}) #{message}"
     end
 
   end
 
-  # A error for improperly specified, inappropriate, or broken paths.
-  class PathError < IMW::Error
-  end
 
 end

data/lib/imw/utils/extensions/array.rb

@@ -1,15 +1,3 @@
-#
-# h2. lib/imw/utils/extensions/array.rb -- array extensions
-#
-# == About
-#
-# Extensions to the +Array+ class.
-#
-# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
-# Copyright:: Copyright (c) 2008 infochimps.org
-# License::   GPL 3.0
-# Website::   http://infinitemonkeywrench.org/
-#
 require 'active_support/core_ext/array/extract_options'
 class Array #:nodoc:
   include ActiveSupport::CoreExtensions::Array::ExtractOptions
@@ -118,8 +106,10 @@ class Array
     terminals.map! {|terminal| yield terminal } if block
     terminals
   end
-  
 
-end
+  # Dump the data in this array to the resource at the given +uri+.
+  def dump uri
+    IMW.open(uri, :mode => 'w').dump(self)
+  end
 
-# puts "#{File.basename(__FILE__)}: I have a loooong list of complaints.  Firstly, ..." # at bottom
+end

data/lib/imw/utils/extensions/hash.rb

@@ -1,16 +1,3 @@
-#
-# h2. lib/imw/utils/extensions/hash.rb -- hash extensions
-#
-# == About
-#
-# Extensions to the built-in +Hash+ class.
-#
-# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
-# Copyright:: Copyright (c) 2008 infochimps.org
-# License::   GPL 3.0
-# Website::   http://infinitemonkeywrench.org/
-#
-
 require 'active_support/core_ext/hash/reverse_merge'
 
 class Hash
@@ -169,6 +156,7 @@ class Hash
   #   search(options.slice(:mass, :velocity, :time))
   # Returns a new hash with only the given keys.
   def slice(*keys)
+    require 'set'
     allowed = Set.new(respond_to?(:convert_key) ? keys.map { |key| convert_key(key) } : keys)
     reject { |key,| !allowed.include?(key) }
   end
@@ -212,7 +200,9 @@ class Hash
     terminals.map! {|terminal| yield terminal } if block
     terminals
   end
-  
-end
 
-# puts "#{File.basename(__FILE__)}: To each improvement there corresponds another, yes?" # at bottom
+  # Dump the data from this Hash into the given +uri+.
+  def dump uri
+    IMW.open(uri).dump(self)
+  end
+end

data/lib/imw/utils/extensions/hpricot.rb

@@ -1,17 +1,3 @@
-#
-# h2. lib/imw/utils/extensions/hpricot.rb -- extensions to hpricot
-#
-# == About
-#
-# Some IMW extensions for Why's Hpricot library.
-#
-# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
-# Copyright:: Copyright (c) 2008 infochimps.org
-# License::   GPL 3.0
-# Website::   http://infinitemonkeywrench.org/
-#
-# puts "#{File.basename(__FILE__)}: Something clever" # at bottom
-
 require 'hpricot'
 
 module Hpricot::IMWExtensions

data/lib/imw/utils/extensions/string.rb

@@ -1,16 +1,3 @@
-#
-# h2. lib/imw/utils/extensions/string.rb -- string extensions
-#
-# == About
-#
-# Implements some useful extensions to the +String+ class.
-#
-# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
-# Copyright:: Copyright (c) 2008 infochimps.org
-# License::   GPL 3.0
-# Website::   http://infinitemonkeywrench.org/
-#
-
 class String
 
   # Does the string end with the specified +suffix+ (stolen from
@@ -44,6 +31,9 @@ class String
     self.downcase.underscore.to_sym
   end
 
-end
+  # Dump this string into the given +uri+.
+  def dump uri
+    IMW.open(uri).dump(self)
+  end
 
-# puts "#{File.basename(__FILE__)}: You tie a long string to your Monkeywrench, place it on the ground, and hide around the corner with the string in your hand, waiting for passersby to try and make a grab for it." # at bottom
+end

data/lib/imw/utils/extensions/symbol.rb

@@ -1,14 +1,3 @@
-#
-# h2. lib/imw/utils/extensions/symbol.rb -- extensions to symbol class
-#
-# == About
-#
-# Author::    (Philip flip Kromer, Dhruv Bansal) for Infinite Monkeywrench Project (mailto:coders@infochimps.org)
-# Copyright:: Copyright (c) 2008 infochimps.org
-# License::   GPL 3.0
-# Website::   http://infinitemonkeywrench.org/
-#
-
 class Symbol
 
   # Turn the symbol into a simple proc (stolen from
@@ -24,5 +13,3 @@ class Symbol
   end
 
 end
-
-# puts "#{File.basename(__FILE__)}: You whisper a word of power and smile as the the Ruby Palace thunders with the sound of falling blocks." # at bottom

data/lib/imw/utils/extensions.rb

@@ -0,0 +1,65 @@
+require 'imw/utils/extensions/string'
+require 'imw/utils/extensions/array'
+require 'imw/utils/extensions/hash'
+require 'imw/utils/extensions/struct'
+require 'imw/utils/extensions/symbol'
+
+require 'active_support/core_ext/object/blank'
+require 'active_support/core_ext/object/misc'
+
+
+module IMW
+  # A replacement for the standard system call which raises an
+  # IMW::SystemCallError if the command fails which prints better
+  # debugging info.
+  #
+  # This function relies upon Kernel.system and obeys the same rules:
+  #
+  # - if +commands+ has only only a single element then no shell
+  #   characters or spaces are escaped -- you have to do it yourself
+  #   or you get to use shell characters, depending on your
+  #   perspective.
+  #
+  # - if +commands+ is a list of elements then the second and further
+  #   elements in the list have their shell characters and spaces
+  #   escaped
+  #
+  # But it also has its own rules:
+  #
+  # - When one of the +commands+ is an empty or blank string,
+  #   Kernel.system honors it and escapes it properly and sends it
+  #   along for evaluation.  This can be a problem for some programs
+  #   and so IMW.system excludes blank (as in <tt>blank?</tt>)
+  #   elements of +commands+.
+  #
+  # - +commands+ will be flattened (see the gotcha below)
+  #
+  # Calling out to the shell like this is often brittle.  Imagine
+  # defining
+  #
+  #   prog  = 'some_prog'
+  #   flags = '-v -f'
+  #   args  = 'file.txt'
+  #
+  # and later calling
+  # 
+  #   IMW.system prog, flags, args
+  #
+  # The space in the second argument ('-v -f') will be escaped and
+  # will therefore not be properly parsed by +some_prog+.  Instead try
+  #
+  #   prog  = 'some_prog'
+  #   flags = ['-v', '-f']
+  #   args = ['file.txt']
+  #   
+  #   IMW.system prog, flags, *args
+  #
+  # which will work fine since +flags+ will automatically be flattend.
+  def self.system *commands
+    stripped_commands = commands.flatten.map { |command| command.to_s unless command.blank? }.compact
+    Kernel.system(*stripped_commands)
+    raise IMW::SystemCallError.new($?.dup, commands.join(' ')) unless $?.success?
+  end
+end
+
+