imw 0.2.18 → 0.3.0

data/lib/imw/tools/transferer.rb

@@ -1,167 +0,0 @@
-module IMW
-  module Tools
-
-    # A class to encapsulate transferring a resource from one URI to
-    # another.
-    class Transferer
-
-      # The action this Transferer is to take.
-      #
-      # @return [:cp, :mv]
-      attr_reader :action
-
-      # Set the action of this Transferer.
-      #
-      # Will raise an error unless +the_action+ is <tt>:cp</tt> or
-      # <tt>:mv</tt>.
-      #
-      # @param [:cp, :mv] the_action
-      def action= the_action
-        @action = case the_action.to_sym
-        when :cp, :copy       then :cp
-        when :mv, :move, :mv! then :mv
-        else raise IMW::ArgumentError.new("action (#{the_action}) must be one of `cp' (or `copy') or `mv' (or `move' or `mv!'")
-        end
-      end
-
-      # The source resource.
-      #
-      # @return [IMW::Resource]
-      attr_reader :source
-
-      # Set the source for this transferer.
-      #
-      # If +the_source+ is local, will check that it exists and raise
-      # an error if not.
-      #
-      # @param [String, IMW::Resource] the_source
-      def source= the_source
-        s = IMW.open(the_source)
-        s.should_exist!("Cannot #{action_verb}") if s.is_local?
-        @source = s
-      end
-      
-      # The destination resource.
-      #
-      # @return [IMW::Resource]
-      attr_reader :destination
-
-      # Set the destination for this transferer.
-      #
-      # If +the_destination+ is local, will check that its parent
-      # directory exists and raise an error if not.
-      def destination= the_destination
-        d = IMW.open(the_destination)
-        d.dir.should_exist!("Cannot #{action_verb}") if d.is_local?
-        @destination = d
-      end
-
-      # Instantiate a new transferer to take the given +action+ on
-      # +source+ and +destination+.
-      #
-      # @param [:cp, :mv] action the action to take
-      # @param [String, IMW::Resource] source
-      # @param [String, IMW::Resource] destination      
-      def initialize action, source, destination
-        self.action      = action
-        self.source      = source
-        self.destination = destination
-        raise IMW::PathError.new("Source and destination have the same URI: #{source}") if self.source.uri.to_s == self.destination.uri.to_s
-      end
-
-      # Transfer source to destination.
-      #
-      # For local transfer, will raise errors unless the necessary
-      # paths exist.
-      def transfer!
-        IMW.announce_if_verbose("#{action_gerund.capitalize} #{source} to #{destination}")
-        send(transfer_method)
-        destination.reopen
-      end
-
-      protected
-
-      # Return the name of the method that should be used to transfer
-      # +source+ to +destination+.
-      #
-      # @return [String]
-      def transfer_method
-        source_scheme      = source.is_local?      ? 'file' : source.scheme
-        destination_scheme = destination.is_local? ? 'file' : destination.scheme
-        method = "#{source_scheme}_to_#{destination_scheme}"
-        raise IMW::NoMethodError.new("Do not know how to #{action_verb} #{source} to #{destination}") unless respond_to?(method)
-        method
-      end
-
-      def action_verb # :nodoc
-        action == :cp ? "copy" : "move"
-      end
-
-      def action_gerund # :nodoc
-        action == :cp ? "copying" : "moving"
-      end
-
-      #
-      # Purely local file
-      #
-
-      def file_to_file
-        fu_action = (action == :cp && source.is_directory?) ? :cp_r : action
-        FileUtils.send(fu_action, source.path, destination.path)
-      end
-
-      #
-      # HTTP
-      #
-
-      def http_to_file
-        File.open(destination.path, 'w') { |f| f.write(source.read) }
-      end
-
-      #
-      # S3
-      #
-
-      def file_to_s3
-        IMW::Schemes::S3.put(source, destination)
-      end
-
-      def http_to_s3
-        IMW::Schemes::S3.put(source, destination)
-      end
-
-      def s3_to_file
-        IMW::Schemes::S3.get(source, destination)
-      end
-
-      def s3_to_s3
-        IMW::Schemes::S3.copy(source, destination)
-      end
-
-      #
-      # HDFS
-      #
-
-      def hdfs_to_hdfs
-        IMW::Schemes::HDFS.fs(action, source.path, destination.path)
-      end
-
-      def file_to_hdfs
-        IMW::Schemes::HDFS.fs(:put, source.path, destination.path)
-      end
-
-      def hdfs_to_file
-        IMW::Schemes::HDFS.fs(:get, source.path, destination.path)
-      end
-
-      def s3_to_hdfs
-        IMW::Schemes::HDFS.fs(action, source.s3n_url, destination.path)
-      end
-
-      def hdfs_to_s3
-        IMW::Schemes::HDFS.fs(action, source.path, destination.s3n_url)
-      end
-
-    end
-  end
-end

data/lib/imw/utils.rb

@@ -1,74 +0,0 @@
-require 'rubygems'
-require 'fileutils'
-require 'active_support/core_ext/object/blank'
-require 'active_support/core_ext/hash/reverse_merge'
-require 'active_support/core_ext/string/starts_ends_with'
-require 'imw/utils/error'
-require 'imw/utils/log'
-require 'imw/utils/paths'
-require 'imw/utils/misc'
-
-module IMW
-
-  # Utility modules.
-  module Utils
-    autoload :DynamicallyExtendable, 'imw/utils/dynamically_extendable'
-    autoload :HasURI,                'imw/utils/has_uri'
-  end
-
-  # A replacement for the standard system call which raises an
-  # IMW::SystemCallError if the command fails which prints better
-  # debugging info.
-  #
-  # This function relies upon Kernel.system and obeys the same rules:
-  #
-  # - if +commands+ has only only a single element then no shell
-  #   characters or spaces are escaped -- you have to do it yourself
-  #   or you get to use shell characters, depending on your
-  #   perspective.
-  #
-  # - if +commands+ is a list of elements then the second and further
-  #   elements in the list have their shell characters and spaces
-  #   escaped
-  #
-  # But it also has its own rules:
-  #
-  # - When one of the +commands+ is an empty or blank string,
-  #   Kernel.system honors it and escapes it properly and sends it
-  #   along for evaluation.  This can be a problem for some programs
-  #   and so IMW.system excludes blank (as in <tt>blank?</tt>)
-  #   elements of +commands+.
-  #
-  # - +commands+ will be flattened (see the gotcha below)
-  #
-  # Calling out to the shell like this is often brittle.  Imagine
-  # defining
-  #
-  #   prog  = 'some_prog'
-  #   flags = '-v -f'
-  #   args  = 'file.txt'
-  #
-  # and later calling
-  # 
-  #   IMW.system prog, flags, args
-  #
-  # The space in the second argument ('-v -f') will be escaped and
-  # will therefore not be properly parsed by +some_prog+.  Instead try
-  #
-  #   prog  = 'some_prog'
-  #   flags = ['-v', '-f']
-  #   args = ['file.txt']
-  #   
-  #   IMW.system prog, flags, *args
-  #
-  # which will work fine since +flags+ will automatically be flattend.
-  def self.system *commands
-    stripped_commands = commands.flatten.map { |command| command.to_s unless command.blank? }.compact
-    IMW.announce_if_verbose(stripped_commands.join(" "))
-    exit_code = Kernel.system(*stripped_commands)
-    raise IMW::SystemCallError.new($?.dup, commands.join(' ')) unless $?.success?
-    exit_code
-  end
-  
-end
-

data/lib/imw/utils/dynamically_extendable.rb

@@ -1,137 +0,0 @@
-module IMW
-  module Utils
-
-    # Provides an including class with a class-level array of
-    # "handlers" that it can use to dynamically extend its instances
-    # with specific modules only if certain conditions are met.
-    #
-    # This allows different instances of a class to implement very
-    # different behavior at runtime.
-    #
-    # An example use case might be a Database class which dynamically
-    # extends its instances with an adaptor module appropriate to the
-    # particular database the object refers to.
-    module DynamicallyExtendable
-
-      def self.included obj
-        obj.extend(ClassMethods)
-      end
-      
-      # Return the modules this object has been extended by.
-      #
-      # @return [Array]
-      def modules
-        @modules ||= []
-      end
-
-      # Works just like Object#extend except it keeps track of the
-      # modules it has extended.
-      #
-      # @see IMW::Utils::DynamicallyExtendable#modules
-      def extend mod
-        modules << mod
-        super mod
-      end
-
-      # Iterate through this object's class's handlers and extend this
-      # object with the module referred to by any matching handlers.
-      def extend_appropriately! options={}
-        self.class.extend_instance! self, options
-      end
-
-      # A collection of methods which provide a class including
-      # IMW::Utils::DynamicallyExtendable with a class-level Array of
-      # handlers that can be applied to instances by calling the
-      # instance's +extend_appropriately!+ method.
-      module ClassMethods
-
-        # The handlers an including class has defined.
-        #
-        # @return [Array<Array>]
-        def handlers
-          @handlers ||= []
-        end
-        
-        # Register a new handler for an including class.
-        #
-        # +handler+ must be one of
-        #
-        # 1. Regexp 
-        # 2. Proc 
-        # 3. +true+
-        #
-        # In case (1), if the regular expression matches the
-        # instance's +to_s+ method then the module (+mod+) will be
-        # used..
-        #
-        # In case (2), if the Proc returns a value other than +false+
-        # or +nil+ after being passed an instance then the module will
-        # be used.
-        #
-        # In case (3), the module will be used.
-        #
-        # @param [String, Module] mod
-        # @param [Regexp, Proc, true] handler
-        def register_handler mod, handler
-          raise IMW::ArgumentError.new("Module must be either a Module or String")       unless mod.is_a?(Module)     || mod.is_a?(String)
-          raise IMW::ArgumentError.new("Handler must be either a Regexp, Proc, or true") unless handler.is_a?(Regexp) || handler.is_a?(Proc) || handler == true
-          handlers << [mod, handler]
-        end
-
-        # Register a collection of handlers.
-        #
-        # @see IMW::Utils::DynamicallyExtendable::ClassMethods#register_handler
-        def register_handlers *pairs
-          pairs.each { |pair| register_handler *pair }
-        end
-        
-        # Iterate through this class's handlers and extend the given
-        # object with modules whose handler conditions match the
-        # instance.
-        #
-        # Passing in <tt>:use_modules</tt> or <tt>:skip_modules</tt>
-        # allows overriding the default behavior of handlers.
-        #
-        # @param [Object] instance
-        # @param [Hash] options
-        # @option options [Array<String,Module>] use_modules a list of modules used regardless of handlers
-        # @option options [Array<String,Module>] skip_modules a list of modules not to be used regardless of handlers
-        # @return [Object] the newly extended object
-        def extend_instance! instance, options={}
-          return if options[:no_modules]
-          options.reverse_merge!(:use_modules => [], :skip_modules => [])
-          handlers.each do |mod_name, handler|
-            case handler
-            when Regexp    then extend_instance_with_mod_or_string!(instance, mod_name, options[:skip_modules]) if handler =~ instance.to_s
-            when Proc      then extend_instance_with_mod_or_string!(instance, mod_name, options[:skip_modules]) if handler.call(instance)
-            when TrueClass then extend_instance_with_mod_or_string!(instance, mod_name, options[:skip_modules])
-            else           raise IMW::TypeError("A handler must be Regexp, Proc, or true")
-            end
-          end
-          options[:use_modules].each { |mod_name| extend_instance_with_mod_or_string!(instance, mod_name, options[:skip_modules]) }
-          instance
-        end
-        
-        # Extend +instance+ with +mod_or_string+.  Will work hard to
-        # try and interpret +mod_or_string+ as a module if it's a
-        # string.
-        #
-        # @param [Object] instance
-        #
-        # @param [Module, String] mod_or_string the module or string
-        # representing a module to extend the instance with
-        #
-        # @param [Array<Module,String>] skip_modules modules to exclude
-        def extend_instance_with_mod_or_string! instance, mod_or_string, skip_modules
-          return if skip_modules.include?(mod_or_string)
-          if mod_or_string.is_a?(Module)
-            instance.extend(mod_or_string)
-          else
-            m = IMW.class_eval(mod_or_string)
-            instance.extend(m) unless skip_modules.include?(m)
-          end
-        end    
-      end
-    end
-  end
-end

data/lib/imw/utils/error.rb

@@ -1,59 +0,0 @@
-module IMW
-
-  # Base error class which all IMW errors subclass.
-  Error = Class.new(StandardError)
-
-  # Method undefined.
-  NoMethodError = Class.new(Error)
-
-  # Type error.
-  TypeError = Class.new(Error)
-
-  # Not implemented (typically because user needs to define a method
-  # when subclassing a base class).
-  NotImplementedError = Class.new(Error)
-
-  # Error during parsing.
-  ParseError = Class.new(Error)
-
-  # Error with a non-existing, invalid, or inaccessible path.
-  PathError = Class.new(Error)
-
-  # Error communicating with a remote entity.
-  NetworkError = Class.new(Error)
-
-  # Raised when a resource is of the wrong scheme.
-  SchemeError = Class.new(Error)
-
-  # Raised when a resource is of the wrong (or malformed) format.
-  FormatError = Class.new(Error)
-
-  # Bad argument.
-  ArgumentError = Class.new(Error)
-
-  # Error in defining or matching a schema.
-  SchemaError = Class.new(Error)
-
-  # An error meant to be used when a system call goes awry.  It will
-  # report exit status and the process id of the offending call.
-  class SystemCallError < IMW::Error
-
-    attr_reader :status, :message
-
-    def initialize(status, message)
-      @status  = status
-      @message = message
-    end
-
-    def display
-      "(error code: #{status.exitstatus}, pid: #{status.pid}) #{message}"
-    end
-
-    def to_s
-      "(error code: #{status.exitstatus}, pid: #{status.pid}) #{message}"
-    end
-
-  end
-
-
-end

data/lib/imw/utils/extensions/hpricot.rb

@@ -1,34 +0,0 @@
-require 'hpricot'
-
-module Hpricot::IMWExtensions
-
-  # Return the contents of the first element to match +path+.
-  def contents_of path
-    cnts = self.at path
-    cnts.inner_html if cnts
-  end
-
-  # Return the value of +attr+ for the first element to match +path+.
-  def path_attr path, attr
-    cnts = self.at path
-    cnts.attributes[attr] if cnts
-  end
-
-  # Return the value of the +class+ attribute of the first element to
-  # match +path+.
-  def class_of path
-    self.path_attr(path, 'class')
-  end
-end
-
-class Hpricot::Elem
-  include Hpricot::IMWExtensions
-end
-
-class Hpricot::Elements
-  include Hpricot::IMWExtensions
-end
-
-class Hpricot::Doc
-  include Hpricot::IMWExtensions
-end