imw 0.1.1 → 0.2.0

data/lib/imw/resources/schemes/hdfs.rb

@@ -0,0 +1,242 @@
+module IMW
+  module Resources
+    module Schemes
+
+      # Defines methods for reading and writing data to/from an
+      # HDFS[http://hadoop.apache.org/common/docs/current/hdfs_design.html]]
+      #
+      # Learn more about Hadoop[http://hadoop.apache.org] and the
+      # {Hadoop Distributed
+      # Filesystem}[http://hadoop.apache.org/common/docs/current/hdfs_design.html].
+      module HDFS
+
+        # Checks to see if this is a file or directory
+        def self.extended obj
+          obj.extend(obj.is_directory? ? HDFSDirectory : HDFSFile)
+        end
+
+        # Is this resource an HDFS resource?
+        #
+        # @return [true, false]
+        def on_hdfs?
+          true
+        end
+        alias_method :is_hdfs?, :on_hdfs?
+
+        # Copy this resource to the +new_uri+.
+        #
+        # @param [String, IMW::Resource] new_uri
+        # @return [IMW::Resource] the new resource
+        def cp new_uri
+          IMW::Transforms::Transferer.new(:cp, self, new_uri).transfer!
+        end
+
+        # Move this resource to the +new_uri+.
+        #
+        # @param [String, IMW::Resource] new_uri
+        # @return [IMW::Resource] the new resource
+        def mv new_uri
+          IMW::Transforms::Transferer.new(:mv, self, new_uri).transfer!
+        end
+
+        # Delete this resource from the HDFS.
+        #
+        # @option options [true,false] :skip_trash
+        def rm options={}
+          should_exist!("Cannot delete.")
+          args = [:rm]
+          args << '-skipTrash' if options[:skip] || options[:skip_trash] || options[:skipTrash]
+          args << path
+          HDFS.fs(*args)
+          self
+        end
+        alias_method :rm!, :rm
+        
+        
+        # Does this path exist on the HDFS?
+        #
+        # @return [true, false]
+        def exist?
+          return @exist unless @exist.nil?
+          refresh!
+          @exist
+        end
+        alias_method :exists?, :exist?
+
+
+        # Return the size (in bytes) of this resource on the HDFS.
+        #
+        # This value is cached.  Call +refresh+ to refresh the cache
+        # manually.
+        #
+        # @return [Fixnum]
+        def size
+          return @size unless @size.nil?
+          refresh!
+          should_exist!("Cannot report size")
+          @size
+        end
+
+        # Return the number of directories contained at or below this
+        # path on the HDFS.
+        #
+        # This value is cached.  Call +refresh+ to refresh the cache
+        # manually.
+        #
+        # @return [Fixnum]
+        def num_dirs
+          return @num_dirs unless @num_dirs.nil?
+          refresh!
+          should_exist!("Cannot report number of directories.")
+          @num_dirs
+        end
+
+        # Return the number of files contained at or below this path
+        # on the HDFS.
+        #
+        # This value is cached.  Call +refresh+ to refresh the cache
+        # manually.
+        #
+        # @return [Fixnum]
+        def num_files
+          return @num_files unless @num_files.nil?
+          refresh!
+          should_exist!("Cannot report number of files.")
+          @num_files
+        end
+
+        # Is this resource an HDFS directory?
+        #
+        # @return [true, false]
+        def is_directory?
+          exist? && num_dirs > 0
+        end
+
+        # Refresh the cached file properties.
+        #
+        # @return [IMW::Resource] this resource
+        def refresh!
+          response = HDFS.fs(:count, path)
+          if response.blank? || response =~ /^Can not find listing for/
+            @exist = false
+            @num_dirs, @num_files, @size, @hdfs_path = false, false, false, false
+          else
+            @exist = true
+            parts = response.split
+            @num_dirs, @num_files, @size = parts[0..2].map(&:to_i)
+            @hdfs_path = parts.last
+          end
+          self
+        end
+
+        # Execute +command+ with +args+ on the Hadoop Distributed
+        # Filesystem (HDFS).
+        #
+        # If passed a block, yield each line of the output from the
+        # command, else just return the output.
+        #
+        # Try running `hadoop fs -help' for more information.
+        #
+        # @param [String, Symbol] command the command to run.
+        # @param [String, Symbol] args the arguments to pass the command
+        # @yield [String] each line of the command's output
+        # @return [String] the command's output
+        def self.fs command, *args
+          command_string = "#{executable} fs -#{command} #{args.compact.map(&:to_str).join(' ')}"
+          command_string += " 2>&1" if command == :count # FIXME or else it just spams the screen when we do HDFS#refresh!
+          output = `#{command_string}`.chomp
+          if block_given?
+            output.split("\n").each do |line|
+              yield line
+            end
+          else
+            output
+          end
+        end
+
+        protected
+        # Returns the path to the Hadoop executable.
+        #
+        # @return [String]
+        def self.executable
+          @executable ||= begin
+                            string = `which hadoop`.chomp
+                            raise IMW::Error.new("Could not find hadoop command.  Is Hadoop installed?") if string.blank?
+                            string
+                          end
+        end
+      end
+
+      # Defines methods for reading data from HDFS files.
+      module HDFSFile
+
+        # Return the contents of this HDFS file as a string.
+        #
+        # Be VERY careful how you use this!
+        #
+        # @return [String]
+        def read
+          HDFS.fs(:cat, path)
+        end
+
+        # Iterate through each line of this HDFS resource.
+        #
+        # @yield [String] each line of the file
+        def each &block
+          HDFS.fs(:cat, path, &block)
+        end
+
+        # Return a handle on a StringIO object representing the
+        # content in this HDFS file.
+        #
+        # Be VERY careful how you use this!  It is a StringIO object
+        # so the whole HDFS file is read into a string before
+        # returning the handle.
+        #
+        # @return [StringIO]
+        def io
+          @io ||= StringIO.new(read)
+        end
+
+        # Map over the lines of this HDFS resource.
+        #
+        # @yield [String] each line of the file
+        # @return [Array] the result of the block on each line
+        def map &block
+          returning([]) do |output|
+            HDFS.fs(:cat, path) do |line|
+              output << block.call(line)
+            end
+          end
+        end
+
+      end
+
+      # Defines methods for listing contents of HDFS directories.
+      module HDFSDirectory
+
+        # Return the paths of all files and directories directly below
+        # this directory on the HDFS.
+        #
+        # @return [Array<String>]
+        def contents
+          returning([]) do |paths|
+            HDFS.fs(:ls, path) do |line|
+              next if line =~ /^Found.*items$/
+              paths << line.split.last
+            end
+          end
+        end
+
+        # Return the resources directly below this directory on the
+        # HDFS.
+        #
+        # @return [Array<IMW::Resource>]
+        def resources
+          contents.map { |path| IMW.open(path) }
+        end
+        
+      end
+    end
+  end
+end

data/lib/imw/resources/schemes/http.rb

@@ -0,0 +1,161 @@
+module IMW
+  module Resources
+    module Schemes
+
+      # Defines methods for accessing a resource over HTTP.  Uses
+      # RestClient to implement the basic HTTP verbs (GET, POST, PUT,
+      # DELETE, HEAD).
+      module HTTP
+
+        # Is this resource being accessed via HTTP?
+        #
+        # @return [true, false]
+        def via_http?
+          true
+        end
+
+        # Copy this resource to the +new_uri+.
+        #
+        # @param [String, IMW::Resource] new_uri
+        # @return [IMW::Resource] the new resource
+        def cp new_uri
+          IMW::Transforms::Transferer.new(:cp, self, new_uri).transfer!
+        end
+        
+
+        # Return the basename of the URI or <tt>_index</tt> if it's
+        # blank, as in the case of <tt>http://www.google.com</tt>.
+        #
+        # @return [String]
+        def effective_basename
+          (basename.blank? || basename =~ %r{^/*$}) ? "_index" : basename
+        end
+
+        # Send a GET request to this resource's URI.
+        #
+        # If the response doesn't have HTTP code 2xx, a RestClient
+        # error will be raised.
+        #
+        # If a block is given then the response will be passed to the
+        # block, even in case of a non-2xx code.
+        #
+        # See the documentation for
+        # RestClient[http://rdoc.info/projects/archiloque/rest-client]
+        # for more information.
+        #
+        # @param [Hash] headers the headers to include in the request
+        # @yield [RestClient::Response] the response from the server
+        # @return [RestClient::Response] the response from the server
+        # @raise [RestClient::NotModified, RestClient::Unauthorized, RestClient::ResourceNotFound, RestClient::RequestFailed] error from RestClient on non-2xx response codes
+        def get headers={}, &block
+          make_restclient_request do
+            RestClient.get(uri.to_s, headers, &block)
+          end
+        end
+
+        # Send a POST request to this resource's URI with data
+        # +payload+.
+        #
+        # If the response doesn't have HTTP code 2xx, a RestClient
+        # error will be raised.
+        #
+        # If a block is given then the response will be passed to the
+        # block, even in case of a non-2xx code.
+        #
+        # See the documentation for
+        # RestClient[http://rdoc.info/projects/archiloque/rest-client]
+        # for more information.
+        #
+        # @param [Hash, String] payload the data to send
+        # @param [Hash] headers the headers to include in the request
+        # @yield [RestClient::Response] the response from the server
+        # @return [RestClient::Response] the response from the server
+        # @raise [RestClient::NotModified, RestClient::Unauthorized, RestClient::ResourceNotFound, RestClient::RequestFailed] error from RestClient on non-2xx response codes
+        def post payload, headers={}, &block
+          make_restclient_request do
+            RestClient.post(uri.to_s, payload, headers, &block)
+          end
+        end
+
+        # Send a PUT request to this resource's URI with data
+        # +payload+.
+        #
+        # If the response doesn't have HTTP code 2xx, a RestClient
+        # error will be raised.
+        #
+        # If a block is given then the response will be passed to the
+        # block, even in case of a non-2xx code.
+        #
+        # See the documentation for
+        # RestClient[http://rdoc.info/projects/archiloque/rest-client]
+        # for more information.
+        #
+        # @param [Hash, String] payload the data to send
+        # @param [Hash] headers the headers to include in the request
+        # @yield [RestClient::Response] the response from the server
+        # @return [RestClient::Response] the response from the server
+        # @raise [RestClient::NotModified, RestClient::Unauthorized, RestClient::ResourceNotFound, RestClient::RequestFailed] error from RestClient on non-2xx response codes
+        def put payload, headers={}, &block
+          make_restclient_request do
+            RestClient.put(uri.to_s, payload, headers, &block)
+          end
+        end
+        
+        # Send a DELETE request to this resource's URI.
+        #
+        # If the response doesn't have HTTP code 2xx, a RestClient
+        # error will be raised.
+        #
+        # If a block is given then the response will be passed to the
+        # block, even in case of a non-2xx code.
+        #
+        # See the documentation for
+        # RestClient[http://rdoc.info/projects/archiloque/rest-client]
+        # for more information.
+        #
+        # @param [Hash] headers the headers to include in the request
+        # @yield [RestClient::Response] the response from the server
+        # @return [RestClient::Response] the response from the server
+        # @raise [RestClient::NotModified, RestClient::Unauthorized, RestClient::ResourceNotFound, RestClient::RequestFailed] error from RestClient on non-2xx response codes
+        def delete headers={}, &block
+          make_restclient_request do
+            RestClient.delete(uri.to_s, headers, &block)
+          end
+        end
+        
+        # Send a HEAD request to this resource's URI.
+        #
+        # If the response doesn't have HTTP code 2xx, a RestClient
+        # error will be raised.
+        #
+        # If a block is given then the response will be passed to the
+        # block, even in case of a non-2xx code.
+        #
+        # See the documentation for
+        # RestClient[http://rdoc.info/projects/archiloque/rest-client]
+        # for more information.
+        #
+        # @param [Hash] headers the headers to include in the request
+        # @yield [RestClient::Response] the response from the server
+        # @return [RestClient::Response] the response from the server
+        # @raise [RestClient::NotModified, RestClient::Unauthorized, RestClient::ResourceNotFound, RestClient::RequestFailed] error from RestClient on non-2xx response codes
+        def head headers={}, &block
+          make_restclient_request do
+            RestClient.head(uri.to_s, headers, &block)
+          end
+        end
+
+        protected
+        def make_restclient_request &block # :nodoc
+          require 'restclient'
+          begin
+            yield
+          rescue RestClient::NotModified, RestClient::Unauthorized, RestClient::ResourceNotFound, RestClient::RequestFailed => e
+            raise IMW::NetworkError.new("#{e.class} -- #{e.message}")
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/imw/resources/schemes/s3.rb

@@ -0,0 +1,137 @@
+module IMW
+  module Resources
+    module Schemes
+
+      # Defines methods for reading and writing data to {Amazon
+      # S3}[http://aws.amazon.com/s3] buckets.
+      #
+      #   IMW.open('s3://my_bucket/path/to/some/file.csv')
+      #
+      # Learn more about {Amazon Web Services}[http://aws.amazon.com].
+      module S3
+
+        # For an S3 resource, the bucket is just the hostname.
+        #
+        # @return [String]
+        def bucket
+          host
+        end
+
+        # Is this resource an S3 resource?
+        #
+        # @return [true, false] 
+        def on_s3?
+          true
+        end
+        alias_method :is_s3?, :on_s3?
+
+        # Copy this resource to the +new_uri+.
+        #
+        # @param [String, IMW::Resource] new_uri
+        # @return [IMW::Resource] the new resource
+        def cp new_uri
+          IMW::Transforms::Transferer.new(:cp, self, new_uri).transfer!
+        end
+
+        # The AWS::S3::S3Object corresponding to this resource.
+        def s3_object
+          self.class.make_connection!
+          @s3_object ||= AWS::S3::S3Object.new(path, bucket)
+        end
+
+        # Does this resource exist on S3?
+        #
+        # @return [true, false]
+        def exist?
+          s3_object.exists?
+        end
+        alias_method :exists?, :exist?
+
+        # Remove this resource from S3.
+        #
+        # @return [IMW::Resource] the deleted object
+        def rm
+          s3_object.delete
+        end
+        alias_method :rm!, :rm
+
+        # Return the S3N URL for this S3 object
+        #
+        #   resource = IMW.open('s3://my_bucket/path/to/some/obj')
+        #   resource.s3n_url
+        #   => 's3n://my_bucket/path/to/some/obj'
+        #
+        # @return [String]
+        def s3n_url
+          uri.to_s.gsub(/^s3:/, 's3n:')
+        end
+
+        # Return the contents of this S3 object.
+        #
+        # @return [String]
+        def read
+          s3_object.value
+        end
+
+        # Store +source+ into +destination+.
+        # 
+        # @param [String, IMW::Resource, #io] source
+        # @param [String, IMW::Resource, #path, #bucket] destination
+        # @return [IMW::Resource] the new S3 object
+        def self.put source, destination
+          source       = IMW.open(source)
+          destintation = IMW.open(destination)
+          raise IMW::ArgumentError.new("destination must be on S3 -- #{destination.uri} given") unless destination.on_s3?
+          make_connection!
+          AWS::S3::S3Object.store(destination.path, source.io, destination.bucket)
+          destination
+        end
+
+        # Download +source+ from S3 into +destination+.
+        #
+        # @param [String, IMW::Resource, #path, #bucket] source
+        # @param [String, IMW::Resource, #write] destination
+        # @return [IMW::Resource] the new resource
+        def self.get source, destination
+          source      = IMW.open(source)
+          destination = IMW.open(destination)
+          make_connection!
+          AWS::S3::Object.stream(source.path, source.bucket) do |chunk|
+            destination.write(chunk)
+          end
+          destination.close
+          destination.reopen
+        end
+
+        # Copy S3 resource +source+ to +destination+.
+        #
+        # @param [String, IMW::Resource, #path, #bucket] source
+        # @param [String, IMW::Resource, #path, #bucket] destination
+        # @return [IMW::Resource] the new resource
+        def self.copy source, destination
+          source      = IMW.open(source)
+          destination = IMW.open(destination)
+          raise IMW::PathError.new("Bucket names must be non-blank and match to 'copy'") unless source.bucket.present? && destination.bucket.present? && source.bucket == destination.bucket
+          make_connection!
+          AWS::S3::Object.copy(source.path, destination.path, destination.bucket)
+          destination
+        end
+
+        protected
+        # Make an S3 connection.
+        #
+        # Uses settings defined in IMW::AWS_CREDENTIALS.
+        #
+        # @return [AWS
+        def self.make_connection!
+          return @connection if @connection
+          raise IMW::Error.new("Must define a constant IMW::AWS_CREDENTIALS with an :access_key_id and a :secret_access_key before using S3 resources") unless defined?(IMW::AWS_CREDENTIALS)
+          require 'aws/s3'
+          @connection = AWS::S3::Base.establish_connection!(IMW::AWS_CREDENTIALS)
+        end
+        
+      end
+    end
+  end
+end
+

data/lib/imw/resources/schemes.rb

@@ -0,0 +1,19 @@
+module IMW
+  module Resources
+    module Schemes
+      autoload :S3,    'imw/resources/schemes/s3'
+      autoload :HTTP,  'imw/resources/schemes/http'
+      autoload :HTTPS, 'imw/resources/schemes/http'
+      autoload :HDFS,  'imw/resources/schemes/hdfs'
+
+      # Handlers which extend a resource with scheme specific methods.
+      SCHEME_HANDLERS = [
+                         ["Schemes::S3",    %r{^s3://}    ],
+                         ["Schemes::HTTP",  %r{^http://}  ],
+                         ["Schemes::HTTPS", %r{^https://} ],
+                         ["Schemes::HDFS",  %r{^hdfs://}  ]
+                        ]
+    end
+  end
+end
+

data/lib/imw/resources.rb

@@ -0,0 +1,118 @@
+require 'imw/resources/formats'
+require 'imw/resources/schemes'
+require 'imw/resources/archives_and_compressed'
+
+module IMW
+
+  # IMW::Resources is a namespace in which all the modules which
+  # define different kinds of behavior for IMW::Resource objects are
+  # defined.
+  #
+  # When an IMW::Resource is instantiated it eventually calls
+  # IMW::Resources#extend_resource! which will iterate through the
+  # handlers in IMW::Resources#handlers, extending the resource with
+  # modules whose handler conditions are satisfied.
+  #
+  # A handler is just an Array with two elements.  The first should be
+  # a module or a string identifying a module.  
+  #
+  # If the second element is a Regexp, the corresponding module will
+  # be used if the regexp matches the resource's URI (as a string)
+  #
+  # If the second element is a Proc, it will be called with the
+  # resource as its only argument and if it returns true then the
+  # module will be used.
+  #
+  # You can define your own handlers by appending them to
+  # IMW::Resources::USER_DEFINED_HANDLERS in your <tt>.imwrc</tt>
+  # file.
+  module Resources
+
+    autoload :LocalObj,        'imw/resources/local'
+    autoload :RemoteObj,       'imw/resources/remote'
+    autoload :StringObj,       'imw/resources/string'
+    autoload :Transferable,    'imw/resources/transferable'
+
+    # Iterate through IMW::Resources#handlers and extend the given
+    # +resource+ with modules whose handler conditions match the
+    # resource.
+    #
+    # @param [IMW::Resource] resource the resource to extend
+    # @return [IMW::Resource] the extended resource
+    def self.extend_resource! resource
+      handlers.each do |mod_name, handler|
+        case handler
+        when Regexp    then extend_resource_with_mod_or_string!(resource, mod_name) if handler =~ resource.uri.to_s
+        when Proc      then extend_resource_with_mod_or_string!(resource, mod_name) if handler.call(resource)
+        when TrueClass then extend_resource_with_mod_or_string!(resource, mod_name)
+        else
+          raise IMW::TypeError("A handler must be Regexp, Proc, or true")
+        end
+      end
+      resource
+    end
+
+    # Basic handlers to determine whether the resource is local,
+    # remote, or a string.
+    BASIC_HANDLERS = [
+                      ["LocalObj",  Proc.new { |resource| resource.scheme == 'file' || resource.scheme.blank?   } ],
+                      ["RemoteObj", Proc.new { |resource| resource.scheme != 'file' && resource.scheme.present? } ],
+                      ["StringObj", Proc.new { |resource| resource.is_stringio?                                 } ]
+                     ]
+
+    # Define this constant in your configuration file to add your own
+    # handlers.
+    USER_DEFINED_HANDLERS = [] unless defined?(USER_DEFINED_HANDLERS)
+
+    # include handlers from other modules
+    include IMW::Resources::Formats
+    include IMW::Resources::Schemes
+    
+    # A list of handlers to try.  Define your own handlers in
+    # IMW::Resources::USER_DEFINED_HANDLERS.
+    #
+    # @return [Array]
+    def self.handlers
+      # order here is important
+      BASIC_HANDLERS + SCHEME_HANDLERS + ARCHIVE_AND_COMPRESSED_HANDLERS + FORMAT_HANDLERS + USER_DEFINED_HANDLERS
+    end
+
+    protected
+
+    # Extend +resource+ with +mod_or_string+.  Will work hard to try
+    # and interpret +mod_or_string+ as a module if it's a string.
+    #
+    # @param [IMW::Resource] resource the resource to extend
+    #
+    # @param [Module, String] mod_or_string the module or string
+    # representing a module to extend the resource with
+    def self.extend_resource_with_mod_or_string! resource, mod_or_string
+      if mod_or_string.is_a?(Module)
+        resource.extend(mod_or_string)
+      else
+        # Given a string "Mod::SubMod::SubSubMod" first split it into
+        # its parts ["Mod", "SubMod", "SubSubMod"] and then begin
+        # class_eval'ing them in order so that each is class_eval'd in
+        # the scope of the one before it.
+        #
+        # There is almost certainly a better way to do this.
+        mod_names = mod_or_string.to_s.split('::')
+        mods = []
+        mod_names.each_with_index do |name, index|
+          if index == 0
+            mods << class_eval(name)
+          else
+            begin
+              mods << class_eval(name)
+            rescue NameError
+              mods << mods[index - 1].class_eval(name)
+            end
+          end
+        end
+        resource.extend(mods.last)
+      end
+    end
+  end
+end
+
+