imw 0.2.0 → 0.2.1

data/lib/imw/resource.rb

@@ -1,32 +1,35 @@
 require 'addressable/uri'
-require 'imw/resources'
 
 module IMW
 
+  # Define this constant in your configuration file to add your own
+  # URI handlers to IMW.
+  USER_DEFINED_HANDLERS = [] unless defined?(USER_DEFINED_HANDLERS)
+
   # A resource can be anything addressable via a URI.  Examples
   # include local files, remote files, webpages, &c.
   #
   # The IMW::Resource class takes a URI as input and then dynamically
-  # extends itself with appropriate modules from IMW::Resources.  As
-  # an example, calling
+  # extends itself with appropriate modules from IMW.  As an example,
+  # calling
   #
   #   my_archive = IMW::Resource.new('/path/to/my/archive.tar.bz2')
   #
   # would return an IMW::Resource extended by
-  # IMW::Resources::Archives::Tarbz2 (among other modules) which
+  # IMW::Archives::Tarbz2 (among other modules) which
   # therefore has methods for extracting, listing, and appending to
   # the archive.
   #
   # Modules are so extended based on handlers defined in the
   # <tt>imw/resources</tt> directory and accessible via
-  # IMW::Resources#handlers.  You can define your own handlers by
-  # defining the constant IMW::Resources::USER_DEFINED_HANDLERS in
-  # your configuration file.
+  # IMW::Resource.handlers.  You can define your own handlers by
+  # defining the constant IMW::Resource::USER_DEFINED_HANDLERS in your
+  # configuration file.
   #
   # The modules extending a particular IMW::Resource instance can be
   # listed as follows
   #
-  #   my_archive.resource_modules #=> [IMW::Resources::LocalObj, IMW::Resources::LocalFile, IMW::Resources::Compressible, IMW::Resources::Archives::Tarbz2]
+  #   my_archive.resource_modules #=> [IMW::Local::Base, IMW::Local::File, IMW::Local::Compressible, IMW::Archives::Tarbz2]
   #
   # By default, resources are opened for reading.  Passing in the
   # appropriate <tt>:mode</tt> option changes this:
@@ -41,6 +44,9 @@ module IMW
   #
   # Read the documentation for modules in IMW::Resources to learn more
   # about the various behaviors an IMW::Resource can acquire.
+  #
+  # You can also instantiate an IMW::Resource using IMW.open, which
+  # accepts all the same arguments as IMW::Resource.new.
   class Resource
 
     attr_reader :uri, :mode
@@ -66,9 +72,9 @@ module IMW
     end
 
     # Extend this resource with modules by passing it through a
-    # collection of handlers defined by IMW::Resources#handlers
+    # collection of handlers defined by IMW::Resource.handlers.
     def extend_appropriately!
-      IMW::Resources.extend_resource!(self)
+      self.class.extend_resource!(self)
     end
 
     # Set the URI of this resource by parsing the given +uri+ (if
@@ -186,5 +192,97 @@ module IMW
         raise IMW::NoMethodError, "undefined method `#{method}' for #{self}, extended by #{resource_modules.join(', ')}"
       end
     end
+
+    # Iterate through IMW::Resource.handlers and extend the given
+    # +resource+ with modules whose handler conditions match the
+    # resource.
+    #
+    # @param [IMW::Resource] resource the resource to extend
+    # @return [IMW::Resource] the extended resource
+    def self.extend_resource! resource
+      handlers.each do |mod_name, handler|
+        case handler
+        when Regexp    then extend_resource_with_mod_or_string!(resource, mod_name) if handler =~ resource.uri.to_s
+        when Proc      then extend_resource_with_mod_or_string!(resource, mod_name) if handler.call(resource)
+        when TrueClass then extend_resource_with_mod_or_string!(resource, mod_name)
+        else
+          raise IMW::TypeError("A handler must be Regexp, Proc, or true")
+        end
+      end
+      resource
+    end
+    
+    # A list of handlers to match against each new resource.
+    # 
+    # When an IMW::Resource is instantiated it eventually calls
+    # IMW::Resource.extend_resource! which will iterate through the
+    # handlers in IMW::Resource.handlers, extending the resource with
+    # modules whose handler conditions are satisfied.
+    #
+    # A handler is just an Array with two elements.  The first should be
+    # a module or a string identifying a module.  
+    #
+    # If the second element is a Regexp, the corresponding module will
+    # be used if the regexp matches the resource's URI (as a string)
+    #
+    # If the second element is a Proc, it will be called with the
+    # resource as its only argument and if it returns true then the
+    # module will be used.
+    #
+    # You can define your own handlers by appending them to
+    # IMW::Resource::USER_DEFINED_HANDLERS in your <tt>.imwrc</tt>
+    # file.
+    #
+    # The order in which handlers appear is significant --
+    # IMW::CompressedFiles::HANDLERS must be _before_
+    # IMW::Archives::HANDLERS, for example, because of (say)
+    # <tt>.tar.bz2</tt> files.
+    # 
+    # @return [Array]
+    def self.handlers
+      # order is important!
+      #
+      # 
+      #
+      #CompressedFiles must come before
+      # Archives because of tar.bz2 type files
+      IMW::Schemes::HANDLERS + IMW::CompressedFiles::HANDLERS + IMW::Archives::HANDLERS + IMW::Formats::HANDLERS + USER_DEFINED_HANDLERS
+    end
+
+    protected
+    # Extend +resource+ with +mod_or_string+.  Will work hard to try
+    # and interpret +mod_or_string+ as a module if it's a string.
+    #
+    # @param [IMW::Resource] resource the resource to extend
+    #
+    # @param [Module, String] mod_or_string the module or string
+    # representing a module to extend the resource with
+    def self.extend_resource_with_mod_or_string! resource, mod_or_string
+      if mod_or_string.is_a?(Module)
+        resource.extend(mod_or_string)
+      else
+        # Given a string "Mod::SubMod::SubSubMod" first split it into
+        # its parts ["Mod", "SubMod", "SubSubMod"] and then begin
+        # class_eval'ing them in order so that each is class_eval'd in
+        # the scope of the one before it.
+        #
+        # There is almost certainly a better way to do this.
+        # mod_names = mod_or_string.to_s.split('::')
+        # mods = []
+        # mod_names.each_with_index do |name, index|
+        #   if index == 0
+        #     mods << IMW.class_eval(name)
+        #   else
+        #     begin
+        #       mods << class_eval(name)
+        #     rescue NameError
+        #       mods << mods[index - 1].class_eval(name)
+        #     end
+        #   end
+        # end
+        # resource.extend(mods.last)
+        resource.extend(IMW.class_eval(mod_or_string))
+      end
+    end    
   end
 end

data/lib/imw/schemes.rb

@@ -0,0 +1,21 @@
+module IMW
+  module Schemes
+    autoload :Local,  'imw/schemes/local'
+    autoload :Remote, 'imw/schemes/remote'
+    autoload :S3,     'imw/schemes/s3'
+    autoload :HTTP,   'imw/schemes/http'
+    autoload :HTTPS,  'imw/schemes/http'
+    autoload :HDFS,   'imw/schemes/hdfs'
+
+    HANDLERS = [
+                ["Schemes::Local::Base",  Proc.new { |resource| resource.scheme == 'file' || resource.scheme.blank?    } ],
+                ["Schemes::Remote::Base", Proc.new { |resource| resource.scheme != 'file' && resource.scheme.present?  } ],
+                ["Schemes::S3",    %r{^s3://}    ],
+                ["Schemes::HTTP",  %r{^http://}  ],
+                ["Schemes::HTTPS", %r{^https://} ],
+                ["Schemes::HDFS",  %r{^hdfs://}  ]
+               ]
+  end
+end
+
+

data/lib/imw/schemes/hdfs.rb

@@ -0,0 +1,240 @@
+module IMW
+  module Schemes
+
+    # Defines methods for reading and writing data to/from an
+    # HDFS[http://hadoop.apache.org/common/docs/current/hdfs_design.html]]
+    #
+    # Learn more about Hadoop[http://hadoop.apache.org] and the
+    # {Hadoop Distributed
+    # Filesystem}[http://hadoop.apache.org/common/docs/current/hdfs_design.html].
+    module HDFS
+
+      # Checks to see if this is a file or directory
+      def self.extended obj
+        obj.extend(obj.is_directory? ? HDFSDirectory : HDFSFile)
+      end
+
+      # Is this resource an HDFS resource?
+      #
+      # @return [true, false]
+      def on_hdfs?
+        true
+      end
+      alias_method :is_hdfs?, :on_hdfs?
+
+      # Copy this resource to the +new_uri+.
+      #
+      # @param [String, IMW::Resource] new_uri
+      # @return [IMW::Resource] the new resource
+      def cp new_uri
+        IMW::Transforms::Transferer.new(:cp, self, new_uri).transfer!
+      end
+
+      # Move this resource to the +new_uri+.
+      #
+      # @param [String, IMW::Resource] new_uri
+      # @return [IMW::Resource] the new resource
+      def mv new_uri
+        IMW::Transforms::Transferer.new(:mv, self, new_uri).transfer!
+      end
+
+      # Delete this resource from the HDFS.
+      #
+      # @option options [true,false] :skip_trash
+      def rm options={}
+        should_exist!("Cannot delete.")
+        args = [:rm]
+        args << '-skipTrash' if options[:skip] || options[:skip_trash] || options[:skipTrash]
+        args << path
+        HDFS.fs(*args)
+        self
+      end
+      alias_method :rm!, :rm
+      
+      
+      # Does this path exist on the HDFS?
+      #
+      # @return [true, false]
+      def exist?
+        return @exist unless @exist.nil?
+        refresh!
+        @exist
+      end
+      alias_method :exists?, :exist?
+
+
+      # Return the size (in bytes) of this resource on the HDFS.
+      #
+      # This value is cached.  Call +refresh+ to refresh the cache
+      # manually.
+      #
+      # @return [Fixnum]
+      def size
+        return @size unless @size.nil?
+        refresh!
+        should_exist!("Cannot report size")
+        @size
+      end
+
+      # Return the number of directories contained at or below this
+      # path on the HDFS.
+      #
+      # This value is cached.  Call +refresh+ to refresh the cache
+      # manually.
+      #
+      # @return [Fixnum]
+      def num_dirs
+        return @num_dirs unless @num_dirs.nil?
+        refresh!
+        should_exist!("Cannot report number of directories.")
+        @num_dirs
+      end
+
+      # Return the number of files contained at or below this path
+      # on the HDFS.
+      #
+      # This value is cached.  Call +refresh+ to refresh the cache
+      # manually.
+      #
+      # @return [Fixnum]
+      def num_files
+        return @num_files unless @num_files.nil?
+        refresh!
+        should_exist!("Cannot report number of files.")
+        @num_files
+      end
+
+      # Is this resource an HDFS directory?
+      #
+      # @return [true, false]
+      def is_directory?
+        exist? && num_dirs > 0
+      end
+
+      # Refresh the cached file properties.
+      #
+      # @return [IMW::Resource] this resource
+      def refresh!
+        response = HDFS.fs(:count, path)
+        if response.blank? || response =~ /^Can not find listing for/
+          @exist = false
+          @num_dirs, @num_files, @size, @hdfs_path = false, false, false, false
+        else
+          @exist = true
+          parts = response.split
+          @num_dirs, @num_files, @size = parts[0..2].map(&:to_i)
+          @hdfs_path = parts.last
+        end
+        self
+      end
+
+      # Execute +command+ with +args+ on the Hadoop Distributed
+      # Filesystem (HDFS).
+      #
+      # If passed a block, yield each line of the output from the
+      # command, else just return the output.
+      #
+      # Try running `hadoop fs -help' for more information.
+      #
+      # @param [String, Symbol] command the command to run.
+      # @param [String, Symbol] args the arguments to pass the command
+      # @yield [String] each line of the command's output
+      # @return [String] the command's output
+      def self.fs command, *args
+        command_string = "#{executable} fs -#{command} #{args.compact.map(&:to_str).join(' ')}"
+        command_string += " 2>&1" if command == :count # FIXME or else it just spams the screen when we do HDFS#refresh!
+        output = `#{command_string}`.chomp
+        if block_given?
+          output.split("\n").each do |line|
+            yield line
+          end
+        else
+          output
+        end
+      end
+
+      protected
+      # Returns the path to the Hadoop executable.
+      #
+      # @return [String]
+      def self.executable
+        @executable ||= begin
+                          string = `which hadoop`.chomp
+                          raise IMW::Error.new("Could not find hadoop command.  Is Hadoop installed?") if string.blank?
+                          string
+                        end
+      end
+    end
+
+    # Defines methods for reading data from HDFS files.
+    module HDFSFile
+
+      # Return the contents of this HDFS file as a string.
+      #
+      # Be VERY careful how you use this!
+      #
+      # @return [String]
+      def read
+        HDFS.fs(:cat, path)
+      end
+
+      # Iterate through each line of this HDFS resource.
+      #
+      # @yield [String] each line of the file
+      def each &block
+        HDFS.fs(:cat, path, &block)
+      end
+
+      # Return a handle on a StringIO object representing the
+      # content in this HDFS file.
+      #
+      # Be VERY careful how you use this!  It is a StringIO object
+      # so the whole HDFS file is read into a string before
+      # returning the handle.
+      #
+      # @return [StringIO]
+      def io
+        @io ||= StringIO.new(read)
+      end
+
+      # Map over the lines of this HDFS resource.
+      #
+      # @yield [String] each line of the file
+      # @return [Array] the result of the block on each line
+      def map &block
+        returning([]) do |output|
+          HDFS.fs(:cat, path) do |line|
+            output << block.call(line)
+          end
+        end
+      end
+
+    end
+
+    # Defines methods for listing contents of HDFS directories.
+    module HDFSDirectory
+
+      # Return the paths of all files and directories directly below
+      # this directory on the HDFS.
+      #
+      # @return [Array<String>]
+      def contents
+        returning([]) do |paths|
+          HDFS.fs(:ls, path) do |line|
+            next if line =~ /^Found.*items$/
+            paths << line.split.last
+          end
+        end
+      end
+
+      # Return the resources directly below this directory on the
+      # HDFS.
+      #
+      # @return [Array<IMW::Resource>]
+      def resources
+        contents.map { |path| IMW.open(path) }
+      end
+      
+    end
+  end
+end

data/lib/imw/schemes/http.rb

@@ -0,0 +1,166 @@
+module IMW
+  module Schemes
+
+    # Defines methods for accessing a resource over HTTP.  Uses
+    # RestClient to implement the basic HTTP verbs (GET, POST, PUT,
+    # DELETE, HEAD).
+    module HTTP
+
+      # Many websites have HTML content without an <tt>.html</tt>
+      # extension so automatically extend +obj+ with
+      # IMW::Resources::Formats::HTML in this case.
+      def self.extended obj
+        obj.extend(IMW::Formats::Html) if obj.extension.blank?
+      end
+
+      # Is this resource being accessed via HTTP?
+      #
+      # @return [true, false]
+      def via_http?
+        true
+      end
+
+      # Copy this resource to the +new_uri+.
+      #
+      # @param [String, IMW::Resource] new_uri
+      # @return [IMW::Resource] the new resource
+      def cp new_uri
+        IMW::Tools::Transferer.new(:cp, self, new_uri).transfer!
+      end
+      
+
+      # Return the basename of the URI or <tt>_index</tt> if it's
+      # blank, as in the case of <tt>http://www.google.com</tt>.
+      #
+      # @return [String]
+      def effective_basename
+        (basename.blank? || basename =~ %r{^/*$}) ? "_index" : basename
+      end
+
+      # Send a GET request to this resource's URI.
+      #
+      # If the response doesn't have HTTP code 2xx, a RestClient
+      # error will be raised.
+      #
+      # If a block is given then the response will be passed to the
+      # block, even in case of a non-2xx code.
+      #
+      # See the documentation for
+      # RestClient[http://rdoc.info/projects/archiloque/rest-client]
+      # for more information.
+      #
+      # @param [Hash] headers the headers to include in the request
+      # @yield [RestClient::Response] the response from the server
+      # @return [RestClient::Response] the response from the server
+      # @raise [RestClient::NotModified, RestClient::Unauthorized, RestClient::ResourceNotFound, RestClient::RequestFailed] error from RestClient on non-2xx response codes
+      def get headers={}, &block
+        make_restclient_request do
+          RestClient.get(uri.to_s, headers, &block)
+        end
+      end
+
+      # Send a POST request to this resource's URI with data
+      # +payload+.
+      #
+      # If the response doesn't have HTTP code 2xx, a RestClient
+      # error will be raised.
+      #
+      # If a block is given then the response will be passed to the
+      # block, even in case of a non-2xx code.
+      #
+      # See the documentation for
+      # RestClient[http://rdoc.info/projects/archiloque/rest-client]
+      # for more information.
+      #
+      # @param [Hash, String] payload the data to send
+      # @param [Hash] headers the headers to include in the request
+      # @yield [RestClient::Response] the response from the server
+      # @return [RestClient::Response] the response from the server
+      # @raise [RestClient::NotModified, RestClient::Unauthorized, RestClient::ResourceNotFound, RestClient::RequestFailed] error from RestClient on non-2xx response codes
+      def post payload, headers={}, &block
+        make_restclient_request do
+          RestClient.post(uri.to_s, payload, headers, &block)
+        end
+      end
+
+      # Send a PUT request to this resource's URI with data
+      # +payload+.
+      #
+      # If the response doesn't have HTTP code 2xx, a RestClient
+      # error will be raised.
+      #
+      # If a block is given then the response will be passed to the
+      # block, even in case of a non-2xx code.
+      #
+      # See the documentation for
+      # RestClient[http://rdoc.info/projects/archiloque/rest-client]
+      # for more information.
+      #
+      # @param [Hash, String] payload the data to send
+      # @param [Hash] headers the headers to include in the request
+      # @yield [RestClient::Response] the response from the server
+      # @return [RestClient::Response] the response from the server
+      # @raise [RestClient::NotModified, RestClient::Unauthorized, RestClient::ResourceNotFound, RestClient::RequestFailed] error from RestClient on non-2xx response codes
+      def put payload, headers={}, &block
+        make_restclient_request do
+          RestClient.put(uri.to_s, payload, headers, &block)
+        end
+      end
+      
+      # Send a DELETE request to this resource's URI.
+      #
+      # If the response doesn't have HTTP code 2xx, a RestClient
+      # error will be raised.
+      #
+      # If a block is given then the response will be passed to the
+      # block, even in case of a non-2xx code.
+      #
+      # See the documentation for
+      # RestClient[http://rdoc.info/projects/archiloque/rest-client]
+      # for more information.
+      #
+      # @param [Hash] headers the headers to include in the request
+      # @yield [RestClient::Response] the response from the server
+      # @return [RestClient::Response] the response from the server
+      # @raise [RestClient::NotModified, RestClient::Unauthorized, RestClient::ResourceNotFound, RestClient::RequestFailed] error from RestClient on non-2xx response codes
+      def delete headers={}, &block
+        make_restclient_request do
+          RestClient.delete(uri.to_s, headers, &block)
+        end
+      end
+      
+      # Send a HEAD request to this resource's URI.
+      #
+      # If the response doesn't have HTTP code 2xx, a RestClient
+      # error will be raised.
+      #
+      # If a block is given then the response will be passed to the
+      # block, even in case of a non-2xx code.
+      #
+      # See the documentation for
+      # RestClient[http://rdoc.info/projects/archiloque/rest-client]
+      # for more information.
+      #
+      # @param [Hash] headers the headers to include in the request
+      # @yield [RestClient::Response] the response from the server
+      # @return [RestClient::Response] the response from the server
+      # @raise [RestClient::NotModified, RestClient::Unauthorized, RestClient::ResourceNotFound, RestClient::RequestFailed] error from RestClient on non-2xx response codes
+      def head headers={}, &block
+        make_restclient_request do
+          RestClient.head(uri.to_s, headers, &block)
+        end
+      end
+
+      protected
+      def make_restclient_request &block # :nodoc
+        require 'restclient'
+        begin
+          yield
+        rescue RestClient::NotModified, RestClient::Unauthorized, RestClient::ResourceNotFound, RestClient::RequestFailed => e
+          raise IMW::NetworkError.new("#{e.class} -- #{e.message}")
+        end
+      end
+    end
+  end
+end
+