imw 0.2.0 → 0.2.1

data/lib/imw/resources/schemes.rb

@@ -1,19 +0,0 @@
-module IMW
-  module Resources
-    module Schemes
-      autoload :S3,    'imw/resources/schemes/s3'
-      autoload :HTTP,  'imw/resources/schemes/http'
-      autoload :HTTPS, 'imw/resources/schemes/http'
-      autoload :HDFS,  'imw/resources/schemes/hdfs'
-
-      # Handlers which extend a resource with scheme specific methods.
-      SCHEME_HANDLERS = [
-                         ["Schemes::S3",    %r{^s3://}    ],
-                         ["Schemes::HTTP",  %r{^http://}  ],
-                         ["Schemes::HTTPS", %r{^https://} ],
-                         ["Schemes::HDFS",  %r{^hdfs://}  ]
-                        ]
-    end
-  end
-end
-

data/lib/imw/resources/schemes/hdfs.rb

@@ -1,242 +0,0 @@
-module IMW
-  module Resources
-    module Schemes
-
-      # Defines methods for reading and writing data to/from an
-      # HDFS[http://hadoop.apache.org/common/docs/current/hdfs_design.html]]
-      #
-      # Learn more about Hadoop[http://hadoop.apache.org] and the
-      # {Hadoop Distributed
-      # Filesystem}[http://hadoop.apache.org/common/docs/current/hdfs_design.html].
-      module HDFS
-
-        # Checks to see if this is a file or directory
-        def self.extended obj
-          obj.extend(obj.is_directory? ? HDFSDirectory : HDFSFile)
-        end
-
-        # Is this resource an HDFS resource?
-        #
-        # @return [true, false]
-        def on_hdfs?
-          true
-        end
-        alias_method :is_hdfs?, :on_hdfs?
-
-        # Copy this resource to the +new_uri+.
-        #
-        # @param [String, IMW::Resource] new_uri
-        # @return [IMW::Resource] the new resource
-        def cp new_uri
-          IMW::Transforms::Transferer.new(:cp, self, new_uri).transfer!
-        end
-
-        # Move this resource to the +new_uri+.
-        #
-        # @param [String, IMW::Resource] new_uri
-        # @return [IMW::Resource] the new resource
-        def mv new_uri
-          IMW::Transforms::Transferer.new(:mv, self, new_uri).transfer!
-        end
-
-        # Delete this resource from the HDFS.
-        #
-        # @option options [true,false] :skip_trash
-        def rm options={}
-          should_exist!("Cannot delete.")
-          args = [:rm]
-          args << '-skipTrash' if options[:skip] || options[:skip_trash] || options[:skipTrash]
-          args << path
-          HDFS.fs(*args)
-          self
-        end
-        alias_method :rm!, :rm
-        
-        
-        # Does this path exist on the HDFS?
-        #
-        # @return [true, false]
-        def exist?
-          return @exist unless @exist.nil?
-          refresh!
-          @exist
-        end
-        alias_method :exists?, :exist?
-
-
-        # Return the size (in bytes) of this resource on the HDFS.
-        #
-        # This value is cached.  Call +refresh+ to refresh the cache
-        # manually.
-        #
-        # @return [Fixnum]
-        def size
-          return @size unless @size.nil?
-          refresh!
-          should_exist!("Cannot report size")
-          @size
-        end
-
-        # Return the number of directories contained at or below this
-        # path on the HDFS.
-        #
-        # This value is cached.  Call +refresh+ to refresh the cache
-        # manually.
-        #
-        # @return [Fixnum]
-        def num_dirs
-          return @num_dirs unless @num_dirs.nil?
-          refresh!
-          should_exist!("Cannot report number of directories.")
-          @num_dirs
-        end
-
-        # Return the number of files contained at or below this path
-        # on the HDFS.
-        #
-        # This value is cached.  Call +refresh+ to refresh the cache
-        # manually.
-        #
-        # @return [Fixnum]
-        def num_files
-          return @num_files unless @num_files.nil?
-          refresh!
-          should_exist!("Cannot report number of files.")
-          @num_files
-        end
-
-        # Is this resource an HDFS directory?
-        #
-        # @return [true, false]
-        def is_directory?
-          exist? && num_dirs > 0
-        end
-
-        # Refresh the cached file properties.
-        #
-        # @return [IMW::Resource] this resource
-        def refresh!
-          response = HDFS.fs(:count, path)
-          if response.blank? || response =~ /^Can not find listing for/
-            @exist = false
-            @num_dirs, @num_files, @size, @hdfs_path = false, false, false, false
-          else
-            @exist = true
-            parts = response.split
-            @num_dirs, @num_files, @size = parts[0..2].map(&:to_i)
-            @hdfs_path = parts.last
-          end
-          self
-        end
-
-        # Execute +command+ with +args+ on the Hadoop Distributed
-        # Filesystem (HDFS).
-        #
-        # If passed a block, yield each line of the output from the
-        # command, else just return the output.
-        #
-        # Try running `hadoop fs -help' for more information.
-        #
-        # @param [String, Symbol] command the command to run.
-        # @param [String, Symbol] args the arguments to pass the command
-        # @yield [String] each line of the command's output
-        # @return [String] the command's output
-        def self.fs command, *args
-          command_string = "#{executable} fs -#{command} #{args.compact.map(&:to_str).join(' ')}"
-          command_string += " 2>&1" if command == :count # FIXME or else it just spams the screen when we do HDFS#refresh!
-          output = `#{command_string}`.chomp
-          if block_given?
-            output.split("\n").each do |line|
-              yield line
-            end
-          else
-            output
-          end
-        end
-
-        protected
-        # Returns the path to the Hadoop executable.
-        #
-        # @return [String]
-        def self.executable
-          @executable ||= begin
-                            string = `which hadoop`.chomp
-                            raise IMW::Error.new("Could not find hadoop command.  Is Hadoop installed?") if string.blank?
-                            string
-                          end
-        end
-      end
-
-      # Defines methods for reading data from HDFS files.
-      module HDFSFile
-
-        # Return the contents of this HDFS file as a string.
-        #
-        # Be VERY careful how you use this!
-        #
-        # @return [String]
-        def read
-          HDFS.fs(:cat, path)
-        end
-
-        # Iterate through each line of this HDFS resource.
-        #
-        # @yield [String] each line of the file
-        def each &block
-          HDFS.fs(:cat, path, &block)
-        end
-
-        # Return a handle on a StringIO object representing the
-        # content in this HDFS file.
-        #
-        # Be VERY careful how you use this!  It is a StringIO object
-        # so the whole HDFS file is read into a string before
-        # returning the handle.
-        #
-        # @return [StringIO]
-        def io
-          @io ||= StringIO.new(read)
-        end
-
-        # Map over the lines of this HDFS resource.
-        #
-        # @yield [String] each line of the file
-        # @return [Array] the result of the block on each line
-        def map &block
-          returning([]) do |output|
-            HDFS.fs(:cat, path) do |line|
-              output << block.call(line)
-            end
-          end
-        end
-
-      end
-
-      # Defines methods for listing contents of HDFS directories.
-      module HDFSDirectory
-
-        # Return the paths of all files and directories directly below
-        # this directory on the HDFS.
-        #
-        # @return [Array<String>]
-        def contents
-          returning([]) do |paths|
-            HDFS.fs(:ls, path) do |line|
-              next if line =~ /^Found.*items$/
-              paths << line.split.last
-            end
-          end
-        end
-
-        # Return the resources directly below this directory on the
-        # HDFS.
-        #
-        # @return [Array<IMW::Resource>]
-        def resources
-          contents.map { |path| IMW.open(path) }
-        end
-        
-      end
-    end
-  end
-end

data/lib/imw/resources/schemes/http.rb

@@ -1,161 +0,0 @@
-module IMW
-  module Resources
-    module Schemes
-
-      # Defines methods for accessing a resource over HTTP.  Uses
-      # RestClient to implement the basic HTTP verbs (GET, POST, PUT,
-      # DELETE, HEAD).
-      module HTTP
-
-        # Is this resource being accessed via HTTP?
-        #
-        # @return [true, false]
-        def via_http?
-          true
-        end
-
-        # Copy this resource to the +new_uri+.
-        #
-        # @param [String, IMW::Resource] new_uri
-        # @return [IMW::Resource] the new resource
-        def cp new_uri
-          IMW::Transforms::Transferer.new(:cp, self, new_uri).transfer!
-        end
-        
-
-        # Return the basename of the URI or <tt>_index</tt> if it's
-        # blank, as in the case of <tt>http://www.google.com</tt>.
-        #
-        # @return [String]
-        def effective_basename
-          (basename.blank? || basename =~ %r{^/*$}) ? "_index" : basename
-        end
-
-        # Send a GET request to this resource's URI.
-        #
-        # If the response doesn't have HTTP code 2xx, a RestClient
-        # error will be raised.
-        #
-        # If a block is given then the response will be passed to the
-        # block, even in case of a non-2xx code.
-        #
-        # See the documentation for
-        # RestClient[http://rdoc.info/projects/archiloque/rest-client]
-        # for more information.
-        #
-        # @param [Hash] headers the headers to include in the request
-        # @yield [RestClient::Response] the response from the server
-        # @return [RestClient::Response] the response from the server
-        # @raise [RestClient::NotModified, RestClient::Unauthorized, RestClient::ResourceNotFound, RestClient::RequestFailed] error from RestClient on non-2xx response codes
-        def get headers={}, &block
-          make_restclient_request do
-            RestClient.get(uri.to_s, headers, &block)
-          end
-        end
-
-        # Send a POST request to this resource's URI with data
-        # +payload+.
-        #
-        # If the response doesn't have HTTP code 2xx, a RestClient
-        # error will be raised.
-        #
-        # If a block is given then the response will be passed to the
-        # block, even in case of a non-2xx code.
-        #
-        # See the documentation for
-        # RestClient[http://rdoc.info/projects/archiloque/rest-client]
-        # for more information.
-        #
-        # @param [Hash, String] payload the data to send
-        # @param [Hash] headers the headers to include in the request
-        # @yield [RestClient::Response] the response from the server
-        # @return [RestClient::Response] the response from the server
-        # @raise [RestClient::NotModified, RestClient::Unauthorized, RestClient::ResourceNotFound, RestClient::RequestFailed] error from RestClient on non-2xx response codes
-        def post payload, headers={}, &block
-          make_restclient_request do
-            RestClient.post(uri.to_s, payload, headers, &block)
-          end
-        end
-
-        # Send a PUT request to this resource's URI with data
-        # +payload+.
-        #
-        # If the response doesn't have HTTP code 2xx, a RestClient
-        # error will be raised.
-        #
-        # If a block is given then the response will be passed to the
-        # block, even in case of a non-2xx code.
-        #
-        # See the documentation for
-        # RestClient[http://rdoc.info/projects/archiloque/rest-client]
-        # for more information.
-        #
-        # @param [Hash, String] payload the data to send
-        # @param [Hash] headers the headers to include in the request
-        # @yield [RestClient::Response] the response from the server
-        # @return [RestClient::Response] the response from the server
-        # @raise [RestClient::NotModified, RestClient::Unauthorized, RestClient::ResourceNotFound, RestClient::RequestFailed] error from RestClient on non-2xx response codes
-        def put payload, headers={}, &block
-          make_restclient_request do
-            RestClient.put(uri.to_s, payload, headers, &block)
-          end
-        end
-        
-        # Send a DELETE request to this resource's URI.
-        #
-        # If the response doesn't have HTTP code 2xx, a RestClient
-        # error will be raised.
-        #
-        # If a block is given then the response will be passed to the
-        # block, even in case of a non-2xx code.
-        #
-        # See the documentation for
-        # RestClient[http://rdoc.info/projects/archiloque/rest-client]
-        # for more information.
-        #
-        # @param [Hash] headers the headers to include in the request
-        # @yield [RestClient::Response] the response from the server
-        # @return [RestClient::Response] the response from the server
-        # @raise [RestClient::NotModified, RestClient::Unauthorized, RestClient::ResourceNotFound, RestClient::RequestFailed] error from RestClient on non-2xx response codes
-        def delete headers={}, &block
-          make_restclient_request do
-            RestClient.delete(uri.to_s, headers, &block)
-          end
-        end
-        
-        # Send a HEAD request to this resource's URI.
-        #
-        # If the response doesn't have HTTP code 2xx, a RestClient
-        # error will be raised.
-        #
-        # If a block is given then the response will be passed to the
-        # block, even in case of a non-2xx code.
-        #
-        # See the documentation for
-        # RestClient[http://rdoc.info/projects/archiloque/rest-client]
-        # for more information.
-        #
-        # @param [Hash] headers the headers to include in the request
-        # @yield [RestClient::Response] the response from the server
-        # @return [RestClient::Response] the response from the server
-        # @raise [RestClient::NotModified, RestClient::Unauthorized, RestClient::ResourceNotFound, RestClient::RequestFailed] error from RestClient on non-2xx response codes
-        def head headers={}, &block
-          make_restclient_request do
-            RestClient.head(uri.to_s, headers, &block)
-          end
-        end
-
-        protected
-        def make_restclient_request &block # :nodoc
-          require 'restclient'
-          begin
-            yield
-          rescue RestClient::NotModified, RestClient::Unauthorized, RestClient::ResourceNotFound, RestClient::RequestFailed => e
-            raise IMW::NetworkError.new("#{e.class} -- #{e.message}")
-          end
-        end
-      end
-    end
-  end
-end
-

data/lib/imw/resources/schemes/s3.rb

@@ -1,137 +0,0 @@
-module IMW
-  module Resources
-    module Schemes
-
-      # Defines methods for reading and writing data to {Amazon
-      # S3}[http://aws.amazon.com/s3] buckets.
-      #
-      #   IMW.open('s3://my_bucket/path/to/some/file.csv')
-      #
-      # Learn more about {Amazon Web Services}[http://aws.amazon.com].
-      module S3
-
-        # For an S3 resource, the bucket is just the hostname.
-        #
-        # @return [String]
-        def bucket
-          host
-        end
-
-        # Is this resource an S3 resource?
-        #
-        # @return [true, false] 
-        def on_s3?
-          true
-        end
-        alias_method :is_s3?, :on_s3?
-
-        # Copy this resource to the +new_uri+.
-        #
-        # @param [String, IMW::Resource] new_uri
-        # @return [IMW::Resource] the new resource
-        def cp new_uri
-          IMW::Transforms::Transferer.new(:cp, self, new_uri).transfer!
-        end
-
-        # The AWS::S3::S3Object corresponding to this resource.
-        def s3_object
-          self.class.make_connection!
-          @s3_object ||= AWS::S3::S3Object.new(path, bucket)
-        end
-
-        # Does this resource exist on S3?
-        #
-        # @return [true, false]
-        def exist?
-          s3_object.exists?
-        end
-        alias_method :exists?, :exist?
-
-        # Remove this resource from S3.
-        #
-        # @return [IMW::Resource] the deleted object
-        def rm
-          s3_object.delete
-        end
-        alias_method :rm!, :rm
-
-        # Return the S3N URL for this S3 object
-        #
-        #   resource = IMW.open('s3://my_bucket/path/to/some/obj')
-        #   resource.s3n_url
-        #   => 's3n://my_bucket/path/to/some/obj'
-        #
-        # @return [String]
-        def s3n_url
-          uri.to_s.gsub(/^s3:/, 's3n:')
-        end
-
-        # Return the contents of this S3 object.
-        #
-        # @return [String]
-        def read
-          s3_object.value
-        end
-
-        # Store +source+ into +destination+.
-        # 
-        # @param [String, IMW::Resource, #io] source
-        # @param [String, IMW::Resource, #path, #bucket] destination
-        # @return [IMW::Resource] the new S3 object
-        def self.put source, destination
-          source       = IMW.open(source)
-          destintation = IMW.open(destination)
-          raise IMW::ArgumentError.new("destination must be on S3 -- #{destination.uri} given") unless destination.on_s3?
-          make_connection!
-          AWS::S3::S3Object.store(destination.path, source.io, destination.bucket)
-          destination
-        end
-
-        # Download +source+ from S3 into +destination+.
-        #
-        # @param [String, IMW::Resource, #path, #bucket] source
-        # @param [String, IMW::Resource, #write] destination
-        # @return [IMW::Resource] the new resource
-        def self.get source, destination
-          source      = IMW.open(source)
-          destination = IMW.open(destination)
-          make_connection!
-          AWS::S3::Object.stream(source.path, source.bucket) do |chunk|
-            destination.write(chunk)
-          end
-          destination.close
-          destination.reopen
-        end
-
-        # Copy S3 resource +source+ to +destination+.
-        #
-        # @param [String, IMW::Resource, #path, #bucket] source
-        # @param [String, IMW::Resource, #path, #bucket] destination
-        # @return [IMW::Resource] the new resource
-        def self.copy source, destination
-          source      = IMW.open(source)
-          destination = IMW.open(destination)
-          raise IMW::PathError.new("Bucket names must be non-blank and match to 'copy'") unless source.bucket.present? && destination.bucket.present? && source.bucket == destination.bucket
-          make_connection!
-          AWS::S3::Object.copy(source.path, destination.path, destination.bucket)
-          destination
-        end
-
-        protected
-        # Make an S3 connection.
-        #
-        # Uses settings defined in IMW::AWS_CREDENTIALS.
-        #
-        # @return [AWS
-        def self.make_connection!
-          return @connection if @connection
-          raise IMW::Error.new("Must define a constant IMW::AWS_CREDENTIALS with an :access_key_id and a :secret_access_key before using S3 resources") unless defined?(IMW::AWS_CREDENTIALS)
-          require 'aws/s3'
-          @connection = AWS::S3::Base.establish_connection!(IMW::AWS_CREDENTIALS)
-        end
-        
-      end
-    end
-  end
-end
-