imw 0.2.0 → 0.2.1

data/lib/imw/schemes/local.rb

@@ -0,0 +1,219 @@
+module IMW
+  module Schemes
+    module Local
+
+      # Defines methods appropriate for any file (or directory) on the
+      # local machine.  Includes methods from the File class like
+      # File#exist?, File#size, &c.
+      #
+      # When extending with this module, it will automatically also
+      # extend with either IMW::Schemes::Local::LocalDirectory or
+      # IMW::Schemes::Local::LocalFile, as appropriate.
+      module Base
+
+        def self.extended obj
+          # also extend with file or directory as appropriate
+          if obj.directory?
+            obj.extend(LocalDirectory)
+          else
+            obj.extend(LocalFile)
+          end
+        end
+        
+        # Steal a bunch of class methods from File which only take a
+        # path as a first argument.
+        [:executable?, :executable_real?, :exist?, :file?, :directory?, :ftype, :owned?, :pipe?, :readable?, :readable_real?, :setgid?, :setuid?, :size, :size?, :socket?, :split, :stat, :sticky?, :writable?, :writable_real?, :zero?].each do |class_method|
+          define_method class_method do
+            File.send(class_method, path)
+          end
+        end
+        alias_method :exists?, :exist?
+
+        # Return the path to this local object.
+        #
+        # @return [String]
+        def path
+          @path ||= File.expand_path(@encoded_uri ? Addressable::URI.decode(uri.to_s) : uri.to_s)
+        end
+
+        # Is this file on the local machine?
+        #
+        # @return [true, false]
+        def is_local?
+          true
+        end
+
+        # Copy this resource to the +new_uri+.
+        #
+        # @param [String, IMW::Resource] new_uri
+        # @return [IMW::Resource] the new resource
+        def cp new_uri
+          IMW::Tools::Transferer.new(:cp, self, new_uri).transfer!
+        end
+
+        # Move this resource to the +new_uri+.
+        #
+        # @param [String, IMW::Resource] new_uri
+        # @return [IMW::Resource] the new resource
+        def mv new_uri
+          IMW::Tools::Transferer.new(:mv, self, new_uri).transfer!
+        end
+        
+      end
+
+      # Defines methods for appropriate for a local file.
+      module LocalFile
+
+        # Is this resource a regular file?
+        #
+        # @return [true, false]
+        def is_file?
+          true
+        end
+
+        # Delete this resource.
+        def rm
+          should_exist!("Cannot delete")
+          FileUtils.rm path
+          self
+        end
+        alias_method :rm!, :rm
+
+        # Return the IO object at this path.
+        #
+        # @return [File]
+        def io
+          @io ||= open(path, mode)
+        end
+
+        # Read from this file.
+        #
+        # @param [Fixnum] length bytes to read
+        # @return [String]
+        def read length=nil
+          io.read(length)
+        end
+
+        # Write to this file
+        #
+        # @param [String, #to_s] text text to write
+        # @return [Fixnum] bytes written
+        def write text
+          io.write text
+        end
+
+        # Return the lines in this file.
+        #
+        # If passed a block, yield each line of the file to the block.
+        #
+        # @yield [String] each line of the file
+        # @return [Array] the lines in the file
+        def load &block
+          if block_given?
+            io.each do |line|
+              yield line
+            end
+          else
+            read.split("\n")
+          end
+        end
+
+        # Map over the lines in this file.
+        #
+        # @yield [String] each line of the file
+        def map &block
+          io.map(&block)
+        end
+
+        # Dump +data+ into this file.
+        #
+        # @param [String, Array, #each] data object to dump
+        # @option options [true, false] :persist (false) Don't close the file after writing
+        def dump data, options={}
+          data.each do |element|  # works if data is an Array or a String
+            io.puts(element.to_s)
+          end
+          io.close unless options[:persist]
+        end
+      end
+
+      # Defines methods for manipulating the contents of a local
+      # directory.
+      module LocalDirectory
+
+        # Is this resource a directory?
+        #
+        # @return [true, false]
+        def is_directory?
+          true
+        end
+
+        # Delete this directory.
+        #
+        # @return [IMW::Resource] the deleted directory
+        def rmdir
+          FileUtils.rmdir path
+          self
+        end
+
+        # Delete this directory recursively.
+        #
+        # @return [IMW::Resource] the deleted directory
+        def rm_rf
+          FileUtils.rm_rf path
+          self
+        end
+
+        # Return a list of paths relative to this directory which match
+        # the +selector+.  Works just like Dir[].
+        #
+        # @param [String] selector
+        # @return [Array] the matched paths
+        def [] selector='*'
+          Dir[File.join(path, selector)]
+        end
+
+        # Return a list of all paths directly within this directory.
+        #
+        # @return [Array]
+        def contents
+          self['*']
+        end
+
+        # Does this directory contain +obj+?
+        #
+        # @param [String, IMW::Resource] obj
+        # @return [true, false]
+        def contains? obj
+          require 'find'
+          obj_path = obj.is_a?(String) ? obj : obj.path
+          Find.find(path) do |sub_path|
+            return true if sub_path.ends_with?(obj_path)
+          end
+          false
+        end
+
+        # Return all paths within this directory, recursively.
+        #
+        # @return [Array<String>]
+        def all_contents
+          self['**/*']
+        end
+
+        # Return all resources within this directory, i.e. - all paths
+        # converted to IMW::Resource objects.
+        #
+        # @return [Array<IMW::Resource>]
+        def resources
+          all_contents.map do |path|
+            IMW.open(path) unless File.directory?(path)
+          end.compact
+        end
+
+      end
+    end
+  end
+end
+
+
+

data/lib/imw/schemes/remote.rb

@@ -0,0 +1,114 @@
+module IMW
+  module Schemes
+
+    # Contains modules which define methods appropriate for remote
+    # resources, no matter the protocol.
+    module Remote
+
+      # Defines methods appropriate for accessing a remote resource,
+      # no matter the protocol.
+      module Base
+        
+        #
+        # TODO -- self.extended should extend by RemoteDirectory when appropriate
+        #
+        
+        def self.extended obj
+          obj.extend(RemoteFile)
+        end
+
+        # Is this resource on a remote host?
+        #
+        # @return [true,false]
+        def is_remote?
+          true
+        end
+
+        # The host of this resource.
+        #
+        # @return [String]
+        def host
+          @host ||= uri.host
+        end
+
+        # Return the query string part of this resource's URI.  Will
+        # likely be +nil+ for local resources.
+        #
+        # @return [String]
+        def query_string
+          @query_string ||= uri.query
+        end
+
+        # Return the fragment part of this resource's URI.  Will likely be
+        # +nil+ for local resources.
+        #
+        # @return [String]
+        def fragment
+          @fragment ||= uri.fragment
+        end
+
+        # Return the path part of this resource's URI.  Will _not_
+        # include the +query_string+ or +fragment+.
+        #
+        # @return [String]
+        def path
+          @path ||= uri.path
+        end
+
+      end
+      
+      module RemoteFile
+
+        # Return the IO object for this remote file.
+        #
+        # The mode of this resource is ignored.
+        #
+        # @return [StringIO]
+        def io
+          require 'open-uri'
+          @io ||= open(uri.to_s)              # ignore mode
+        end
+
+        # Read the contents of this remote file.
+        #
+        # @return [String]
+        def read
+          io.read
+        end
+
+        # Return the lines of this remote file.
+        #
+        # If passed a block then yield each line to the block.
+        #
+        # @return [Array] the lines of this remote file
+        # @yield [String] each line of this remote file
+        def load &block
+          if block_given?
+            io.each do |line|
+              yield line
+            end
+          else
+            read.split("\n")
+          end
+        end
+
+        # Map over the lines in this remote file.
+        #
+        # @yield [String] each line of the file
+        def map &block
+          io.map(&block)
+        end
+      end
+
+
+      module RemoteDirectory
+
+        #
+        # TODO -- bloody everything
+        #
+
+        
+      end
+    end
+  end
+end

data/lib/imw/schemes/s3.rb

@@ -0,0 +1,135 @@
+module IMW
+  module Schemes
+
+    # Defines methods for reading and writing data to {Amazon
+    # S3}[http://aws.amazon.com/s3] buckets.
+    #
+    #   IMW.open('s3://my_bucket/path/to/some/file.csv')
+    #
+    # Learn more about {Amazon Web Services}[http://aws.amazon.com].
+    module S3
+
+      # For an S3 resource, the bucket is just the hostname.
+      #
+      # @return [String]
+      def bucket
+        host
+      end
+
+      # Is this resource an S3 resource?
+      #
+      # @return [true, false] 
+      def on_s3?
+        true
+      end
+      alias_method :is_s3?, :on_s3?
+
+      # Copy this resource to the +new_uri+.
+      #
+      # @param [String, IMW::Resource] new_uri
+      # @return [IMW::Resource] the new resource
+      def cp new_uri
+        IMW::Tools::Transferer.new(:cp, self, new_uri).transfer!
+      end
+
+      # The AWS::S3::S3Object corresponding to this resource.
+      def s3_object
+        self.class.make_connection!
+        @s3_object ||= AWS::S3::S3Object.new(path, bucket)
+      end
+
+      # Does this resource exist on S3?
+      #
+      # @return [true, false]
+      def exist?
+        s3_object.exists?
+      end
+      alias_method :exists?, :exist?
+
+      # Remove this resource from S3.
+      #
+      # @return [IMW::Resource] the deleted object
+      def rm
+        s3_object.delete
+      end
+      alias_method :rm!, :rm
+
+      # Return the S3N URL for this S3 object
+      #
+      #   resource = IMW.open('s3://my_bucket/path/to/some/obj')
+      #   resource.s3n_url
+      #   => 's3n://my_bucket/path/to/some/obj'
+      #
+      # @return [String]
+      def s3n_url
+        uri.to_s.gsub(/^s3:/, 's3n:')
+      end
+
+      # Return the contents of this S3 object.
+      #
+      # @return [String]
+      def read
+        s3_object.value
+      end
+
+      # Store +source+ into +destination+.
+      # 
+      # @param [String, IMW::Resource, #io] source
+      # @param [String, IMW::Resource, #path, #bucket] destination
+      # @return [IMW::Resource] the new S3 object
+      def self.put source, destination
+        source       = IMW.open(source)
+        destintation = IMW.open(destination)
+        raise IMW::ArgumentError.new("destination must be on S3 -- #{destination.uri} given") unless destination.on_s3?
+        make_connection!
+        AWS::S3::S3Object.store(destination.path, source.io, destination.bucket)
+        destination
+      end
+
+      # Download +source+ from S3 into +destination+.
+      #
+      # @param [String, IMW::Resource, #path, #bucket] source
+      # @param [String, IMW::Resource, #write] destination
+      # @return [IMW::Resource] the new resource
+      def self.get source, destination
+        source      = IMW.open(source)
+        destination = IMW.open(destination)
+        make_connection!
+        AWS::S3::Object.stream(source.path, source.bucket) do |chunk|
+          destination.write(chunk)
+        end
+        destination.close
+        destination.reopen
+      end
+
+      # Copy S3 resource +source+ to +destination+.
+      #
+      # @param [String, IMW::Resource, #path, #bucket] source
+      # @param [String, IMW::Resource, #path, #bucket] destination
+      # @return [IMW::Resource] the new resource
+      def self.copy source, destination
+        source      = IMW.open(source)
+        destination = IMW.open(destination)
+        raise IMW::PathError.new("Bucket names must be non-blank and match to 'copy'") unless source.bucket.present? && destination.bucket.present? && source.bucket == destination.bucket
+        make_connection!
+        AWS::S3::Object.copy(source.path, destination.path, destination.bucket)
+        destination
+      end
+
+      protected
+      # Make an S3 connection.
+      #
+      # Uses settings defined in IMW::AWS_CREDENTIALS.
+      #
+      # @return [AWS
+      def self.make_connection!
+        return @connection if @connection
+        raise IMW::Error.new("Must define a constant IMW::AWS_CREDENTIALS with an :access_key_id and a :secret_access_key before using S3 resources") unless defined?(IMW::AWS_CREDENTIALS)
+        require 'aws/s3'
+        @connection = AWS::S3::Base.establish_connection!(IMW::AWS_CREDENTIALS)
+      end
+      
+    end
+  end
+end
+