imw 0.2.6 → 0.2.7

data/VERSION

@@ -1 +1 @@
-0.2.6
+0.2.7

data/lib/imw/archives.rb

@@ -5,13 +5,13 @@ module IMW
 
     # Handlers for archives.
     HANDLERS = [
-                ["Archives::Tarbz2",     Proc.new { |r| r.is_local? && r.path =~ /\.tar\.bz2$/  } ],
-                ["Archives::Tarbz2",     Proc.new { |r| r.is_local? && r.path =~ /\.tbz2$/      } ],
-                ["Archives::Targz",      Proc.new { |r| r.is_local? && r.path =~ /\.tar\.gz$/   } ],
-                ["Archives::Targz",      Proc.new { |r| r.is_local? && r.path =~ /\.tgz$/       } ],
-                ["Archives::Tar",        Proc.new { |r| r.is_local? && r.path =~ /\.tar$/       } ],
-                ["Archives::Rar",        Proc.new { |r| r.is_local? && r.path =~ /\.rar$/       } ],
-                ["Archives::Zip",        Proc.new { |r| r.is_local? && r.path =~ /\.zip$/       } ]
+                ["Archives::Tarbz2",     Proc.new { |r| r.is_local? && r.path =~ /\.tar\.bz2$/i  } ],
+                ["Archives::Tarbz2",     Proc.new { |r| r.is_local? && r.path =~ /\.tbz2$/i      } ],
+                ["Archives::Targz",      Proc.new { |r| r.is_local? && r.path =~ /\.tar\.gz$/i   } ],
+                ["Archives::Targz",      Proc.new { |r| r.is_local? && r.path =~ /\.tgz$/i       } ],
+                ["Archives::Tar",        Proc.new { |r| r.is_local? && r.path =~ /\.tar$/i       } ],
+                ["Archives::Rar",        Proc.new { |r| r.is_local? && r.path =~ /\.rar$/i       } ],
+                ["Archives::Zip",        Proc.new { |r| r.is_local? && r.path =~ /\.zip$/i       } ]
                ]
 
     autoload :Rar,    'imw/archives/rar'
@@ -37,13 +37,6 @@ module IMW
         true
       end
 
-      # Archives should not return snippets.
-      #
-      # @return [nil]
-      def snippet
-        nil
-      end
-
       # Create an archive of the given +input_paths+.
       #
       # The input paths must be strings and will be shell-escaped

data/lib/imw/compressed_files.rb

@@ -10,9 +10,9 @@ module IMW
     # well as the IMW::CompressedFiles::Compressible module for
     # compressing regular files.
     HANDLERS = [
-                ["CompressedFiles::Compressible", Proc.new { |r| r.is_local? && r.is_file? && r.path != /\.(bz2|gz|tgz|tbz2)$/                       } ],
-                ["CompressedFiles::Gz",           Proc.new { |r| r.is_local? && r.path =~ /\.gz$/  && r.path !~ /\.tar\.gz$/  && r.path !~ /\.tgz$/  } ],
-                ["CompressedFiles::Bz2",          Proc.new { |r| r.is_local? && r.path =~ /\.bz2$/ && r.path !~ /\.tar\.bz2$/ && r.path !~ /\.tbz2$/ } ]    
+                ["CompressedFiles::Compressible", Proc.new { |r| r.is_local? && r.is_file? && r.path != /\.(bz2|gz|tgz|tbz2)$/i                         } ],
+                ["CompressedFiles::Gz",           Proc.new { |r| r.is_local? && r.path =~ /\.gz$/i  && r.path !~ /\.tar\.gz$/i  && r.path !~ /\.tgz$/i  } ],
+                ["CompressedFiles::Bz2",          Proc.new { |r| r.is_local? && r.path =~ /\.bz2$/i && r.path !~ /\.tar\.bz2$/i && r.path !~ /\.tbz2$/i } ]    
                ]
 
     # Defines methods for decompressing a compressed file.  This
@@ -71,13 +71,6 @@ module IMW
         IMW.open(decompressed_path)
       end
 
-      # Compressed files should not return snippets.
-      #
-      # @return [nil]
-      def snippet
-        nil
-      end
-
       # Decompress this file in its present directory, overwriting any
       # existing files while keeping the original compressed file.
       #

data/lib/imw/formats.rb

@@ -14,16 +14,16 @@ module IMW
     # Handlers which augment a resource with data format specific
     # methods.
     HANDLERS = [
-                [ "Formats::Csv",   /\.csv$/    ],
-                [ "Formats::Tsv",   /\.tsv$/    ],
-                [ "Formats::Excel", /\.xslx?$/  ],
-                [ "Formats::Json",  /\.json$/   ],
-                [ "Formats::Xml",   /\.xml$/    ],
-                [ "Formats::Xsl",   /\.xsl$/    ],                         
-                [ "Formats::Html",  /\.html?$/  ],
-                [ "Formats::Xhtml", /\.xhtml?$/ ],
-                [ "Formats::Rdf",   /\.rdf?$/   ],
-                [ "Formats::Yaml",  /\.ya?ml$/  ]
+                [ "Formats::Csv",   /\.csv$/i    ],
+                [ "Formats::Tsv",   /\.tsv$/i    ],
+                [ "Formats::Excel", /\.xslx?$/i  ],
+                [ "Formats::Json",  /\.json$/i   ],
+                [ "Formats::Xml",   /\.xml$/i    ],
+                [ "Formats::Xsl",   /\.xsl$/i    ],                         
+                [ "Formats::Html",  /\.html?$/i  ],
+                [ "Formats::Xhtml", /\.xhtml?$/i ],
+                [ "Formats::Rdf",   /\.rdf?$/i   ],
+                [ "Formats::Yaml",  /\.ya?ml$/i  ]
                ]
   end
 end

data/lib/imw/resource.rb

@@ -74,7 +74,14 @@ module IMW
   # accepts all the same arguments as IMW::Resource.new.
   class Resource
 
-    attr_reader :uri, :mode
+    # The URI object associated with this resource.
+    attr_reader :uri
+
+    # The mode in which to access this resource.
+    attr_accessor :mode
+
+    # A copy of the options passed to this resource on initialization.
+    attr_accessor :resource_options
 
     # Create a new resource representing +uri+.
     #
@@ -102,8 +109,9 @@ module IMW
     # @option options [String] mode the mode to open the resource in (will be ignored when inapplicable)
     # @return [IMW::Resource]
     def initialize uri, options={}
-      self.uri = uri
-      @mode    = options[:mode] || 'r'
+      self.uri              = uri
+      self.resource_options = options
+      self.mode             = options[:mode] || 'r'
       extend_appropriately!(options) unless options[:no_modules]
     end
 
@@ -192,6 +200,13 @@ module IMW
       @name ||= extname ? basename[0,basename.length - extname.length] : basename
     end
 
+    # Returns the user associated with the host of this URI.
+    #
+    # @return [String]
+    def user
+      @user ||= uri.user
+    end
+
     def to_s
       uri.to_s
     end

data/lib/imw/schemes.rb

@@ -6,14 +6,14 @@ module IMW
     autoload :HTTP,   'imw/schemes/http'
     autoload :HTTPS,  'imw/schemes/http'
     autoload :HDFS,   'imw/schemes/hdfs'
-
+    
     HANDLERS = [
                 ["Schemes::Local::Base",  Proc.new { |resource| resource.scheme == 'file' || resource.scheme.blank?    } ],
                 ["Schemes::Remote::Base", Proc.new { |resource| resource.scheme != 'file' && resource.scheme.present?  } ],
-                ["Schemes::S3",    %r{^s3://}    ],
-                ["Schemes::HTTP",  %r{^http://}  ],
-                ["Schemes::HTTPS", %r{^https://} ],
-                ["Schemes::HDFS",  %r{^hdfs://}  ]
+                ["Schemes::S3",        %r{^s3://}i    ],
+                ["Schemes::HTTP",      %r{^http://}i  ],
+                ["Schemes::HTTPS",     %r{^https://}i ],
+                ["Schemes::HDFS",      %r{^hdfs://}i  ]
                ]
   end
 end

data/lib/imw/schemes/local.rb

@@ -151,20 +151,17 @@ module IMW
         # - extension
         # - snippet
         def summary
-          {
+          data = {
             :basename  => basename,
             :size      => size,
-            :extension => extension,
-            :snippet   => snippet
+            :extension => extension
           }
+          if respond_to?(:snippet)
+            data[:snippet] = snippet
+          end
+          data
         end
 
-        # Return a 1024-char snippet from this local file.
-        #
-        # @return [String]
-        def snippet
-          io.read(1024)
-        end
       end
 
       # Defines methods for manipulating the contents of a local

data/lib/imw/tools/summarizer.rb

@@ -44,7 +44,7 @@ module IMW
       def summary
         @summary ||= inputs.map do |input|
           if input.respond_to?(:summary)
-            input.summary
+            input.summary rescue {}
           else
             {}
           end

data/lib/imw/tools/transferer.rb

@@ -1,14 +1,72 @@
 module IMW
   module Tools
+
+    # A class to encapsulate transferring a resource from one URI to
+    # another.
     class Transferer
 
-      attr_accessor :action, :source, :destination
+      # The action this Transferer is to take.
+      #
+      # @return [:cp, :mv]
+      attr_reader :action
+
+      # Set the action of this Transferer.
+      #
+      # Will raise an error unless +the_action+ is <tt>:cp</tt> or
+      # <tt>:mv</tt>.
+      #
+      # @param [:cp, :mv] the_action
+      def action= the_action
+        @action = case the_action.to_sym
+        when :cp, :copy       then :cp
+        when :mv, :move, :mv! then :mv
+        else raise IMW::ArgumentError.new("action (#{the_action}) must be one of `cp' (or `copy') or `mv' (or `move' or `mv!'")
+        end
+      end
+
+      # The source resource.
+      #
+      # @return [IMW::Resource]
+      attr_reader :source
+
+      # Set the source for this transferer.
+      #
+      # If +the_source+ is local, will check that it exists and raise
+      # an error if not.
+      #
+      # @param [String, IMW::Resource] the_source
+      def source= the_source
+        s = IMW.open(the_source)
+        s.should_exist!("Cannot #{action_verb}") if s.is_local?
+        @source = s
+      end
       
+      # The destination resource.
+      #
+      # @return [IMW::Resource]
+      attr_reader :destination
+
+      # Set the destination for this transferer.
+      #
+      # If +the_destination+ is local, will check that its parent
+      # directory exists and raise an error if not.
+      def destination= the_destination
+        d = IMW.open(the_destination)
+        d.dir.should_exist!("Cannot #{action_verb}") if d.is_local?
+        @destination = d
+      end
+
+      # Instantiate a new transferer to take the given +action+ on
+      # +source+ and +destination+.
+      #
+      # @param [:cp, :mv] action the action to take
+      # @param [String, IMW::Resource] source
+      # @param [String, IMW::Resource] destination      
       def initialize action, source, destination
-        @action      = normalize_action(action)
-        @source      = IMW.open(source)
-        @destination = IMW.open(destination)
-        raise IMW::PathError.new("Source and destination have the same URI: #{@source.uri}") if @source.uri.to_s == @destination.uri.to_s
+        self.action      = action
+        self.source      = source
+        self.destination = destination
+        raise IMW::PathError.new("Source and destination have the same URI: #{source}") if self.source.uri.to_s == self.destination.uri.to_s
       end
 
       # Transfer source to destination.
@@ -16,38 +74,31 @@ module IMW
       # For local transfer, will raise errors unless the necessary
       # paths exist.
       def transfer!
-        if source.is_local? 
-          source.should_exist!("Cannot #{action}") 
-          source_scheme = 'file'                   
-        else
-          source_scheme = source.scheme
-        end
-        
-        if destination.is_local?
-          destination.dir.should_exist!("Cannot #{action}")          
-          destination_scheme = 'file'
-        else
-          destination_scheme = destination.scheme
-        end
-        
-        method             = "#{source_scheme}_to_#{destination_scheme}"
-        
-        if respond_to?(method)
-          send(method)
-        else
-          raise IMW::NoMethodError.new("Do not know how to #{action} #{source.uri} => #{destination.uri} (#{source_scheme.inspect} => #{destination_scheme.inspect})")
-        end
+        IMW.announce_if_verbose("#{action_gerund.capitalize} #{source} to #{destination}")
+        send(transfer_method)
         destination.reopen
       end
 
       protected
 
-      def normalize_action action # :nodoc:
-        case action.to_sym
-        when :cp, :copy then :cp
-        when :mv, :move, :mv! then :mv
-        else raise IMW::ArgumentError.new("action (#{action}) must be one of `cp' (or `copy') or `mv' (or `move' or `mv!'")
-        end
+      # Return the name of the method that should be used to transfer
+      # +source+ to +destination+.
+      #
+      # @return [String]
+      def transfer_method
+        source_scheme      = source.is_local?      ? 'file' : source.scheme
+        destination_scheme = destination.is_local? ? 'file' : destination.scheme
+        method = "#{source_scheme}_to_#{destination_scheme}"
+        raise IMW::NoMethodError.new("Do not know how to #{action_verb} #{source} to #{destination}") unless respond_to?(method)
+        method
+      end
+
+      def action_verb # :nodoc
+        action == :cp ? "copy" : "move"
+      end
+
+      def action_gerund # :nodoc
+        action == :cp ? "copying" : "moving"
       end
 
       #

data/spec/imw/schemes/sql_spec.rb

@@ -0,0 +1,3 @@
+require File.join(File.dirname(__FILE__),'../../spec_helper')
+
+

data/spec/imw/tools/transferer_spec.rb

@@ -7,65 +7,77 @@ describe IMW::Tools::Transferer do
     @nested  = IMW.open('new_dir/nested.txt')
     @nested_dir = IMW.open('new_dir/nested')
     @local   = IMW.open("foobar.txt")
+    @dest    = IMW.open("barbaz.txt")
     @http    = IMW.open("http://www.google.com")
     @hdfs    = IMW.open("hdfs:///path/to/foobar.txt")
     @s3      = IMW.open("s3://mybucket/foo/bar")
+    IMWTest::Random.file(@local.path)
   end
 
   it "should raise an error unless the action is one of :cp, :copy, :mv :move, or :mv!" do
-    IMW::Tools::Transferer.new(:cp,   @local, @http).should be
-    IMW::Tools::Transferer.new(:copy, @local, @http).should be
-    IMW::Tools::Transferer.new(:mv,   @local, @http).should be
-    IMW::Tools::Transferer.new(:move, @local, @http).should be
-    IMW::Tools::Transferer.new(:mv!,  @local, @http).should be
-    lambda { IMW::Tools::Transferer.new(:foobar, @local, @http) }.should raise_error(IMW::ArgumentError)
+    @transferer = IMW::Tools::Transferer.new(:cp, @local, @http)
+    @transferer.action = :cp
+    @transferer.action = :copy
+    @transferer.action = :mv
+    @transferer.action = :mv!    
+    @transferer.action = :move
+    lambda { @transferer.action = :foobar }.should raise_error(IMW::ArgumentError)
   end
 
   it "should raise an error if the source and the destination have the same URI" do
     lambda { IMW::Tools::Transferer.new(:cp, @local, @local) }.should raise_error(IMW::PathError)
   end
 
+  it "should print a log message when IMW is verbose" do
+    IMW.stub!(:verbose).and_return(:true)
+    IMW.should_receive(:announce_if_verbose).with("Copying #{@local} to #{@dest}")
+    IMW::Tools::Transferer.new(:cp, @local, @dest).transfer!
+  end
+  
   describe "transfering local files" do
 
     before do
       IMWTest::Random.file @local.path
+      @transferer = IMW::Tools::Transferer.new(:cp, @local, @dest)
     end
     
     it "should raise an error if the source doesn't exist" do
       @local.rm!
-      lambda { IMW::Tools::Transferer.new(:cp, @local, 'barbaz.txt').transfer! }.should raise_error(IMW::PathError)
+      lambda { @transferer.source = @local }.should raise_error(IMW::PathError)
     end
 
     it "should raise an error if the directory of the destination doesn't exist" do
-      lambda { IMW::Tools::Transferer.new(:cp, @local, @nested).transfer! }.should raise_error(IMW::PathError)
+      lambda { @transferer.destination = @nested }.should raise_error(IMW::PathError)
     end
 
     it "can copy a local file" do
-      IMW::Tools::Transferer.new(:cp, @local, 'barbaz.txt').transfer!
+      @transferer.transfer!
       @local.exist?.should be_true
-      IMW.open('barbaz.txt').exist?.should be_true
+      @dest.exist?.should  be_true
     end
 
     it "can copy a local file to a directory" do
       FileUtils.mkdir(@dir.path)
-      IMW::Tools::Transferer.new(:cp, @local, @dir).transfer!
+      @transferer.destination = @dir
+      @transferer.transfer!
       IMW.open(File.join(@dir.path, @local.basename)).exist?.should be_true
     end
 
     it "can move a local file" do
-      IMW::Tools::Transferer.new(:mv, @local, 'barbaz.txt').transfer!
+      @transferer.action = :mv
+      @transferer.transfer!
       @local.exist?.should be_false
-      IMW.open('barbaz.txt').exist?.should be_true
+      @dest.exist?.should  be_true
     end
 
     it "can move a local file to a directory" do
       FileUtils.mkdir(@dir.path)
-      IMW::Tools::Transferer.new(:mv, @local, @dir).transfer!
+      @transferer.action = :mv
+      @transferer.destination = @dir
+      @transferer.transfer!
       IMW.open(File.join(@dir.path, @local.basename)).exist?.should be_true
       @local.exist?.should be_false
     end
-    
-
   end
 
   describe "transfering local directories" do

metadata

@@ -1,7 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: imw
 version: !ruby/object:Gem::Version 
-  version: 0.2.6
+  hash: 25
+  prerelease: false
+  segments: 
+  - 0
+  - 2
+  - 7
+  version: 0.2.7
 platform: ruby
 authors: 
 - Dhruv Bansal
@@ -10,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-06-15 00:00:00 -05:00
+date: 2010-07-09 00:00:00 -05:00
 default_executable: imw
 dependencies: []
 
@@ -135,6 +141,7 @@ files:
 - spec/support/paths_matcher.rb
 - spec/support/random.rb
 - spec/support/without_regard_to_order_matcher.rb
+- spec/imw/schemes/sql_spec.rb
 has_rdoc: true
 homepage: http://github.com/infochimps/imw
 licenses: []
@@ -145,21 +152,27 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.5
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: The Infinite Monkeywrench (IMW) makes acquiring, extracting, transforming, loading, and packaging data easy.
@@ -188,6 +201,7 @@ test_files:
 - spec/imw/archives_spec.rb
 - spec/imw/schemes/http_spec.rb
 - spec/imw/schemes/local_spec.rb
+- spec/imw/schemes/sql_spec.rb
 - spec/imw/schemes/remote_spec.rb
 - spec/imw/schemes/hdfs_spec.rb
 - spec/imw/schemes/s3_spec.rb