imw 0.2.0 → 0.2.1

data/spec/imw/tools/transferer_spec.rb

@@ -0,0 +1,113 @@
+require File.dirname(__FILE__) + "/../../spec_helper"
+
+describe IMW::Tools::Transferer do
+  before do
+    @local  = IMW.open("foobar.txt")
+    @http   = IMW.open("http://www.google.com")
+    @hdfs   = IMW.open("hdfs:///path/to/foobar.txt")
+    @s3     = IMW.open("s3://mybucket/foo/bar")
+  end
+
+  it "should raise an error unless the action is one of :cp, :copy, :mv :move, or :mv!" do
+    IMW::Tools::Transferer.new(:cp,   @local, @http).should be
+    IMW::Tools::Transferer.new(:copy, @local, @http).should be
+    IMW::Tools::Transferer.new(:mv,   @local, @http).should be
+    IMW::Tools::Transferer.new(:move, @local, @http).should be
+    IMW::Tools::Transferer.new(:mv!,  @local, @http).should be
+    lambda { IMW::Tools::Transferer.new(:foobar, @local, @http) }.should raise_error(IMW::ArgumentError)
+  end
+
+  it "should raise an error if the source and the destination have the same URI" do
+    lambda { IMW::Tools::Transferer.new(:cp, @local, @local) }.should raise_error(IMW::PathError)
+  end
+
+  describe "transfering local files" do
+    it "should raise an error if the source doesn't exist" do
+      lambda { IMW::Tools::Transferer.new(:cp, @local, 'barbaz.txt').transfer! }.should raise_error(IMW::PathError)
+    end
+
+    it "can copy a local file" do
+      IMWTest::Random.file @local.path
+      IMW::Tools::Transferer.new(:cp, @local, 'barbaz.txt').transfer!
+      @local.exist?.should be_true
+      IMW.open('barbaz.txt').exist?.should be_true
+    end
+
+    it "can move a local file" do
+      IMWTest::Random.file @local.path 
+      IMW::Tools::Transferer.new(:mv, @local, 'barbaz.txt').transfer!
+      @local.exist?.should be_false
+      IMW.open('barbaz.txt').exist?.should be_true
+    end
+
+  end
+
+  describe "transferring HTTP files" do
+    it "can copy a remote file to a local path" do
+      IMW::Tools::Transferer.new(:cp, @http, @local).transfer!
+      @local.exist?.should be_true
+    end
+  end
+
+  describe "transferring S3 files" do
+
+    it "can copy an S3 file to a local path" do
+      IMW::Schemes::S3.should_receive(:get).with(@s3, @local)
+      IMW::Tools::Transferer.new(:cp, @s3, @local).transfer!
+    end
+
+    it "can copy a local path to an S3 file" do
+      IMWTest::Random.file @local.path
+      IMW::Schemes::S3.should_receive(:put).with(@local, @s3)
+      IMW::Tools::Transferer.new(:cp, @local, @s3).transfer!
+    end
+
+    it "can copy between S3 files" do
+      @new_s3 = IMW.open('s3://mybucket/new/path')
+      IMW::Schemes::S3.should_receive(:copy).with(@s3, @new_s3)
+      IMW::Tools::Transferer.new(:cp, @s3, @new_s3).transfer!
+    end
+  end
+
+  describe "transferring HDFS files" do
+    before do
+      IMW::Schemes::HDFS.stub!(:fs)
+    end
+
+    it "can copy a local file to an HDFS path" do
+      IMWTest::Random.file @local.path
+
+      IMW::Schemes::HDFS.should_receive(:fs).with(:put, @local.path, @hdfs.path)
+      IMW::Tools::Transferer.new(:cp, @local, @hdfs).transfer!
+    end
+
+    it "can copy an HDFS file to a local path" do
+      IMW::Schemes::HDFS.should_receive(:fs).with(:get, @hdfs.path, @local.path)
+      IMW::Tools::Transferer.new(:cp, @hdfs, @local).transfer!
+    end
+
+    it "can copy between HDFS paths" do
+      @new_hdfs = IMW.open('hdfs:///a/new/path')
+      IMW::Schemes::HDFS.should_receive(:fs).with(:cp, @hdfs.path, @new_hdfs.path)
+      IMW::Tools::Transferer.new(:cp, @hdfs, @new_hdfs).transfer!
+    end
+
+    it "can move between HDFS paths" do
+      @new_hdfs = IMW.open('hdfs:///a/new/path')
+      IMW::Schemes::HDFS.should_receive(:fs).with(:mv, @hdfs.path, @new_hdfs.path)
+      IMW::Tools::Transferer.new(:mv, @hdfs, @new_hdfs).transfer!
+    end
+
+    it "can copy from S3 to HDFS" do
+      IMW::Schemes::HDFS.should_receive(:fs).with(:cp, @s3.s3n_url, @hdfs.path)
+      IMW::Tools::Transferer.new(:cp, @s3, @hdfs).transfer!
+    end
+
+    it "can copy from HDFS to S3" do
+      IMW::Schemes::HDFS.should_receive(:fs).with(:cp, @hdfs.path, @s3.s3n_url)
+      IMW::Tools::Transferer.new(:cp, @hdfs, @s3).transfer!
+    end
+  end
+end
+
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: imw
 version: !ruby/object:Gem::Version 
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors: 
 - Dhruv Bansal
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-12 00:00:00 -05:00
+date: 2010-05-14 00:00:00 -05:00
 default_executable: imw
 dependencies: []
 
@@ -36,10 +36,26 @@ files:
 - etc/imwrc.rb
 - examples/dataset.rb
 - lib/imw.rb
+- lib/imw/archives.rb
+- lib/imw/archives/rar.rb
+- lib/imw/archives/tar.rb
+- lib/imw/archives/tarbz2.rb
+- lib/imw/archives/targz.rb
+- lib/imw/archives/zip.rb
 - lib/imw/boot.rb
+- lib/imw/compressed_files.rb
+- lib/imw/compressed_files/bz2.rb
+- lib/imw/compressed_files/compressible.rb
+- lib/imw/compressed_files/gz.rb
 - lib/imw/dataset.rb
 - lib/imw/dataset/paths.rb
 - lib/imw/dataset/workflow.rb
+- lib/imw/formats.rb
+- lib/imw/formats/delimited.rb
+- lib/imw/formats/excel.rb
+- lib/imw/formats/json.rb
+- lib/imw/formats/sgml.rb
+- lib/imw/formats/yaml.rb
 - lib/imw/parsers.rb
 - lib/imw/parsers/html_parser.rb
 - lib/imw/parsers/html_parser/matchers.rb
@@ -47,34 +63,16 @@ files:
 - lib/imw/parsers/regexp_parser.rb
 - lib/imw/repository.rb
 - lib/imw/resource.rb
-- lib/imw/resources.rb
-- lib/imw/resources/archive.rb
-- lib/imw/resources/archives_and_compressed.rb
-- lib/imw/resources/archives_and_compressed/bz2.rb
-- lib/imw/resources/archives_and_compressed/gz.rb
-- lib/imw/resources/archives_and_compressed/rar.rb
-- lib/imw/resources/archives_and_compressed/tar.rb
-- lib/imw/resources/archives_and_compressed/tarbz2.rb
-- lib/imw/resources/archives_and_compressed/targz.rb
-- lib/imw/resources/archives_and_compressed/zip.rb
-- lib/imw/resources/compressed_file.rb
-- lib/imw/resources/compressible.rb
-- lib/imw/resources/formats.rb
-- lib/imw/resources/formats/delimited.rb
-- lib/imw/resources/formats/excel.rb
-- lib/imw/resources/formats/json.rb
-- lib/imw/resources/formats/sgml.rb
-- lib/imw/resources/formats/yaml.rb
-- lib/imw/resources/local.rb
-- lib/imw/resources/remote.rb
-- lib/imw/resources/schemes.rb
-- lib/imw/resources/schemes/hdfs.rb
-- lib/imw/resources/schemes/http.rb
-- lib/imw/resources/schemes/s3.rb
 - lib/imw/runner.rb
-- lib/imw/transforms.rb
-- lib/imw/transforms/archiver.rb
-- lib/imw/transforms/transferer.rb
+- lib/imw/schemes.rb
+- lib/imw/schemes/hdfs.rb
+- lib/imw/schemes/http.rb
+- lib/imw/schemes/local.rb
+- lib/imw/schemes/remote.rb
+- lib/imw/schemes/s3.rb
+- lib/imw/tools.rb
+- lib/imw/tools/archiver.rb
+- lib/imw/tools/transferer.rb
 - lib/imw/utils.rb
 - lib/imw/utils/error.rb
 - lib/imw/utils/extensions.rb
@@ -97,32 +95,32 @@ files:
 - spec/data/sample.txt
 - spec/data/sample.xml
 - spec/data/sample.yaml
+- spec/imw/archive_spec.rb
+- spec/imw/archives/rar_spec.rb
+- spec/imw/archives/tar_spec.rb
+- spec/imw/archives/tarbz2_spec.rb
+- spec/imw/archives/targz_spec.rb
+- spec/imw/archives/zip_spec.rb
+- spec/imw/compressed_files/bz2_spec.rb
+- spec/imw/compressed_files/compressible_spec.rb
+- spec/imw/compressed_files/gz_spec.rb
+- spec/imw/compressed_files_spec.rb
 - spec/imw/dataset/paths_spec.rb
 - spec/imw/dataset/workflow_spec.rb
+- spec/imw/formats/delimited_spec.rb
+- spec/imw/formats/json_spec.rb
+- spec/imw/formats/sgml_spec.rb
+- spec/imw/formats/yaml_spec.rb
 - spec/imw/parsers/line_parser_spec.rb
 - spec/imw/parsers/regexp_parser_spec.rb
 - spec/imw/resource_spec.rb
-- spec/imw/resources/archive_spec.rb
-- spec/imw/resources/archives_and_compressed/bz2_spec.rb
-- spec/imw/resources/archives_and_compressed/gz_spec.rb
-- spec/imw/resources/archives_and_compressed/rar_spec.rb
-- spec/imw/resources/archives_and_compressed/tar_spec.rb
-- spec/imw/resources/archives_and_compressed/tarbz2_spec.rb
-- spec/imw/resources/archives_and_compressed/targz_spec.rb
-- spec/imw/resources/archives_and_compressed/zip_spec.rb
-- spec/imw/resources/compressed_file_spec.rb
-- spec/imw/resources/compressible_spec.rb
-- spec/imw/resources/formats/delimited_spec.rb
-- spec/imw/resources/formats/json_spec.rb
-- spec/imw/resources/formats/sgml_spec.rb
-- spec/imw/resources/formats/yaml_spec.rb
-- spec/imw/resources/local_spec.rb
-- spec/imw/resources/remote_spec.rb
-- spec/imw/resources/schemes/hdfs_spec.rb
-- spec/imw/resources/schemes/http_spec.rb
-- spec/imw/resources/schemes/s3_spec.rb
-- spec/imw/transforms/archiver_spec.rb
-- spec/imw/transforms/transferer_spec.rb
+- spec/imw/schemes/hdfs_spec.rb
+- spec/imw/schemes/http_spec.rb
+- spec/imw/schemes/local_spec.rb
+- spec/imw/schemes/remote_spec.rb
+- spec/imw/schemes/s3_spec.rb
+- spec/imw/tools/archiver_spec.rb
+- spec/imw/tools/transferer_spec.rb
 - spec/imw/utils/paths_spec.rb
 - spec/imw/utils/shared_paths_spec.rb
 - spec/rcov.opts
@@ -163,32 +161,32 @@ signing_key:
 specification_version: 3
 summary: The Infinite Monkeywrench (IMW) makes acquiring, extracting, transforming, loading, and packaging data easy.
 test_files: 
-- spec/imw/resources/compressed_file_spec.rb
-- spec/imw/resources/archives_and_compressed/targz_spec.rb
-- spec/imw/resources/archives_and_compressed/tar_spec.rb
-- spec/imw/resources/archives_and_compressed/zip_spec.rb
-- spec/imw/resources/archives_and_compressed/tarbz2_spec.rb
-- spec/imw/resources/archives_and_compressed/bz2_spec.rb
-- spec/imw/resources/archives_and_compressed/rar_spec.rb
-- spec/imw/resources/archives_and_compressed/gz_spec.rb
-- spec/imw/resources/archive_spec.rb
-- spec/imw/resources/local_spec.rb
-- spec/imw/resources/remote_spec.rb
-- spec/imw/resources/compressible_spec.rb
-- spec/imw/resources/formats/json_spec.rb
-- spec/imw/resources/formats/yaml_spec.rb
-- spec/imw/resources/formats/delimited_spec.rb
-- spec/imw/resources/formats/sgml_spec.rb
-- spec/imw/resources/schemes/http_spec.rb
-- spec/imw/resources/schemes/hdfs_spec.rb
-- spec/imw/resources/schemes/s3_spec.rb
+- spec/imw/archives/targz_spec.rb
+- spec/imw/archives/tar_spec.rb
+- spec/imw/archives/zip_spec.rb
+- spec/imw/archives/tarbz2_spec.rb
+- spec/imw/archives/rar_spec.rb
+- spec/imw/tools/archiver_spec.rb
+- spec/imw/tools/transferer_spec.rb
+- spec/imw/archive_spec.rb
+- spec/imw/compressed_files/compressible_spec.rb
+- spec/imw/compressed_files/bz2_spec.rb
+- spec/imw/compressed_files/gz_spec.rb
 - spec/imw/dataset/paths_spec.rb
 - spec/imw/dataset/workflow_spec.rb
+- spec/imw/compressed_files_spec.rb
 - spec/imw/parsers/line_parser_spec.rb
 - spec/imw/parsers/regexp_parser_spec.rb
 - spec/imw/resource_spec.rb
-- spec/imw/transforms/archiver_spec.rb
-- spec/imw/transforms/transferer_spec.rb
+- spec/imw/formats/json_spec.rb
+- spec/imw/formats/yaml_spec.rb
+- spec/imw/formats/delimited_spec.rb
+- spec/imw/formats/sgml_spec.rb
+- spec/imw/schemes/http_spec.rb
+- spec/imw/schemes/local_spec.rb
+- spec/imw/schemes/remote_spec.rb
+- spec/imw/schemes/hdfs_spec.rb
+- spec/imw/schemes/s3_spec.rb
 - spec/imw/utils/paths_spec.rb
 - spec/imw/utils/shared_paths_spec.rb
 - spec/spec_helper.rb

data/lib/imw/resources.rb

@@ -1,118 +0,0 @@
-require 'imw/resources/formats'
-require 'imw/resources/schemes'
-require 'imw/resources/archives_and_compressed'
-
-module IMW
-
-  # IMW::Resources is a namespace in which all the modules which
-  # define different kinds of behavior for IMW::Resource objects are
-  # defined.
-  #
-  # When an IMW::Resource is instantiated it eventually calls
-  # IMW::Resources#extend_resource! which will iterate through the
-  # handlers in IMW::Resources#handlers, extending the resource with
-  # modules whose handler conditions are satisfied.
-  #
-  # A handler is just an Array with two elements.  The first should be
-  # a module or a string identifying a module.  
-  #
-  # If the second element is a Regexp, the corresponding module will
-  # be used if the regexp matches the resource's URI (as a string)
-  #
-  # If the second element is a Proc, it will be called with the
-  # resource as its only argument and if it returns true then the
-  # module will be used.
-  #
-  # You can define your own handlers by appending them to
-  # IMW::Resources::USER_DEFINED_HANDLERS in your <tt>.imwrc</tt>
-  # file.
-  module Resources
-
-    autoload :LocalObj,        'imw/resources/local'
-    autoload :RemoteObj,       'imw/resources/remote'
-    autoload :StringObj,       'imw/resources/string'
-    autoload :Transferable,    'imw/resources/transferable'
-
-    # Iterate through IMW::Resources#handlers and extend the given
-    # +resource+ with modules whose handler conditions match the
-    # resource.
-    #
-    # @param [IMW::Resource] resource the resource to extend
-    # @return [IMW::Resource] the extended resource
-    def self.extend_resource! resource
-      handlers.each do |mod_name, handler|
-        case handler
-        when Regexp    then extend_resource_with_mod_or_string!(resource, mod_name) if handler =~ resource.uri.to_s
-        when Proc      then extend_resource_with_mod_or_string!(resource, mod_name) if handler.call(resource)
-        when TrueClass then extend_resource_with_mod_or_string!(resource, mod_name)
-        else
-          raise IMW::TypeError("A handler must be Regexp, Proc, or true")
-        end
-      end
-      resource
-    end
-
-    # Basic handlers to determine whether the resource is local,
-    # remote, or a string.
-    BASIC_HANDLERS = [
-                      ["LocalObj",  Proc.new { |resource| resource.scheme == 'file' || resource.scheme.blank?   } ],
-                      ["RemoteObj", Proc.new { |resource| resource.scheme != 'file' && resource.scheme.present? } ],
-                      ["StringObj", Proc.new { |resource| resource.is_stringio?                                 } ]
-                     ]
-
-    # Define this constant in your configuration file to add your own
-    # handlers.
-    USER_DEFINED_HANDLERS = [] unless defined?(USER_DEFINED_HANDLERS)
-
-    # include handlers from other modules
-    include IMW::Resources::Formats
-    include IMW::Resources::Schemes
-    
-    # A list of handlers to try.  Define your own handlers in
-    # IMW::Resources::USER_DEFINED_HANDLERS.
-    #
-    # @return [Array]
-    def self.handlers
-      # order here is important
-      BASIC_HANDLERS + SCHEME_HANDLERS + ARCHIVE_AND_COMPRESSED_HANDLERS + FORMAT_HANDLERS + USER_DEFINED_HANDLERS
-    end
-
-    protected
-
-    # Extend +resource+ with +mod_or_string+.  Will work hard to try
-    # and interpret +mod_or_string+ as a module if it's a string.
-    #
-    # @param [IMW::Resource] resource the resource to extend
-    #
-    # @param [Module, String] mod_or_string the module or string
-    # representing a module to extend the resource with
-    def self.extend_resource_with_mod_or_string! resource, mod_or_string
-      if mod_or_string.is_a?(Module)
-        resource.extend(mod_or_string)
-      else
-        # Given a string "Mod::SubMod::SubSubMod" first split it into
-        # its parts ["Mod", "SubMod", "SubSubMod"] and then begin
-        # class_eval'ing them in order so that each is class_eval'd in
-        # the scope of the one before it.
-        #
-        # There is almost certainly a better way to do this.
-        mod_names = mod_or_string.to_s.split('::')
-        mods = []
-        mod_names.each_with_index do |name, index|
-          if index == 0
-            mods << class_eval(name)
-          else
-            begin
-              mods << class_eval(name)
-            rescue NameError
-              mods << mods[index - 1].class_eval(name)
-            end
-          end
-        end
-        resource.extend(mods.last)
-      end
-    end
-  end
-end
-
-

data/lib/imw/resources/archives_and_compressed.rb

@@ -1,32 +0,0 @@
-module IMW
-  module Resources
-    autoload :Compressible,    'imw/resources/compressible'
-    autoload :CompressedFile,  'imw/resources/compressed_file'
-    autoload :Archive,         'imw/resources/archive'
-    autoload :Archives,        'imw/resources/archive'
-    autoload :CompressedFiles, 'imw/resources/compressed_file'
-
-    # Handlers which augment the resource with methods for archiving,
-    # extracting, compressing, decompressing...
-    ARCHIVE_AND_COMPRESSED_HANDLERS = [
-
-                                       # try compressible first -- compressed files below will override it
-                                       ["Compressible",         Proc.new { |r| r.is_local? } ],
-
-                                       # order is important! -- tar.bz2 must come before .bz2, &c.
-                                       ["Archives::Tarbz2",     Proc.new { |r| r.is_local? && r.path =~ /\.tar\.bz2$/                                              } ],
-                                       ["Archives::Tarbz2",     Proc.new { |r| r.is_local? && r.path =~ /\.tbz2$/                                                  } ],
-                                       ["CompressedFiles::Bz2", Proc.new { |r| r.is_local? && r.path =~ /\.bz2$/ && r.path !~ /\.tar\.bz2$/ && r.path !~ /\.tbz2$/ } ],
-                                       ["Archives::Targz",      Proc.new { |r| r.is_local? && r.path =~ /\.tar\.gz$/                                               } ],
-                                       ["Archives::Targz",      Proc.new { |r| r.is_local? && r.path =~ /\.tgz$/                                                   } ],
-                                       ["CompressedFiles::Gz",  Proc.new { |r| r.is_local? && r.path =~ /\.gz$/  && r.path !~ /\.tar\.gz$/  && r.path !~ /\.tgz$/  } ],
-                                       ["Archives::Tar",        Proc.new { |r| r.is_local? && r.path =~ /\.tar$/                                                   } ],
-                                       ["Archives::Rar",        Proc.new { |r| r.is_local? && r.path =~ /\.rar$/                                                   } ],
-                                       ["Archives::Zip",        Proc.new { |r| r.is_local? && r.path =~ /\.zip$/                                                   } ]
-
-                                      ]
-
-    
-  end
-end
-