imw 0.2.18 → 0.3.0

data/spec/imw/tools/extension_analyzer_spec.rb

@@ -1,153 +0,0 @@
-require File.dirname(__FILE__) + "/../../spec_helper"
-
-describe IMW::Tools::ExtensionAnalyzer do
-
-  before do
-    class Analyzer
-      attr_accessor :dir, :resources
-      include IMW::Tools::ExtensionAnalyzer
-      def initialize dir
-        self.dir = File.expand_path(dir)
-        @resources = IMW.open(self.dir).all_resources
-      end
-      def total_size
-        @total_size ||= resources.map(&:size).inject(0) { |e, sum| sum += e }
-      end
-    end
-  end
-
-  describe 'working with an empty directory' do
-    before do
-      @analyzer = Analyzer.new(IMWTest::TMP_DIR)
-    end
-
-    %w[most_common_extension_by_count most_common_extension_by_size most_common_extension].each do |method|
-      it "should return 'flat' when asked for its '#{method}'" do
-        @analyzer.send(method).should == 'flat'
-      end
-    end
-    
-    %w[extension_counts normalized_extension_counts extension_sizes normalized_extension_sizes].each do |method|
-      it "should return an empty hash when asked for its '#{method}'" do
-        @analyzer.send(method).should == {}
-      end
-    end
-  end
-
-  describe 'working with files that lack extensions' do
-    
-    before do
-      @dir = File.join(IMWTest::TMP_DIR, 'ext_dir')
-      FileUtils.mkdir_p(@dir)
-
-      @f1  = "foobar1"
-      @f2  = "foobar2"
-      @f3  = "foobar1"
-      @files = [@f1, @f2, @f3]
-      
-      @files.each do |basename|
-        IMWTest::Random.file File.join(@dir, basename)
-      end
-      
-      @analyzer = Analyzer.new(IMWTest::TMP_DIR)
-    end
-
-    %w[most_common_extension_by_count most_common_extension_by_size most_common_extension].each do |method|
-      it "should return 'flat' when asked for its '#{method}'" do
-        @analyzer.send(method).should == 'flat'
-      end
-    end
-  end
-  
-  describe 'working with a directory of files' do
-    before do
-      @dir = File.join(IMWTest::TMP_DIR, 'ext_dir')
-      FileUtils.mkdir_p(@dir)
-
-      @csv1  = "foobar1.csv"
-      @csv2  = "foobar2.csv"
-      @xml  = "foobar1.xml"
-      @txt  = "foobar1.txt"
-      @files = [@csv1, @csv2, @xml, @txt]
-      
-      @files.each do |basename|
-        IMWTest::Random.file File.join(@dir, basename)
-      end
-
-      def bloat basename
-        File.open(File.join(@dir, basename), 'a') do |f|
-          1000.times do
-            f.write( 'hello ' * 100)
-          end
-        end
-      end
-
-      @analyzer = Analyzer.new @dir
-    end
-
-    describe "working with extension counts" do
-      it "should be able to return counts by extension" do
-        @analyzer.extension_counts.should == {'xml' => 1, 'txt' => 1, 'csv' => 2 }
-      end
-
-      it "should be able to return the most common extension by count" do
-        @analyzer.most_common_extension_by_count.should == 'csv'
-      end
-
-      it "should be able to calculate extension weighted by number of files" do
-        @analyzer.normalized_extension_counts.should == { 'csv' => 0.5, 'xml' => 0.25, 'txt' => 0.25 }
-      end
-    end
-
-    describe "working with extension sizes" do
-      it "should be able to calculate extension sizes" do
-        csv_size = File.size(File.join(@dir, @csv1)) + File.size(File.join(@dir, @csv2))
-        xml_size = File.size(File.join(@dir, @xml))
-        txt_size = File.size(File.join(@dir, @txt))
-        @analyzer.extension_sizes.should == { 'csv' => csv_size, 'xml' => xml_size, 'txt' => txt_size }
-      end
-
-      it "should be able to return the most common extension by size" do
-        bloat @txt
-        @analyzer.most_common_extension_by_size.should == 'txt'
-      end
-
-      it "should be able to calculate extension sizes" do
-        csv_size = File.size(File.join(@dir, @csv1)) + File.size(File.join(@dir, @csv2))
-        xml_size = File.size(File.join(@dir, @xml))
-        txt_size = File.size(File.join(@dir, @txt))
-        total_size = csv_size + xml_size + txt_size
-        @analyzer.normalized_extension_sizes.should == { 'csv' => csv_size.to_f / total_size.to_f, 'xml' => xml_size.to_f / total_size.to_f, 'txt' => txt_size.to_f / total_size.to_f }
-      end
-    end
-
-    describe "determining the most common extension" do
-
-      it "should obviously return an extension if it is the most common by count as well as the most common by size" do
-        bloat @csv1
-        @analyzer.most_common_extension.should == 'csv'
-      end
-
-      it "should return the most common extension by count if the count fraction is half or greater and the size fraction is less than half" do
-        bloat @txt
-        bloat @xml
-        @analyzer.most_common_extension.should == 'csv'
-      end
-
-      it "should return the most common extension by size if the size fraction is half or greater and the count fraction is less than half" do
-        # need to add an xml file
-        @new_xml = File.join(@dir, 'xml2.xml')
-        IMWTest::Random.file(@new_xml)
-        bloat @txt
-        @analyzer = Analyzer.new @dir
-        @analyzer.most_common_extension.should == 'txt'
-      end
-
-      it "should return the most common extension by size if no other conditions are met" do
-        bloat @txt
-        @analyzer.most_common_extension.should == 'txt'
-      end
-      
-    end
-  end
-end

data/spec/imw/tools/summarizer_spec.rb

@@ -1,8 +0,0 @@
-require File.dirname(__FILE__) + "/../../spec_helper"
-
-describe IMW::Tools::Summarizer do
-
-  
-end
-
-

data/spec/imw/tools/transferer_spec.rb

@@ -1,195 +0,0 @@
-require File.dirname(__FILE__) + "/../../spec_helper"
-
-describe IMW::Tools::Transferer do
-  before do
-    @dir     = IMW.open("dir")
-    @new_dir = IMW.open("new_dir")
-    @nested  = IMW.open('new_dir/nested.txt')
-    @nested_dir = IMW.open('new_dir/nested')
-    @local   = IMW.open("foobar.txt")
-    @dest    = IMW.open("barbaz.txt")
-    @http    = IMW.open("http://www.google.com")
-    @hdfs    = IMW.open("hdfs:///path/to/foobar.txt")
-    @s3      = IMW.open("s3://mybucket/foo/bar")
-    IMWTest::Random.file(@local.path)
-  end
-
-  it "should raise an error unless the action is one of :cp, :copy, :mv :move, or :mv!" do
-    @transferer = IMW::Tools::Transferer.new(:cp, @local, @http)
-    @transferer.action = :cp
-    @transferer.action = :copy
-    @transferer.action = :mv
-    @transferer.action = :mv!    
-    @transferer.action = :move
-    lambda { @transferer.action = :foobar }.should raise_error(IMW::ArgumentError)
-  end
-
-  it "should raise an error if the source and the destination have the same URI" do
-    lambda { IMW::Tools::Transferer.new(:cp, @local, @local) }.should raise_error(IMW::PathError)
-  end
-
-  it "should print a log message when IMW is verbose" do
-    IMW.stub!(:verbose).and_return(:true)
-    IMW.should_receive(:announce_if_verbose).with("Copying #{@local} to #{@dest}")
-    IMW::Tools::Transferer.new(:cp, @local, @dest).transfer!
-  end
-  
-  describe "transfering local files" do
-
-    before do
-      IMWTest::Random.file @local.path
-      @transferer = IMW::Tools::Transferer.new(:cp, @local, @dest)
-    end
-    
-    it "should raise an error if the source doesn't exist" do
-      @local.rm!
-      lambda { @transferer.source = @local }.should raise_error(IMW::PathError)
-    end
-
-    it "should raise an error if the directory of the destination doesn't exist" do
-      lambda { @transferer.destination = @nested }.should raise_error(IMW::PathError)
-    end
-
-    it "can copy a local file" do
-      @transferer.transfer!
-      @local.exist?.should be_true
-      @dest.exist?.should  be_true
-    end
-
-    it "can copy a local file to a directory" do
-      FileUtils.mkdir(@dir.path)
-      @transferer.destination = @dir
-      @transferer.transfer!
-      IMW.open(File.join(@dir.path, @local.basename)).exist?.should be_true
-    end
-
-    it "can move a local file" do
-      @transferer.action = :mv
-      @transferer.transfer!
-      @local.exist?.should be_false
-      @dest.exist?.should  be_true
-    end
-
-    it "can move a local file to a directory" do
-      FileUtils.mkdir(@dir.path)
-      @transferer.action = :mv
-      @transferer.destination = @dir
-      @transferer.transfer!
-      IMW.open(File.join(@dir.path, @local.basename)).exist?.should be_true
-      @local.exist?.should be_false
-    end
-  end
-
-  describe "transfering local directories" do
-
-    before do
-      IMWTest::Random.directory_with_files @dir.path
-      @dir = @dir.reopen
-    end
-    
-    it "should raise an error if the source doesn't exist" do
-      @dir.rm_rf!
-      lambda { IMW::Tools::Transferer.new(:cp, @dir, @new_dir).transfer! }.should raise_error(IMW::PathError)
-    end
-
-    it "should raise an error if the directory of the destination doesn't exist" do
-      lambda { IMW::Tools::Transferer.new(:cp, @dir, @nested_dir).transfer! }.should raise_error(IMW::PathError)
-    end
-
-    it "can copy a local directory" do
-      IMW::Tools::Transferer.new(:cp, @dir, @new_dir).transfer!
-      @dir.exist?.should be_true
-      @new_dir.exist?.should be_true
-    end
-
-    it "can move a local directory" do
-      IMW::Tools::Transferer.new(:mv, @dir, @new_dir).transfer!
-      @dir.exist?.should be_false
-      @new_dir.exist?.should be_true
-    end
-
-    it "can copy a local directory to an existing directory" do
-      FileUtils.mkdir(@new_dir.path)
-      IMW::Tools::Transferer.new(:cp, @dir, @nested_dir).transfer!
-      @dir.exist?.should be_true
-      @nested_dir.exist?.should be_true
-    end
-
-    it "can move a local directory to an existing directory" do
-      FileUtils.mkdir(@new_dir.path)
-      IMW::Tools::Transferer.new(:mv, @dir, @nested_dir).transfer!
-      @dir.exist?.should_not be_true
-      @nested_dir.exist?.should be_true
-    end
-  end
-  
-
-  describe "transferring HTTP files" do
-    it "can copy a remote file to a local path" do
-      IMW::Tools::Transferer.new(:cp, @http, @local).transfer!
-      @local.exist?.should be_true
-    end
-  end
-
-  describe "transferring S3 files" do
-
-    it "can copy an S3 file to a local path" do
-      IMW::Schemes::S3.should_receive(:get).with(@s3, @local)
-      IMW::Tools::Transferer.new(:cp, @s3, @local).transfer!
-    end
-
-    it "can copy a local path to an S3 file" do
-      IMWTest::Random.file @local.path
-      IMW::Schemes::S3.should_receive(:put).with(@local, @s3)
-      IMW::Tools::Transferer.new(:cp, @local, @s3).transfer!
-    end
-
-    it "can copy between S3 files" do
-      @new_s3 = IMW.open('s3://mybucket/new/path')
-      IMW::Schemes::S3.should_receive(:copy).with(@s3, @new_s3)
-      IMW::Tools::Transferer.new(:cp, @s3, @new_s3).transfer!
-    end
-  end
-
-  describe "transferring HDFS files" do
-    before do
-      IMW::Schemes::HDFS.stub!(:fs)
-    end
-
-    it "can copy a local file to an HDFS path" do
-      IMWTest::Random.file @local.path
-
-      IMW::Schemes::HDFS.should_receive(:fs).with(:put, @local.path, @hdfs.path)
-      IMW::Tools::Transferer.new(:cp, @local, @hdfs).transfer!
-    end
-
-    it "can copy an HDFS file to a local path" do
-      IMW::Schemes::HDFS.should_receive(:fs).with(:get, @hdfs.path, @local.path)
-      IMW::Tools::Transferer.new(:cp, @hdfs, @local).transfer!
-    end
-
-    it "can copy between HDFS paths" do
-      @new_hdfs = IMW.open('hdfs:///a/new/path')
-      IMW::Schemes::HDFS.should_receive(:fs).with(:cp, @hdfs.path, @new_hdfs.path)
-      IMW::Tools::Transferer.new(:cp, @hdfs, @new_hdfs).transfer!
-    end
-
-    it "can move between HDFS paths" do
-      @new_hdfs = IMW.open('hdfs:///a/new/path')
-      IMW::Schemes::HDFS.should_receive(:fs).with(:mv, @hdfs.path, @new_hdfs.path)
-      IMW::Tools::Transferer.new(:mv, @hdfs, @new_hdfs).transfer!
-    end
-
-    it "can copy from S3 to HDFS" do
-      IMW::Schemes::HDFS.should_receive(:fs).with(:cp, @s3.s3n_url, @hdfs.path)
-      IMW::Tools::Transferer.new(:cp, @s3, @hdfs).transfer!
-    end
-
-    it "can copy from HDFS to S3" do
-      IMW::Schemes::HDFS.should_receive(:fs).with(:cp, @hdfs.path, @s3.s3n_url)
-      IMW::Tools::Transferer.new(:cp, @hdfs, @s3).transfer!
-    end
-  end
-end
-
-

data/spec/imw/utils/dynamically_extendable_spec.rb

@@ -1,69 +0,0 @@
-require File.dirname(__FILE__) + "/../../spec_helper"
-
-describe "extending resources with specific modules" do
-  before do
-    @class  = Class.new
-    @class.send(:include, IMW::Utils::DynamicallyExtendable)
-    @instance = @class.new
-  end
-
-  it "should raise an error when registering a malformed handler" do
-    lambda { @class.register_handler("Foo", 3) }.should raise_error(IMW::ArgumentError)
-  end
-
-  it "should store in instances modules they've been extended by" do
-    @foo = Module.new
-    @instance.extend(@foo)
-    @instance.modules.should include(@foo)
-  end
-
-  describe "evaluating handlers" do
-    before do
-      @proccer  = Module.new
-      @class.send(:attr_accessor, :prop)
-      @class.register_handler(@proccer, Proc.new { |instance| instance.prop })
-
-      @regexper = Module.new
-      @class.send(:define_method, :to_s) { 'whoa' }
-      @class.register_handler(@regexper, /whoa/)
-    end
-
-    it "should extend an instance with a matching proc handler" do
-      @instance.prop = true
-      @instance.extend_appropriately!
-      @instance.modules.should include(@proccer)
-    end
-
-    it "should not extend an instance with a non-matching proc handler" do
-      @instance.prop = false
-      @instance.extend_appropriately!
-      @instance.modules.should_not include(@proccer)
-    end
-
-    it "should extend an instance with a matching regexp handler" do
-      @instance.extend_appropriately!
-      @instance.modules.should include(@regexper)
-    end
-
-    it "should not extend an instance with a non-matching regexp handler" do
-      @class.send(:define_method, :to_s) { 'fowl' }
-      @instance.extend_appropriately!
-      @instance.modules.should_not include(@regexper)
-    end
-
-    it "should not extend an instance with a module it was asked to skip" do
-      @instance.extend_appropriately!(:skip_modules => [@regexper])
-      @instance.modules.should_not include(@regexper)
-    end
-
-    it "should not extend with any modules if asked" do
-      @instance.extend_appropriately!(:no_modules => true)
-      @instance.modules.should_not include(@regexper)
-    end
-
-    it "should use a module if asked to do so even if it's handler didn't match" do
-      @instance.extend_appropriately!(:use_modules => [@proccer])
-      @instance.modules.should include(@proccer)
-    end
-  end
-end

data/spec/imw/utils/has_uri_spec.rb

@@ -1,61 +0,0 @@
-require File.dirname(__FILE__) + "/../../spec_helper"
-
-class Klass
-  include IMW::Utils::HasURI
-end
-
-def new_obj uri
-  obj = Klass.new
-  obj.uri = uri
-  obj
-end
-
-describe IMW::Utils::HasURI do
-  
-  it "local file path" do
-    obj = new_obj("/home/foo.txt")
-    obj.stub!(:path).and_return("/home/foo.txt")
-
-    obj.scheme.should    be_nil
-    obj.dirname.should   == '/home'
-    obj.basename.should  == 'foo.txt'
-    obj.extname.should   == '.txt'
-    obj.extension.should == 'txt'
-    obj.name.should      == 'foo'
-  end
-
-  it "local file path with spaces in the name" do
-    obj = new_obj("/home/foo bar.txt")
-    obj.stub!(:path).and_return("/home/foo bar.txt")
-    obj.name.should == 'foo bar'
-  end
-
-  it "local file path with an explicit file:// scheme" do
-    obj = new_obj("file:///home/foo.txt")
-    obj.scheme.should == 'file'
-  end
-
-  it "web URL with query and fragment" do
-    obj = new_obj("http://mysite.com/some/page?param=value#frag")
-    obj.stub!(:path).and_return("/some/page")
-    obj.scheme.should        == 'http'
-    obj.dirname.should       == '/some'
-    obj.basename.should      == 'page'
-    obj.extname.should       == ''
-    obj.extension.should     == ''
-    obj.name.should          == 'page'
-  end
-
-  it "should be able to strip URIs" do
-    new_obj('/path/to/something').stripped_uri.to_s.should == '/path/to/something'
-    new_obj('http://user:pass@example.com:8080/path/to/some/script.php?param=value#frag').stripped_uri.to_s.should == 'http://user:pass@example.com:8080/path/to/some/script.php'
-  end
-
-  it "should be able to return raw paths" do
-    new_obj('s3://bucket/crazy url with # some dumb naming convention').raw_path.should == '/crazy url with # some dumb naming convention'
-    new_obj('s3://bucket/crazy url with ?some dumb naming convention').raw_path.should  == '/crazy url with ?some dumb naming convention'
-    new_obj('s3://bucket/crazy url with ?some dumb naming #convention').raw_path.should == '/crazy url with ?some dumb naming #convention'
-    new_obj('s3://bucket/crazy url with #some dumb naming ?convention').raw_path.should == '/crazy url with #some dumb naming ?convention'
-  end
-
-end