imw 0.2.18 → 0.3.0

data/spec/imw/resource_spec.rb

@@ -1,32 +0,0 @@
-require File.dirname(__FILE__) + "/../spec_helper"
-
-describe IMW::Resource do
-
-  describe "handling missing methods" do
-    before do
-      @resource = IMW::Resource.new('/home/foof.txt', :no_modules => true)
-    end
-
-    it "should return false when querying with a method that isn't defined" do
-      @resource.is_remote?.should be_false
-    end
-
-    it "should raise an IMW::NoMethodError in any other case" do
-      lambda { @resource.do_seomthing }.should raise_error(IMW::NoMethodError)
-    end
-
-    it "should print the modules it's been extended by when raising an IMW::NoMethodError" do
-      begin
-        @resource.extend(IMW::Schemes::Local::Base)
-        @resource.do_something
-      rescue IMW::NoMethodError => e
-        e.message.should match(/extended by IMW::Schemes::Local::Base/)
-      end
-    end
-  end
-
-end
-
-
-
-

data/spec/imw/schemes/hdfs_spec.rb

@@ -1,67 +0,0 @@
-require File.join(File.dirname(__FILE__),'../../spec_helper')
-
-describe IMW::Schemes::HDFS do
-  before do
-    def fake_hdfs_resource path, options={}
-      if options == false
-        response = ""
-      else        
-        response = "           #{options[:dirs] || 0}            #{options[:files] || 1}              #{options[:size] || 1000} hdfs://localhost#{path}"
-      end
-      IMW::Schemes::HDFS.should_receive(:fs).with(:count, path).at_least(:once).and_return(response)
-      IMW.open("hdfs://#{path}")
-    end
-    @path = '/path/to/myfile'    
-  end
-  
-  describe "refreshing its properties" do
-    it "should correctly get properties for a resource which exists" do
-      resource = fake_hdfs_resource(@path, :dirs => 2, :files => 3, :size => 1000)
-      resource.exist?.should    be_true
-      resource.num_dirs.should  == 2
-      resource.num_files.should == 3
-      resource.size.should      == 1000
-    end
-
-    it "should gracefully handle a resource which doesn't exist" do
-      resource = fake_hdfs_resource(@path, false)
-      resource.exist?.should be_false
-    end
-
-    it "should execute the correct command to delete the path" do
-      resource = fake_hdfs_resource(@path)
-      IMW::Schemes::HDFS.should_receive(:fs).with(:rm, resource.path)
-      resource.rm
-    end
-
-    it "should execute the correct command to delete the path when skipping the trash" do
-      resource = fake_hdfs_resource(@path)
-      IMW::Schemes::HDFS.should_receive(:fs).with(:rm, '-skipTrash', resource.path)
-      resource.rm :skip_trash => true
-    end
-
-    it "should recognize a file and extend it properly" do
-      resource = fake_hdfs_resource(@path)
-      resource.num_dirs.should == 0      
-      resource.num_files.should == 1
-      resource.exist?.should be_true
-      resource.is_directory?.should be_false
-      resource.modules.should include(IMW::Schemes::HDFSFile)
-    end
-
-    it "should recognize a directory and extend it properly" do
-      resource = fake_hdfs_resource(@path, :dirs => 2, :files => 1)
-      resource.num_dirs.should == 2
-      resource.num_files.should == 1
-      resource.exist?.should be_true
-      resource.is_directory?.should be_true
-      resource.modules.should include(IMW::Schemes::HDFSDirectory)
-    end
-
-    it "should be able to join path segments to a directory" do
-      resource     = fake_hdfs_resource(@path, :dirs => 2)
-      sub_resource = fake_hdfs_resource("#{@path}/a/b/c")
-      resource.join('a', 'b/c').to_s.should == sub_resource.to_s
-    end
-  end
-end

data/spec/imw/schemes/http_spec.rb

@@ -1,19 +0,0 @@
-require File.join(File.dirname(__FILE__),'../../spec_helper')
-
-describe IMW::Schemes::HTTP do
-
-  describe "finding its effective basename" do
-    it "should use the real basename when present" do
-      IMW.open('http://www.google.com/foobar').effective_basename.should == 'foobar'
-    end
-
-    it "should use '_index' when at the root (without a slash)" do
-      IMW.open('http://www.google.com').effective_basename.should == '_index'
-    end
-
-    it "should use '_index' when at the root (even when a slash is given)" do
-      IMW.open('http://www.google.com/').effective_basename.should == '_index'
-    end
-    
-  end
-end

data/spec/imw/schemes/local_spec.rb

@@ -1,165 +0,0 @@
-require File.join(File.dirname(__FILE__),'../../spec_helper')
-
-describe IMW::Schemes::Local::Base do
-
-  it "should not extend a local file with LocalDirectory" do
-    @file = IMW::Resource.new('foo.txt', :no_modules => true)
-    @file.should_not_receive(:extend).with(IMW::Schemes::Local::LocalDirectory)
-    IMW::Resource.extend_instance!(@file)
-  end
-
-  it "should not extend a local directory with LocalFile" do
-    @dir = IMW::Resource.new(IMWTest::TMP_DIR, :no_modules => true)
-    @dir.should_not_receive(:extend).with(IMW::Schemes::Local::LocalFile)
-    IMW::Resource.extend_instance!(@dir)
-  end
-
-  it "should correctly resolve relative paths" do
-    IMW.open('foobar').dirname.should == IMWTest::TMP_DIR
-  end
-
-  it "should be able to return its directory as an IMW object" do
-    IMW.open('/path/to/file').dir.path.should == '/path/to'
-    IMW.open('/').dir.path.should == '/'
-  end
-
-  
-end
-
-describe IMW::Schemes::Local::LocalFile do
-  before do
-    IMWTest::Random.file('original.txt')
-    @file = IMW::Resource.new('original.txt')
-  end
-
-  it "can delete the file" do
-    @file.rm
-    @file.exist?.should be_false
-  end
-
-  it "can read a file" do
-    @file.read.size.should > 0
-  end
-
-  it "can load the lines of a file" do
-    data = @file.load
-    data.size.should > 0
-    data.class.should == Array
-  end
-
-  it "can iterate over the lines of a file" do
-    @file.load do |line|
-      line.class.should == String
-      break
-    end
-  end
-
-  it "can map the lines of a file" do
-    @file.map do |line|
-      line[0..5]
-    end.class.should == Array
-  end
-
-  it "can produce a snippet" do
-    path = IMWTest::DATA_DIR + "/formats/none/sample"
-    # FIXME only look at the first 100 bytes b/c of subsequent non-ascii chars...
-    IMW.open(path).snippet[0..100].should == File.new(path).read(101)
-  end
-
-  it "can produce a summary with size and line count" do
-    @file.summary[:num_lines].should > 0
-    @file.summary[:size].should > 0
-  end
-end
-
-describe IMW::Schemes::Local::LocalDirectory do
-  before do
-    FileUtils.mkdir_p('dir')
-    FileUtils.mkdir_p('dir/subdir')
-    FileUtils.cd('dir') do
-      IMWTest::Random.file('file1.tsv')
-      IMWTest::Random.file('file2.tsv')
-      FileUtils.cd('subdir') do
-        IMWTest::Random.file('file3.csv')
-      end
-    end
-    @dir = IMW::Resource.new('dir')
-  end
-
-  it "can delete an empty directory" do
-    FileUtils.mkdir('empty')
-    dir = IMW.open('empty')
-    dir.rmdir
-    dir.exist?.should be_false
-  end
-
-  it "can recursively delete a directory" do
-    @dir.rm_rf
-    @dir.exist?.should be_false
-  end
-
-  it "can list its contents" do
-    @dir.contents.size.should == 3
-  end
-
-  it "can list its contents recursively" do
-    @dir.all_contents.size.should == 4
-  end
-
-  it "can list its contents as IMW::Resource objects" do
-    @dir.resources.map(&:class).uniq.first.should == IMW::Resource
-  end
-
-  describe "checking whether it contains other resources" do
-
-    it "should return false for remote paths" do
-      @dir.contains?("http://google.com").should be_false
-    end
-
-    it "should return true for its own path" do
-      @dir.contains?(@dir.path).should be_true
-    end
-
-    it "should return false for a path that doesn't start with its path" do
-      @dir.contains?(File.expand_path('foo')).should be_false
-    end
-
-    it "should return false for a path that starts with its path but doesn't exist" do
-      @dir.contains?(File.expand_path('dir/foo/baz')).should be_false
-    end
-
-    it "should return true for a path that starts with its path and exists" do
-      FileUtils.mkdir_p('dir/foo/baz')
-      @dir.contains?(File.expand_path('dir/foo/baz')).should be_true
-    end
-
-  end
-
-  it "can join with a path" do
-    @dir.join("a", "b/c").to_s.should == File.join(@dir.path, 'a/b/c')
-  end
-
-  it "can create a subdirectory" do
-    @dir.join("mallaco").exist?.should be_false
-    subdir = @dir.subdir!("mallaco")
-    subdir.exist?.should be_true
-    subdir.directory?.should be_true
-  end
-  
-  describe 'can package itself to' do
-    ['tar', 'tar.bz2', 'tar.gz', 'zip', 'rar'].each do |extension|
-      it "a #{extension} archive" do
-        @dir.package("package.#{extension}").exist?.should be_true # FIXME should explicitly check paths are correct in archive
-      end
-    end
-  end
-
-  it "can produce a summary with size and number of files" do
-    @dir.create
-    @dir.summary[:num_files].should == @dir.contents.size
-    @dir.summary[:size].should > 0
-  end
-  
-end
-
-

data/spec/imw/schemes/remote_spec.rb

@@ -1,38 +0,0 @@
-require File.join(File.dirname(__FILE__),'../../spec_helper')
-
-describe IMW::Schemes::Remote::Base do
-end
-
-describe IMW::Schemes::Remote::RemoteFile do
-
-  before do
-    # skip modules or else it will automatically become HTML!
-    @file = IMW.open('http://www.google.com', :skip_modules => ["Schemes::HTTP", "Formats::HTML"])
-  end
-
-  describe 'with the file' do
-
-    it "can read a remote file" do
-      @file.read.size.should > 0
-    end
-
-    it "can load the lines of a remote file" do
-      data = @file.load
-      data.size.should > 0
-      data.class.should == Array
-    end
-
-    it "can iterate over the lines of a remote file" do
-      @file.load do |line|
-        line.class.should == String
-        break
-      end
-    end
-
-    it "can map the lines of a remote file" do
-      @file.map do |line|
-        line[0..5]
-      end.class.should == Array
-    end
-  end
-end

data/spec/imw/schemes/s3_spec.rb

@@ -1,31 +0,0 @@
-require File.join(File.dirname(__FILE__),'../../spec_helper')
-
-describe IMW::Schemes::S3 do
-
-  describe 'manipulating S3 paths' do
-    before do
-      @resource = IMW::Resource.new('s3://mybucket/foobar/foo.txt')
-    end
-
-    it "should set the bucket" do
-      @resource.bucket.should == 'mybucket'
-    end
-
-    it "can generate an S3N url" do
-      @resource.s3n_url.should == 's3n://mybucket/foobar/foo.txt'
-    end
-
-    it "can join path segments" do
-      @resource.join('a', 'b/c').to_s.should == File.join(@resource.to_s, 'a/b/c')
-    end
-  end
-
-  describe "reading S3 files" do
-    before { IMW::Schemes::S3.make_connection! }
-    ['file', 'file with spaces', 'file with # fragment'].each do |f|
-      it "can read a file named '#{f}' from S3" do
-        IMW::Resource.new("s3://imw.infinitemonkeys.info/spec/schemes/s3/#{f}").read.chomp.should == 'ok'
-      end
-    end
-  end
-end

data/spec/imw/schemes/sql_spec.rb

@@ -1,3 +0,0 @@
-require File.join(File.dirname(__FILE__),'../../spec_helper')
-
-

data/spec/imw/tools/aggregator_spec.rb

@@ -1,71 +0,0 @@
-require File.dirname(__FILE__) + "/../../spec_helper"
-
-describe IMW::Tools::Aggregator do
-  before do
-    @dir = 'agg_dir'
-    FileUtils.mkdir_p(@dir)
-
-    # remote files
-    @homepage = "http://www.google.com"
-    @website  = "http://www.google.com/support/"
-    @remote_files = [@homepage, @website]
-
-    # regular files
-    @csv  = "foobar-csv.csv"
-    @xml  = "foobar-xml.xml"
-    @txt  = "foobar-txt.txt"
-    @blah = "foobar"
-    @files = [@csv, @xml, @txt, @blah]
-
-    # compressed files
-    @bz2  = "foobar-bz2.bz2"
-    @gz   = "foobar-gz.gz"
-    @compressed_files = [@bz2, @gz]
-
-    # archives
-    @zip      = "foobar-zip.zip"
-    @tarbz2   = "foobar-tarbz2.tar.bz2"
-    @targz    = "foobar-targz.tar.gz"
-    @tar      = "foobar-tar.tar"
-    @rar      = "foobar-rar.rar"
-    @archives = [@zip, @tarbz2, @targz, @rar, @tar]
-
-    @local_files = @files + @compressed_files + @archives
-
-    @all_files = @remote_files + @local_files
-    
-    @local_files.each do |path|
-      IMWTest::Random.file path
-    end
-
-    @aggregator = IMW::Tools::Aggregator.new @dir
-  end
-    
-  it "should copy regular files to its directory" do
-    @aggregator.aggregate *@files
-    @aggregator.dir.path.should contain(*@files)
-    @files.each { |path| IMW.open(path).exist?.should be_true }
-  end
-
-  it "should copy remote files to its archive directory" do
-    @aggregator.aggregate *@remote_files
-    @aggregator.dir.path.should contain('_index', 'support') # _index from Http#effective_basename on http://www.google.com
-  end
-
-  it "should uncompress compressed files to its directory" do
-    @aggregator.aggregate *@compressed_files
-    @aggregator.dir.path.should     contain('foobar-bz2', 'foobar-gz')
-    @aggregator.dir.path.should_not contain(*@compressed_files)
-  end
-    
-  it "should copy the content of archive files to its archive directory (but not the actual archives)" do
-    @aggregator.aggregate *@archives
-    @archives.each do |archive|
-      @aggregator.dir.path.should_not contain(archive)
-      @aggregator.dir.path.should     contain(*IMW.open(archive).contents)
-    end
-  end
-  
-end
-
-

data/spec/imw/tools/archiver_spec.rb

@@ -1,120 +0,0 @@
-require File.dirname(__FILE__) + "/../../spec_helper"
-
-describe IMW::Tools::Archiver do
-  before do
-    @name = 'foobar'
-
-    # remote files
-    @homepage = "http://www.google.com"
-    @website  = "http://www.google.com/support/"
-    @remote_files = [@homepage, @website]
-
-    # regular files
-    @csv  = "foobar-csv.csv"
-    @xml  = "foobar-xml.xml"
-    @txt  = "foobar-txt.txt"
-    @blah = "foobar"
-    @files = [@csv, @xml, @txt, @blah]
-
-    # compressed files
-    @bz2  = "foobar-bz2.bz2"
-    @gz   = "foobar-gz.gz"
-    @compressed_files = [@bz2, @gz]
-
-    # archives
-    @zip      = "foobar-zip.zip"
-    @tarbz2   = "foobar-tarbz2.tar.bz2"
-    @targz    = "foobar-targz.tar.gz"
-    @tar      = "foobar-tar.tar"
-    @rar      = "foobar-rar.rar"
-    @archives = [@zip, @tarbz2, @targz, @rar, @tar]
-
-    @local_files = @files + @compressed_files + @archives
-
-    @all_files = @remote_files + @local_files
-    
-    @local_files.each do |path|
-      IMWTest::Random.file path
-    end
-
-    @archiver = IMW::Tools::Archiver.new @name, @all_files
-  end
-
-  after do
-    @archiver.clean!
-  end
-  
-  describe "preparing input files" do
-    
-    describe "before preparing input files" do
-      it "should not be prepared when initialized" do
-        @archiver.prepared?.should be_false
-      end
-    end
-
-    describe "after preparing files" do
-      before { @archiver.prepare! }
-
-      it "should be prepared" do
-        @archiver.prepared?.should be_true
-      end
-
-      it "should name its archive directory properly" do
-        @archiver.tmp_dir.should contain(@name)
-      end
-      
-      it "should copy regular files to its archive directory" do
-        @archiver.dir.should contain(*@files)
-        @local_files.each { |path| IMW.open(path).exist?.should be_true }
-      end
-
-      it "should copy remote files to its archive directory" do
-        @archiver.dir.should contain('_index', 'support') # _index from Http#effective_basename on http://www.google.com
-      end
-
-      it "should uncompress compressed files to its archive directory" do
-        @archiver.dir.should     contain('foobar-bz2', 'foobar-gz')
-        @archiver.dir.should_not contain(*@compressed_files)
-      end
-      
-      it "should copy the content of archive files to its archive directory (but not the actual archives)" do
-        @archives.each do |archive|
-          @archiver.dir.should_not contain(archive)
-          @archiver.dir.should     contain(*IMW.open(archive).contents)
-        end
-      end
-
-    end
-  end
-  
-  describe "when packaging files" do
-    @packages = ["package.tar.bz2", "package.zip", "package.tar.gz", "package.tar", "package.rar"]
-
-    @packages.each do |package|    
-      it "should create a #{package} file containing all the files and return it" do
-        output = @archiver.package!(package)
-        output.basename.should == package
-        @archiver.tmp_dir.should contain(IMW.open(package).contents)
-      end
-    end
-
-    describe 'when packaging into multiple output formats' do
-
-      it "should prepare input files without being asked" do
-        @archiver.prepared?.should be_false
-        @archiver.package! 'package.tar.bz2'
-        @archiver.prepared?.should be_true
-      end
-      
-      it "should not prepare input files once they've already been prepared" do
-        @archiver.prepared?.should be_false
-        @archiver.package! 'package.tar.bz2'
-        @archiver.prepared?.should be_true        
-        @archiver.should_not_receive(:prepare!)
-        @archiver.package! 'package.tar.gz'
-      end
-    end
-  end
-end
-
-