imw 0.2.7 → 0.2.8

data/spec/imw/formats/excel_spec.rb

@@ -0,0 +1,55 @@
+require File.join(File.dirname(__FILE__),'../../spec_helper')
+
+describe IMW::Formats::Excel do
+  
+  before do
+    @sample = IMW.open(File.join(IMWTest::DATA_DIR, 'formats/excel/sample.xls'))
+  end
+
+  it "should be able to parse the Excel document" do
+    @sample.load[1].last.should == 'lemurinus'
+  end
+
+  it "should be able to create a snippet" do
+    @sample.snippet[1].last.should == 'lemurinus'
+  end
+
+  # it "should be able to write CSV" do
+  #   data = [['foobar', 1, 2], ['bazbooz', 3, 4]]
+  #   IMW.open!('test.csv').emit(data)
+  #   IMW.open('test.csv').load[1].last.should == "4"
+  # end
+
+  # it "should raise an error on an invalid schema" do
+  #   lambda { @sample.schema = [{:name => :foobar, :has_many => {:associations => [:foo, :bar]}}] }.should raise_error(IMW::SchemaError)
+  # end
+
+  # it "should accept a valid schema" do
+  #   @sample.schema = [:foo, :bar, :baz]
+  #   @sample.schema.should == [{:name => 'foo'}, {:name => 'bar'}, {:name => 'baz'}]
+  # end
+
+  # describe "guessing a schema" do
+
+  #   Dir[File.join(IMWTest::DATA_DIR, 'formats/delimited/with_schema/*')].each do |path|
+  #     it "should correctly guess that with_schema/#{File.basename(path)} has headers in its first row" do
+  #       IMW.open(path).headers_in_first_line?.should == true
+  #     end
+  #   end
+
+  #   Dir[File.join(IMWTest::DATA_DIR, 'formats/delimited/without_schema/*')].each do |path|
+  #     it "should correctly guess that without_schema/#{File.basename(path)} does not have headers in its first row" do
+  #       IMW.open(path).headers_in_first_line?.should == false
+  #     end
+  #   end
+
+  #   it "should automatically set the headers on a source with guessed headers" do
+  #     resource = IMW.open(Dir[File.join(IMWTest::DATA_DIR, 'formats/delimited/with_schema/*')].first)
+  #     resource.guess_schema!
+  #     resource.delimited_options[:headers].class.should == Array
+  #     resource.schema.should_not be_empty
+  #   end
+
+  # end
+  
+end

data/spec/imw/formats/json_spec.rb

@@ -3,15 +3,15 @@ require File.join(File.dirname(__FILE__),'../../spec_helper')
 describe IMW::Formats::Json do
 
   before do
-    @sample = IMW.open(File.join(IMWTest::DATA_DIR, 'sample.json'))
+    @sample = IMW.open(File.join(IMWTest::DATA_DIR, 'formats/json/sample.json'))
   end
 
   it "should be able to parse the JSON" do
-    @sample.load.first['id'].should == 1
+    @sample.load["Aotus"].first['id'].should == 1
   end
 
   it "should be able to write JSON" do
-    IMW.open!('test.json').dump({ 'foobar' => 3, 'bazbooz' => 4 })
+    IMW.open!('test.json') { |f| f.emit({ 'foobar' => 3, 'bazbooz' => 4 }) }
     IMW.open('test.json').load['foobar'].should == 3
   end
   

data/spec/imw/formats/sgml_spec.rb

@@ -4,21 +4,21 @@ describe IMW::Formats::Xml do
   # just spec Xml now as the others are identical
 
   before do
-    @sample = IMW.open(File.join(IMWTest::DATA_DIR, 'sample.xml'))
+    @sample = IMW.open(File.join(IMWTest::DATA_DIR, 'formats/sgml/sample.xml'))
   end
 
   it "should be able to load the XML" do
-    ((@sample.load/"monkey").first/"genus").inner_text.should == 'Aotus'
+    ((@sample.load/"genus").first/"name").first.inner_text.should == 'Mandrillus'
   end
 
   it "should yield the XML when load is given a block" do
     @sample.load do |xml|
-      ((xml/"monkey").first/"genus").inner_text.should == 'Aotus'
+      ((xml/"genus").first/"name").first.inner_text.should == 'Mandrillus'
     end
   end
 
   it "should parse the XML" do
-    @sample.parse(:monkeys => ['monkey'])[:monkeys].size.should == 130
+    @sample.parse(:species => ['species[@id]'])[:species].size.should == 130
   end
 end
 

data/spec/imw/formats/yaml_spec.rb

@@ -3,16 +3,16 @@ require File.join(File.dirname(__FILE__),'../../spec_helper')
 describe IMW::Formats::Yaml do
 
   before do
-    @sample = IMW.open(File.join(IMWTest::DATA_DIR, 'sample.yaml'))
+    @sample = IMW.open(File.join(IMWTest::DATA_DIR, 'formats/yaml/sample.yaml')).load
   end
 
   it "should be able to parse the YAML" do
-    @sample.first['id'].should == 1
+    @sample['Lophocebus'].first[:id].should == 94
   end
 
   it "should be able to write YAML" do
     data = { 'foobar' => 3, 'bazbooz' => 4 }
-    IMW.open!('test.yaml').dump(data)
+    IMW.open!('test.yaml') { |f| f.emit(data) }
     IMW.open('test.yaml').load['foobar'].should == 3
   end
   

data/spec/imw/metadata/field_spec.rb

@@ -0,0 +1,26 @@
+require File.dirname(__FILE__) + "/../../spec_helper"
+
+describe IMW::Metadata::Field do
+
+  describe "initializing" do
+    it "should parse a symbol or string into a hash" do
+      IMW::Metadata::Field.new(:foobar).should  == { :name => "foobar" }
+      IMW::Metadata::Field.new('foobar').should == { :name => 'foobar' }
+    end
+
+    it "should raise an error on a Hash without a :name key" do
+      lambda { IMW::Metadata::Field.new(:foo => 'bar') }.should raise_error(IMW::ArgumentError)
+    end
+
+    it "should accept a Hash with a :name key" do
+      data = { :name => :foobar, :title => "Bazbooz", :unit => "m" }
+      IMW::Metadata::Field.new(data).should == data
+    end
+
+    it "should dup a field if given one" do
+      orig_field = IMW::Metadata::Field.new('foobar')
+      IMW::Metadata::Field.new(orig_field).should == orig_field
+    end
+  end
+  
+end

data/spec/imw/metadata/schema_spec.rb

@@ -0,0 +1,27 @@
+require File.dirname(__FILE__) + "/../../spec_helper"
+
+describe IMW::Metadata::Schema do
+
+  describe "initializing" do
+    it "should accept an array" do
+      IMW::Metadata::Schema.new([1,2,3]).should == [{:name => '1'}, {:name => '2'}, {:name => '3'}]
+    end
+
+    it "should dup a Schema if given one" do
+      orig_schema = IMW::Metadata::Schema.new([1,2,3])
+      IMW::Metadata::Schema.new(orig_schema).should == orig_schema
+    end
+  end
+
+  describe 'loading' do
+    it "should load an Array in a resource" do
+      resource = IMW.open('some_resource')
+      resource.should_receive(:load).and_return(%w[foo bar baz])
+      IMW.should_receive(:open).and_return(resource)
+      IMW::Metadata::Schema.load(resource.to_s).map { |field| field[:name] }.should == %w[foo bar baz]
+    end
+
+  end
+  
+end
+

data/spec/imw/metadata_spec.rb

@@ -0,0 +1,39 @@
+require File.dirname(__FILE__) + "/../spec_helper"
+
+describe IMW::Metadata do
+
+  describe "initalizing" do
+
+    it "should accept a hash" do
+      IMW::Metadata.new('a' => ['a', 'b']).should == { 'a' => [{:name => 'a'}, {:name => 'b'}] }
+    end
+  end
+
+  describe 'loading' do
+
+    it "should accept a Hash in a resource" do
+      data = {'a' => ['a', 'b']}
+      resource = IMW.open('some_resource')
+      IMW.should_receive(:open).with(resource).and_return(resource)
+      resource.should_receive(:load).and_return(data)
+      IMW::Metadata.load(resource).should == { 'a' => [{:name => 'a'}, {:name => 'b'}] }
+    end
+  end
+
+  describe "constructing absolute URIs" do
+
+    before { @metadata = IMW::Metadata.new }
+    
+    it "should return the resource given without a base" do
+      @metadata.send(:absolute_uri, 'path/to/something').should == 'path/to/something'
+    end
+
+    it "should return the absolute URI with a base" do
+      path = File.join(IMWTest::TMP_DIR, 'metadata.yaml')
+      FileUtils.mkdir_p(path)
+      @metadata.base = path
+      @metadata.send(:absolute_uri, 'path/to/something').should == File.join(IMWTest::TMP_DIR, '/path/to/something')
+    end
+  end
+  
+end

data/spec/imw/parsers/line_parser_spec.rb

@@ -4,7 +4,7 @@ require 'ostruct'
 describe IMW::Parsers::LineParser do
 
   before do
-    @path = File.dirname(__FILE__) + "/../../data/sample.csv"
+    @path = File.join(IMWTest::DATA_DIR, 'formats/delimited/sample.csv')
     @file = File.new(@path)
     @fields = [:id, :name, :genus, :species]
   end

data/spec/imw/resource_spec.rb

@@ -25,106 +25,6 @@ describe IMW::Resource do
     end
   end
 
-  describe "parsing various and sundry URIs should correctly parse a" do
-    
-    before do
-      IMW::Resource.should_receive(:extend_resource!).with(an_instance_of(IMW::Resource), {})
-    end
-
-    it "local file path" do
-      resource = IMW::Resource.new("/home/foo.txt")
-      resource.stub!(:path).and_return("/home/foo.txt")
-
-      resource.scheme.should    be_nil
-      resource.dirname.should   == '/home'
-      resource.basename.should  == 'foo.txt'
-      resource.extname.should   == '.txt'
-      resource.extension.should == 'txt'
-      resource.name.should      == 'foo'
-    end
-
-    it "local file path with spaces in the name" do
-      resource = IMW::Resource.new("/home/foo bar.txt")
-      resource.stub!(:path).and_return("/home/foo bar.txt")
-      resource.name.should == 'foo bar'
-    end
-
-    it "local file path with an explicit file:// scheme" do
-      resource = IMW::Resource.new("file:///home/foo.txt")
-      resource.scheme.should == 'file'
-    end
-
-    it "web URL with query and fragment" do
-      resource = IMW::Resource.new("http://mysite.com/some/page?param=value#frag")
-      resource.stub!(:path).and_return("/some/page")
-      resource.scheme.should        == 'http'
-      resource.dirname.should       == '/some'
-      resource.basename.should      == 'page'
-      resource.extname.should       == ''
-      resource.extension.should     == ''
-      resource.name.should          == 'page'
-    end
-
-  end
-
-  it "should open a URI without attempting to extend with modules if so asked" do
-    IMW::Resource.should_not_receive(:extend_resource!)
-    IMW::Resource.new("/path/to/some/file.txt", :no_modules => true)
-  end
-
-  describe "extending resources with specific modules" do
-    before do 
-      @resource = IMW::Resource.new('http://www.infochimps.com/data', :no_modules => true)
-    end
-
-    it "should use a specific module when asked with a string" do
-      IMW::Resource.extend_resource!(@resource, :use_modules => ["Formats::Csv"])
-      @resource.resource_modules.should include(IMW::Formats::Csv)
-    end
-
-    it "should use a specific module when asked with a module" do
-      IMW::Resource.extend_resource!(@resource, :use_modules => [IMW::Formats::Csv])
-      @resource.resource_modules.should include(IMW::Formats::Csv)
-    end
-
-    it "should not use a specific module when asked with a string" do
-      IMW::Resource.extend_resource!(@resource, :skip_modules => ["Schemes::HTTP"])
-      @resource.resource_modules.should_not include(IMW::Schemes::HTTP)
-    end
-
-    it "should not use a specific module when asked with a module" do
-      IMW::Resource.extend_resource!(@resource, :skip_modules => [IMW::Schemes::HTTP])
-      @resource.resource_modules.should_not include(IMW::Schemes::HTTP)
-    end
-    
-  end
-
-  describe "registering a new handler" do
-    after do
-      IMW::USER_DEFINED_HANDLERS.delete_if { true }
-    end
-
-    it "should raise an error if the module given isn't a module or string" do
-      lambda { IMW.register_handler 3,     // }.should     raise_error(IMW::ArgumentError)
-      lambda { IMW.register_handler "IMW", // }.should_not raise_error(IMW::ArgumentError)
-      lambda { IMW.register_handler IMW,   // }.should_not raise_error(IMW::ArgumentError)
-    end
-
-    it "should raise an error if the handler given isn't a Regexp, Proc, or true" do
-      lambda { IMW.register_handler IMW, 3                     }.should     raise_error(IMW::ArgumentError)
-      lambda { IMW.register_handler IMW, /foo/                 }.should_not raise_error(IMW::ArgumentError)
-      lambda { IMW.register_handler IMW, Proc.new { |r| true } }.should_not raise_error(IMW::ArgumentError)
-      lambda { IMW.register_handler IMW, true                  }.should_not raise_error(IMW::ArgumentError)      
-    end
-
-    it "should use a valid handler when appropriate" do
-      NewModule = Module.new
-      IMW.register_handler NewModule, /\.foo$/
-      IMW.open('/path/to/something.foo').resource_modules.should include(NewModule)
-    end
-
-  end
-
 end
 
 

data/spec/imw/schemes/hdfs_spec.rb

@@ -2,13 +2,13 @@ require File.join(File.dirname(__FILE__),'../../spec_helper')
 
 describe IMW::Schemes::HDFS do
   before do
-    def fake_hdfs_resource path, num_dirs=nil, num_files=nil, size=nil
-      if num_dirs && num_files && size
-        response = "           #{num_dirs}            #{num_files}              #{size} hdfs://localhost#{path}"
-      else
+    def fake_hdfs_resource path, options={}
+      if options == false
         response = ""
+      else        
+        response = "           #{options[:dirs] || 0}            #{options[:files] || 1}              #{options[:size] || 1000} hdfs://localhost#{path}"
       end
-      IMW::Schemes::HDFS.should_receive(:fs).with(:count, path).and_return(response)
+      IMW::Schemes::HDFS.should_receive(:fs).with(:count, path).at_least(:once).and_return(response)
       IMW.open("hdfs://#{path}")
     end
     @path = '/path/to/myfile'    
@@ -16,7 +16,7 @@ describe IMW::Schemes::HDFS do
   
   describe "refreshing its properties" do
     it "should correctly get properties for a resource which exists" do
-      resource = fake_hdfs_resource(@path, 2, 3, 1000)
+      resource = fake_hdfs_resource(@path, :dirs => 2, :files => 3, :size => 1000)
       resource.exist?.should    be_true
       resource.num_dirs.should  == 2
       resource.num_files.should == 3
@@ -24,38 +24,44 @@ describe IMW::Schemes::HDFS do
     end
 
     it "should gracefully handle a resource which doesn't exist" do
-      resource = fake_hdfs_resource(@path)
+      resource = fake_hdfs_resource(@path, false)
       resource.exist?.should be_false
     end
 
     it "should execute the correct command to delete the path" do
-      resource = fake_hdfs_resource(@path, 2, 3, 1000)
+      resource = fake_hdfs_resource(@path)
       IMW::Schemes::HDFS.should_receive(:fs).with(:rm, resource.path)
       resource.rm
     end
 
     it "should execute the correct command to delete the path when skipping the trash" do
-      resource = fake_hdfs_resource(@path, 2, 3, 1000)      
+      resource = fake_hdfs_resource(@path)
       IMW::Schemes::HDFS.should_receive(:fs).with(:rm, '-skipTrash', resource.path)
       resource.rm :skip_trash => true
     end
 
     it "should recognize a file and extend it properly" do
-      resource = fake_hdfs_resource(@path, 0, 1, 1000)
+      resource = fake_hdfs_resource(@path)
       resource.num_dirs.should == 0      
       resource.num_files.should == 1
       resource.exist?.should be_true
       resource.is_directory?.should be_false
-      resource.resource_modules.should include(IMW::Schemes::HDFSFile)
+      resource.modules.should include(IMW::Schemes::HDFSFile)
     end
 
     it "should recognize a directory and extend it properly" do
-      resource = fake_hdfs_resource(@path, 2, 1, 1000)
+      resource = fake_hdfs_resource(@path, :dirs => 2, :files => 1)
       resource.num_dirs.should == 2
       resource.num_files.should == 1
       resource.exist?.should be_true
       resource.is_directory?.should be_true
-      resource.resource_modules.should include(IMW::Schemes::HDFSDirectory)
+      resource.modules.should include(IMW::Schemes::HDFSDirectory)
+    end
+
+    it "should be able to join path segments to a directory" do
+      resource     = fake_hdfs_resource(@path, :dirs => 2)
+      sub_resource = fake_hdfs_resource("#{@path}/a/b/c")
+      resource.join('a', 'b/c').to_s.should == sub_resource.to_s
     end
   end
 end

data/spec/imw/schemes/local_spec.rb

@@ -5,13 +5,13 @@ describe IMW::Schemes::Local::Base do
   it "should not extend a local file with LocalDirectory" do
     @file = IMW::Resource.new('foo.txt', :no_modules => true)
     @file.should_not_receive(:extend).with(IMW::Schemes::Local::LocalDirectory)
-    @file.extend_appropriately!
+    IMW::Resource.extend_instance!(@file)
   end
 
   it "should not extend a local directory with LocalFile" do
     @dir = IMW::Resource.new(IMWTest::TMP_DIR, :no_modules => true)
     @dir.should_not_receive(:extend).with(IMW::Schemes::Local::LocalFile)
-    @dir.extend_appropriately!
+    IMW::Resource.extend_instance!(@dir)
   end
 
   it "should correctly resolve relative paths" do
@@ -37,7 +37,6 @@ describe IMW::Schemes::Local::LocalFile do
     @file.exist?.should be_false
   end
 
-
   it "can read a file" do
     @file.read.size.should > 0
   end
@@ -60,6 +59,12 @@ describe IMW::Schemes::Local::LocalFile do
       line[0..5]
     end.class.should == Array
   end
+
+  it "can produce a snippet" do
+    path = IMWTest::DATA_DIR + "/formats/none/sample"
+    # FIXME only look at the first 100 bytes b/c of subsequent non-ascii chars...
+    IMW.open(path).snippet[0..100].should == File.new(path).read(101)
+  end
 end
 
 describe IMW::Schemes::Local::LocalDirectory do
@@ -100,6 +105,57 @@ describe IMW::Schemes::Local::LocalDirectory do
     @dir.resources.map(&:class).uniq.first.should == IMW::Resource
   end
 
+  describe "checking whether it contains other resources" do
+
+    it "should return false for remote paths" do
+      @dir.contains?("http://google.com").should be_false
+    end
+
+    it "should return true for its own path" do
+      @dir.contains?(@dir.path).should be_true
+    end
+
+    it "should return false for a path that doesn't start with its path" do
+      @dir.contains?(File.expand_path('foo')).should be_false
+    end
+
+    it "should return false for a path that starts with its path but doesn't exist" do
+      @dir.contains?(File.expand_path('dir/foo/baz')).should be_false
+    end
+
+    it "should return true for a path that starts with its path and exists" do
+      FileUtils.mkdir_p('dir/foo/baz')
+      @dir.contains?(File.expand_path('dir/foo/baz')).should be_true
+    end
+
+  end
+
+  describe "handling schemata" do
+
+    it "should recognize a YAML schema file" do
+      schemata_path = File.join(@dir.path, 'schema.yaml')
+      IMWTest::Random.file(schemata_path)
+      @dir.schemata_path.should == schemata_path
+    end
+
+    it "should recognize a JSON schema file" do
+      schemata_path = File.join(@dir.path, 'schema.json')
+      IMWTest::Random.file(schemata_path)
+      @dir.schemata_path.should == schemata_path
+    end
+
+    it "should recognize a funny-named YAML schema file" do
+      schemata_path = File.join(@dir.path, 'schema-1838293.yml')
+      IMWTest::Random.file(schemata_path)
+      @dir.schemata_path.should == schemata_path
+    end
+    
+  end
+
+  it "can join with a path" do
+    @dir.join("a", "b/c").to_s.should == File.join(@dir.path, 'a/b/c')
+  end
+
   describe 'can package itself to' do
     ['tar', 'tar.bz2', 'tar.gz', 'zip', 'rar'].each do |extension|
       it "a #{extension} archive" do