imw 0.2.16 → 0.2.17

data/lib/imw/resource.rb

@@ -73,23 +73,20 @@ module IMW
     #
     # @param [String, Addressable::URI] uri
     # @param [Hash] options
-    # @option options [true, false] no_modules
     # @option options [String] mode the mode to open the resource in (will be ignored when inapplicable)
-    # @option options [IMW::Metadata::Record, Array] schema the schema of this resource
     # @return [IMW::Resource]
     def initialize uri, options={}
       self.uri              = uri
       self.resource_options = options
       self.mode             = options[:mode] || 'r'
-      self.schema           = options[:schema]   if options[:schema]
       extend_appropriately!(options)
     end
 
     # Provides resources with a wrapped Addressable::URI object.
     include IMW::Utils::HasURI
 
-    # Provides resources with a schema.
-    include IMW::Metadata::Schematized
+    # Provides resources with a summary, metadata, & schema.
+    include IMW::Metadata::HasSummary
     
     # Gives IMW::Resource instances with the ability to dynamically
     # extend themselves with modules chosen from a set of handlers

data/lib/imw/schemes/http.rb

@@ -35,7 +35,7 @@ module IMW
       def effective_basename
         (basename.blank? || basename =~ %r{^/*$}) ? "_index" : basename
       end
-
+      
       # Send a GET request to this resource's URI.
       #
       # If the response doesn't have HTTP code 2xx, a RestClient

data/lib/imw/schemes/local.rb

@@ -71,6 +71,8 @@ module IMW
       # Defines methods for appropriate for a local file.
       module LocalFile
 
+        include IMW::Metadata::HasMetadata
+        
         # Is this resource a regular file?
         #
         # @return [true, false]
@@ -173,7 +175,7 @@ module IMW
         def snippet
           returning([]) do |snip|
             (io.read(1024) || '').bytes.each do |byte|
-                                        # CR            LF          SPACE            ~
+              # CR            LF          SPACE            ~
               snip << byte.chr if byte == 13 || byte == 10 || byte >= 32 && byte <= 126
             end
           end.join
@@ -206,19 +208,17 @@ module IMW
         # - basename
         # - size
         # - extension
-        # - snippet
-        def summary
-          data = {
-            :basename  => basename,
-            :size      => size,
-            :extension => extension,
-            :num_lines => num_lines
-          }
-          data[:snippet] = snippet if respond_to?(:snippet)
-          data[:schema]  = schema  if respond_to?(:schema)
-          data
+        # - num_lines
+        def external_summary
+          super().merge({
+                          :size      => size,
+                          :num_lines => num_lines
+                        })
         end
 
+
+        
+
         protected
         
         # Return a triple of line, word, and character counts for this
@@ -231,7 +231,7 @@ module IMW
           @wc ||= begin
                     `wc #{path}`.chomp.strip.split.map(&:to_i)
                   rescue
-                    [0,0,0] # FIXME
+                    [nil,nil,nil] # FIXME
                   end
         end
 
@@ -388,44 +388,16 @@ module IMW
           end
         end
         
-        # Return a hash summarizing this directory with a key
-        # <tt>:contents</tt> containing an array of hashes summarizing
-        # this directories contents.
-        #
         # The directory summary includes the following information
-        # - basename
         # - size
         # - num_files
-        # - contents
         #
         # @return [Hash]
-        def summary
-          {
-            :basename  => basename,
-            :size      => size,
-            :num_files => contents.length,
-            :contents  => resources.map do |resource|
-              resource.guess_schema! if guess_schema? && resource.respond_to?(:guess_schema!)
-              resource_summary = resource.summary
-              resource_summary[:schema] = metadata[resource] if metadata && metadata.describe?(resource) # this should be handled by 'resources' method above
-              resource_summary
-            end
-          }
-        end
-
-        # Whether or not to have this directory's resources guess
-        # their schemas when none is provided.
-        #
-        # @return [true, false]
-        def guess_schema?
-          (!! @guess_schema)
-        end
-
-        # Force this directory's resources to guess at their schema.
-        #
-        # @return [true]
-        def guess_schema!
-          @guess_schema = true
+        def external_summary
+          super().merge({
+                          :size      => size,
+                          :num_files => contents.length,
+                        })
         end
 
       end

data/lib/imw/schemes/sql.rb

@@ -46,6 +46,18 @@ module IMW
           end
         end
 
+        # Return a summary of this database.
+        #
+        # Purposefully does not call +super+.
+        #
+        # @return [Hash]
+        def external_summary
+          {
+            :uri      => uri.to_s,
+            :database => database
+          }
+        end
+
         # The (cached) database connection for this resource.
         #
         # @return [DBI::DatabaseHandle]

data/lib/imw/tools/summarizer.rb

@@ -43,26 +43,25 @@ module IMW
 
       # Return a summary of the +inputs+ to this Summarizer.
       #
-      # Delegates to the +summary+ method of each constituent
-      # IMW::Resource in +inputs+.
+      # Will swallow errors.
       #
       # @return [Array<Hash>]
       def summary
-        @summary ||= inputs.map do |input|
-          #input.guess_schema! if input.schema.nil? && input.respond_to?(:guess_schema!)
-          (input.respond_to?(:summary) ? input.summary : {}) rescue {}
-        end
+        @summary ||= summary! rescue []
       end
 
-      # The metadata employed by this Summarizer.
-      #
-      # It can be set by setting <tt>options[:metadata]</tt>.
+      # Return a summary of the +inputs+ to this summarizer.
+      # 
+      # Delegates to the +summary+ method of each constituent
+      # IMW::Resource in +inputs+.
       #
-      # @return [IMW::Metadata, nil]
-      def metadata
-        @metadata ||= options[:metadata] && IMW::Metadata.load(options[:metadata])
+      # @return [Array]
+      def summary!
+        inputs.map do |input|
+          (input.respond_to?(:summary) ? input.summary : {})
+        end
       end
-
+      
       protected
       # Set new inputs for this summarizer.
       #

data/spec/imw/formats/delimited_spec.rb

@@ -18,32 +18,23 @@ describe IMW::Formats::Csv do
     IMW.open('test.csv').load[1].last.should == "4"
   end
 
-  it "should raise an error on an invalid schema" do
-    lambda { @sample.schema = [{:name => :foobar, :has_many => {:associations => [:foo, :bar]}}] }.should raise_error(IMW::SchemaError)
-  end
-
-  it "should accept a valid schema" do
-    @sample.schema = [:foo, :bar, :baz]
-    @sample.schema.should == [{:name => 'foo'}, {:name => 'bar'}, {:name => 'baz'}]
-  end
-
   describe "guessing a schema" do
 
     Dir[File.join(IMWTest::DATA_DIR, 'formats/delimited/with_schema/*')].each do |path|
       it "should correctly guess that with_schema/#{File.basename(path)} has headers in its first row" do
-        IMW.open(path).headers_in_first_line?.should == true
+        IMW.open(path).fields_in_first_line?.should == true
       end
     end
 
     Dir[File.join(IMWTest::DATA_DIR, 'formats/delimited/without_schema/*')].each do |path|
       it "should correctly guess that without_schema/#{File.basename(path)} does not have headers in its first row" do
-        IMW.open(path).headers_in_first_line?.should == false
+        IMW.open(path).fields_in_first_line?.should == false
       end
     end
 
     it "should automatically set the headers on a source with guessed headers" do
       resource = IMW.open(Dir[File.join(IMWTest::DATA_DIR, 'formats/delimited/with_schema/*')].first)
-      resource.guess_schema!
+      resource.guess_fields!
       resource.delimited_options[:headers].class.should == Array
       resource.schema.should_not be_empty
     end

data/spec/imw/metadata/contains_metadata_spec.rb

@@ -0,0 +1,56 @@
+require File.dirname(__FILE__) + "/../../spec_helper"
+
+describe IMW::Metadata::ContainsMetadata do
+
+  before do
+    class Foo
+      attr_accessor :contents
+      def path     ;  IMWTest::TMP_DIR ; end
+      def basename ;  File.basename(IMWTest::TMP_DIR) ; end
+      include IMW::Metadata::ContainsMetadata
+    end
+    @foo = Foo.new
+    @foo.contents = []
+  end
+
+  describe 'finding the default metadata URI' do
+    it "should return the default metadata URI when 'contents' is empty" do
+      @foo.default_metadata_uri.should == File.join(IMWTest::TMP_DIR, File.basename(IMWTest::TMP_DIR) + ".icss.yaml")
+    end
+
+    it "should return the default metadata URI when 'contents' doesn't contain any metadata files" do
+      @foo.contents.concat ['bar.txt', 'crazy_file.yaml', 'foo.json'].map { |p| File.join(IMWTest::TMP_DIR, p) }
+      @foo.default_metadata_uri.should == File.join(IMWTest::TMP_DIR, File.basename(IMWTest::TMP_DIR) + ".icss.yaml")
+    end
+
+    %w[my-projects.icss.yaml stupid-crazy-fool-of-a-dataset-icss.json foobar-25.metadata.buzz.yml].each do |basename|
+      it "should return the metadata URI when 'contents' contains a URI matching '#{basename}'" do
+        @foo.contents.concat ['bar.txt', 'crazy_file.yaml', 'foo.json', basename].map { |p| File.join(IMWTest::TMP_DIR, p) }
+        @foo.default_metadata_uri.should == File.join(IMWTest::TMP_DIR, basename)
+      end
+    end
+  end
+
+
+  describe 'returning its metadata' do
+    it "should return 'nil' when no metadata exists on disk" do
+      @foo.metadata.should be_nil
+    end
+
+    it "should return Metadata when metadata exists on disk" do
+      IMW.open!(@foo.default_metadata_uri) do |f|
+        f.write <<YAML
+---
+foo:
+  description: bar
+  fields: baz
+YAML
+      end
+      @foo.metadata.class.should == IMW::Metadata
+      @foo.metadata['foo']['description'].should == 'bar'
+    end
+  end
+  
+  
+
+end

data/spec/imw/metadata/field_spec.rb

@@ -3,17 +3,16 @@ require File.dirname(__FILE__) + "/../../spec_helper"
 describe IMW::Metadata::Field do
 
   describe "initializing" do
-    it "should parse a symbol or string into a hash" do
-      IMW::Metadata::Field.new(:foobar).should  == { :name => "foobar" }
-      IMW::Metadata::Field.new('foobar').should == { :name => 'foobar' }
+    it "should parse a string into a hash" do
+      IMW::Metadata::Field.new('foobar').should == { "name" => 'foobar' }
     end
 
     it "should raise an error on a Hash without a :name key" do
-      lambda { IMW::Metadata::Field.new(:foo => 'bar') }.should raise_error(IMW::ArgumentError)
+      lambda { IMW::Metadata::Field.new('foo' => 'bar') }.should raise_error(IMW::ArgumentError)
     end
 
     it "should accept a Hash with a :name key" do
-      data = { :name => :foobar, :title => "Bazbooz", :unit => "m" }
+      data = { 'name' => :foobar, 'title' => "Bazbooz", 'unit' => "m" }
       IMW::Metadata::Field.new(data).should == data
     end
 

data/spec/imw/metadata/has_metadata_spec.rb

@@ -0,0 +1,58 @@
+require File.dirname(__FILE__) + "/../../spec_helper"
+
+describe IMW::Metadata::HasMetadata do
+
+  before do
+    class Foo
+      def uri ; File.join(IMWTest::TMP_DIR, 'test', 'subdir', 'foobar.csv') ; end
+      def basename ; File.basename(uri) ; end
+      def extension ; 'csv' ; end
+      def dir ; IMW.open(File.join(IMWTest::TMP_DIR, 'test', 'subdir')) ; end
+      include IMW::Metadata::HasMetadata
+    end
+    @foo = Foo.new
+  end
+
+  it "should be able to build a schema" do
+    @foo.schema.should include(:type, :namespace, :name, :doc, :fields, :non_avro)
+  end
+  
+  describe "finding its metadata" do
+
+    before do
+      FileUtils.mkdir_p(@foo.dir.path)
+      IMWTest::Random.file(File.join(@foo.dir.path, 'foobar.csv'))
+    end
+
+    it "should return 'nil' when it can't find any metadata" do
+      @foo.metadata.should be_nil
+    end
+
+    it "should return 'nil' when a metadata file is found that doesn't describe it" do
+      IMW.open!("has_metadata_test.icss.yaml") do |f|
+        f.write <<YAML
+---
+foobar.csv:
+  description: bar
+  fields: ["baz", "booz"]
+YAML
+      end
+      @foo.metadata.should be_nil
+    end
+
+#     it "should return the metadata when a metadata file is found that does describe it" do
+#       IMW.open!("has_metadata_test.icss.yaml") do |f|
+#         f.write <<YAML
+# ---
+# #{IMWTest::TMP_DIR}/test/subdir/foobar.csv:
+#   description: bar
+#   fields: ["baz", "booz"]
+# YAML
+#       end
+#       @foo.metadata.class.should == IMW::Metadata
+#       @foo.metadata[@foo]['description'].should == 'bar'
+#     end
+    
+  end
+  
+end

data/spec/imw/metadata/has_summary_spec.rb

@@ -0,0 +1,32 @@
+require File.dirname(__FILE__) + "/../../spec_helper"
+
+describe IMW::Metadata::HasSummary do
+
+  before do
+    class Foo
+      def initialize(*args) ; @args = args ; end
+      def uri ;               File.join(IMWTest::TMP_DIR, *@args) ; end
+      def basename ;          File.basename(uri) ; end
+      def extension ;         File.extname(@args.last || '').gsub(/^\./,'') ; end
+      include IMW::Metadata::HasSummary
+    end
+    @foo = Foo.new('foo', 'bar.csv')
+  end
+
+  it "should build a summary from an external summary" do
+    @foo.summary.should include(:uri, :basename, :extension)
+  end
+
+  it "should build a summary from an external summary and a schema when possible" do
+    @foo.stub!(:schema).and_return({:foo => 'bar'})
+    @foo.summary[:schema].should == {:foo => 'bar'}
+  end
+
+  it "should be able to build an external summary describing how it's situated in the world" do
+    @foo.summary[:uri].should       == File.join(IMWTest::TMP_DIR, 'foo', 'bar.csv')
+    @foo.summary[:basename].should  == 'bar.csv'
+    @foo.summary[:extension].should == 'csv'
+  end
+
+end
+

data/spec/imw/metadata/schema_spec.rb

@@ -3,25 +3,22 @@ require File.dirname(__FILE__) + "/../../spec_helper"
 describe IMW::Metadata::Schema do
 
   describe "initializing" do
-    it "should accept an array" do
-      IMW::Metadata::Schema.new([1,2,3]).should == [{:name => '1'}, {:name => '2'}, {:name => '3'}]
+    it "should merge with a Hash" do
+      IMW::Metadata::Schema.new({:foo => 'foobar'}).should == { :foo => 'foobar' }
     end
 
-    it "should dup a Schema if given one" do
-      orig_schema = IMW::Metadata::Schema.new([1,2,3])
-      IMW::Metadata::Schema.new(orig_schema).should == orig_schema
+    it "should merge with a Schema" do
+      IMW::Metadata::Schema.new(IMW::Metadata::Schema.new({:foo => 'foobar'})).should == { :foo => 'foobar' }
     end
-  end
 
-  describe 'loading' do
-    it "should load an Array in a resource" do
-      resource = IMW.open('some_resource')
-      resource.should_receive(:load).and_return(%w[foo bar baz])
-      IMW.should_receive(:open).and_return(resource)
-      IMW::Metadata::Schema.load(resource.to_s).map { |field| field[:name] }.should == %w[foo bar baz]
+    it "should ignore anything else" do
+      IMW::Metadata::Schema.new('foobar').should == {}
     end
 
+    it "should accept empty args" do
+      IMW::Metadata::Schema.new.should == {}
+    end
+    
   end
   
 end
-