imw 0.2.16 → 0.2.17

data/VERSION

@@ -1 +1 @@
-0.2.16
+0.2.17

data/lib/imw/dataset.rb

@@ -119,7 +119,7 @@ module IMW
 
     # Provides this dataset with DSL like methods to construct a
     # schema in an IMW file.
-    include IMW::Metadata::DSL
+    # include IMW::Metadata::DSL
 
   end
 end

data/lib/imw/formats/delimited.rb

@@ -11,12 +11,6 @@ module IMW
     # @abstract
     module Delimited
 
-      # Ensure that this delimited resource is described by a an
-      # ordered collection of flat fields.
-      def validate_schema!
-        raise IMW::SchemaError.new("#{self.class} resources must be described by an ordered set of flat fields") if schema.any?(&:nested?)
-      end
-
       # Default options to be passed to
       # FasterCSV[http://fastercsv.rubyforge.org/]; see its
       # documentation for more information.
@@ -24,7 +18,7 @@ module IMW
       # @return [Hash]
       def delimited_options
         @delimited_options ||= {
-          :headers        => schema && schema.map { |field| field['name'] }
+          :headers        => fields && fields.map { |field| field['name'] }
         }.merge(resource_options_compatible_with_faster_csv)
       end
 
@@ -68,7 +62,7 @@ module IMW
       # of this delimited data is a row of headers.
       #
       # @return [true, false]
-      def headers_in_first_line?
+      def fields_in_first_line?
         # grab the header and up to 10 body rows
         require 'fastercsv'
         copy  = FasterCSV.new(io, resource_options_compatible_with_faster_csv.merge(:headers => false))
@@ -93,15 +87,16 @@ module IMW
         determinant && determinant >= 0.05
       end
 
-      # If it seems like there are headers in the first line of this
-      # data then go ahead and use them to define a schema.
+      # If it seems like there are fields in the first line of this
+      # data then go ahead and use them to define this resource's
+      # fields.
       #
-      # Will overwrite a schema already present for this resource.
-      def guess_schema!
-        return unless headers_in_first_line?
+      # Will overwrite any fields already present for this resource.
+      def guess_fields!
+        return unless fields_in_first_line?
         copy                        = FasterCSV.new(io, resource_options_compatible_with_faster_csv.merge(:headers => false))
         names                       = (copy.shift || []) rescue []
-        self.schema                 = IMW::Metadata::Schema.new(names)
+        self.fields                 = names.map { |n| { 'name' => n } }
         delimited_options[:headers] = names
       end
 

data/lib/imw/metadata.rb

@@ -4,13 +4,13 @@ module IMW
   # with a dataset's fields.
   class Metadata < Hash
     
-    autoload :Field,       'imw/metadata/field'
-    autoload :Schema,      'imw/metadata/schema'
-    autoload :Schematized, 'imw/metadata/schematized'
-    autoload :DSL,         'imw/metadata/dsl'
+    autoload :Field,            'imw/metadata/field'
+    autoload :Schema,           'imw/metadata/schema'
     autoload :ContainsMetadata, 'imw/metadata/contains_metadata'
+    autoload :HasSummary,       'imw/metadata/has_summary'
+    autoload :HasMetadata,      'imw/metadata/has_metadata'
 
-    # The resource this Schema is anchored to.
+    # The resource this metadata is anchored to.
     #
     # This attribute is useful for letting relative paths in a
     # schema file refer to a common base URL.
@@ -18,12 +18,12 @@ module IMW
     # @return [IMW::Resource]
     attr_reader :base
     
-    # Set the resource this Schema is anchored to.
+    # Set the base resource this metdata is anchored to.
     #
     # @param [IMW::Resource, String, Addressable::URI] new_base
     def base= new_base
       base_resource = IMW.open(new_base)
-      base_resource.should_exist!("Metdata base directory must exist")
+      base_resource.should_exist!("Metadata base directory must exist")
       raise IMW::PathError.new("Metadata base must be a directory") unless base_resource.is_directory?
       @base = base_resource
     end
@@ -31,34 +31,51 @@ module IMW
     def initialize obj=nil, options={}
       super()
       self.base = options[:base] if options[:base]
-      obj.each_pair { |resource, schema| self[resource] = Schema.new(schema) } if obj
+      if obj
+        obj.each_pair do |resource, metadata|
+          self[resource] = metadata
+        end
+      end
     end
 
-    def self.load metadata_resource, options
-      resource = IMW.open(metadata_resource)
+    def self.load obj, options={}
+      resource = IMW.open(obj)
       new(resource.load, {:base => resource.dirname}.merge(options))
     end
 
-    def []= resource_spec, schema_spec
-      schema = schema_spec.is_a?(Schema) ? schema_spec : Schema.new(schema_spec)
-      super(absolute_uri(resource_spec), schema_spec)
+    def []= resource, metadata
+      super(absolute_uri(resource), metadata)
+    end
+
+    def [] resource
+      super(absolute_uri(resource))
+    end
+
+    def describe? resource
+      self[(absolute_uri(resource))]
     end
+    alias_method :describes?, :describe?
 
-    def [] resource_spec
-      super(absolute_uri(resource_spec))
+    def description_for resource
+      return unless describes?(resource)
+      self[resource]['description']
     end
 
-    def describe? resource_spec
-      has_key?(absolute_uri(resource_spec))
+    def fields_for resource
+      return unless describes?(resource)
+      (self[resource]['fields'] || []).map { |f| Metadata::Field.new(f) }
     end
 
     protected
 
-    def absolute_uri resource_spec
-      if base && resource_spec.to_s !~ %r{(^/|://)} # relative path
-        base.join(resource_spec).to_s
+    def absolute_uri resource
+      obj = IMW.open(resource)
+      if base && obj.uri.to_s !~ %r{(^/|://)} # relative path
+        s = base.join(obj.uri.to_s).uri.to_s
+        s
       else
-        resource_spec.to_s
+        s = obj.uri.to_s
+        s
       end
     end
     

data/lib/imw/metadata/contains_metadata.rb

@@ -1,44 +1,54 @@
-module IMW
+ module IMW
   class Metadata
 
-    # A module that can be mixed into any class defining a +contents+
-    # methods which returns an Array of URI strings.
+    # A module for finding metadata describing the sub-resources of a
+    # given resource.
+    #
+    # An including class describing the parent resource must define
+    # the +contents+ method which must return an Array of Strings
+    # contained within the parent .  These objects will be matched
+    # against possible metadata URIs and the corresponding
+    # IMW::Metadata class created on the fly.
+    #
+    # In case no such object is found, the class should also define
+    # the +basename+ and +path+ methods which will be used to generate
+    # a default URI where metadata about the parent's resources should
+    # live.
     module ContainsMetadata
       
-      # The path at which this resource's metadata file lives.
+      # The URI containing the metadata for this resource and its
+      # contents.
       #
-      # Will default to any file beginning with +metadata+ and ending
-      # with a +yaml+ or +json+ extension contained in this resource's
-      # +contents+.
+      # Looks for an existing JSON or YAML file containing the strings
+      # "icss" or "metadata" directly contained within this resource.
+      #
+      # If none are found, defaults to a URI named after this
+      # resource's basename with the string ".icss.yaml" appended.
       #
       # @return [String, nil]
-      def metadata_uri
-        @metadata_uri ||= contents.detect { |path| path =~ /metadata.*\.(ya?ml|json)$/ }
+      def default_metadata_uri
+        contents.detect { |path| path =~ /(icss|metadata).*\.(ya?ml|json)$/i } || File.join(path, "#{basename}.icss.yaml")
       end
 
-      # Explicitly set the path to the metadata for this resource.
-      attr_writer :metadata_uri
-      
-      # Does this resource contain metadata for other resources it
-      # contains?
+      # Return the metadata for this resource if it exists.
       #
-      # @return [true, false]
-      def metadata?
-        (!! metadata_uri)
-      end
-
-      # Return the metadata for this resource.
+      # Will look for an existing resource at +default_metadata_uri+.
       #
       # @return [IMW::Metadata, nil]
       def metadata
-        @metadata ||= metadata? && IMW::Metadata.load(metadata_uri)
+        return @metadata if @metadata
+        obj = IMW.open(default_metadata_uri)
+        self.metadata=(obj) if obj.exist?
+        @metadata
       end
 
-      # Explicitly set the metadata for this resource.
-      attr_writer :metadata
+      # Set the metadata for this resource to +obj+.
+      #
+      # @param [String, Addressable::URI, IMW::Resource] obj
+      def metadata= obj
+        @metadata = IMW::Metadata.load(obj)
+      end
       
     end
   end
 end
-
-

data/lib/imw/metadata/field.rb

@@ -16,22 +16,6 @@ module IMW
     #
     #   IMW::Metadata::Field.new 'name' => 'id', 'type' => :integer, 'title' => "ID", 'description' => "Auto-incremented."
     #   #=> { 'name' => 'id', 'type' => :integer, 'title' > "ID", 'description' => "Auto-incremented." }
-    #
-    # Some properties make a field special:
-    #
-    # <tt>has_many</tt>::
-    #   Denotes that this record is in a "has_many" relationship with
-    #   one or more other records.  The corresponding value should be
-    #   an array
-    #   
-    # <tt>has_one</tt>::
-    #   Denotes that this record is in a "has_one" relationship with
-    #   one or more other records.  The corresponding value should be
-    #   an Array in which each key names the joined record and each
-    #   value is an Array of fields describing the joined record..
-    #
-    # @see IMW::Metadata::Record for more usage of the
-    # <tt>:has_many</tt> and <tt>:has_one</tt> properties.
     class Field < Hash
 
       def initialize obj
@@ -43,23 +27,11 @@ module IMW
           self['name'] = obj.to_s.strip
         end
       end
-      
-      def hierarchical?
-        has_key?('has_many') || has_key?('has_one')
-      end
-      alias_method :nested?, :hierarchical?
-
-      def flat?
-        ! hierarchical?
-      end
 
       def titleize
         self['title'] || self['name'].capitalize # FIXME we can do better than this!
       end
 
-      def associations
-      end
-      
     end
   end
 end

data/lib/imw/metadata/has_metadata.rb

@@ -0,0 +1,93 @@
+module IMW
+  class Metadata
+
+
+    # A module which defines how a resource finds Metadata that it can
+    # look up metadata about itself.
+    #
+    # "metadata" in this context is defined as accessors for
+    # +metadata+ (IMW::Metadata), +schema+ (IMW::Metadata::Schema),
+    # +fields+ (IMW::Metadata::Field), and +description+ (String).
+    #
+    # An including class should define a method +dir+ which should
+    # return an object that might contain Metadata, i.e. - that
+    # includes the IMW::Metadata::ContainsMetadata module.
+    #
+    # An including class can optionally define the methods +snippet+
+    # which returns a snippet of the resource as well as
+    # +record_count+ to return a count of how many records the
+    # resource contains.
+    module HasMetadata
+
+      # The schema for this object.
+      #
+      # @return [Hash]
+      def schema
+        return @schema if @schema
+        @schema             = IMW::Metadata::Schema.new
+        @schema[:type]      = "record"
+        @schema[:namespace] = "schema.imw.resource"
+        @schema[:name]      = (basename || '')
+        @schema[:doc]       = description
+        @schema[:fields]    = fields
+        
+        @schema[:non_avro ] = {}
+        @schema[:non_avro][:snippet]      = snippet      if respond_to?(:snippet)
+        @schema[:non_avro][:record_count] = record_count if respond_to?(:record_count)
+        @schema
+      end
+
+      # Return the metadata object that contains metadata for this
+      # resource.
+      #
+      # Will look in this resource's directory and recursively upward
+      # till the root directory is reached or a metadata file is
+      # discovered.
+      #
+      # @return [IMW::Metadata, nil]
+      def metadata
+        return @metadata if @metadata
+        d = dir
+        while d.path != '/'
+          break if d.metadata && d.metadata.describes?(self)
+          d = d.dir
+        end
+        @metadata = d.metadata
+      end
+
+      # The fields for this resource's data.
+      #
+      # Each field will be a Hash of information.
+      #
+      # @return [Array<Hash>]
+      def fields
+        @fields ||= metadata && metadata.fields_for(self)
+      end
+
+      # Set the fields for this resource.
+      #
+      # @param [Array<Hash>] new_fields
+      # @return [Array<Hash>]
+      def fields= new_fields
+        @fields = new_fields.map { |f| Metadata::Field.new(f) }
+      end
+
+      # A description for this Resource.
+      #
+      # @return [String]
+      def description
+        @description ||= metadata && metadata.description_for(self)
+      end
+
+      # Set the description of this Resource.
+      #
+      # @param [String] new_description
+      # @return [String]
+      def description= new_description
+        @description = new_description
+      end
+      
+    end
+  end
+end
+

data/lib/imw/metadata/has_summary.rb

@@ -0,0 +1,51 @@
+module IMW
+  class Metadata
+
+    # A module for generating a summary & schema of a resource.
+    #
+    # The including class should define methods +uri+, +basename+, +extension+.
+    module HasSummary
+      
+      # Return a full summary of this Resource.
+      #
+      # The summary will include "external" information about how this
+      # resource appears to the world (via its URI), "internal"
+      # metadata about this resource (its description, &c.), as well
+      # as the structure of this resource's data (it's schema's fields
+      # and a snippet).
+      #
+      # Will return a Hash, with a <tt>:schema</tt> key which maps to
+      # a well-formed AVRO schema for this resource.
+      #
+      # @return [Hash]
+      def summary
+        return @summary if @summary
+        @summary            = external_summary
+        @summary[:schema]   = schema                   if respond_to?(:schema)
+        @summary[:contents] = resources.map(&:summary) if respond_to?(:resources)
+        @summary
+      end
+
+      # Return information (usually scheme-dependent) on how this
+      # resource is situated in the world, i.e. - its URI, its size,
+      # how many lines it has, &c.
+      #
+      # Modules which override this should chain with +super+:
+      #
+      #   # in my_scheme.rb
+      #   def external_summary
+      #     super().merge(:user => 'bob', :password => 'smith')
+      #   end
+      #
+      # @return [Hash]
+      def external_summary
+        {
+          :uri       => uri.to_s,
+          :basename  => basename,
+          :extension => extension
+        }
+      end
+    end
+
+  end
+end

data/lib/imw/metadata/schema.rb

@@ -1,227 +1,17 @@
 module IMW
+
   class Metadata
     
-    # A class to describe the schema of a resource.
-    #
-    # A Schema is built on top of an Array because it is often
-    # important to have an ordering for a record's fields.
-    #
-    # For fields with no such ordering, an Array also works because
-    # each of its element will be a field with a +name+ that can be
-    # used to index the corresponding field.
-    #
-    # A Schema is instantiated with a basic Ruby data structure.
-    #
-    # == Tabular Data
-    #
-    # Tabular data formats (CSV, TSV, &c.) contain flat records
-    # consisting of repeating rows with the same fields in the same
-    # position.  A sample of delimited data looks like
-    #
-    #   ID,Name,Genus,Species					   
-    #   001,Gray-bellied Night Monkey,Aotus,lemurinus		   
-    #   002,Panamanian Night Monkey,Aotus,zonalis		   
-    #   003,Hernández-Camacho's Night Monkey,Aotus,jorgehernandezi 
-    #   004,Gray-handed Night Monkey,Aotus,griseimembra		   
-    #   005,Hershkovitz's Night Monkey,Aotus,hershkovitzi
-    #   ...
-    #
-    # The schema of these records is summarized as a Ruby data
-    # structure in the following way
-    #
-    #   [
-    #     { :name => :id,      :type => :integer                         },
-    #     { :name => :name,    :type => :string, :title => "Common Name" },
-    #     { :name => :genus,   :type => :string, :title => "Genus"       },
-    #     { :name => :species, :type => :string, :title => "Species"     }
-    #   ]
-    #
-    # The outer-most Array represents each row and each Hash in the
-    # Array represents one of the fields in a row.  A Schema
-    # initialized with the above Ruby code can be thought of and
-    # played with as an Array of Hashes even though it really is a
-    # Schema object of Field objects.
-    #
-    # == Hierarchical Data
-    #
-    # Hierarchical data formats (JSON, YAML, XML, &c.) can have
-    # arbitrarily complex records with fields within fields and so on.
-    # A sample of hierarchical XML data looks like
-    #
-    #   <genera>				      
-    #     <genus>				      
-    #       <name>Mandrillus</name>		      
-    #       <species>				      
-    #         <species id="113">		      
-    #           <name>sphinx</name>		      
-    #           <common_name>Mandrill</common_name>   
-    #         </species>			      
-    #         <species id="114">		      
-    #           <name>leucophaeus</name>	      
-    #           <common_name>Drill</common_name>      
-    #         </species>			      
-    #       </species>				      
-    #     </genus>				      
-    #     <genus>				      
-    #       <name>Rungwecebus</name>		      
-    #       <species>				      
-    #         <species id="100">		      
-    #           <name>kipunji</name>		      
-    #           <common_name>Kipunji</common_name>    
-    #         </species>			      
-    #       </species>				      
-    #     </genus>                                    
-    #
-    # These records are described by the following Ruby data structure
-    #
-    #   [
-    #     { :name     => :genera,
-    #       :has_many => [
-    #         { :name => 'name',    :type => :string, title => "Genus" },
-    #         { :name => 'species',
-    #           :has_many => [
-    #             { :name => :id,          :type => :integer                         },
-    #             { :name => :name,        :type => :string, :title => "Species"     },
-    #             { :name => :common_name, :type => :string, :title => "Common Name" }
-    #           ]
-    #         }
-    #       ]
-    #     }
-    #   ]
-    #
-    # By IMW convention, the outer-most element of the Schema is still
-    # an Array describing a collection of identical records even
-    # though XML data must have a single root node, limiting the
-    # collection to a single record.
-    #
-    # The first field of the Schema is named +genera+ and it uses the
-    # special field property +has_many+ to denote that the field
-    # points to a collection of sub-records.
+    # Represents a schema for data.
     #
-    # Each of these sub-records has its own sub-schema defined by the
-    # Array that the +has_many+ property keys to.  In this case, the
-    # two fields are +name+ and +species+.  +name+ is a simple String
-    # value while +species+ itself points at another collection of
-    # objects.
-    #
-    # This second-level nested record (a particular species) is itself
-    # composed of the three (flat) fields +id+, +name+, and
-    # +common_name+.  Note that the Schema doesn't know (or care) that
-    # the +id+ field is contained in an XML attribute while the +name+
-    # and +common_name+ fields are contained as text within daughter
-    # nodes.
-    #
-    # A different way of structure the same information, this time
-    # expressed in YAML:
-    #
-    #   ---                      
-    #   Mandrillus:              
-    #   - :species: sphinx       
-    #     :name: Mandrill        
-    #     :id: "113"             
-    #   - :species: leucophaeus  
-    #     :name: Drill           
-    #     :id: "114"             
-    #   Rungwecebus:             
-    #   - :species: kipunji      
-    #     :name: Kipunji         
-    #     :id: "100"
-    #
-    # Would lead to a different Schema
-    # 
-    #   [
-    #     { :name => :genus, :title => "Genus",
-    #       :has_many => [
-    #         { :name => :id,          :type => :integer                         },
-    #         { :name => :name,        :type => :string, :title => "Common Name" },
-    #         { :name => :species,     :type => :string, :title => "Species"     }
-    #       ]
-    #     }
-    #   ]
-    #
-    # Where the unnecessary outer wrapper field +genera+ has been
-    # dispensed with.
-    #
-    # In addition to "has many" relationships a record can have a
-    # "has_one" relationship.  The above data might be expressed
-    #
-    #   ---                      
-    #   Mandrillus:
-    #     - species: sphinx       
-    #       name: Mandrill        
-    #       id: "113"
-    #       discoverer:
-    #         name: Dr. Monkeypants
-    #         year: 1838
-    #     - species: leucophaeus  
-    #       name: Drill           
-    #       id: "114"
-    #       discoverer:
-    #         name: Ms. Cecelia Apefingers
-    #         year: 1921
-    #
-    # would result in the following Schema:
-    #
-    #   [
-    #     { :name => :genus, :title => "Genus",
-    #       :has_many => [
-    #         { :name => :id,         :type => :integer                         },
-    #         { :name => :name,       :type => :string, :title => "Common Name" },
-    #         { :name => :species,    :type => :string                          },
-    #         { :name => :discoverer,
-    #           :has_one => [
-    #             { :name => 'name', :type => :string  },
-    #             { :name => 'year', :type => :integer }
-    #           ]
-    #         }
-    #       ]
-    #     }
-    #   ]
-    #
-    # The +discoverer+ field is marked as +has_one+ which means the
-    # +name+ and +year+ fields in the corresponding Array will be
-    # interpreted as fields in a single attached sub-record.
-    #
-    # = Compact Schemas
-    #
-    # The internal hashes in a Schema specification are really Field
-    # objects and the initializer will promote Strings and Symbols to
-    # Field objects automatically.  This means that the above Schema
-    # specification could be replaced by
-    #
-    #   [
-    #     { :name => :genus
-    #       :has_many => [
-    #         :id,
-    #         :name,
-    #         :species,
-    #         { :name => :discoverer,
-    #           :has_one => [
-    #             :name,
-    #             :year
-    #           ]
-    #         }
-    #       ]
-    #     }
-    #   ]
-    #
-    # though there is an accompanying loss of metadata about each
-    # field.
-    class Schema < Array
+    # FIXME add methods that help couple nicely with Avro schemata.
+    class Schema < Hash
 
-      def initialize input=nil
+      def initialize obj=nil
         super()
-        concat(input.map { |field| IMW::Metadata::Field.new(field) }) if input
-      end
-
-      def self.load resource
-        new(IMW.open(resource).load)
+        merge!(obj) if obj.is_a?(Hash) || obj.is_a?(Schema)
       end
 
-      def [] index
-        [Integer, Range].include?(index.class) ? super(index) : detect { |field| field[:name].to_s == index.to_s }
-      end
-      
     end
   end
 end