imw 0.2.7 → 0.2.8

data/lib/imw/formats/json.rb

@@ -27,15 +27,13 @@ module IMW
         load(&block)
       end
 
-      # Dump the +data+ into this resource.  It must be opened for
+      # Emit the +data+ into this resource.  It must be opened for
       # writing.
       #
-      # @param [Hash, String, Array, Fixnum] data the Ruby object to dump
-      # @option options [true, false] :persist (false) Don't close the IO object after writing
-      def dump data, options={}
+      # @param [Hash, String, Array, Fixnum] data the Ruby object to emit
+      def emit data, options={}
         require 'json'
         write(data.to_json)
-        io.close unless options[:persist]
         self
       end
     end

data/lib/imw/formats/pdf.rb

@@ -0,0 +1,71 @@
+module IMW
+  module Formats
+
+    # Defines methods for parsing and generating PDF.
+    #
+    # Uses PDF::Reader for parsing and Prawn for generating.
+    module Pdf
+
+      # Return a snippet of text from this PDF.
+      #
+      # @return [String]
+      def snippet
+        begin
+          require 'pdf/reader'
+          snippetizer = Snippetizer.new
+          PDF::Reader.file(path, snippetizer)
+          snippetizer.snippet
+        rescue Snippetizer::SnippetEndError
+          snippetizer.snippet
+        rescue
+          ''
+        end
+      end
+
+      # A receiver class used by PDF::Reader which agglomerates text
+      # up to 1024 bytes and then bails.
+      class Snippetizer
+
+        # A custom error class that can be thrown while receiving text
+        # from PDF::Reader to cut-short walking large PDF documents.
+        SnippetEndError = Class.new(IMW::Error)
+
+        # The snippet being built by this snippetizer.
+        attr_accessor :snippet
+
+        def initialize
+          @snippet = ''
+        end
+
+        # Agglomerates text from PDF::Reader up to a fixed size of
+        # 1024 bytes.
+        #
+        # Will convert a single-space line from PDF::Reader as a
+        # newline character.
+        #
+        # FIXME How does the receiver ask PDF::Reader to abort walking
+        # the document now that enough text has been returned?  Till a
+        # more graceful way is found this method simply raises an
+        # error, creating a GOTO...
+        def show_text *params
+          params.each do |string|
+            if @snippet.size < 1024
+              if string == ' '
+                @snippet += "\n"
+              else
+                @snippet += string[0..1024]
+              end
+            else
+              raise SnippetEndError.new
+            end
+          end
+        end
+        alias_method :show_text_with_positioning,      :show_text
+        alias_method :move_to_next_line_and_show_text, :show_text
+        alias_method :set_spacing_next_line_show_text, :show_text
+      end
+      
+    end
+  end
+end
+

data/lib/imw/formats/yaml.rb

@@ -27,15 +27,13 @@ module IMW
         load(&block)
       end
       
-      # Dump the +data+ into this resource.  It must be opened for
+      # Emit the +data+ into this resource.  It must be opened for
       # writing.
       #
-      # @param [Hash, String, Array, Fixnum] data the Ruby object to dump
-      # @option options [true, false] :persist (false) Don't close the IO object after writing
-      def dump data, options={}
+      # @param [Hash, String, Array, Fixnum] data the Ruby object to emit
+      def emit data, options={}
         require 'yaml'
         write(data.to_yaml)
-        io.close unless options[:persist]
         self
       end
     end

data/lib/imw/metadata.rb

@@ -0,0 +1,66 @@
+module IMW
+
+  # A collection of classes for describing the metadata associated
+  # with a dataset's fields.
+  class Metadata < Hash
+    
+    autoload :Field,       'imw/metadata/field'
+    autoload :Schema,      'imw/metadata/schema'
+    autoload :Schematized, 'imw/metadata/schematized'
+    autoload :DSL,         'imw/metadata/dsl'
+    autoload :ContainsMetadata, 'imw/metadata/contains_metadata'
+
+    # The resource this Schema is anchored to.
+    #
+    # This attribute is useful for letting relative paths in a
+    # schema file refer to a common base URL.
+    #
+    # @return [IMW::Resource]
+    attr_reader :base
+    
+    # Set the resource this Schema is anchored to.
+    #
+    # @param [IMW::Resource, String, Addressable::URI] new_base
+    def base= new_base
+      base_resource = IMW.open(new_base)
+      base_resource.should_exist!("Metdata base directory must exist")
+      raise IMW::PathError.new("Metadata base must be a directory") unless base_resource.is_directory?
+      @base = base_resource
+    end
+
+    def initialize obj=nil, options={}
+      super()
+      self.base = options[:base] if options[:base]
+      obj.each_pair { |resource, schema| self[resource] = Schema.new(schema) } if obj
+    end
+
+    def self.load metadata_resource, options
+      resource = IMW.open(metadata_resource)
+      new(resource.load, {:base => resource.dirname}.merge(options))
+    end
+
+    def []= resource_spec, schema_spec
+      schema = schema_spec.is_a?(Schema) ? schema_spec : Schema.new(schema_spec)
+      super(absolute_uri(resource_spec), schema_spec)
+    end
+
+    def [] resource_spec
+      super(absolute_uri(resource_spec))
+    end
+
+    def describe? resource_spec
+      has_key?(absolute_uri(resource_spec))
+    end
+
+    protected
+
+    def absolute_uri resource_spec
+      if base && resource_spec.to_s !~ %r{(^/|://)} # relative path
+        base.join(resource_spec).to_s
+      else
+        resource_spec.to_s
+      end
+    end
+    
+  end
+end

data/lib/imw/metadata/contains_metadata.rb

@@ -0,0 +1,44 @@
+module IMW
+  class Metadata
+
+    # A module that can be mixed into any class defining a +contents+
+    # methods which returns an Array of URI strings.
+    module ContainsMetadata
+      
+      # The path at which this resource's metadata file lives.
+      #
+      # Will default to any file beginning with +metadata+ and ending
+      # with a +yaml+ or +json+ extension contained in this resource's
+      # +contents+.
+      #
+      # @return [String, nil]
+      def metadata_uri
+        @metadata_uri ||= contents.detect { |path| path =~ /metadata.*\.(ya?ml|json)$/ }
+      end
+
+      # Explicitly set the path to the metadata for this resource.
+      attr_writer :metadata_uri
+      
+      # Does this resource contain metadata for other resources it
+      # contains?
+      #
+      # @return [true, false]
+      def metadata?
+        (!! metadata_uri)
+      end
+
+      # Return the metadata for this resource.
+      #
+      # @return [IMW::Metadata, nil]
+      def metadata
+        @metadata ||= metadata? && IMW::Metadata.load(metadata_uri)
+      end
+
+      # Explicitly set the metadata for this resource.
+      attr_writer :metadata
+      
+    end
+  end
+end
+
+

data/lib/imw/metadata/dsl.rb

@@ -0,0 +1,111 @@
+module IMW
+  class Metadata
+
+    # A module which defines a DSL that can be used to define metadata
+    # for an object.
+    module DSL
+      
+      # Open a new resource at the given URI.
+      #
+      # If this dataset has metadata and it describes the resource
+      # then configure the resource to understand its schema..
+      #
+      # The +schema+ property passed via the options hash will
+      # override this.
+      # 
+      # @param [String, Addressable::Uri, IMW::Resource] uri
+      # @param [Hash] options
+      # @return [IMW::Resource]
+      # @see IMW.open
+      def open uri, options={}, &block
+        schema_options = (options[:schema].nil? && metadata && metadata.describe?(uri)) ? {:schema => metadata[uri]} : {}
+        IMW.open(uri, options.merge(schema_options), &block)
+      end
+
+      def open! uri, options={}, &block
+        self.open(uri, options.merge(:mode => 'w'), &block)
+      end
+
+      # When called without a block return this object's metadata.
+      # 
+      #   metadata
+      #   #=> { '/path/to/file' => [...], '/path/to/other/file' => [...], ... }
+      #
+      # When called with a block, accumulate schema and fields into
+      # this object's metadata
+      #
+      #   metadata do
+      #
+      #     schema "/path/to/file" do
+      #       # ...
+      #     end
+      #
+      #     schema "/path/to/other/file" do
+      #       # ...
+      #     end
+      #   end
+      #
+      # @see [IMW::Metadata::Schema]
+      # @see [IMW::Metadata::Field]      
+      # @return [IMW::Metadata]
+      def metadata arg=nil, options={}, &block
+        case arg
+        when Hash 
+          @metadata ||= Metadata.new(arg, options)
+        when nil
+          @metadata ||= Metadata.new nil, options
+        else
+          @metadata ||= Metadata.load(arg, options)
+        end
+        @metadata.base = options[:base] if options[:base]
+        return @metadata unless block_given?
+        yield
+      end
+
+      def schema resource, options={}, &block
+        new_field_accumulator!
+        yield
+        metadata[resource] = Schema.new(last_field_accumulator!)
+      end
+
+      def field name, options={}
+        accumulate_field Field.new(options.merge(:name => name))
+      end
+
+      def has_one name, options={}, &block
+        new_field_accumulator!
+        yield
+        accumulate_field Field.new(options.merge(:name => name, :has_one => last_field_accumulator!))
+      end
+
+      def has_many name, options={}, &block
+        new_field_accumulator!
+        yield
+        accumulate_field Field.new(options.merge(:name => name, :has_many => last_field_accumulator!))
+      end
+
+      protected
+
+      def field_accumulators    # :nodoc:
+        @field_accumulators ||= []
+      end
+      
+      def new_field_accumulator!    # :nodoc:
+        field_accumulators.push([])
+      end
+
+      def last_field_accumulator!    # :nodoc:
+        field_accumulators.pop
+      end
+      
+      def field_accumulator?    # :nodoc:
+        ! field_accumulators.empty?
+      end
+
+      def accumulate_field f    # :nodoc:
+        # raise IMW::SchemaError.new("No record or sub-record to accumulate fields in!") unless field_accumulator?
+        field_accumulators.last << f if field_accumulator?
+      end
+    end
+  end
+end

data/lib/imw/metadata/field.rb

@@ -0,0 +1,65 @@
+module IMW
+
+  class Metadata
+    
+    # Conceptually, a field is a "slot" for which "records" can have
+    # values.
+    #
+    # An IMW::Metadata::Field is essentially a Hash that has one required
+    # property: a name.
+    #
+    #   IMW::Metadata::Field.new('id')
+    #   #=> { 'name' => 'id' }
+    #
+    # But you can declare as many other properties as you want (as long
+    # as you include a +name+):
+    #
+    #   IMW::Metadata::Field.new 'name' => 'id', 'type' => :integer, 'title' => "ID", 'description' => "Auto-incremented."
+    #   #=> { 'name' => 'id', 'type' => :integer, 'title' > "ID", 'description' => "Auto-incremented." }
+    #
+    # Some properties make a field special:
+    #
+    # <tt>has_many</tt>::
+    #   Denotes that this record is in a "has_many" relationship with
+    #   one or more other records.  The corresponding value should be
+    #   an array
+    #   
+    # <tt>has_one</tt>::
+    #   Denotes that this record is in a "has_one" relationship with
+    #   one or more other records.  The corresponding value should be
+    #   an Array in which each key names the joined record and each
+    #   value is an Array of fields describing the joined record..
+    #
+    # @see IMW::Metadata::Record for more usage of the
+    # <tt>:has_many</tt> and <tt>:has_one</tt> properties.
+    class Field < Hash
+
+      def initialize obj
+        super()
+        if obj.is_a?(Hash) || obj.is_a?(Field)
+          merge!(obj)
+          raise IMW::ArgumentError.new("A field must have a name") if obj['name'].blank?
+        else
+          self['name'] = obj.to_s.strip
+        end
+      end
+      
+      def hierarchical?
+        has_key?('has_many') || has_key?('has_one')
+      end
+      alias_method :nested?, :hierarchical?
+
+      def flat?
+        ! hierarchical?
+      end
+
+      def titleize
+        self['title'] || self['name'].capitalize # FIXME we can do better than this!
+      end
+
+      def associations
+      end
+      
+    end
+  end
+end

data/lib/imw/metadata/schema.rb

@@ -0,0 +1,227 @@
+module IMW
+  class Metadata
+    
+    # A class to describe the schema of a resource.
+    #
+    # A Schema is built on top of an Array because it is often
+    # important to have an ordering for a record's fields.
+    #
+    # For fields with no such ordering, an Array also works because
+    # each of its element will be a field with a +name+ that can be
+    # used to index the corresponding field.
+    #
+    # A Schema is instantiated with a basic Ruby data structure.
+    #
+    # == Tabular Data
+    #
+    # Tabular data formats (CSV, TSV, &c.) contain flat records
+    # consisting of repeating rows with the same fields in the same
+    # position.  A sample of delimited data looks like
+    #
+    #   ID,Name,Genus,Species					   
+    #   001,Gray-bellied Night Monkey,Aotus,lemurinus		   
+    #   002,Panamanian Night Monkey,Aotus,zonalis		   
+    #   003,Hernández-Camacho's Night Monkey,Aotus,jorgehernandezi 
+    #   004,Gray-handed Night Monkey,Aotus,griseimembra		   
+    #   005,Hershkovitz's Night Monkey,Aotus,hershkovitzi
+    #   ...
+    #
+    # The schema of these records is summarized as a Ruby data
+    # structure in the following way
+    #
+    #   [
+    #     { :name => :id,      :type => :integer                         },
+    #     { :name => :name,    :type => :string, :title => "Common Name" },
+    #     { :name => :genus,   :type => :string, :title => "Genus"       },
+    #     { :name => :species, :type => :string, :title => "Species"     }
+    #   ]
+    #
+    # The outer-most Array represents each row and each Hash in the
+    # Array represents one of the fields in a row.  A Schema
+    # initialized with the above Ruby code can be thought of and
+    # played with as an Array of Hashes even though it really is a
+    # Schema object of Field objects.
+    #
+    # == Hierarchical Data
+    #
+    # Hierarchical data formats (JSON, YAML, XML, &c.) can have
+    # arbitrarily complex records with fields within fields and so on.
+    # A sample of hierarchical XML data looks like
+    #
+    #   <genera>				      
+    #     <genus>				      
+    #       <name>Mandrillus</name>		      
+    #       <species>				      
+    #         <species id="113">		      
+    #           <name>sphinx</name>		      
+    #           <common_name>Mandrill</common_name>   
+    #         </species>			      
+    #         <species id="114">		      
+    #           <name>leucophaeus</name>	      
+    #           <common_name>Drill</common_name>      
+    #         </species>			      
+    #       </species>				      
+    #     </genus>				      
+    #     <genus>				      
+    #       <name>Rungwecebus</name>		      
+    #       <species>				      
+    #         <species id="100">		      
+    #           <name>kipunji</name>		      
+    #           <common_name>Kipunji</common_name>    
+    #         </species>			      
+    #       </species>				      
+    #     </genus>                                    
+    #
+    # These records are described by the following Ruby data structure
+    #
+    #   [
+    #     { :name     => :genera,
+    #       :has_many => [
+    #         { :name => 'name',    :type => :string, title => "Genus" },
+    #         { :name => 'species',
+    #           :has_many => [
+    #             { :name => :id,          :type => :integer                         },
+    #             { :name => :name,        :type => :string, :title => "Species"     },
+    #             { :name => :common_name, :type => :string, :title => "Common Name" }
+    #           ]
+    #         }
+    #       ]
+    #     }
+    #   ]
+    #
+    # By IMW convention, the outer-most element of the Schema is still
+    # an Array describing a collection of identical records even
+    # though XML data must have a single root node, limiting the
+    # collection to a single record.
+    #
+    # The first field of the Schema is named +genera+ and it uses the
+    # special field property +has_many+ to denote that the field
+    # points to a collection of sub-records.
+    #
+    # Each of these sub-records has its own sub-schema defined by the
+    # Array that the +has_many+ property keys to.  In this case, the
+    # two fields are +name+ and +species+.  +name+ is a simple String
+    # value while +species+ itself points at another collection of
+    # objects.
+    #
+    # This second-level nested record (a particular species) is itself
+    # composed of the three (flat) fields +id+, +name+, and
+    # +common_name+.  Note that the Schema doesn't know (or care) that
+    # the +id+ field is contained in an XML attribute while the +name+
+    # and +common_name+ fields are contained as text within daughter
+    # nodes.
+    #
+    # A different way of structure the same information, this time
+    # expressed in YAML:
+    #
+    #   ---                      
+    #   Mandrillus:              
+    #   - :species: sphinx       
+    #     :name: Mandrill        
+    #     :id: "113"             
+    #   - :species: leucophaeus  
+    #     :name: Drill           
+    #     :id: "114"             
+    #   Rungwecebus:             
+    #   - :species: kipunji      
+    #     :name: Kipunji         
+    #     :id: "100"
+    #
+    # Would lead to a different Schema
+    # 
+    #   [
+    #     { :name => :genus, :title => "Genus",
+    #       :has_many => [
+    #         { :name => :id,          :type => :integer                         },
+    #         { :name => :name,        :type => :string, :title => "Common Name" },
+    #         { :name => :species,     :type => :string, :title => "Species"     }
+    #       ]
+    #     }
+    #   ]
+    #
+    # Where the unnecessary outer wrapper field +genera+ has been
+    # dispensed with.
+    #
+    # In addition to "has many" relationships a record can have a
+    # "has_one" relationship.  The above data might be expressed
+    #
+    #   ---                      
+    #   Mandrillus:
+    #     - species: sphinx       
+    #       name: Mandrill        
+    #       id: "113"
+    #       discoverer:
+    #         name: Dr. Monkeypants
+    #         year: 1838
+    #     - species: leucophaeus  
+    #       name: Drill           
+    #       id: "114"
+    #       discoverer:
+    #         name: Ms. Cecelia Apefingers
+    #         year: 1921
+    #
+    # would result in the following Schema:
+    #
+    #   [
+    #     { :name => :genus, :title => "Genus",
+    #       :has_many => [
+    #         { :name => :id,         :type => :integer                         },
+    #         { :name => :name,       :type => :string, :title => "Common Name" },
+    #         { :name => :species,    :type => :string                          },
+    #         { :name => :discoverer,
+    #           :has_one => [
+    #             { :name => 'name', :type => :string  },
+    #             { :name => 'year', :type => :integer }
+    #           ]
+    #         }
+    #       ]
+    #     }
+    #   ]
+    #
+    # The +discoverer+ field is marked as +has_one+ which means the
+    # +name+ and +year+ fields in the corresponding Array will be
+    # interpreted as fields in a single attached sub-record.
+    #
+    # = Compact Schemas
+    #
+    # The internal hashes in a Schema specification are really Field
+    # objects and the initializer will promote Strings and Symbols to
+    # Field objects automatically.  This means that the above Schema
+    # specification could be replaced by
+    #
+    #   [
+    #     { :name => :genus
+    #       :has_many => [
+    #         :id,
+    #         :name,
+    #         :species,
+    #         { :name => :discoverer,
+    #           :has_one => [
+    #             :name,
+    #             :year
+    #           ]
+    #         }
+    #       ]
+    #     }
+    #   ]
+    #
+    # though there is an accompanying loss of metadata about each
+    # field.
+    class Schema < Array
+
+      def initialize input=nil
+        super()
+        concat(input.map { |field| IMW::Metadata::Field.new(field) }) if input
+      end
+
+      def self.load resource
+        new(IMW.open(resource).load)
+      end
+
+      def [] index
+        [Integer, Range].include?(index.class) ? super(index) : detect { |field| field[:name].to_s == index.to_s }
+      end
+      
+    end
+  end
+end