mida 0.3.0 → 0.3.1

data/CHANGELOG.rdoc

@@ -1,3 +1,8 @@
+== 0.3.1 (5th July 2011)
+* Add bin/mida exectuable
+* Changed <tt>Item#to_h</tt> to only return keys with values
+* Add validate option to +Document+ and +Item+ to turn on/off validation
+
 == 0.3.0 (29th June 2011)
 * Merge +VocabularyDesc+ into +Vocabulary+
 * Vocabularies are now auto registered using +inherited+ hook

data/README.rdoc

@@ -18,7 +18,20 @@ Mida keeps RubyGems[http://rubygems.org/gems/mida] up-to-date with its latest ve
 
 * +Nokogiri+
 
-== Usage
+== Command Line Usage
+
+To use the command line tool, supply it with the urls or filenames that you
+would like to be parsed (by default each item is output as yaml):
+  mida http://lawrencewoodman.github.com/mida/news/
+
+If you want to search for specific types you can use the <tt>-t</tt> switch
+followed by a Regular Expression:
+  mida -t /person/i http://lawrencewoodman.github.com/mida/news/
+
+For more information look at <tt>mida</tt>'s help:
+  mida -h
+
+== Library Usage
 The following examples assume that you have required +mida+ and
 <tt>open-uri</tt>.
 

data/Rakefile

@@ -1,24 +1,25 @@
 task :default => :spec
 
 desc "Create Gem"
-require 'rake/gempackagetask'
+require 'rubygems/package_task'
 spec = Gem::Specification.new do |s|
   s.name        = "mida"
   s.summary     = "A Microdata parser/extractor library"
   s.description = "A Microdata parser and extractor library, based on the latest published version of the Microdata Specification, dated 5th April 2011."
-  s.version     = "0.3.0"
+  s.version     = "0.3.1"
   s.author      = "Lawrence Woodman"
   s.email       = "lwoodman@vlifesystems.com"
   s.homepage    = %q{http://lawrencewoodman.github.com/mida/}
   s.platform    = Gem::Platform::RUBY
   s.required_ruby_version = '>=1.9'
   s.files       = Dir['lib/**/*.rb'] + Dir['spec/**/*.rb'] + Dir['*.rdoc'] + Dir['Rakefile']
+  s.executables = ['mida']
   s.extra_rdoc_files = ['README.rdoc', 'LICENSE.rdoc', 'CHANGELOG.rdoc']
   s.rdoc_options << '--main' << 'README.rdoc'
-  s.add_dependency('nokogiri')
+  s.add_dependency('nokogiri', '>= 1.5')
   s.add_development_dependency('rspec', '>= 2.0' )
 end
-Rake::GemPackageTask.new(spec).define
+Gem::PackageTask.new(spec).define
 
 desc "Run Specs"
 require 'rspec/core/rake_task'

data/bin/mida

@@ -0,0 +1,110 @@
+#!/usr/bin/env ruby
+## Mida: Microdata parser/extractor
+##
+## Usage: mida [options...] [sources...]
+##
+## Find the Microdata in the given 'sources', which can be urls or files.
+## Urls must be prefixed with: http://
+##
+
+require 'open-uri'
+require 'yaml'
+require 'optparse'
+
+# Displays comment at top of file
+def banner
+  File.readlines(__FILE__).
+  grep(/^##.*/).
+  map { |line| line.chomp[3..-1] }.
+  join("\n")+"\n"+"  Options:\n"
+end
+
+begin
+  require 'mida'
+rescue LoadError
+  raise if $!.to_s !~ /mida/
+  libdir = File.expand_path("../../lib", __FILE__).sub(/^#{Dir.pwd}/, '.')
+  if !$:.include?(libdir)
+    warn "warn: #{$!.to_s}. trying again with #{libdir} on load path."
+    $:.unshift libdir
+    retry
+  end
+  raise
+end
+
+options = {sourcename: true, validate: true}
+ARGV.options do |option|
+  option.banner = banner
+  option.on('-c','--count', 'Display the counts of each Microdata Type') do
+    options[:count] = true
+  end
+  option.on('-n','--no-sourcename', "Don't display the source name") do
+    options[:sourcename] = false
+  end
+  option.on('-t','--type TYPE', Regexp,
+            'A regexp to match the itemtypes against') do |type|
+    options[:type] = type
+  end
+  option.on('-v','--no-validate',
+            "Don't validate the items against known Vocabularies") do
+    options[:validate] = false
+  end
+  option.on_tail('-h','--help', 'This help message') {puts option; exit}
+
+  begin
+    option.parse!
+  rescue OptionParser::InvalidOption => error
+    puts "#{error}\n#{option}"; exit
+  end
+  if ARGV.empty? then puts option; exit end
+end
+
+# Get the url from the source if there is one
+def get_url
+  ARGV.first =~ %r{^http://.*} ? ARGV.first : nil
+end
+
+# Display each item as yaml
+def display_items(items)
+  items.each {|item| puts item.to_h.to_yaml} 
+end
+
+# Returns a hash {type => count}
+def count_types(types)
+  types.each_with_object(Hash.new(0)) {|type,count| count[type] += 1}
+end
+
+# Display the number of each type of item
+def display_count(items)
+  types = items.collect {|item| item.type}
+  count_types(types).each {|type, count| puts "Found #{count} #{type}"}
+end
+
+def parse_source(source, options)
+  url = get_url  
+  begin
+    open(source) do |f|
+      doc = Mida::Document.new(f, url, options[:validate])
+      type = options[:type] || %r{}
+      items = doc.search(type)
+      if items.empty?
+        puts "No microdata found in this document."; exit
+      else
+        if options.include?(:count)
+          display_count(items)
+        else
+          display_items(items)
+        end
+      end
+    end
+  rescue
+    puts "Failed to parse: #{source}"
+    exit
+  end
+end
+
+ARGV.each do |source|
+  puts "Parsing: #{source}" if options[:sourcename]
+  parse_source(source, options)
+  puts
+end  

data/lib/mida/document.rb

@@ -15,10 +15,11 @@ module Mida
     # [target] The string containing the html that you want to parse.
     # [page_url] The url of target used for form absolute urls. This must
     #            include the filename, e.g. index.html.
-    def initialize(target, page_url=nil)
+    # [validate] Whether to validate the items against known vocabularies.
+    def initialize(target, page_url=nil, validate=true)
       @doc = Nokogiri(target)
       @page_url = page_url
-      @items = extract_items
+      @items = extract_items(validate)
     end
 
     # Implements method for Enumerable
@@ -45,13 +46,13 @@ module Mida
     end
 
   private
-    def extract_items
+    def extract_items(validate)
       itemscopes = @doc.search('//*[@itemscope and not(@itemprop)]')
       return nil unless itemscopes
 
       itemscopes.collect do |itemscope|
         itemscope = Itemscope.new(itemscope, @page_url)
-        Item.new(itemscope)
+        Item.new(itemscope, validate)
       end
     end
 

data/lib/mida/item.rb

@@ -24,12 +24,13 @@ module Mida
     # its +properties+
     #
     # [itemscope] The itemscope that has been parsed by +Itemscope+
-    def initialize(itemscope)
+    # [validate] Whether to validate the item against known vocabularies
+    def initialize(itemscope, validate=true)
       @type = itemscope.type
       @id = itemscope.id
       @vocabulary = Mida::Vocabulary.find(@type)
       @properties = itemscope.properties
-      validate_properties
+      validate_properties if validate
     end
 
     # Return a Hash representation
@@ -39,7 +40,12 @@ module Mida
     #     properties: {'a name' => 'avalue' }
     #   }
     def to_h
-      {type: @type, id: @id, properties: properties_to_h(@properties)}
+      # Only fill hash with non-nil values
+      hash = {}
+      @type and hash[:type] = @type
+      @id and hash[:id] = @id
+      @properties.any? and hash[:properties] = properties_to_h(@properties)
+      hash
     end
 
     def to_s
@@ -58,7 +64,11 @@ module Mida
       @properties =
       @properties.each_with_object({}) do |(property, values), hash|
         valid_values = validate_values(property, values)
-        hash[property] = valid_values unless valid_values.nil?
+        if valid_values.respond_to?(:any?)
+          hash[property] = valid_values if valid_values.any?
+        else
+          hash[property] = valid_values
+        end
       end
     end
 
@@ -77,9 +87,9 @@ module Mida
     # Return valid values, converted to the correct +DataType+
     # or +Item+ and number if necessary
     def validate_values(property, values)
-      return nil unless valid_property?(property, values)
+      return [] unless valid_property?(property, values)
       prop_num = property_number(property)
-      return nil unless valid_num_values?(prop_num, values)
+      return [] unless valid_num_values?(prop_num, values)
       prop_types = property_types(property)
 
       valid_values = values.each_with_object([]) do |value, valid_values|

data/spec/document_spec.rb

@@ -30,16 +30,6 @@ def match_array(value_array, expected_results)
   end
 end
 
-shared_examples_for 'one root itemscope' do
-  it 'should not match itemscopes with different names' do
-    @md.search(%r{nothing}).size.should == 0
-  end
-
-  it 'should find the correct number of itemscopes' do
-    @md.items.size.should == 1
-  end
-end
-
 describe Mida::Document do
   before do
     html = '
@@ -78,6 +68,49 @@ describe Mida::Document do
   end
 end
 
+describe Mida::Document, 'when initialized' do
+  before do
+    @html = '
+      <html><body>
+        <div itemscope itemtype="http://data-vocabulary.org/Review">
+          <span itemprop="itemreviewed">Romeo Pizza</span>
+          <span itemprop="itemreviewed">Some Other Pizza</span>
+        </div>
+      </body></html>
+    '
+
+    class Review < Mida::Vocabulary
+      itemtype %r{http://data-vocabulary.org/Review}
+      has_one 'item_reviewed'
+    end
+  end
+
+  context 'with validation on' do
+    before do
+      @md = Mida::Document.new(@html)
+    end
+
+    it 'should reject properties for items of known vocabularies that are not valid' do
+      @md.items[0].properties.should == {}
+    end
+  end
+
+  context 'with validation off' do
+    before do
+      @md = Mida::Document.new(@html, false)
+    end
+
+    it 'should accept properties for items of known vocabularies even if not valid' do
+      @md.items[0].properties['itemreviewed'].should ==
+        ['Romeo Pizza', 'Some Other Pizza']
+    end
+  end
+
+  after do
+    Mida::Vocabulary.unregister(Review)
+  end
+end
+
 describe Mida::Document, 'when run against a full html document containing itemscopes with and without itemtypes' do
 
   before do
@@ -161,7 +194,6 @@ describe Mida::Document, 'when run against a full html document containing two n
   it 'should return all the properties and types with the correct values for 1st itemscope' do
     expected_results = [{
       type: 'http://data-vocabulary.org/Review',
-      id: nil,
       properties: {
         'itemreviewed' => ['Romeo Pizza'],
         'rating' => ['4.5']
@@ -173,7 +205,6 @@ describe Mida::Document, 'when run against a full html document containing two n
   it 'should return all the properties from the text for 2nd itemscope' do
     expected_results = [{
       type: 'http://data-vocabulary.org/Organization',
-      id: nil,
       properties: {
         'name' => ['An org name'],
         'url' => ['http://example.com']
@@ -209,7 +240,13 @@ describe Mida::Document, 'when run against a full html document containing one
     @md = Mida::Document.new(html)
   end
 
-  it_should_behave_like 'one root itemscope'
+  it 'should not match itemscopes with different names' do
+    @md.search(%r{nothing}).size.should == 0
+  end
+
+  it 'should find the correct number of itemscopes' do
+    @md.items.size.should == 1
+  end
 
   it 'should return the correct number of itemscopes' do
     vocabularies = [
@@ -224,13 +261,11 @@ describe Mida::Document, 'when run against a full html document containing one
     it 'should return all the properties from the text with the correct values' do
       expected_results = [{
         type: 'http://data-vocabulary.org/Product',
-        id: nil,
         properties: {
           'name' => ['DC07'],
           'brand' => ['Dyson'],
           'review' => [{
             type: 'http://data-vocabulary.org/Review-aggregate',
-            id: nil,
             properties: {
               'count' => ['1'],
               'rating' => ['5.0']
@@ -287,14 +322,12 @@ describe Mida::Document, 'when run against a document containing an itemscope
       pending("get the contains: feature working")
       expected_result = {
         type: 'http://data-vocabulary.org/Product',
-        id: nil,
         properties: {
           'name' => 'DC07',
           'brand' => 'Dyson'
         },
         contains: {
           type: 'http://data-vocabulary.org/Review-aggregate',
-          id: nil,
           properties: {
             'count' => '1',
             'rating' => '5.0'

data/spec/item_spec.rb

@@ -27,12 +27,8 @@ describe Mida::Item, 'when initialized with an incomplete itemscope' do
     @item.properties.should == {}
   end
 
-  it '#to_h should return the correct type and properties' do
-    @item.to_h.should == {
-      type: nil,
-      id: nil,
-      properties: {}
-    }
+  it '#to_h should return an empty hash' do
+    @item.to_h.should == {}
   end
 end
 
@@ -66,7 +62,7 @@ describe Mida::Item, 'when initialized with a complete itemscope of an unknown t
     }
   end
 
-  it '#to_h should return the correct type and properties' do
+  it '#to_h should return the correct type, id and properties' do
     @item.to_h.should == {
       type: 'book',
       id: "urn:isbn:978-1-849510-50-9",
@@ -113,7 +109,7 @@ describe Mida::Item, 'when initialized with an itemscope of a known type' do
     @item.properties['url'].should == ['http://example.com/user/lorry']
   end
 
-  it 'should reject datatypes that are not valid' do
+  it 'should accept datatypes that are valid' do
     @item.properties['date'][0].should == '2nd October 2009'
   end
 
@@ -121,10 +117,6 @@ describe Mida::Item, 'when initialized with an itemscope of a known type' do
     @item.properties['date'][1].should == Date.iso8601('2009-10-02')
   end
 
-  it 'should reject datatypes that are not valid' do
-    @item.properties['date'][1].should == Date.iso8601('2009-10-02')
-  end
-
   it '#properties should return the same properties as the itemscope' do
     @item.properties.should == {
       'name' => 'Lorry Woodman',
@@ -136,7 +128,6 @@ describe Mida::Item, 'when initialized with an itemscope of a known type' do
   it '#to_h should return the correct type and properties' do
     @item.to_h.should == {
       type: 'http://example.com/vocab/person',
-      id: nil,
       properties: {
         'name' => 'Lorry Woodman',
         'date' => ['2nd October 2009', Date.iso8601('2009-10-02')],
@@ -157,31 +148,58 @@ describe Mida::Item, 'when initialized with an itemscope of a known type that do
       itemtype %r{http://example.com/vocab/person}
       has_one 'name', 'tel'
       has_many 'url', 'city'
+      has_one 'dob' do
+        extract Mida::DataType::ISO8601Date
+      end
     end
 
-    itemscope = mock(Mida::Itemscope)
-    itemscope.stub!(:type).and_return("http://example.com/vocab/person")
-    itemscope.stub!(:id).and_return(nil)
-    itemscope.stub!(:properties).and_return(
+    @itemscope = mock(Mida::Itemscope)
+    @itemscope.stub!(:type).and_return("http://example.com/vocab/person")
+    @itemscope.stub!(:id).and_return(nil)
+    @itemscope.stub!(:properties).and_return(
       { 'name' => ['Lorry Woodman'],
         'tel' => ['000004847582', '111111857485'],
         'url' => ['http://example.com/user/lorry'],
-        'city' => ['Bristol']
+        'city' => ['Bristol'],
+        'dob' => 'When I was born'
       }
     )
-    @item = Mida::Item.new(itemscope)
   end
 
-  it '#vocabulary should return the correct vocabulary' do
-    @item.vocabulary.should == Person
-  end
+  context 'when validation selected' do
+    before do
+      @item = Mida::Item.new(@itemscope)
+    end
+
+    it '#vocabulary should return the correct vocabulary' do
+      @item.vocabulary.should == Person
+    end
+
+    it 'should not keep properties that have too many values' do
+      @item.properties.should_not have_key('tel')
+    end
 
-  it 'should not keep properties that have too many values' do
-    @item.properties.should_not have_key('tel')
+    it 'should not keep properties that have the wrong DataType' do
+      @item.properties.should_not have_key('dob')
+    end
   end
 
-  it 'should keep have_many properties even if they have only one value' do
-    @item.properties.should have_key('city')
+  context 'when validation not selected' do
+    before do
+      @item = Mida::Item.new(@itemscope, false)
+    end
+
+    it '#vocabulary should return the correct vocabulary' do
+      @item.vocabulary.should == Person
+    end
+
+    it 'should keep properties even if they have too many values' do
+      @item.properties.should have_key('tel')
+    end
+
+    it 'should keep properties even if they have the wrong DataType' do
+      @item.properties.should have_key('dob')
+    end
   end
 
 end

metadata

@@ -2,7 +2,7 @@
 name: mida
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors: 
 - Lawrence Woodman
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-06-29 00:00:00 Z
+date: 2011-07-05 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: nokogiri
@@ -20,7 +20,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: "0"
+        version: "1.5"
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
@@ -36,8 +36,8 @@ dependencies:
   version_requirements: *id002
 description: A Microdata parser and extractor library, based on the latest published version of the Microdata Specification, dated 5th April 2011.
 email: lwoodman@vlifesystems.com
-executables: []
-
+executables: 
+- mida
 extensions: []
 
 extra_rdoc_files: 
@@ -78,6 +78,7 @@ files:
 - README.rdoc
 - LICENSE.rdoc
 - Rakefile
+- bin/mida
 homepage: http://lawrencewoodman.github.com/mida/
 licenses: []