mida 0.0.0

data/LICENSE.rdoc

@@ -0,0 +1,21 @@
+= The MIT License
+
+Copyright (c) 2011 Lawrence Woodman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,68 @@
+= Mida
+
+* {Mida Project Page}[https://github.com/LawrenceWoodman/mida]
+* {Mida Bug Tracker}[https://github.com/LawrenceWoodman/mida/issues]
+
+== Description
+A Microdata[http://en.wikipedia.org/wiki/Microdata_(HTML5)] parser and
+extractor library for ruby.
+This is based on the latest Published version of the Microdata Specification
+dated {5th April 2011}[http://www.w3.org/TR/2011/WD-microdata-20110405/].
+
+== Installation
+With Ruby and Rubygems:
+  gem install mida
+
+=== Requirements:
+
+* +Nokogiri+
+
+== Usage
+The following examples assume that you have required +mida+ and
++open-uri+.
+
+=== Extracting Microdata from a page
+All the Microdata is extracted from a page when a new <tt>Mida::Document</tt> instance
+is created.
+
+To extract all the Microdata from a webpage:
+  url = 'http://example.com'
+  open(url) {|f| doc = Mida::Document.new(f, url)}
+
+The top-level +Items+ will be held in an array accessible via
+<tt>doc.items</tt>.
+
+To simply list all the top-level +Items+ that have been found:
+	puts doc.items
+
+=== Searching
+If you want to search for an +Item+ that has a specific +itemtype+/vocabulary
+this can be done with the +search+ method.
+
+To return all the +Items+ that use one of Google's Review vocabularies:
+  doc.search(%r{http://data-vocabulary\.org.*?review.*?}i)
+
+=== Inspecting an +Item+
+Each +Item+ is a <tt>Mida::Item</tt> instance and has three main methods of
+interest, +type+, +properties+ and +id+.
+
+To find out the +itemtype+ of the +Item+:
+  puts doc.items.first.type
+
+To find out the +itemid+ of the +Item+:
+  puts doc.items.first.id
+
+Properties are returned as a hash containing name/values pairs.  The
+values will be an array of either +String+ or <tt>Mida::Item</tt> instances.
+
+To see the +properties+ of the +Item+:
+  puts doc.items.first.properties
+
+== Bugs/Feature Requests
+If you find a bug or want to make a feature request, please report it at the
+Mida project's {issues tracker}[https://github.com/LawrenceWoodman/mida/issues]
+on github.
+
+== License
+Copyright (c) 2011 Lawrence Woodman.
+This software is licensed under the MIT License.  Please see the file, LICENSE.rdoc, for details.

data/Rakefile

@@ -0,0 +1,26 @@
+task :default => :spec
+
+desc "Create Gem"
+require 'rake/gempackagetask'
+spec = Gem::Specification.new do |s|
+  s.name        = "mida"
+  s.summary     = "A Microdata parser"
+  s.description = File.read(File.join(File.dirname(__FILE__), 'README.rdoc'))
+  s.version     = "0.0.0"
+  s.author      = "Lawrence Woodman"
+  s.email       = "lwoodman@vlifesystems.com"
+  s.homepage    = %q{http://github.com/LawrenceWoodman/mida}
+  s.platform    = Gem::Platform::RUBY
+  s.required_ruby_version = '>=1.9'
+  s.files       = Dir['lib/**/*.rb'] + Dir['spec/**/*.rb'] + Dir['*.rdoc'] + Dir['Rakefile']
+  s.has_rdoc    = true
+  s.extra_rdoc_files = ['README.rdoc', 'LICENSE.rdoc']
+  s.rdoc_options << '--main' << 'README.rdoc'
+  s.add_dependency('nokogiri')
+  s.add_development_dependency('rspec')
+end
+Rake::GemPackageTask.new(spec).define
+
+desc "Run Specs"
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)

data/TODO.rdoc

@@ -0,0 +1,6 @@
+= Todo List
+
+* Support img rating in alt for google?
+* Look further in extra complications of microdata, e.g. alt tag for img rating and different size ratings
+  http://www.google.com/support/webmasters/bin/answer.py?answer=172705 
+* Put nested itemscopes, that are not a property of its parent into the parents hash using [:nested]

data/lib/mida.rb

@@ -0,0 +1,6 @@
+$LOAD_PATH.unshift File.dirname(__FILE__)
+Dir[File.dirname(__FILE__) + '/mida/*.rb'].each { |f| require f }
+
+# Mida is a Microdata parser and extractor.
+module Mida
+end

data/lib/mida/document.rb

@@ -0,0 +1,61 @@
+require 'nokogiri'
+
+module Mida
+
+  # Class that holds the extracted Microdata
+  class Document
+
+    # An Array of Mida::Item objects.  These are all top-level
+    # and hence not properties of other Items
+    attr_reader :items
+
+    # Create a new Microdata object
+    #
+    # [target] The string containing the html that you want to parse
+    # [page_url] The url of target used for form absolute urls. This must
+    #            include the filename, e.g. index.html.
+    def initialize(target, page_url=nil)
+      @doc = Nokogiri(target)
+      @page_url = page_url
+      @items = extract_items
+    end
+
+    # Returns an array of matching Mida::Item objects
+    #
+    # [vocabulary] A regexp to match the item types against
+    def search(vocabulary, items=@items)
+      found_items = []
+      items.each do |item|
+        # Allows matching against empty string, otherwise couldn't match
+        # as item.type can be nil
+        if (item.type.nil? && "" =~ vocabulary) || (item.type =~ vocabulary)
+          found_items << item
+        end
+        found_items += search_values(item.properties.values, vocabulary)
+      end
+      found_items
+    end
+
+  private
+    def extract_items
+      items_doc = @doc.search('//*[@itemscope and not(@itemprop)]')
+      return nil unless items_doc
+
+      items_doc.collect do |item_doc|
+        Item.new(item_doc, @page_url)
+      end
+    end
+
+    def search_values(values, vocabulary)
+      items = []
+      values.each do |value|
+        if value.is_a?(Mida::Item) then items += search(vocabulary, [value])
+        elsif value.is_a?(Array) then items += search_values(value, vocabulary)
+        end
+      end
+      items
+    end
+
+  end
+
+end

data/lib/mida/item.rb

@@ -0,0 +1,100 @@
+require 'nokogiri'
+
+module Mida
+
+  # Class that holds each item/itemscope
+  class Item
+    # The Type of the item
+    attr_reader :type
+
+    # The Global Identifier of the item
+    attr_reader :id
+
+    # A Hash representing the properties as name/values paris
+    # The values will be an array containing either +String+
+    # or <tt>Mida::Item</tt> instances
+    attr_reader :properties
+
+    # Create a new Item object
+    #
+    # [itemscope] The itemscope that you want to parse
+    # [page_url] The url of target used for form absolute urls
+    def initialize(itemscope, page_url=nil)
+      @itemscope, @page_url = itemscope, page_url
+      @type, @id = extract_attribute('itemtype'), extract_attribute('itemid')
+      @properties = {}
+      add_itemref_properties
+      traverse_elements(extract_elements(itemscope))
+    end
+
+    # Return a Hash representation
+    # of the form {type: 'The item type', properties: {'a name' => 'avalue' }}
+    def to_h
+      {type: @type, id: @id, properties: properties_to_h(@properties)}
+    end
+
+    def to_s
+      to_h.to_s
+    end
+
+    def ==(other)
+      @type == other.type and @id == other.id and @properties == other.properties
+    end
+
+  private
+
+    def extract_attribute(attribute)
+      (value = @itemscope.attribute(attribute)) ? value.value : nil
+    end
+
+    def extract_elements(itemscope)
+      itemscope.search('./*')
+    end
+
+    # Find an element with a matching id
+    def find_with_id(id)
+      @itemscope.search("//*[@id='#{id}']")
+    end
+
+    # The value as it should appear in to_h()
+    def value_to_h(value)
+      case
+      when value.is_a?(Array) then value.collect {|element| value_to_h(element)}
+      when value.is_a?(Item) then value.to_h
+      else value
+      end
+    end
+
+    def properties_to_h(properties)
+      hash = {}
+      properties.each { |name, value| hash[name] = value_to_h(value) }
+      hash
+    end
+
+    # Add any properties referred to by 'itemref'
+    def add_itemref_properties
+      itemref = extract_attribute('itemref')
+      if itemref
+        itemref.split.each {|id| traverse_elements(find_with_id(id))}
+      end
+    end
+
+    def traverse_elements(elements)
+      elements.each do |element|
+        internal_elements = extract_elements(element)
+        if internal_elements.empty? || element.attribute('itemscope')
+          add_itemprop(element)
+        else
+          traverse_elements(internal_elements)
+        end
+      end
+    end
+
+    def add_itemprop(itemprop)
+      properties = Property.parse(itemprop, @page_url)
+      properties.each { |name, value| (@properties[name] ||= []) << value }
+    end
+
+  end
+
+end

data/lib/mida/property.rb

@@ -0,0 +1,70 @@
+require 'nokogiri'
+require 'uri'
+
+module Mida
+
+  # Module that parses itemprop elements
+  module Property
+
+    # Returns a Hash representing the property.
+    # Hash is of the form {'property name' => 'value'}
+    # [element] The itemprop element to be parsed
+    # [page_url] The url of the page, including the filename, used to form absolute urls
+    def self.parse(element, page_url=nil)
+      hash = {}
+      extract_property_names(element).each do |name|
+        hash[name] = extract_property(element, page_url)
+      end
+      hash
+    end
+
+    NON_TEXTCONTENT_ELEMENTS = {
+      'a' => 'href',        'area' => 'href',
+      'audio' => 'src',     'embed' => 'src',
+      'iframe' => 'src',    'img' => 'src',
+      'link' => 'href',     'meta' => 'content',
+      'object' => 'data',   'source' => 'src',
+      'time' => 'datetime', 'track' => 'src',
+      'video' => 'src'
+    }
+
+    URL_ATTRIBUTES = ['data', 'href', 'src']
+
+    # This returns an empty string if can't form a valid
+    # absolute url as per the Microdata spec.
+    def self.make_absolute_url(url, page_url)
+      return url unless URI.parse(url).relative?
+      begin
+        URI.parse(page_url).merge(url).to_s
+      rescue URI::Error
+        ''
+      end
+    end
+
+    def self.extract_property_names(itemprop)
+      itemprop_attr = itemprop.attribute('itemprop')
+      itemprop_attr ? itemprop_attr.value.split() : []
+    end
+
+    def self.extract_property_value(itemprop, page_url)
+      element = itemprop.name
+      if NON_TEXTCONTENT_ELEMENTS.has_key?(element)
+        attribute = NON_TEXTCONTENT_ELEMENTS[element]
+        value = itemprop.attribute(attribute).value
+        (URL_ATTRIBUTES.include?(attribute)) ? make_absolute_url(value, page_url) : value
+      else
+        itemprop.inner_text
+      end
+    end
+
+    def self.extract_property(itemprop, page_url)
+      if itemprop.attribute('itemscope')
+        Mida::Item.new(itemprop, page_url)
+      else
+        extract_property_value(itemprop, page_url)
+      end
+    end
+
+  end
+
+end

data/spec/document_spec.rb

@@ -0,0 +1,684 @@
+require_relative 'spec_helper'
+require_relative '../lib/mida'
+
+def test_parsing(md, vocabulary, expected_results)
+  items = md.search(vocabulary)
+  expected_results.each_with_index do |expected_result,i|
+    item = items[i]
+    test_to_h(item, expected_result)
+    test_properties(item, expected_result)
+  end
+end
+
+def test_to_h(item, expected_result)
+  item.to_h.should == expected_result
+end
+
+def test_properties(item, expected_result)
+  item.properties.each do |name, value|
+    match_array(value, expected_result[:properties][name])
+  end
+end
+
+def match_array(value_array, expected_results)
+  value_array.each_with_index do |element, i|
+    if element.is_a?(Mida::Item)
+      test_properties(element, expected_results[i])
+    else
+      element.should == expected_results[i]
+    end
+  end
+end
+
+shared_examples_for 'one root itemscope' do
+  it 'should not match itemscopes with different names' do
+    @md.search(%r{nothing}).size.should == 0
+  end
+
+  it 'should find the correct number of itemscopes' do
+    @md.items.size.should == 1
+  end
+end
+
+describe Mida::Document, 'when run with a document containing textContent and non textContent itemprops' do
+  before do
+    @html = '
+      <html>
+        <head itemscope>
+          <link itemprop="link_field" rel="stylesheet" type="text/css" href="stylesheet.css" />
+        </head>
+        <body>
+          There is some text here
+          <div>
+            and also some here
+            <div itemscope>
+              <span itemprop="span_field">Some span content</span>
+              <time itemprop="dtreviewed" datetime="2009-01-06">Jan 6</time>.
+              <meta itemprop="meta_field" content="Some meta content">
+              <a itemprop="a_field1" href="http://example.com">non content</a>
+              <a itemprop="a_field2" href="welcome/index.html">non content</a>
+              <a itemprop="a_field3" href="/intro">non content</a>
+              <a itemprop="a_field4" href="/intro/index.html">non content</a>
+              <map name="somemap">
+                <area shape="rect" coords="0,0,50,120" href="left.html" />
+                <area itemprop="area_right" shape="rect" coords="51,0,120,120" href="right.html" />
+              </map>
+              <audio itemprop="audio_field" src="asound.ogg" controls="controls">
+                Audio tag not supported by your browser.
+              </audio>
+
+              <embed itemprop="embed_field" src="helloworld.swf" />
+              <iframe itemprop="iframe_field" src="http://www.example.com/iframe_test"></iframe>
+              <img itemprop="img_field" src="animage.png" width="120" height="120" usemap="#planetmap" />
+              <object itemprop="object_field" data="object.png" type="image/png" />
+              <audio controls="controls">
+                <source itemprop="source_field" src="song.ogg" type="audio/ogg" />
+                <track itemprop="track_field" src="atrack.ogg" />
+                Audio tag not supported by your browser.
+              </audio>
+              <video itemprop="video_field" src="movie.ogg" controls="controls">
+                Video tag not supported by your browser.
+              </video>
+            </div>
+          </div>
+        </body>
+      </html>
+    '
+  end
+
+
+  context 'when not given a page_url' do
+    before do
+      @md = Mida::Document.new(@html)
+    end
+
+    it 'should return all the properties and types with the correct values' do
+      expected_results = [
+        { type: nil, id: nil, properties: {'link_field' => ['']} },
+        { type: nil,
+          id: nil,
+          properties: {
+            'span_field' => ['Some span content'],
+            'dtreviewed' => ['2009-01-06'],
+            'meta_field' => ['Some meta content'],
+            'a_field1' => ['http://example.com'],
+            'a_field2' => [''],
+            'a_field3' => [''],
+            'a_field4' => [''],
+            'area_right' => [''],
+            'audio_field' => [''],
+            'embed_field' => [''],
+            'iframe_field' => ['http://www.example.com/iframe_test'],
+            'img_field' => [''],
+            'object_field' => [''],
+            'source_field' => [''],
+            'track_field' => [''],
+            'video_field' => ['']
+          }
+        }
+      ]
+
+      test_parsing(@md, %r{}, expected_results)
+
+    end
+  end
+
+  context 'when given a page_url' do
+    before do
+      @md = Mida::Document.new(@html, 'http://example.com/start/')
+    end
+
+    it 'should return all the properties and types with the correct values' do
+      expected_results = [
+        { type: nil, id: nil, properties: {
+          'link_field' => ['http://example.com/start/stylesheet.css']
+          }
+        },
+        { type: nil,
+          id: nil,
+          properties: {
+            'span_field' => ['Some span content'],
+            'dtreviewed' => ['2009-01-06'],
+            'meta_field' => ['Some meta content'],
+            'a_field1' => ['http://example.com'],
+            'a_field2' => ['http://example.com/start/welcome/index.html'],
+            'a_field3' => ['http://example.com/intro'],
+            'a_field4' => ['http://example.com/intro/index.html'],
+            'area_right' => ['http://example.com/start/right.html'],
+            'audio_field' => ['http://example.com/start/asound.ogg'],
+            'embed_field' => ['http://example.com/start/helloworld.swf'],
+            'iframe_field' => ['http://www.example.com/iframe_test'],
+            'img_field' => ['http://example.com/start/animage.png'],
+            'object_field' => ['http://example.com/start/object.png'],
+            'source_field' => ['http://example.com/start/song.ogg'],
+            'track_field' => ['http://example.com/start/atrack.ogg'],
+            'video_field' => ['http://example.com/start/movie.ogg']
+          }
+        }
+      ]
+
+      test_parsing(@md, %r{}, expected_results)
+    end
+  end
+
+end
+
+describe Mida::Document, 'when run against a full html document containing one itemscope with no itemtype' do
+
+  before do
+    html = '
+      <html><body>
+        There is some text here
+        <div>
+          and also some here
+          <div itemscope>
+            <span itemprop="itemreviewed">Romeo Pizza</span>
+            Reviewed by <span itemprop="reviewer">Ulysses Grant</span> on
+            <time itemprop="dtreviewed" datetime="2009-01-06">Jan 6</time>.
+            <meta itemprop="fielda" content="a5482">
+
+            <span itemprop="summary">Delicious, tasty pizza in Eastlake!</span>
+            <span itemprop="description">This is a very nice pizza place.</span>
+            Rating: <span itemprop="rating">4.5</span>
+          </div>
+        </div>
+      </body></html>
+    '
+    @md = Mida::Document.new(html)
+
+  end
+
+  it_should_behave_like 'one root itemscope'
+
+  it 'should return all the properties and types with the correct values' do
+    expected_results = [{
+      type: nil,
+      id: nil,
+      properties: {
+        'itemreviewed' => ['Romeo Pizza'],
+        'reviewer' => ['Ulysses Grant'],
+        'dtreviewed' => ['2009-01-06'],
+        'fielda' => ['a5482'],
+        'summary' => ['Delicious, tasty pizza in Eastlake!'],
+        'description' => ['This is a very nice pizza place.'],
+        'rating' => ['4.5']
+      }
+    }]
+
+    test_parsing(@md, %r{}, expected_results)
+  end
+
+end
+
+describe Mida::Document, 'when run against a full html document containing one itemscope nested within another' do
+
+  before do
+    html = '
+      <html><body>
+        There is some text here
+        <div>
+          and also some here
+          <div itemscope>
+            <span itemprop="itemreviewed">Romeo Pizza</span>
+            <div itemprop="address" itemscope>
+              <span itemprop="firstline">237 Italian Way</span>
+              <span itemprop="country">United Kingdom</span>
+            </div>
+            Rating: <span itemprop="rating">4.5</span>
+          </div>
+        </div>
+      </body></html>
+    '
+
+    @md = Mida::Document.new(html)
+
+  end
+
+  it_should_behave_like 'one root itemscope'
+
+  it 'should return all the properties and types with the correct values' do
+    expected_results = [{
+      type: nil,
+      id: nil,
+      properties: {
+        'itemreviewed' => ['Romeo Pizza'],
+        'address' => [{
+          type: nil, id: nil, properties: {
+            'firstline' => ['237 Italian Way'],
+            'country' => ['United Kingdom']
+           }
+        }],
+        'rating' => ['4.5']
+      }
+    }]
+
+    test_parsing(@md, %r{}, expected_results)
+  end
+
+end
+
+describe Mida::Document, 'when run against a full html document containing one itemscope nested within another within another' do
+
+  before do
+    html = '
+      <html><body>
+        There is some text here
+        <div>
+          and also some here
+          <div itemscope>
+            <span itemprop="itemreviewed">Romeo Pizza</span>
+            <div itemprop="address" itemscope>
+              <div itemprop="firstline" itemscope>
+                <span itemprop="number">237</span>
+                <span itemprop="road">Italian Way</span>
+              </div>
+              <span itemprop="country">United Kingdom</span>
+            </div>
+            Rating: <span itemprop="rating">4.5</span>
+          </div>
+        </div>
+      </body></html>
+    '
+
+    @md = Mida::Document.new(html)
+  end
+
+  it_should_behave_like 'one root itemscope'
+
+  it 'should return all the properties and types with the correct values' do
+    expected_results = [{
+      type: nil,
+      id: nil,
+      properties: {
+        'itemreviewed' => ['Romeo Pizza'],
+        'address' => [{
+          type: nil,
+          id: nil,
+          properties: {
+            'firstline' => [{
+              type: nil,
+              id: nil,
+              properties: {
+                'number' => ['237'],
+                'road' => ['Italian Way']
+              },
+            }],
+            'country' => ['United Kingdom']
+          },
+        }],
+        'rating' => ['4.5']
+      }
+    }]
+
+    test_parsing(@md, %r{^$}, expected_results)
+  end
+
+end
+
+describe Mida::Document, 'when run against a full html document containing one itemscope with an itemtype' do
+
+  before do
+    html = '
+      <html><body>
+        There is some text here
+        <div>
+          and also some here
+          <div itemscope itemtype="http://data-vocabulary.org/Review">
+            <span itemprop="itemreviewed">Romeo Pizza</span>
+            Reviewed by <span itemprop="reviewer">Ulysses Grant</span> on
+            <time itemprop="dtreviewed" datetime="2009-01-06">Jan 6</time>.
+            <span itemprop="summary">Delicious, tasty pizza in Eastlake!</span>
+            <span itemprop="description">This is a very nice pizza place.</span>
+            Rating: <span itemprop="rating">4.5</span>
+          </div>
+        </div>
+      </body></html>
+    '
+
+    @md = Mida::Document.new(html)
+
+  end
+
+  it_should_behave_like 'one root itemscope'
+
+  it 'should find the correct number of itemscopes if outer specified' do
+    @md.search(%r{http://data-vocabulary.org/Review}).size.should == 1
+  end
+
+  it 'should specify the correct type' do
+    @md.search(%r{http://data-vocabulary.org/Review}).first.type.should == 'http://data-vocabulary.org/Review'
+  end
+
+  it 'should return all the properties and types with the correct values' do
+    expected_results = [{
+      type: 'http://data-vocabulary.org/Review',
+      id: nil,
+      properties: {
+        'itemreviewed' => ['Romeo Pizza'],
+        'reviewer' => ['Ulysses Grant'],
+        'dtreviewed' => ['2009-01-06'],
+        'summary' => ['Delicious, tasty pizza in Eastlake!'],
+        'description'  => ['This is a very nice pizza place.'],
+        'rating' => ['4.5']
+      }
+    }]
+    test_parsing(@md, %r{http://data-vocabulary.org/Review}, expected_results)
+  end
+
+end
+
+describe Mida::Document, 'when run against a full html document containing two non-nested itemscopes with itemtypes' do
+
+  before do
+    html = '
+      <html><body>
+        There is some text here
+        <div>
+          and also some here
+          <div itemscope itemtype="http://data-vocabulary.org/Review">
+            <span itemprop="itemreviewed">Romeo Pizza</span>
+            Rating: <span itemprop="rating">4.5</span>
+          </div>
+          <div itemscope itemtype="http://data-vocabulary.org/Organization">
+            <span itemprop="name">An org name</span>
+            <span itemprop="url">http://example.com</span>
+          </div>
+        </div>
+      </body></html>
+    '
+
+    @md = Mida::Document.new(html)
+
+  end
+
+  it 'should return all the itemscopes' do
+    @md.items.size.should == 2
+  end
+
+  it 'should give the type of each itemscope if none specified' do
+    itemscope_names = {
+      'http://data-vocabulary.org/Review' => 0,
+      'http://data-vocabulary.org/Organization' => 0
+    }
+
+    @md.items.each do |item|
+      itemscope_names[item.type] += 1
+    end
+
+    itemscope_names.size.should eq 2
+    itemscope_names.each { |name, num| num.should == 1 }
+  end
+
+
+  it 'should return all the properties and types with the correct values for 1st itemscope' do
+    expected_results = [{
+      type: 'http://data-vocabulary.org/Review',
+      id: nil,
+      properties: {
+        'itemreviewed' => ['Romeo Pizza'],
+        'rating' => ['4.5']
+      }
+    }]
+    test_parsing(@md, %r{http://data-vocabulary.org/Review}, expected_results)
+  end
+
+  it 'should return all the properties from the text for 2nd itemscope' do
+    expected_results = [{
+      type: 'http://data-vocabulary.org/Organization',
+      id: nil,
+      properties: {
+        'name' => ['An org name'],
+        'url' => ['http://example.com']
+      }
+    }]
+    test_parsing(@md, %r{http://data-vocabulary.org/Organization}, expected_results)
+  end
+
+end
+
+describe Mida::Document, 'when run against a full html document containing one
+  itemscope nested within another and the inner block is
+  surrounded with another non itemscope block' do
+
+  before do
+    html = '
+      <html><body>
+        <div itemscope itemtype="http://data-vocabulary.org/Product">
+          <ul class="reviews">
+            <li id="model" itemprop="name">DC07</li>
+            <li id="make" itemprop="brand">Dyson</li>
+            <li itemprop="review" itemscope itemtype="http://data-vocabulary.org/Review-aggregate">
+              <span class="ratingDetails">
+                <span itemprop="count">1</span> Review,
+                Average: <span itemprop="rating">5.0</span>
+              </span>
+            </li>
+          </ul>
+        </div>
+      </body></html>
+    '
+
+    @md = Mida::Document.new(html)
+  end
+
+  it_should_behave_like 'one root itemscope'
+
+  it 'should return the correct number of itemscopes' do
+    vocabularies = [
+      %r{http://data-vocabulary.org/Product},
+      %r{http://data-vocabulary.org/Review-aggregate}
+    ]
+    vocabularies.each {|vocabulary| @md.search(vocabulary).size.should == 1}
+  end
+
+  context "when looking at the outer vocabulary" do
+    it 'should return all the properties from the text with the correct values' do
+      expected_results = [{
+        type: 'http://data-vocabulary.org/Product',
+        id: nil,
+        properties: {
+          'name' => ['DC07'],
+          'brand' => ['Dyson'],
+          'review' => [{
+            type: 'http://data-vocabulary.org/Review-aggregate',
+            id: nil,
+            properties: {
+              'count' => ['1'],
+              'rating' => ['5.0']
+            }
+          }]
+        }
+      }]
+
+      test_parsing(@md, %r{http://data-vocabulary.org/Product}, expected_results)
+    end
+  end
+
+end
+
+describe Mida::Document, 'when run against a document containing an itemscope
+  that contains another non-linked itemscope' do
+
+  before do
+    html = '
+      <html><body>
+        <div itemscope itemtype="http://data-vocabulary.org/Product">
+          <ul class="reviews">
+            <li id="model" itemprop="name">DC07</li>
+            <li id="make" itemprop="brand">Dyson</li>
+            <li itemscope itemtype="http://data-vocabulary.org/Review-aggregate">
+              <span class="ratingDetails">
+                <span itemprop="count">1</span> Review,
+                Average: <span itemprop="rating">5.0</span>
+              </span>
+            </li>
+          </ul>
+        </div>
+      </body></html>
+    '
+
+    @md = Mida::Document.new(html)
+  end
+
+  it 'should return the correct number of itemscopes when search used' do
+    vocabularies = {
+      %r{} => 2,
+      %r{http://data-vocabulary.org/Product} => 1,
+      %r{http://data-vocabulary.org/Review-aggregate} => 1
+    }
+    vocabularies.each {|vocabulary, num| @md.search(vocabulary).size.should == num}
+  end
+
+  it 'should return the correct number of items' do
+    @md.items.size.should == 2
+  end
+
+  context "when no vocabulary specified or looking at the outer vocabulary" do
+    it 'should return all the properties from the text with the correct values' do
+      pending("get the contains: feature working")
+      expected_result = {
+        type: 'http://data-vocabulary.org/Product',
+        id: nil,
+        properties: {
+          'name' => 'DC07',
+          'brand' => 'Dyson'
+        },
+        contains: {
+          type: 'http://data-vocabulary.org/Review-aggregate',
+          id: nil,
+          properties: {
+            'count' => '1',
+            'rating' => '5.0'
+          }
+        }
+      }
+
+      @md.search('http://data-vocabulary.org/Product').first.should == expected_result
+    end
+  end
+end
+
+describe Mida::Document, 'when run against a document using itemrefs' do
+
+  before do
+    html = '
+      <html><body>
+        <div itemscope id="amanda" itemref="a b">
+          <span itemprop="age">30</span>
+        </div>
+        <p id="a">Name: <span itemprop="name">Amanda</span></p>
+        <div id="b" itemprop="band" itemscope itemref="c"></div>
+        <div id="c">
+          <p>Band: <span itemprop="name">Jazz Band</span></p>
+          <p>Size: <span itemprop="size">12</span> players</p>
+        </div>
+      </body></html>
+    '
+
+    @md = Mida::Document.new(html)
+  end
+
+  it 'should return all the properties from the text with the correct values' do
+    expected_results = [{
+      type: nil,
+      id: nil,
+      properties: {
+        'name' => ['Amanda'],
+        'band' => [{
+          type: nil,
+          id: nil,
+          properties: {
+            'name' => ['Jazz Band'],
+            'size' => ['12']
+          }
+        }],
+        'age' => ['30']
+      }
+    }]
+
+    test_parsing(@md, %r{}, expected_results)
+  end
+end
+
+describe Mida::Document, 'when run against a document using multiple itemprops with the same name' do
+
+  before do
+    html = '
+      <html><body>
+        <div itemscope itemtype="icecreams">
+          <p>Flavours in my favourite ice cream:</p>
+          <ul>
+            <li itemprop="flavour">Lemon sorbet</li>
+            <li itemprop="flavour">Apricot sorbet</li>
+            <li itemprop="flavour" itemscope itemtype="icecream-type">
+              <span itemprop="fruit">Strawberry</span>
+              <span itemprop="style">Homemade</span>
+            </li>
+          </ul>
+        </div>
+      </body></html>
+    '
+
+    @md = Mida::Document.new(html)
+  end
+
+  it_should_behave_like 'one root itemscope'
+
+  it 'should return the correct number of itemscopes' do
+    vocabularies = [
+      %r{icecreams},
+      %r{icecream-type}
+    ]
+    vocabularies.each {|vocabulary| @md.search(vocabulary).size.should == 1}
+  end
+
+  it 'should return all the properties from the text with the correct values' do
+    expected_results = [{
+      type: 'icecreams',
+      id: nil,
+      properties: {
+        'flavour' => [
+          'Lemon sorbet',
+          'Apricot sorbet',
+          { type: 'icecream-type',
+            id: nil,
+            properties: {
+              'fruit' => ['Strawberry'],
+              'style' => ['Homemade']
+            }
+          }
+        ]
+      }
+    }]
+
+    test_parsing(@md, %r{icecreams}, expected_results)
+  end
+end
+
+describe Mida::Document, 'when run against a document using an itemprop with multiple properties' do
+
+  before do
+    html = '
+      <html><body>
+        <div itemscope>
+          <span itemprop="favourite-colour favourite-fruit">orange</span>
+        </div>
+      </body></html>
+    '
+
+    @md = Mida::Document.new(html)
+  end
+
+  it 'should return all the properties from the text with the correct values' do
+    expected_results = [{
+      type: nil,
+      id: nil,
+      properties: {
+        'favourite-colour' => ['orange'],
+        'favourite-fruit' => ['orange']
+      }
+    }]
+
+    test_parsing(@md, %r{}, expected_results)
+  end
+end