hypermicrodata 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 31c2827d5ef761811004b563f135127def94a625
-  data.tar.gz: 229d52f8eee1e15347d72171d37f6ebda948df84
+  metadata.gz: bc45691f143d5cfcd20199463e9b549565c04b69
+  data.tar.gz: 2f0c44acf87fa98cc606bf3f292d3c940a2cbf57
 SHA512:
-  metadata.gz: b1caaf52ab0fe4dc1c3285b04b9a1db8b3b1debe0d2966cc3c903d88a72e3900d3d2815521c95d5d98217e0c81cffe8f1d63fa9977e53d2ee1db361f8ea70de0
-  data.tar.gz: 3f2531028ea38dc3d082e84a14aa426406d7ebee8667e3459611ccfd05c7857eb965daf73715c5c390ef272989566b761386512955e940d4b6032bb821332dfb
+  metadata.gz: 2c3b43e8c4ca94990f23827ccd3cb4111e7f98143c5291bb166a4bb3c621dc65065184e826caac6d792ec5b504d0f141dbc5cf6cb9956a90314ad5deb625c824
+  data.tar.gz: 71df574ab4d757fed7e67e39f3d6e88afdf76a4f416955ebce4c4884d7ca271e4c9cd6f713d5072bad004c154121116034efa1514bed3f7236ce4f42eae02e2a

data/.travis.yml

@@ -2,7 +2,8 @@ language: ruby
 rvm:
   - 1.9.3
   - 2.0.0
-  - 2.1.1
+  - 2.1
+  - 2.2
   - jruby-19mode # JRuby in 1.9 mode
 # uncomment this line if your project needs to run something other than `rake`:
 script: rake test

data/README.md

@@ -42,6 +42,14 @@ Supported formats are
 
 When you use this in Rails, you don't need to extract data manually.
 
+/config/mime_types.rb
+
+```
+Mime::Type.register 'application/vnd.amundsen-uber+json', :uberjson
+# or if you want HAL
+Mime::Type.register 'application/hal+json', :haljson
+```
+
 /app/controllers/people_controller.rb
 
 ```
@@ -55,8 +63,7 @@ end
 /app/views/people/show.html.haml
 
 ```
-.person{itemscope: true, itemtype: 'http://schema.org/Person',
-        itemid: person_url(@person), data: {main_item: true}}
+%main.person{itemscope: true, itemtype: 'http://schema.org/Person', itemid: person_url(@person)}
   .media
     .media-image.pull-left
       = image_tag @person.picture_path, alt: '', itemprop: 'image'
@@ -66,6 +73,11 @@ end
   = link_to 'collection', people_path, rel: 'collection', itemprop: 'isPartOf'
 ```
 
+`<main>` elements is considered root nodes of the extraction into JSON.
+
+If you don't want use `<main>`, you can use elements with `data-main-item` attribute instead.
+
+
 And you can serve following JSON:
 
 ```

data/hypermicrodata.gemspec

@@ -22,6 +22,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "mechanize"
   spec.add_dependency "halibut"
   spec.add_dependency "multi_json"
+  spec.add_dependency "addressable"
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"

data/lib/hypermicrodata.rb

@@ -15,7 +15,7 @@ require "hypermicrodata/extract"
 require "hypermicrodata/rails/html_based_json_renderer"
 require 'open-uri'
 require 'json'
-require 'uri'
+require 'addressable/uri'
 
 module Hypermicrodata
 

data/lib/hypermicrodata/document.rb

@@ -12,17 +12,29 @@ module Hypermicrodata
     end
 
     def extract_items
-      itemscopes = []
-      if @filter_xpath_attr
-        itemscopes = @doc.xpath("//*[#{@filter_xpath_attr} and @itemscope]")
-        puts "XPath //*[#{@filter_xpath_attr}] is not found. root node is used." if itemscopes.empty?
-      end
-      itemscopes = @doc.xpath('self::*[@itemscope] | .//*[@itemscope and not(@itemprop)]') if itemscopes.empty?
-
       itemscopes.collect do |itemscope|
-        Item.new(itemscope, @page_url)
+        Item.parse(itemscope, @page_url)
       end
     end
 
+    private
+
+    def itemscopes
+      items_xpath = 'self::*[@itemscope] | .//*[@itemscope and not(@itemprop)] | .//form[not(@itemprop)]'
+      if @filter_xpath_attr
+        filtered_doc = @doc.xpath("//*[#{@filter_xpath_attr}]")
+        unless filtered_doc.empty?
+          return filtered_doc.xpath(items_xpath)
+        end
+      end
+      print "XPath //*[#{@filter_xpath_attr}] is not found. "
+      filtered_doc = @doc.xpath('//main')
+      unless filtered_doc.empty?
+        print "main node is used.\n"
+        return filtered_doc.xpath(items_xpath)
+      end
+      print "root node is used.\n"
+      @doc.xpath(items_xpath)
+    end
   end
 end

data/lib/hypermicrodata/item.rb

@@ -2,6 +2,14 @@ module Hypermicrodata
   class Item
     attr_reader :type, :properties, :links, :id
 
+    def self.parse(top_node, page_url)
+      if top_node.name == 'form'
+        FormItem.new(top_node, page_url)
+      else
+        Item.new(top_node, page_url)
+      end
+    end
+
     def initialize(top_node, page_url)
       @top_node = top_node
       @type = extract_itemtype
@@ -68,8 +76,7 @@ module Hypermicrodata
       itemscope = element.attribute('itemscope')
       itemprop = element.attribute('itemprop')
       internal_elements = extract_elements(element)
-      add_itemprop(element) if itemscope || itemprop || ItempropParser::LINK_ELEMENTS.include?(element.name)
-      add_form(element) if element.name == 'form'
+      add_itemprop(element) if itemprop || ItempropParser::LINK_ELEMENTS.include?(element.name)
       parse_elements(internal_elements) if internal_elements && !itemscope
     end
 
@@ -77,7 +84,10 @@ module Hypermicrodata
     def add_itemprop(element)
       property = ItempropParser.parse(element, @page_url)
       if property.link? && property.names.empty? && property.rels.empty?
-        (@links['link'] ||= []) << property
+        href = property.value.to_s.strip
+        unless href.empty? || href == '#' # href which doesn't work as link is ignored
+          (@links[element.name] ||= []) << property
+        end
       else
         property.names.each { |name| (@properties[name] ||= []) << property }
         property.rels.each { |rel| (@links[rel] ||= []) << property }
@@ -92,22 +102,64 @@ module Hypermicrodata
       end
     end
 
-    def add_form(element)
-      submit_buttons = FormParser.parse(element, @page_url)
-      submit_buttons.each do |submit_button|
-        submit_button.names.each { |name| (@properties[name] ||= []) << submit_button }
-        if submit_button.rels.empty?
-          (@links['submit'] ||= []) << submit_button
-        else
-          submit_button.rels.each { |rel| (@links[rel] ||= []) << submit_button }
+    # Find an element with a matching id
+    def find_with_id(id)
+      @top_node.search("//*[@id='#{id}']")
+    end
+  end
+
+  class FormItem < Item
+    attr_reader :submit_buttons
+
+    def initialize(top_node, page_url)
+      form = Mechanize::Form.new(top_node)
+      @submit_buttons = form.submits.map do |button|
+        SubmitButton.new(button, form)
+      end
+      super
+    end
+
+    private
+
+    def extract_itemtype
+      super || ['http://schema.org/Action']
+    end
+
+    # TODO: Make it DRY
+    def parse_element(element)
+      itemscope = element.attribute('itemscope')
+      itemprop = element.attribute('itemprop')
+      internal_elements = extract_elements(element)
+      add_itemprop(element) if itemprop || ItempropParser::LINK_ELEMENTS.include?(element.name) || submit_button_include?(element)
+      parse_elements(internal_elements) if internal_elements && !itemscope
+    end
+
+    def add_itemprop(element)
+      return super unless submit_button_include?(element)
+      property = @submit_buttons.find {|b| b.node == element }
+      if property.names.empty? && property.rels.empty?
+        href = property.value.to_s.strip
+        unless href.empty? || href == '#' # href which doesn't work as link is ignored
+          (@links[element.name] ||= []) << property
         end
+      else
+        property.names.each { |name| (@properties[name] ||= []) << property }
+        property.rels.each { |rel| (@links[rel] ||= []) << property }
       end
     end
 
-    # Find an element with a matching id
-    def find_with_id(id)
-      @top_node.search("//*[@id='#{id}']")
+    def submit_button_include?(element)
+      @submit_buttons.any? {|b| b.node == element }
     end
+  end
+end
 
+# Patch for bug
+Mechanize::Form.class_eval do
+  # Returns all buttons of type Submit
+  def submits
+    @submits ||= buttons.select {|f|
+      f.class == Mechanize::Form::Submit || (f.class == Mechanize::Form::Button && (f.type.nil? || f.type == 'submit'))
+    }
   end
 end

data/lib/hypermicrodata/itemprop_parser.rb

@@ -65,10 +65,10 @@ module Hypermicrodata
     # This returns an empty string if can't form a valid
     # absolute url as per the Microdata spec.
     def make_absolute_url(url)
-      return url unless URI.parse(url).relative?
+      return url unless Addressable::URI.parse(url).relative?
       begin
-        URI.parse(@page_url).merge(url).to_s
-      rescue URI::Error
+        Addressable::URI.parse(@page_url).merge(url).to_s
+      rescue
         url
       end
     end
@@ -104,7 +104,7 @@ module Hypermicrodata
 
     def extract_property
       if @element.attribute('itemscope')
-        Item.new(@element, @page_url)
+        Item.parse(@element, @page_url)
       else
         extract_property_value
       end

data/lib/hypermicrodata/submit_button.rb

@@ -26,11 +26,11 @@ module Hypermicrodata
     end
 
     def names
-      (@button.node['itemprop'] || '').split(' ')
+      (node['itemprop'] || '').split(' ')
     end
 
     def rels
-      rel = (@button.node['rel'] || @button.node['data-rel'] || @button.dom_class || '')
+      rel = (node['rel'] || node['data-rel'] || @button.dom_class || '')
       rel.split(' ')
     end
 
@@ -46,6 +46,10 @@ module Hypermicrodata
       true
     end
 
+    def node
+      @button.node
+    end
+
     private
     def setup!
       if method_field = @form.fields.find { |f| f.name == '_method' }
@@ -76,30 +80,4 @@ module Hypermicrodata
       end.compact.join('&')
     end
   end
-
-  class FormParser
-    attr_reader :submit_buttons
-
-    def initialize(element, page_url = nil)
-      @element, @page_url = element, page_url
-      form = Mechanize::Form.new(element)
-      @submit_buttons = form.submits.map do |button|
-        SubmitButton.new(button, form)
-      end
-    end
-
-    def self.parse(element, page_url = nil)
-      self.new(element, page_url).submit_buttons
-    end
-  end
-end
-
-# Patch for bug
-Mechanize::Form.class_eval do
-  # Returns all buttons of type Submit
-  def submits
-    @submits ||= buttons.select {|f|
-      f.class == Mechanize::Form::Submit || (f.class == Mechanize::Form::Button && (f.type.nil? || f.type == 'submit'))
-    }
-  end
 end

data/lib/hypermicrodata/version.rb

@@ -1,3 +1,3 @@
 module Hypermicrodata
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hypermicrodata
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Jason Ronallo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-13 00:00:00.000000000 Z
+date: 2015-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -67,6 +67,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -154,7 +168,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.2
+rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
 summary: Ruby library for extracting HTML5 Microdata with Hypermedia