scrappy 0.3.4 → 0.3.5

data/History.txt

@@ -1,3 +1,7 @@
+=== 0.3.5 2011-03-29
+
+* Added nofollow support to NewUriSelectors
+
 === 0.3.4 2011-03-25
 
 * Fixed gem dependency

data/lib/scrappy.rb

@@ -24,5 +24,5 @@ require 'scrappy/agent/blind_agent'
 require 'scrappy/agent/agent'
 
 module Scrappy
-  VERSION = '0.3.4'
+  VERSION = '0.3.5'
 end

data/lib/scrappy/agent/agent.rb

@@ -71,17 +71,22 @@ module Scrappy
       else
         []
       end
+
+      # Recently created URIs are not followed
+      nofollow = triples.select { |s,p,o| p==ID("rdf:type") and o==ID("sc:NewUri") }.map{|s,p,o| s}.select{|n| n.is_a?(Symbol)}
+      pages -= nofollow
+      uris  -= nofollow
       
       items = ( pages.map { |uri| {:uri=>uri.to_s, :depth=>[-1, depth].max} } +
                 uris.map  { |uri| {:uri=>uri.to_s, :depth=>[-1, depth-1].max} } ).
-                uniq.select{ |item| !RDF::ID.bnode?(item[:uri]) }
+                uniq.select { |item| !RDF::ID.bnode?(item[:uri]) }
       
-      items.each { |item| puts "Enqueuing (depth = #{item[:depth]}): #{item[:uri]}" } if options.debug
+      items.each { |item| puts "Enqueuing (depth = #{item[:depth]}): #{item[:uri]}" if !queue or !(queue.history + queue.items).include?(item) } if options.debug
       
-      if queue.nil?
-        triples += process items
-      else
+      if queue
         items.each { |item| queue.push_unless_done item }
+      else
+        triples += process items
       end
 
       triples unless options.dump
@@ -153,7 +158,7 @@ module Scrappy
     end
 
     def clean triples
-      triples.uniq.select { |s,p,o| p!=ID('rdf:type') or ![ID('sc:Index'), ID('sc:Page')].include?(o) }
+      triples.uniq.select { |s,p,o| p!=ID('rdf:type') or ![ID('sc:Index'), ID('sc:Page'), ID('sc:NewUri')].include?(o) }
     end
     
     # Do the extraction using RDF repository

data/lib/scrappy/agent/map_reduce.rb

@@ -6,6 +6,8 @@ module MapReduce
   class Queue
     include MonitorMixin
     
+    attr_reader :history, :items
+    
     def initialize
       super
       @items   = []
@@ -17,7 +19,7 @@ module MapReduce
       item = nil
       synchronize do
         item = @items.shift
-        @history << item
+        @history << item if item
         if @items.empty?
           yield item if (block_given? and item)
           yielded = true
@@ -36,7 +38,7 @@ module MapReduce
     end
 
     def push_unless_done value
-      synchronize { @items << value unless @history.include?(value) }
+      synchronize { @items << value if !@history.include?(value) and !@items.include?(value) }
     end
     
     def empty?

data/lib/scrappy/extractor/fragment.rb

@@ -2,20 +2,19 @@ module Sc
   class Fragment
     include RDF::NodeProxy
 
-    # Extracts data out of a document and returns an RDF::Graph
-    def extract_graph options={}
-      graph = RDF::Graph.new
-      extract(options).each { |node| graph << node }
-      graph
-    end
-
     # Extracts data out of a document and returns an array of nodes
     def extract options={}
+      all_mappings(options).map { |mapping| mapping[:node] }
+    end
+
+    # Returns all mappings of a fragment by
+    # recursively processing all submappings.
+    def all_mappings options={}
       # Extracts all the mappings and any subfragment
-      mappings(options).map do |result|
-        node         = result[:node]
-        subfragments = result[:subfragments]
-        doc          = result[:doc]
+      mappings(options).map do |mapping|
+        node         = mapping[:node]
+        subfragments = mapping[:subfragments]
+        doc          = mapping[:doc]
 
         # Process subfragments
         consistent = true
@@ -23,18 +22,19 @@ module Sc
           # Get subfragment object
           subfragment = subfragment.proxy Node('sc:Fragment')
           
-          # Extract data from the subfragment
-          subnodes  = subfragment.extract(options.merge(:doc=>doc))
+          # Add triples from submappings
+          submappings = subfragment.all_mappings(options.merge(:doc=>doc))
 
           # Add relations
-          subnodes.each do |subnode|
+          submappings.each do |submapping|
+            subnode = submapping[:node]
             node.graph << subnode if subnode.is_a?(RDF::Node)
             subfragment.sc::relation.each { |relation| node[relation] += [subnode] }
           end
           
           # Check consistency
-          consistent = false if subfragment.sc::min_cardinality.first and subnodes.size < subfragment.sc::min_cardinality.first.to_i
-          consistent = false if subfragment.sc::max_cardinality.first and subnodes.size > subfragment.sc::max_cardinality.first.to_i
+          consistent = false if subfragment.sc::min_cardinality.first and submappings.size < subfragment.sc::min_cardinality.first.to_i
+          consistent = false if subfragment.sc::max_cardinality.first and submappings.size > subfragment.sc::max_cardinality.first.to_i
         end
 
         # Skip the node if it has inconsistent relations
@@ -42,11 +42,12 @@ module Sc
         # violate the constraint sc:min_cardinality = 1
         next if !consistent
         
-        node
+        { :node=>node, :subfragments=>subfragments, :doc=>doc }
       end.compact
     end
 
-    # Returns all the mappings between this fragment and RDF nodes
+    # Returns the mappings between this fragment
+    # and the RDF nodes it matches
     def mappings options
       # Identify the fragment's mappings
       docs = sc::selector.map { |s| graph.node(s).select options[:doc] }.flatten
@@ -66,7 +67,7 @@ module Sc
           value = doc[:value].to_s.strip
           if options[:referenceable]
             node.rdf::value = value
-            node.rdf::type  = Node('rdf:Literal')
+            node.rdf::type += [Node('rdf:Literal')]
             node
           else
             value
@@ -117,6 +118,7 @@ module Sc
           node.graph  << uri_node
           node.sc::uri = uri_node
         end
+        node.rdf::type += [Node("sc:NewUri")] if d[:nofollow]
         
         node
       end.first

data/lib/scrappy/extractor/selectors/new_uri.rb

@@ -13,6 +13,8 @@ module Sc
       prefix = (prefix =~ /\Ahttp\:/ or prefix =~ /\Ahttps\:/) ? URI::parse(doc[:uri]).merge(prefix).to_s : "#{doc[:uri]}#{prefix}"
       suffix = sc::suffix.first.to_s
       
+      nofollow = (sc::follow.first != "false")
+      
       contents.map do |content, attribute|
         new_uri = if (content.to_s =~ /\Ahttp\:/ or content.to_s =~ /\Ahttps\:/)
           "#{content}#{suffix}"
@@ -29,7 +31,7 @@ module Sc
           "#{prefix}#{variable}#{suffix}"
         end
         
-        { :uri=>new_uri, :content=>doc[:content], :value=>new_uri, :attribute=>attribute }
+        { :uri=>new_uri, :content=>doc[:content], :value=>new_uri, :attribute=>attribute, :nofollow=>nofollow }
       end
     end
   end

data/lib/scrappy/learning/optimizer.rb

@@ -2,10 +2,7 @@ module Scrappy
   module Optimizer
     # Iterates through a knowledge base and tries to merge and generalize
     # selectors whenever the output of the resulting kb is the same
-    def optimize kb, sample
-      # Get the output only once
-      output = RDF::Graph.new extract(sample[:uri], sample[:html], kb)
-
+    def optimize_patterns kb, sample
       # Build an array of fragments
       root_fragments = kb.find(nil, Node('rdf:type'), Node('sc:Fragment')) - kb.find([], Node('sc:subfragment'), nil)
       fragments = []; root_fragments.each { |f| fragments << kb.node(Node(f.id, RDF::Graph.new(f.all_triples))) }
@@ -13,89 +10,119 @@ module Scrappy
       # Parse the document
       doc = { :uri=>sample[:uri], :content=>Nokogiri::HTML(sample[:html], nil, 'utf-8') }
 
-      begin
-        changed, fragments = optimize_once fragments, doc, output
-      end until !changed
+      # Optimize the fragment
+      fragments = optimize fragments, :docs=>[doc]
       
       graph = RDF::Graph.new
       fragments.each { |fragment| graph << fragment }
-
-      puts graph.serialize(:yarf)
-      exit
       
       graph
     end
     
     protected
-    # Tries to optimize a set of fragments.
-    # Returns true if there were changes, false otherwise,
-    # and the new fragments as the second array element
-    def optimize_once fragments, doc, output
+    # Tries to optimize a set of fragments
+    def optimize fragments, options
+      # Tries to iterate until no changes are made
+      @tried = []
+      new_fragments = fragments.map{ |f| f.proxy(Node('sc:Fragment')) }
+      begin
+        fragments = new_fragments
+        new_fragments = optimize_once fragments, options
+      end until fragments == new_fragments
+      fragments
+    end
+    
+    # Tries to perform one optimization of two fragments out of a set of fragments
+    def optimize_once fragments, options
+      docs = options[:docs]
       fragments.each do |fragment1|
         fragments.each do |fragment2|
           next if fragment1 == fragment2
-          new_fragment = mix_if_gain(fragment1, fragment2, doc, output)
+          # Won't get gain if the fragment does not produce the same kind of RDF resource
+          next if !fragment1.sc::type.equivalent?(fragment2.sc::type) or
+                  !fragment1.sc::relation.equivalent?(fragment2.sc::relation) or
+                  !fragment1.sc::superclass.equivalent?(fragment2.sc::superclass) or
+                  !fragment1.sc::sameas.equivalent?(fragment2.sc::sameas) or
+                   fragment1.sc::identifier.size != fragment2.sc::identifier.size
+
+          next if @tried.include?([fragment1, fragment2])
+          next if @tried.include?([fragment2, fragment1])
+
+          @tried << [fragment1, fragment2]
+
+          # Get mappings without mixing fragments
+          old_mappings = []
+          docs.each do |doc|
+            old_mappings += fragment1.all_mappings(:doc=>doc)
+            old_mappings += fragment2.all_mappings(:doc=>doc)
+          end
+          old_docs = old_mappings.map { |mapping| mapping[:doc] }
+          
+          # Get mixed fragment
+          new_fragment = mix(fragment1, fragment2, options)
           
-          # End if a new fragment was created
-          if new_fragment
-            return [true, fragments - [fragment1] - [fragment2] + [new_fragment]]
+          # Get new mappings
+          new_mappings = []
+          docs.each { |doc| new_mappings += new_fragment.mappings(:doc=>doc) }
+          new_docs = new_mappings.map { |mapping| mapping[:doc] }
+
+          # Optimize subfragments
+          subfragments = optimize(fragment1.sc::subfragment + fragment2.sc::subfragment, options.merge(:docs=>new_docs))
+          subfragments.each { |subfragment| new_fragment.graph << subfragment }
+          new_fragment.sc::subfragment = subfragments.map &:node
+
+          # End if the new fragment returns the same results
+          if true
+            return fragments - [fragment1] - [fragment2] + [new_fragment]
           end
         end
       end
-      [false, fragments]
+      fragments
     end
     
-    def mix_if_gain fragment1, fragment2, doc, output
-      # Won't get gain if the fragment does not produce the same kind of RDF resource
-      return if !fragment1.sc::type.equivalent?(fragment2.sc::type) or
-                !fragment1.sc::relation.equivalent?(fragment2.sc::relation) or
-                !fragment1.sc::superclass.equivalent?(fragment2.sc::superclass) or
-                !fragment1.sc::sameas.equivalent?(fragment2.sc::sameas) or
-                 fragment1.sc::identifier.size != fragment2.sc::identifier.size
+    def mix fragment1, fragment2, options
+      docs = options[:docs]
       
       # Build new fragment
-      new_fragment = Node(nil)
-      new_fragment.rdf::type      = fragment1.rdf::type
+      new_fragment = Node(nil).proxy(Node('sc:Fragment'))
+      new_fragment.rdf::type      = Node('sc:Fragment')
       new_fragment.sc::type       = fragment1.sc::type
       new_fragment.sc::relation   = fragment1.sc::relation
       new_fragment.sc::superclass = fragment1.sc::superclass
       new_fragment.sc::sameas     = fragment1.sc::sameas
+
+      # If fragments share the same parent, cardinality has to increase
+      # Otherwise, they might map the same subdocument, so cardinality
+      # limits are made more general.
+      if fragment1.sc::superfragment.first and fragment1.sc::superfragment == fragment2.sc::superfragment
+        new_fragment.sc::min_cardinality = (fragment1.sc::min_cardinality.first.to_i + fragment2.sc::min_cardinality.first.to_i).to_s
+        new_fragment.sc::max_cardinality = (fragment1.sc::max_cardinality.first.to_i + fragment2.sc::max_cardinality.first.to_i).to_s
+      else
+        new_fragment.sc::min_cardinality = [fragment1.sc::min_cardinality.first.to_i, + fragment2.sc::min_cardinality.first.to_i].min.to_s
+        new_fragment.sc::max_cardinality = [fragment1.sc::max_cardinality.first.to_i, + fragment2.sc::max_cardinality.first.to_i].max.to_s
+      end
       
       # sc:selector
-      selector = generalize_selectors(fragment1.sc::selector + fragment2.sc::selector)
+      selector = generalize(fragment1.sc::selector + fragment2.sc::selector)
       new_fragment.graph       << selector
       new_fragment.sc::selector = selector
       
       # sc:identifier
       if fragment1.sc::identifier.first
-        selector = generalize_selectors(fragment1.sc::identifier + fragment2.sc::identifier)
+        selector = generalize(fragment1.sc::identifier + fragment2.sc::identifier)
         new_fragment.graph         << selector
         new_fragment.sc::identifier = selector
       end
-      
-      # sc:subfragment
-      all_subfragments = fragment1.sc::subfragment + fragment2.sc::subfragment
-      all_subfragments.map { |sf| [sf.sc::type.sort_by(&:to_s), sf.sc::relation.sort_by(&:to_s)] }.uniq.each do |types, relations|
-        subfragments = all_subfragments.select do |sf|
-          sf.sc::type.sort_by(&:to_s)     == types and
-          sf.sc::relation.sort_by(&:to_s) == relations
-        end
-      end
-      
-      # Check new output
-      separate_output1 = fragment1.extract_graph :doc=>doc
-      separate_output2 = fragment2.extract_graph :doc=>doc
-      separate_output  = separate_output1.merge separate_output2
-      new_output       = new_fragment.proxy.extract_graph :doc=>doc
-      
-      # Check if the output with the new fragment is a subset of the full output
-      # and if the output of the fragments alone is a subset of the output of the new
-      # fragment. This way we ensure the output is the same without using all the
-      # fragments that are available in the knowledge base.
-      new_fragment.proxy # if output.contains?(new_output) and new_output.contains?(separate_output)
+
+      # All new nodes are expected to be inconsistent after performing
+      # subfragments' extractions. Otherwise, if new nodes are consistent, it means
+      # the output from the mixed fragment is different from the separate fragments
+      # and therefore the generalization has failed, so no mixed fragment is returned
+      new_fragment
     end
-    
-    def generalize_selectors selectors
+
+    # Generalize a set of selectors
+    def generalize selectors
       selector = Node(nil)
       selector.rdf::type           = Node('sc:VisualSelector')
       selector.sc::min_relative_x  = selectors.map { |s| s.sc::min_relative_x.map(&:to_i) }.flatten.min.to_s

data/lib/scrappy/server/admin.rb

@@ -58,8 +58,9 @@ module Scrappy
       # Patterns
       
       app.get '/patterns' do
-        @uris = Scrappy::Kb.patterns.find(nil, Node('rdf:type'), Node('sc:Fragment')).
-                map { |node| node.sc::type }.flatten.map(&:to_s).sort
+        @uris = ( Scrappy::Kb.patterns.find(nil, Node('rdf:type'), Node('sc:Fragment')) -
+                  Scrappy::Kb.patterns.find([], Node('sc:subfragment'), nil) ).
+                  map { |node| node.sc::type }.flatten.map(&:to_s).sort
         haml :patterns
       end
 
@@ -95,7 +96,7 @@ module Scrappy
       end
       
       app.post '/samples/:id/optimize' do |id|
-        Scrappy::Kb.patterns = agent.optimize(Scrappy::Kb.patterns, Scrappy::App.samples[id.to_i])
+        Scrappy::Kb.patterns = agent.optimize_patterns(Scrappy::Kb.patterns, Scrappy::App.samples[id.to_i])
         Scrappy::App.save_patterns Scrappy::Kb.patterns
         flash[:notice] = "Optimization completed"
         redirect "#{settings.base_uri}/samples"

data/scrappy.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{scrappy}
-  s.version = "0.3.4"
+  s.version = "0.3.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jose Ignacio"]
-  s.date = %q{2011-03-25}
+  s.date = %q{2011-03-29}
   s.default_executable = %q{scrappy}
   s.description = %q{RDF web scraper}
   s.email = %q{joseignacio.fernandez@gmail.com}

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 3
-  - 4
-  version: 0.3.4
+  - 5
+  version: 0.3.5
 platform: ruby
 authors: 
 - Jose Ignacio
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-03-25 00:00:00 +01:00
+date: 2011-03-29 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency