scrappy 0.3.5 → 0.4.0

data/lib/scrappy/learning/trainer.rb

@@ -3,43 +3,79 @@ module Scrappy
     # Generates visual patterns
     def train *samples
       RDF::Graph.new( samples.inject([]) do |triples, sample|
-        triples + train_sample(sample).triples
+        triples + train_sample( sample ).triples
       end )
     end
-    
+
+    # Generate XPath fragments
+    def train_xpath *samples
+      RDF::Graph.new( samples.inject([]) do |triples, sample|
+        triples + train_sample(sample, true).triples
+      end )
+    end
+
     private
-    def train_sample sample
-      results = RDF::Graph.new extract(sample[:uri], sample[:html], Scrappy::Kb.extractors, :minimum)
+    def train_sample sample, xpath=false
+      sample  = sample.merge(:content=>Nokogiri::HTML(sample[:html], nil, 'utf-8'))
+      results = RDF::Graph.new extract(sample[:uri], sample[:html], xpath ? Scrappy::Kb.patterns : Scrappy::Kb.extractors, :minimum)
 
       typed_nodes     = results.find(nil, Node("rdf:type"), [])
       non_root_nodes  = results.find([], [], nil)
 
       nodes = typed_nodes - non_root_nodes
-      
+
+      superfragment = Node(nil)
+      selector      = Node(nil)
+      identifier    = Node(nil)
+      selector.rdf::type    = Node('sc:UriSelector')
+      selector.rdf::value   = sample[:uri]
+      identifier.rdf::type  = Node('sc:BaseUriSelector')
+      superfragment.rdf::type       = Node('sc:Fragment')
+      superfragment.sc::selector    = selector
+      superfragment.sc::identifier  = identifier
+      superfragment.graph << selector
+      superfragment.graph << identifier
+
       RDF::Graph.new( nodes.inject([]) do |triples, node|
-        triples + fragment_for(node).graph.triples
-      end )
+        fragment = fragment_for(node, sample, xpath)
+        # Include a superfragment that limits the fragment to a specified URI
+        if xpath
+          other_triples = [ [superfragment.id, ID('sc:subfragment'), fragment.id] ]
+        else
+          other_triples = []
+        end
+        
+        triples + fragment.graph.triples + other_triples
+      end + (xpath ? superfragment.graph.triples : []) )
     end
     
-    def fragment_for node, parent=nil
+    def fragment_for node, sample, xpath, parent=nil, parent_path=nil
       fragment = Node(nil)
+      node_path = node.sc::source.first.sc::selector.first.sc::path.first
       node.keys.each do |predicate|
         case predicate
         when ID("sc:source") then
-          selector = selector_for(node.sc::source.first, parent)
+          selector = selector_for(node.sc::source.first, sample, xpath, parent, parent_path)
           fragment.graph << selector
           fragment.sc::selector = selector
         when ID("sc:uri") then
-          selector = selector_for(node.sc::uri.first.sc::source.first, node)
+          selector = selector_for(node.sc::uri.first.sc::source.first, sample, xpath, node, node_path)
           fragment.graph << selector
           fragment.sc::identifier = selector
         when ID("rdf:type") then
           fragment.sc::type = node.rdf::type
         else
           if node[predicate].map(&:class).uniq.first == RDF::Node
+            done = []
             node[predicate].map do |subnode|
-              subfragment = fragment_for(subnode, node)
+              selector = subnode.sc::source.first.sc::selector.first
+              next if done.include?( {}.merge(selector) )
+              done << {}.merge(selector)
+
+              subfragment = fragment_for(subnode, sample, xpath, node, node_path)
               subfragment.sc::relation = Node(predicate)
+              subfragment.sc::min_cardinality = "1"
+              subfragment.sc::max_cardinality = "1"
           
               fragment.graph << subfragment
               fragment.sc::subfragment += [subfragment]
@@ -48,47 +84,83 @@ module Scrappy
         end
       end
       fragment.rdf::type = Node("sc:Fragment")
-      fragment.sc::min_cardinality = "1"
-      fragment.sc::max_cardinality = "1"
+      
       fragment
     end
     
-    def selector_for fragment, parent=nil
+    def selector_for fragment, sample, xpath=false, parent=nil, parent_path=nil
       fragment_selector = fragment.sc::selector.first
       presentation = fragment.sc::presentation.first
       
       selector = Node(nil)
-      selector.rdf::type = Node("sc:VisualSelector")
-
-      origin_x = parent ? parent.sc::source.first.sc::presentation.first.sc::x.first.to_i : 0
-      origin_y = parent ? parent.sc::source.first.sc::presentation.first.sc::y.first.to_i : 0
-      
-      relative_x = presentation.sc::x.first.to_i - origin_x
-      relative_y = presentation.sc::y.first.to_i - origin_y
       
-      selector.sc::min_relative_x  = relative_x.to_s
-      selector.sc::max_relative_x  = relative_x.to_s
-      selector.sc::min_relative_y  = relative_y.to_s
-      selector.sc::max_relative_y  = relative_y.to_s
-      selector.sc::min_x           = presentation.sc::x
-      selector.sc::max_x           = presentation.sc::x
-      selector.sc::min_y           = presentation.sc::y
-      selector.sc::max_y           = presentation.sc::y
-
-      selector.sc::min_width       = presentation.sc::width
-      selector.sc::max_width       = presentation.sc::width
-      selector.sc::min_height      = presentation.sc::height
-      selector.sc::max_height      = presentation.sc::height
-      selector.sc::min_font_size   = presentation.sc::font_size
-      selector.sc::max_font_size   = presentation.sc::font_size
-      selector.sc::min_font_weight = presentation.sc::font_weight
-      selector.sc::max_font_weight = presentation.sc::font_weight
-      selector.sc::font_family     = presentation.sc::font_family
+      if xpath
+        selector.rdf::type  = Node("sc:XPathSelector")
+        selector.sc::text   = presentation.sc::text
+        selector.rdf::value = path_for fragment_selector.sc::path.first, parent_path, sample
+      else
+        selector.rdf::type = Node("sc:VisualSelector")
+        
+        origin_x = parent ? parent.sc::source.first.sc::presentation.first.sc::x.first.to_i : 0
+        origin_y = parent ? parent.sc::source.first.sc::presentation.first.sc::y.first.to_i : 0
+        
+        relative_x = presentation.sc::x.first.to_i - origin_x
+        relative_y = presentation.sc::y.first.to_i - origin_y
+        
+        selector.sc::min_relative_x  = relative_x.to_s
+        selector.sc::max_relative_x  = relative_x.to_s
+        selector.sc::min_relative_y  = relative_y.to_s
+        selector.sc::max_relative_y  = relative_y.to_s
+        selector.sc::min_x           = presentation.sc::x
+        selector.sc::max_x           = presentation.sc::x
+        selector.sc::min_y           = presentation.sc::y
+        selector.sc::max_y           = presentation.sc::y
+  
+        selector.sc::min_width       = presentation.sc::width
+        selector.sc::max_width       = presentation.sc::width
+        selector.sc::min_height      = presentation.sc::height
+        selector.sc::max_height      = presentation.sc::height
+        selector.sc::min_font_size   = presentation.sc::font_size
+        selector.sc::max_font_size   = presentation.sc::font_size
+        selector.sc::min_font_weight = presentation.sc::font_weight
+        selector.sc::max_font_weight = presentation.sc::font_weight
+        selector.sc::font_family     = presentation.sc::font_family
+  
+        selector.sc::tag = ["text"] if presentation.sc::font_family.first
+        special_tag      = fragment_selector.sc::tag.select { |tag| ["a","img"].include?(tag) }
+        selector.sc::tag = special_tag if special_tag.size > 0
+      end
 
-      selector.sc::tag       = fragment_selector.sc::tag.select { |tag| ["a","img"].include?(tag) }
       selector.sc::attribute = fragment_selector.sc::attribute
       
       selector
     end
+    
+    def path_for path, parent_path, sample
+      return "./." if path == parent_path
+      return path if ["", "/html", "/html/body"].include?(path)
+      
+      node = sample[:content].search(path).first
+      conditions = []
+      if node[:class]
+        conditions += node[:class].split(" ").map {|c| "contains(concat(' ',normalize-space(@class),' '),concat(' ','#{c.strip}',' '))" }
+      else
+        conditions += ["not(@class)"]
+      end
+      if node[:id]
+        conditions += ["contains(@id,'#{node[:id].strip}')"]
+      else
+        conditions += ["not(@id)"]
+      end
+      selector = "/#{node.name}[#{conditions * " and "}]"
+      index = nil
+      results = node.parent.search(".#{selector}")
+      results.each_with_index { |n,i| index = i+1 if n.path == path }
+      
+      previous_path = path.split("/")[0..-2] * "/"
+      suffix = results.size > 1  ? "[#{index}]" : ""
+      
+      path_for(previous_path, parent_path, sample) + selector + suffix
+    end
   end
 end

data/lib/scrappy/server/admin.rb

@@ -37,11 +37,13 @@ module Scrappy
       app.post '/extractors' do
         if params[:html]
           # Generate extractor automatically
-          iconv = Iconv.new(params[:encoding], 'UTF-8')
-          html  = iconv.iconv(params[:html]) 
-          puts params[:html]
-          puts params[:uri]
-          raise Exception, "Automatic generation of extractors is not supported yet"
+          html = Iconv.iconv('UTF-8', params[:encoding], params[:html]).first
+          samples = [{ :html=>html, :uri=>params[:uri] }]
+          extractor = agent.train_xpath(*samples)
+          # Train
+          Scrappy::App.add_extractor extractor
+          # Optimize
+          Scrappy::App.replace_extractor agent.optimize_extractors(Scrappy::Kb.extractors, samples), samples
         else
           # Store the given extractor
           Scrappy::App.add_extractor RDF::Parser.parse(:yarf,params[:rdf])
@@ -58,14 +60,30 @@ module Scrappy
       # Patterns
       
       app.get '/patterns' do
-        @uris = ( Scrappy::Kb.patterns.find(nil, Node('rdf:type'), Node('sc:Fragment')) -
-                  Scrappy::Kb.patterns.find([], Node('sc:subfragment'), nil) ).
-                  map { |node| node.sc::type }.flatten.map(&:to_s).sort
+        @patterns = ( Scrappy::Kb.patterns.find(nil, Node('rdf:type'), Node('sc:Fragment')) -
+                      Scrappy::Kb.patterns.find([], Node('sc:subfragment'), nil) )
         haml :patterns
       end
 
-      app.delete '/patterns/*' do |uri|
-        Scrappy::App.delete_pattern uri
+      app.get '/patterns/visual' do
+        @patterns = ( Scrappy::Kb.patterns.find(nil, Node('rdf:type'), Node('sc:Fragment')) -
+                      Scrappy::Kb.patterns.find([], Node('sc:subfragment'), nil) )
+        html = @patterns.map { |pattern| render_fragment(pattern) } * ""
+        "<html><body>#{html}</body></html>"
+      end
+
+      app.get '/patterns/*' do |id|
+        "<html><body>#{render_fragment(Scrappy::Kb.patterns[id])}</body></html>"
+      end
+
+      app.delete '/patterns' do
+        Scrappy::App.delete_patterns
+        flash[:notice] = "Patterns deleted"
+        redirect "#{settings.base_uri}/patterns"
+      end
+      
+      app.delete '/patterns/*' do |id|
+        Scrappy::App.delete_pattern id
         flash[:notice] = "Pattern deleted"
         redirect "#{settings.base_uri}/patterns"
       end
@@ -78,9 +96,34 @@ module Scrappy
       end
       
       app.get '/samples/:id' do |id|
+        Nokogiri::HTML(Scrappy::App.samples[id.to_i][:html], nil, 'utf-8').search("*").map do |node|
+          next if node.text?
+          text  = node.children.map { |n| n.content if n.text? } * " "
+          x = node[:vx].to_i
+          y = node[:vy].to_i
+          w = node[:vw].to_i
+          h = node[:vh].to_i
+          font = node[:vfont]
+          size = node[:vsize].to_i
+          weight = node[:vweight].to_i
+          color  = "#555"
+          color  = "#55f" if node.name == "a"
+          style  = "position: absolute; left: #{x}px; top: #{y}px; width: #{w}px; height: #{h}px; font-family: #{font}; font-size: #{size}px; font-weight: #{weight}; border: 1px solid gray; color: #{color};"
+          style += "background-color: #f00; opacity: 0.2;" if node.name == "img"
+          style += "text-decoration: underline;" if node.name == "a"
+          "<div style='#{style}'>#{text}</div>"
+        end * ""
+      end
+
+      app.get '/samples/:id/raw' do |id|
         Scrappy::App.samples[id.to_i][:html]
       end
-      
+
+      app.get '/samples/:id/annotations' do |id|
+        headers 'Content-Type' => 'text/plain'
+        RDF::Graph.new(Scrappy::App.samples[id.to_i][:output] || []).serialize(:yarf)
+      end
+
       app.get '/samples/:id/:kb_type' do |id,kb_type|
         kb = (kb_type == "patterns" ? Scrappy::Kb.patterns : Scrappy::Kb.extractors)
         sample = Scrappy::App.samples[id.to_i]
@@ -88,19 +131,109 @@ module Scrappy
         RDF::Graph.new(agent.extract(sample[:uri], sample[:html], kb, Agent::Options.referenceable)).serialize(:yarf)
       end
       
-      app.post '/samples/:id/train' do |id|
-        new_extractor = agent.train Scrappy::App.samples[id.to_i]
-        Scrappy::App.add_pattern new_extractor
+      app.post '/samples/annotate' do
+        samples = (params['samples'] || []).map { |i| Scrappy::App.samples[i.to_i] }.each do |sample|
+          sample[:output] = agent.extract(sample[:uri], sample[:html], Scrappy::Kb.extractors)
+        end
+        Scrappy::App.save_samples
+        flash[:notice] = "Samples annotated"
+        redirect "#{settings.base_uri}/samples"
+      end
+      
+      app.post '/samples/train/:kb_type' do |kb_type|
+        kb = (kb_type == "patterns" ? Scrappy::Kb.patterns : Scrappy::Kb.extractors)
+        samples = (params['samples'] || []).map { |i| Scrappy::App.samples[i.to_i] }
+        if kb_type == "patterns"
+          Scrappy::App.add_patterns agent.train(*samples)
+        else
+          Scrappy::App.add_extractor agent.train_xpath(*samples)
+        end
         flash[:notice] = "Training completed"
         redirect "#{settings.base_uri}/samples"
       end
       
-      app.post '/samples/:id/optimize' do |id|
-        Scrappy::Kb.patterns = agent.optimize_patterns(Scrappy::Kb.patterns, Scrappy::App.samples[id.to_i])
-        Scrappy::App.save_patterns Scrappy::Kb.patterns
+      app.post '/samples/optimize/:kb_type' do |kb_type|
+        kb = (kb_type == "patterns" ? Scrappy::Kb.patterns : Scrappy::Kb.extractors)
+        samples = (params['samples'] || []).map { |i| Scrappy::App.samples[i.to_i] }
+        if kb_type == "patterns"
+          Scrappy::App.save_patterns agent.optimize_patterns(kb, samples)
+        else
+          Scrappy::App.replace_extractor agent.optimize_extractors(kb, samples), samples
+        end
         flash[:notice] = "Optimization completed"
         redirect "#{settings.base_uri}/samples"
       end
+      
+      app.post '/samples/test/:kb_type' do |kb_type|
+        kb = (kb_type == "patterns" ? Scrappy::Kb.patterns : Scrappy::Kb.extractors)
+        @results   = {}
+        @missing   = []
+        @wrong     = []
+        output     = RDF::Parser.parse(:ntriples, params["output"].to_s).triples
+        extraction = []
+        (params['samples'] || []).each do |i|
+          sample      = Scrappy::App.samples[i.to_i]
+          output     += sample[:output] || []
+          extraction += agent.extract(sample[:uri], sample[:html], kb)
+        end
+        
+        output     = output.uniq
+        extraction = extraction.uniq
+
+        predicates = output.map { |s,p,o| p }.uniq
+        types      = output.map { |s,p,o| o if p == ID('rdf:type') }.compact.uniq
+
+        predicates.each do |predicate|
+          new_output     = output.select     { |s,p,o| p==predicate }
+          new_extraction = extraction.select { |s,p,o| p==predicate }
+          precision, recall, fscore = agent.send :metrics, new_output, new_extraction
+          @results[predicate] ||= Hash.new(0.0)
+          @results[predicate][:count]     += 1
+          @results[predicate][:fscore]    += fscore
+          @results[predicate][:precision] += precision
+          @results[predicate][:recall]    += recall
+        end
+        
+        types.each do |type|
+          new_output     = output.select     { |s,p,o| p==ID("rdf:type") and o==type }
+          new_extraction = extraction.select { |s,p,o| p==ID("rdf:type") and o==type }
+
+          precision, recall, fscore = agent.send :metrics, new_output, new_extraction
+          @results[type] ||= Hash.new(0.0)
+          @results[type][:count]     += 1
+          @results[type][:fscore]    += fscore
+          @results[type][:precision] += precision
+          @results[type][:recall]    += recall
+        end
+        
+        precision, recall, fscore = agent.send :metrics, output, extraction
+        @results[:total] ||= Hash.new(0.0)
+        @results[:total][:count]     += 1
+        @results[:total][:fscore]    += fscore
+        @results[:total][:precision] += precision
+        @results[:total][:recall]    += recall
+
+        @missing += output     - extraction
+        @wrong   += extraction - output
+
+        # Here we get sth like: { :'dc:title'=>{:fscore=>0.3, ...}, :total=>{:fscore=>0.4, ...} }
+        @results.each do |key, result|
+          count = result[:count]
+          result.each do |k,v|
+            result[k] /= count
+          end
+        end
+        
+        @total      = output.size
+        @extracted  = extraction.size
+        @correct    = @extracted - @wrong.size
+        
+        @missing    = RDF::Graph.new(@missing)
+        @wrong      = RDF::Graph.new(@wrong)
+        
+        flash.now[:notice] = "Testing completed"
+        haml :test
+      end
 
       app.post '/samples' do
         html   = Iconv.iconv('UTF-8', params[:encoding], params[:html]).first
@@ -114,6 +247,36 @@ module Scrappy
         flash[:notice] = "Sample deleted"
         redirect "#{settings.base_uri}/samples"
       end
+      
+      def render_fragment fragment, selected_branch=nil
+        label = if fragment.sc::relation.first
+          fragment.sc::relation.map {|id| RDF::ID.compress(id)} * ', '
+        else
+          fragment.sc::type.map {|id| RDF::ID.compress(id)} * ', '
+        end
+        subfragments = [selected_branch || [:min, :max]].flatten.map do |branch|
+          fragment.sc::subfragment.map { |f| render_fragment(f, branch) } * ""
+        end * ""
+
+        [selected_branch || [:min, :max]].flatten.map do |branch|
+          fragment.sc::selector.map do |selector|
+            x,y,w,h,font,size,weight,color = case branch
+            when :min then
+              [selector.sc::min_relative_x.first, selector.sc::min_relative_y.first, selector.sc::min_width.first, selector.sc::min_height.first, selector.sc::font_family.first, selector.sc::min_font_size.first, selector.sc::min_font_weight.first, :blue]
+            when :max then
+              [selector.sc::max_relative_x.first, selector.sc::max_relative_y.first, selector.sc::max_width.first, selector.sc::max_height.first, selector.sc::font_family.first, selector.sc::max_font_size.first, selector.sc::max_font_weight.first, :red]
+            end
+            style = "position: absolute; left: #{x}px; top: #{y}px; width: #{w}px; height: #{h}px; font-family: #{font}; font-size: #{size}px; font-weight: #{weight}; border: 1px solid #{color}; color: #555;"
+            "<div style='#{style}'>#{label}#{subfragments}</div>"
+          end * ""
+        end * ""
+      end
+      
+      def percentage value
+        "%.2f%" % (value * 100.0)
+      end
+      
+      app.helpers Admin
     end
   end
 end

data/lib/scrappy/support.rb

@@ -29,28 +29,4 @@ class String
     tr("-", "_").
     downcase
   end
-end
-
-class Array
-  # Return true if a given array has the same elements as this one
-  def equivalent? array
-    self.all?  { |i| array.include?(i) } and
-    array.all? { |i| self.include?(i)  }
-  end
-end
-
-module RDF
-  class Node
-    def self.mix *nodes
-      id = nodes.first
-      graph = RDF::Graph.new( nodes.inject([]) do |triples, node|
-        triples + node.graph.triples.map do |s,p,o|
-          [ s==node.id ? id : s,
-            p==node.id ? id : p,
-            o==node.id ? id : o ]
-        end
-      end )
-      graph[id]
-    end
-  end
 end