scrappy 0.3.5 → 0.4.0

data/History.txt

@@ -1,6 +1,11 @@
+=== 0.4.0 2011-06-30
+
+* Added machine learning of patterns that allow creating extractors automatically
+
 === 0.3.5 2011-03-29
 
 * Added nofollow support to NewUriSelectors
+* Fix in queue management
 
 === 0.3.4 2011-03-25
 

data/Rakefile

@@ -10,7 +10,7 @@ Echoe.new('scrappy', Scrappy::VERSION) do |p|
   p.author         = "Jose Ignacio"
   p.email          = "joseignacio.fernandez@gmail.com"
   p.ignore_pattern = ["pkg/*"]
-  p.dependencies = [['activesupport','>= 2.3.5'], ['sinatra', '>= 1.1.2'], ['thin', '>= 1.2.7'], ['nokogiri', '>= 1.4.1'], ['mechanize','>= 1.0.0'], ['lightrdf','>= 0.3.7'], ['i18n', '>= 0.4.2'], ['rest-client', '>=1.6.1'], ['haml', '>= 3.0.24'], ['rack-flash', '>= 0.1.1']]
+  p.dependencies = [['activesupport','>= 2.3.5'], ['sinatra', '>= 1.1.2'], ['thin', '>= 1.2.7'], ['nokogiri', '>= 1.4.1'], ['mechanize','>= 1.0.0'], ['lightrdf','>= 0.3.9'], ['i18n', '>= 0.4.2'], ['rest-client', '>=1.6.1'], ['haml', '>= 3.0.24'], ['rack-flash', '>= 0.1.1']]
 end
 
 Rake::RDocTask.new(:rdoc) do |rdoc|

data/bin/scrappy

@@ -33,7 +33,7 @@ module Scrappy
         opts.on('-g URI', '--get URI')          { |uri| Options.uri = uri; Options.http_method=:get }
         opts.on('-p URI', '--post URI')         { |uri| Options.uri = uri; Options.http_method=:post }
         opts.on('-D', '--dump')                 { Agent::Options.dump = true; Agent::Options.format = :rdf }
-        opts.on('-u', '--debug')                { Agent::Options.debug = true }
+        opts.on('-u [KEY]', '--debug [KEY]')    { |key| Agent::Options.debug = true; Agent::Options.debug_key = key.downcase if key }
         opts.on('-s', '--server [BASE_URI]')    { |uri| Options.server = true; Options.base_uri = uri }
         opts.on('-a', '--admin [BASE_URI]')     { |uri| Options.admin = true; Options.base_uri = uri }
         opts.on('-P P', '--port P')             { |p| Options.port = p }
@@ -71,7 +71,7 @@ module Scrappy
     end
 
     def self.quit
-      puts "\"#{Quotes.sort_by{rand}.first}\"" unless Options.quiet
+      puts "\"#{Quotes[rand(Quotes.length)]}\"" unless Options.quiet
       exit
     end
     
@@ -103,26 +103,61 @@ module Scrappy
     def self.editable_kb?
       @editable_kb
     end
-    def self.add_pattern graph
+    def self.add_patterns graph
       new_patterns = Scrappy::Kb.patterns.merge graph
       save_patterns new_patterns
       onload
     end
     def self.save_patterns new_patterns
-      open(@patterns_file, "w") { |f| f.write new_patterns.serialize(:yarf) }
+      fragments = case new_patterns
+      when Array then
+        new_patterns
+      when RDF::Graph then
+        ( new_patterns.find(nil, Node('rdf:type'), Node('sc:Fragment')) - new_patterns.find([], Node('sc:subfragment'), nil) ).
+          map { |f| RDF::Graph.new(f.all_triples) }
+      end
+
+      content  = RDF::ID.ns.map{|k,v| "#{k}: #{v}\n"} * ''
+      fragments.each { |f| content += f.serialize(:yarf, false) }
+      open(@patterns_file, "w") { |f| f.write content }
+    end
+    def self.delete_patterns
+      graph         = Scrappy::Kb.patterns
+      graph.triples = []
+      content       = graph.serialize(:yarf)
+      open(@patterns_file, "w") { |f| f.write content }
+      onload
     end
-    def self.delete_pattern uri
-      graph = Scrappy::Kb.patterns
-      fragments = graph.find(nil, Node('rdf:type'), Node('sc:Fragment')).
-                        select { |fragment| fragment.sc::type.include?(Node(uri)) }
-      fragments.each { |fragment| graph.triples -= fragment.all_triples }
-      open(@patterns_file, "w") { |f| f.write graph.serialize(:yarf) }
+    def self.delete_pattern id
+      graph          = Scrappy::Kb.patterns
+      fragment       = graph[id]
+      graph.triples -= fragment.all_triples
+      content        = graph.serialize(:yarf)
+      open(@patterns_file, "w") { |f| f.write content }
       onload
     end
     def self.add_extractor graph
       open(File.join(@extractors_folder,"extractor_#{Dir[File.join(@extractors_folder,'*')].size}.yarf"), "w") { |f| f.write graph.serialize(:yarf) }
       onload
     end
+    def self.replace_extractor graph, samples
+      kb = Scrappy::Kb.extractors
+      
+      all_fragments = kb.find(nil, Node('rdf:type'), Node('sc:Fragment')) - kb.find([], Node('sc:subfragment'), nil)
+      all_fragments.each do |fragment|
+        fragment.sc::selector.each do |selector|
+          next unless ( selector.rdf::type.include?(Node('sc:UriSelector')) or
+                        selector.rdf::type.include?(Node('sc:UriPatternSelector')) )
+          
+          samples.each do |sample|
+            selector.rdf::value.each do |uri|
+              delete_extractor(uri) if !kb.node(selector).filter(:uri=>sample[:uri]).empty?
+            end
+          end
+        end
+      end
+      add_extractor graph
+    end
     def self.delete_extractor uri
       Dir[File.join(@extractors_folder, '*')].each do |file|
         format = file.split('.').last.to_sym
@@ -133,9 +168,12 @@ module Scrappy
                           flatten.select do |uri_selector|
                             uri_selector.rdf::value.include?(uri)
         end
+        next if uri_selectors.empty?
         fragments = uri_selectors.map { |uri_selector| graph.find(nil, Node('sc:selector'), uri_selector) }.flatten
         fragments.each { |fragment| graph.triples -= fragment.all_triples }
-        open(file, "w") { |f| f.write graph.serialize(format) } if fragments.any?
+        text = graph.serialize(format)
+        open(file, "w") { |f| f.write text } if fragments.any?
+        File.delete(file) if text==""
       end
       onload
     end
@@ -165,7 +203,7 @@ Options
   -l, --levels VALUE       Sets recursion levels for resource crawling (default is infinite crawling)
   -d, --delay VALUE        Sets delay (in ms) between requests (default is 0)
   -D, --dump               Dumps RDF data to disk
-  -u, --debug              Shows debugging traces
+  -u, --debug [KEYWORD]    Shows debugging traces. Use optional keyword to filter selectors' output
   -o, --observe URLs       Observes the specified URLs storing their data into the repository
   -s, --server [ROOT]      Runs web server (optionally specify server's root url)
   -a, --admin [ROOT]       Runs admin web server (optionally specify server's root url)
@@ -234,15 +272,15 @@ Copyright
       # Load knowledge base
       Agent::Options.kb ||= RDF::Graph.new
       
-      Kb.extractors, Kb.patterns = if File.exists?(@cache_file) and File.mtime(@cache_file) >= Dir["#{@extractors_folder}/*",@extractors_folder,@patterns_file].map{ |f| File.mtime(f) }.max
+      Kb.extractors, Kb.patterns, RDF::ID::count = if File.exists?(@cache_file) and File.mtime(@cache_file) > Dir["#{@extractors_folder}/*",@extractors_folder,@patterns_file].map{ |f| File.mtime(f) }.max
         # Just load kb from cache
         open(@cache_file) { |f| Marshal.load(f) }
       else
         # Load YARF files and cache kb
         extractors = load_files_from(@extractors_folder)
         patterns   = File.exists?(@patterns_file) ? RDF::Parser.parse(:yarf, open(@patterns_file).read) : RDF::Graph.new
-        open(@cache_file, "w") { |f| Marshal.dump([extractors, patterns], f) }
-        [extractors, patterns]
+        open(@cache_file, "w") { |f| Marshal.dump([extractors, patterns, RDF::ID::count], f) }
+        [extractors, patterns, RDF::ID::count]
       end
 
       # Sets new kb
@@ -315,7 +353,7 @@ Dogs have owners, cats have staff
 I put all my genius into my life; I put only my talent into my works
 It is better to be beautiful than to be good, but it is better to be good than to be ugly
 All human beings, by nature, desire to know
-All life is an experiment
+All life is an experiment. The more experiments you make the better
 An investment in knowledge always pays the best interest
 An optimist is a person who sees a green light everywhere. The pessimist sees only the red light. But the truly wise person is color blind
 Chance favors only those who court her
@@ -347,7 +385,27 @@ The man who does things makes many mistakes, but he never makes the biggest mist
 The man who makes no mistakes does not usually make anything
 The results you achieve will be in direct proportion to the effort you apply
 The reward of a thing well done is to have done it
-Don’t argue with idiots. They will bring you down to their level and beat you with experience""".split("\n")
+Don’t argue with idiots. They will bring you down to their level and beat you with experience
+Choose a work you love, and you will never have to work a day in your life
+The secret of creativity is knowing how to hide your sources
+I never think of the future. It comes soon enough
+If you want to go quick, go alone. If you want to go far, go together
+The only thing that interferes with my learning is my education
+Excesive literary production is a social offense
+A man who dares to waste one hour of time has not discovered the value of life
+Any idiot can face a crisis -- it's day to day living that wears you out
+Every man dies. Not every man really lives
+After two weeks of working on a project, you know whether it will work or not
+All things are difficult before they are easy
+Sport is hard work for which you do not get paid
+Do not hire a man who does your work for money, but him who does it for love of it
+Failure is success if we learn from it
+Formal education will make you a living; self-education will make you a fortune
+Lost time is never found again
+Men talk of killing time, while time quietly kills them
+Only entropy comes easy
+Any man can make mistakes, but only an idiot persists in his error
+Managing is getting paid for home runs someone else hits""".split("\n")
 end
 
 Scrappy::App.run

data/lib/scrappy.rb

@@ -24,5 +24,5 @@ require 'scrappy/agent/blind_agent'
 require 'scrappy/agent/agent'
 
 module Scrappy
-  VERSION = '0.3.5'
+  VERSION = '0.4.0'
 end

data/lib/scrappy/extractor/extractor.rb

@@ -21,19 +21,15 @@ module Scrappy
         
         # Extract each fragment
         options = { :doc => { :uri=>uri, :content=>content }, :referenceable=>referenceable }
-        triples = []
-        fragments_for(kb, uri).each do |fragment|
-          kb.node(fragment).extract(options).each do |node|
-            triples += node.graph.triples
-          end
-        end
+        output  = extract_graph(fragments_for(kb, uri), options)
 
         puts "done!" if self.options.debug
 
-        triples
+        output.triples
       end
     end
     
+    # Returns a list of fragments that have mappings in a given URI
     def fragments_for kb, uri
       root_fragments = kb.find(nil, Node('rdf:type'), Node('sc:Fragment')) - kb.find([], Node('sc:subfragment'), nil)
 
@@ -52,7 +48,14 @@ module Scrappy
 
       visual_selectors = selectors.select { |selector| selector.rdf::type.include?(Node('sc:VisualSelector')) }
 
-      (uri_selectors + visual_selectors).map { |selector| fragments[selector] }
+      (uri_selectors + visual_selectors).map { |selector| fragments[selector].proxy }
+    end
+
+    # Extracts all mappings from a fragment and returns a graph
+    def extract_graph fragments, options
+      output = RDF::Graph.new
+      fragments.each { |fragment| fragment.extract(options).each { |result| output << result } }
+      output
     end
   end
 end

data/lib/scrappy/extractor/fragment.rb

@@ -1,7 +1,7 @@
 module Sc
   class Fragment
     include RDF::NodeProxy
-
+    
     # Extracts data out of a document and returns an array of nodes
     def extract options={}
       all_mappings(options).map { |mapping| mapping[:node] }

data/lib/scrappy/extractor/selector.rb

@@ -4,7 +4,9 @@ module Sc
     include Scrappy::Formats
     
     def select doc
-      if sc::debug.first=="true" and Scrappy::Agent::Options.debug
+      if sc::debug.first=="true" and Scrappy::Agent::Options.debug and
+        (Scrappy::Agent::Options.debug_key.nil? or doc[:value].downcase.include?(Scrappy::Agent::Options.debug_key) )
+        
         puts '== DEBUG'
         puts '== Selector:'
         puts node.serialize(:yarf, false)
@@ -18,7 +20,9 @@ module Sc
       # Filter method is defined in each subclass
       results = filter doc
 
-      if sc::debug.first=="true" and Scrappy::Agent::Options.debug
+      if sc::debug.first=="true" and Scrappy::Agent::Options.debug and
+        (Scrappy::Agent::Options.debug_key.nil? or doc[:value].downcase.include?(Scrappy::Agent::Options.debug_key) )
+        
         puts "== No results" if results.empty?
         results.each_with_index do |result, i|
           puts "== Result ##{i}:"

data/lib/scrappy/extractor/selectors/uri_pattern.rb

@@ -2,7 +2,7 @@ module Sc
   class UriPatternSelector < Selector
     def filter doc
       # Check if the uri fits the pattern
-      if rdf::value.any? { |v| doc[:uri] =~ /\A#{v.gsub('.','\.').gsub('*', '.+')}\Z/ }
+      if rdf::value.any? { |v| doc[:uri] =~ /\A#{v.gsub('.','\.').gsub('*', '.+').gsub('?', '\?')}\Z/ }
         [ { :uri=>doc[:uri], :content=>doc[:content], :value=>format(doc[:value], sc::format, doc[:uri]) } ]
       else
         []

data/lib/scrappy/extractor/selectors/visual.rb

@@ -1,60 +1,74 @@
 module Sc
   class VisualSelector < Selector
+
+    def initialize args={}
+      super
+      @cache = {}
+    end
+
     def filter doc
-      # By initializing variables, we avoid getting data from a hash (slow)
-      min_relative_x  = (sc::min_relative_x.first.to_i if sc::min_relative_x.first)
-      max_relative_x  = (sc::max_relative_x.first.to_i if sc::max_relative_x.first)
-      min_relative_y  = (sc::min_relative_y.first.to_i if sc::min_relative_y.first)
-      max_relative_y  = (sc::max_relative_y.first.to_i if sc::max_relative_y.first)
-      min_x           = (sc::min_x.first.to_i if sc::min_x.first)
-      max_x           = (sc::max_x.first.to_i if sc::max_x.first)
-      min_y           = (sc::min_y.first.to_i if sc::min_y.first)
-      max_y           = (sc::max_y.first.to_i if sc::max_y.first)
-      min_width       = (sc::min_width.first.to_i if sc::min_width.first)
-      max_width       = (sc::max_width.first.to_i if sc::max_width.first)
-      min_height      = (sc::min_height.first.to_i if sc::min_height.first)
-      max_height      = (sc::max_height.first.to_i if sc::max_height.first)
-      min_font_size   = (sc::min_font_size.first.to_i if sc::min_font_size.first)
-      max_font_size   = (sc::max_font_size.first.to_i if sc::max_font_size.first)
-      min_font_weight = (sc::min_font_weight.first.to_i if sc::min_font_weight.first)
-      max_font_weight = (sc::max_font_weight.first.to_i if sc::max_font_weight.first)
-      font_family     =  sc::font_family.first
-      attributes      =  sc::attribute
-      formats         =  sc::format
-      
-      doc[:content].search(sc::tag.first || "*").select do |node|
-        relative_x = node['vx'].to_i - doc[:content]['vx'].to_i
-        relative_y = node['vy'].to_i - doc[:content]['vy'].to_i
-        
-        !node.text? and
-        ( !min_relative_x  or relative_x          >= min_relative_x) and
-        ( !max_relative_x  or relative_x          <= max_relative_x) and
-        ( !min_relative_y  or relative_y          >= min_relative_y) and
-        ( !max_relative_y  or relative_y          <= max_relative_y) and
-        
-        ( !min_x           or node['vx'].to_i      >= min_x) and
-        ( !max_x           or node['vx'].to_i      <= max_x) and
-        ( !min_y           or node['vy'].to_i      >= min_y) and
-        ( !max_y           or node['vy'].to_i      <= max_y) and
+      @cache[doc] ||= begin
+        # By initializing variables, we avoid getting data from a hash (slow)
+        min_relative_x  = (sc::min_relative_x.first.to_i if sc::min_relative_x.first)
+        max_relative_x  = (sc::max_relative_x.first.to_i if sc::max_relative_x.first)
+        min_relative_y  = (sc::min_relative_y.first.to_i if sc::min_relative_y.first)
+        max_relative_y  = (sc::max_relative_y.first.to_i if sc::max_relative_y.first)
+        min_x           = (sc::min_x.first.to_i if sc::min_x.first)
+        max_x           = (sc::max_x.first.to_i if sc::max_x.first)
+        min_y           = (sc::min_y.first.to_i if sc::min_y.first)
+        max_y           = (sc::max_y.first.to_i if sc::max_y.first)
+        min_width       = (sc::min_width.first.to_i if sc::min_width.first)
+        max_width       = (sc::max_width.first.to_i if sc::max_width.first)
+        min_height      = (sc::min_height.first.to_i if sc::min_height.first)
+        max_height      = (sc::max_height.first.to_i if sc::max_height.first)
+        min_font_size   = (sc::min_font_size.first.to_i if sc::min_font_size.first)
+        max_font_size   = (sc::max_font_size.first.to_i if sc::max_font_size.first)
+        min_font_weight = (sc::min_font_weight.first.to_i if sc::min_font_weight.first)
+        max_font_weight = (sc::max_font_weight.first.to_i if sc::max_font_weight.first)
+        font_family     =  sc::font_family.first
+        attributes      =  sc::attribute
+        formats         =  sc::format
+        tag             =  sc::tag
         
-        ( !min_width       or node['vw'].to_i      >= min_width) and
-        ( !max_width       or node['vw'].to_i      <= max_width) and
-        ( !min_height      or node['vh'].to_i      >= min_height) and
-        ( !max_height      or node['vh'].to_i      <= max_height) and
+        elements  = doc[:content].search((tag - ["text"]).first || "*")
+        elements += Nokogiri::XML::NodeSet.new(doc[:content].document, [doc[:content]]) if tag.include?(doc[:content].name)
         
-        ( !min_font_size   or node['vsize'].to_i   >= min_font_size) and
-        ( !max_font_size   or node['vsize'].to_i   <= max_font_size) and
-        ( !min_font_weight or node['vweight'].to_i >= min_font_weight) and
-        ( !max_font_weight or node['vweight'].to_i <= max_font_weight) and
-        ( !font_family     or node['vfont']        == font_family)
-      end.map do |content|
-        if attributes.first
-          # Select node's attribute if given
-          attributes.map { |attribute| { :uri=>doc[:uri], :content=>content, :value=>content[attribute], :attribute=>attribute } }
-        else
-          [ { :uri=>doc[:uri], :content=>content, :value=>format(content, formats, doc[:uri]) } ]
-        end
-      end.flatten
+        elements.select do |node|
+          relative_x = node['vx'].to_i - doc[:content]['vx'].to_i
+          relative_y = node['vy'].to_i - doc[:content]['vy'].to_i
+          
+          !node.text? and
+          ( (node['vfont']   and node.name!="a" and node.name!="img") or !tag.include?("text") ) and
+          ( !min_relative_x  or relative_x           >= min_relative_x) and
+          ( !max_relative_x  or relative_x           <= max_relative_x) and
+          ( !min_relative_y  or relative_y           >= min_relative_y) and
+          ( !max_relative_y  or relative_y           <= max_relative_y) and
+          
+          ( !min_x           or node['vx'].to_i      >= min_x) and
+          ( !max_x           or node['vx'].to_i      <= max_x) and
+          ( !min_y           or node['vy'].to_i      >= min_y) and
+          ( !max_y           or node['vy'].to_i      <= max_y) and
+          
+          ( !min_width       or node['vw'].to_i      >= min_width) and
+          ( !max_width       or node['vw'].to_i      <= max_width) and
+          ( !min_height      or node['vh'].to_i      >= min_height) and
+          ( !max_height      or node['vh'].to_i      <= max_height) and
+          
+          ( !min_font_size   or node['vsize'].to_i   >= min_font_size) and
+          ( !max_font_size   or node['vsize'].to_i   <= max_font_size) and
+          ( !min_font_weight or node['vweight'].to_i >= min_font_weight) and
+          ( !max_font_weight or node['vweight'].to_i <= max_font_weight) and
+          ( !font_family     or node['vfont']        == font_family)
+        end.map do |content|
+          if attributes.first
+            # Select node's attribute if given
+            attributes.map { |attribute| { :uri=>doc[:uri], :content=>content, :value=>content[attribute], :attribute=>attribute } }
+          else
+            [ { :uri=>doc[:uri], :content=>content, :value=>format(content, formats, doc[:uri]) } ]
+          end
+        end.flatten
+      end
     end
+    
   end
 end

data/lib/scrappy/learning/optimizer.rb

@@ -1,151 +1,399 @@
+require 'set'
+
 module Scrappy
   module Optimizer
     # Iterates through a knowledge base and tries to merge and generalize
     # selectors whenever the output of the resulting kb is the same
-    def optimize_patterns kb, sample
+    def optimize_extractors kb, samples
       # Build an array of fragments
-      root_fragments = kb.find(nil, Node('rdf:type'), Node('sc:Fragment')) - kb.find([], Node('sc:subfragment'), nil)
-      fragments = []; root_fragments.each { |f| fragments << kb.node(Node(f.id, RDF::Graph.new(f.all_triples))) }
+      all_fragments = kb.find(nil, Node('rdf:type'), Node('sc:Fragment')) - kb.find([], Node('sc:subfragment'), nil)
 
-      # Parse the document
-      doc = { :uri=>sample[:uri], :content=>Nokogiri::HTML(sample[:html], nil, 'utf-8') }
+      root_superfragments = all_fragments.select do |fragment|
+        fragment.sc::selector.any? do |selector|
+          ( selector.rdf::type.include?(Node('sc:UriSelector')) or
+            selector.rdf::type.include?(Node('sc:UriPatternSelector')) ) and
+          samples.any? { |sample| !kb.node(selector).filter(:uri=>sample[:uri]).empty? }
+        end
+      end
+      root_fragments = root_superfragments.map { |f| f.sc::subfragment }.flatten
 
-      # Optimize the fragment
-      fragments = optimize fragments, :docs=>[doc]
+      # Optimize the fragments
+      fragments = optimize_all root_fragments, samples, :extractors
       
-      graph = RDF::Graph.new
-      fragments.each { |fragment| graph << fragment }
+      # Build a graph by adding all fragments to a common URI-selected superfragment
+      superfragment = Node(nil)
+      identifier    = Node(nil)
+      selector      = uri_selector_for(samples.map { |sample| sample[:uri] })
+      identifier.rdf::type  = Node('sc:BaseUriSelector')
+      superfragment.rdf::type       = Node('sc:Fragment')
+      superfragment.sc::selector    = selector
+      superfragment.sc::identifier  = identifier
+      superfragment.graph << selector
+      superfragment.graph << identifier
+
+      triples = fragments.inject([]) do |triples, fragment|
+        triples << [superfragment.id, ID('sc:subfragment'), fragment.id]
+        triples += fragment.all_triples
+      end
+      triples += superfragment.all_triples
+            
+      RDF::Graph.new(triples)
+    end
+    
+    # Iterates through a knowledge base and tries to merge and generalize
+    # selectors whenever the output of the resulting kb is the same
+    def optimize_patterns kb, samples
+      # Build an array of fragments
+      root_fragments = kb.find(nil, Node('rdf:type'), Node('sc:Fragment')) - kb.find([], Node('sc:subfragment'), nil)
+
+      # Optimize the fragments
+      fragments = optimize_all root_fragments, samples, :patterns
       
-      graph
+      # Build a graph
+      RDF::Graph.new(fragments.inject([]) { |triples, fragment| triples += fragment.all_triples })
     end
     
     protected
-    # Tries to optimize a set of fragments
-    def optimize fragments, options
-      # Tries to iterate until no changes are made
+    # Optimizes a set of fragments
+    def optimize_all root_fragments, samples, kb_type
+      # Parse the documents
+      docs = samples.map do |sample|
+        output  = kb_type==:patterns ? samples[:output] : extract(sample[:uri], sample[:html], Scrappy::Kb.extractors)
+        content = Nokogiri::HTML(sample[:html], nil, 'utf-8')
+        { :uri=>sample[:uri], :content=>content, :output=>output }
+      end
+      
+      # Fragment cloning to use a new common pool for caching intermediate results
+      fragments = []
+      pool      = {}
+      root_fragments.each do |f|
+        fragment = Node(f.id, RDF::Graph.new(f.all_triples))
+        fragment.graph.pool = pool
+        fragments << fragment
+      end
+      
+      # Iterates until no changes are made
       @tried = []
-      new_fragments = fragments.map{ |f| f.proxy(Node('sc:Fragment')) }
+      @distances = {}
+      new_fragments = fragments
+      score         = 0.0
+      i             = 0
+      last_save     = 0
       begin
-        fragments = new_fragments
-        new_fragments = optimize_once fragments, options
-      end until fragments == new_fragments
-      fragments
-    end
-    
-    # Tries to perform one optimization of two fragments out of a set of fragments
-    def optimize_once fragments, options
-      docs = options[:docs]
-      fragments.each do |fragment1|
-        fragments.each do |fragment2|
-          next if fragment1 == fragment2
-          # Won't get gain if the fragment does not produce the same kind of RDF resource
-          next if !fragment1.sc::type.equivalent?(fragment2.sc::type) or
-                  !fragment1.sc::relation.equivalent?(fragment2.sc::relation) or
-                  !fragment1.sc::superclass.equivalent?(fragment2.sc::superclass) or
-                  !fragment1.sc::sameas.equivalent?(fragment2.sc::sameas) or
-                   fragment1.sc::identifier.size != fragment2.sc::identifier.size
-
-          next if @tried.include?([fragment1, fragment2])
-          next if @tried.include?([fragment2, fragment1])
+        new_score = score(new_fragments, docs, kb_type)
+        
+        if new_score >= score # Improvement after optimization?
+          puts 'Successful optimization' if i > 0
+          score     = new_score
+          fragments = new_fragments
 
-          @tried << [fragment1, fragment2]
+          # Save to disk
+          if (Time.now - last_save).to_i > 60 and i > 0 and kb_type == :patterns
+            print "Saving..."; $stdout.flush
+            Scrappy::App.save_patterns fragments
+            puts "done!"
 
-          # Get mappings without mixing fragments
-          old_mappings = []
-          docs.each do |doc|
-            old_mappings += fragment1.all_mappings(:doc=>doc)
-            old_mappings += fragment2.all_mappings(:doc=>doc)
+            last_save = Time.now
           end
-          old_docs = old_mappings.map { |mapping| mapping[:doc] }
+        else
+          puts 'Unsuccessful optimization, rolling back...'
+        end
+        puts
+        puts "Fragments: #{fragments.size}, score: #{score}"
+        puts "Trying optimization #{i+=1}..."
+        new_fragments = optimize fragments
+      end while new_fragments
+      puts 'Optimization finished'
+      
+      fragments
+    end
+    
+    # Tries to perform one optimization in a set of fragments
+    def optimize fragments
+      fragments.each_with_index do |fragment1, index|
+        fragments[0...index].sort_by { |fragment2| distance(fragment1, fragment2) }.each do |fragment2|
+          next if @tried.include?([fragment1, fragment2]) or @tried.include?([fragment2, fragment1])
           
-          # Get mixed fragment
-          new_fragment = mix(fragment1, fragment2, options)
+          new_fragment = group fragment1, fragment2
           
-          # Get new mappings
-          new_mappings = []
-          docs.each { |doc| new_mappings += new_fragment.mappings(:doc=>doc) }
-          new_docs = new_mappings.map { |mapping| mapping[:doc] }
-
-          # Optimize subfragments
-          subfragments = optimize(fragment1.sc::subfragment + fragment2.sc::subfragment, options.merge(:docs=>new_docs))
-          subfragments.each { |subfragment| new_fragment.graph << subfragment }
-          new_fragment.sc::subfragment = subfragments.map &:node
-
-          # End if the new fragment returns the same results
-          if true
-            return fragments - [fragment1] - [fragment2] + [new_fragment]
-          end
+          @tried << [fragment1, fragment2]
+
+          # End by including the new fragment in the list and returning it
+          return fragments - [fragment1] - [fragment2] + [new_fragment] if new_fragment
         end
       end
-      fragments
+      return
     end
-    
-    def mix fragment1, fragment2, options
-      docs = options[:docs]
+
+    # Groups two fragments into one
+    def group fragment1, fragment2, siblings=true
+      return unless signature(fragment1) == signature(fragment2)
       
-      # Build new fragment
-      new_fragment = Node(nil).proxy(Node('sc:Fragment'))
-      new_fragment.rdf::type      = Node('sc:Fragment')
+      new_fragment                = Node(nil)
+      new_fragment.rdf::type      = Node("sc:Fragment")
+      new_fragment.graph.pool     = fragment1.graph.pool
       new_fragment.sc::type       = fragment1.sc::type
       new_fragment.sc::relation   = fragment1.sc::relation
       new_fragment.sc::superclass = fragment1.sc::superclass
       new_fragment.sc::sameas     = fragment1.sc::sameas
 
-      # If fragments share the same parent, cardinality has to increase
-      # Otherwise, they might map the same subdocument, so cardinality
-      # limits are made more general.
-      if fragment1.sc::superfragment.first and fragment1.sc::superfragment == fragment2.sc::superfragment
-        new_fragment.sc::min_cardinality = (fragment1.sc::min_cardinality.first.to_i + fragment2.sc::min_cardinality.first.to_i).to_s
-        new_fragment.sc::max_cardinality = (fragment1.sc::max_cardinality.first.to_i + fragment2.sc::max_cardinality.first.to_i).to_s
-      else
-        new_fragment.sc::min_cardinality = [fragment1.sc::min_cardinality.first.to_i, + fragment2.sc::min_cardinality.first.to_i].min.to_s
-        new_fragment.sc::max_cardinality = [fragment1.sc::max_cardinality.first.to_i, + fragment2.sc::max_cardinality.first.to_i].max.to_s
+      if fragment1.sc::min_cardinality.first and fragment2.sc::min_cardinality.first
+        if siblings
+          new_fragment.sc::min_cardinality = (fragment1.sc::min_cardinality.first.to_i + fragment2.sc::min_cardinality.first.to_i).to_s
+        else
+          new_fragment.sc::min_cardinality = [fragment1.sc::min_cardinality.first.to_i, + fragment2.sc::min_cardinality.first.to_i].min.to_s
+        end
       end
-      
+      if fragment1.sc::max_cardinality.first and fragment2.sc::max_cardinality.first
+        if siblings
+          new_fragment.sc::max_cardinality = (fragment1.sc::max_cardinality.first.to_i + fragment2.sc::max_cardinality.first.to_i).to_s
+        else
+          new_fragment.sc::max_cardinality = [fragment1.sc::max_cardinality.first.to_i, + fragment2.sc::max_cardinality.first.to_i].max.to_s
+        end
+      end
+
       # sc:selector
-      selector = generalize(fragment1.sc::selector + fragment2.sc::selector)
-      new_fragment.graph       << selector
-      new_fragment.sc::selector = selector
+      new_selector = merge(*(fragment1.sc::selector + fragment2.sc::selector))
+      return unless new_selector
+      new_fragment.sc::selector = new_selector
+      new_fragment.graph << new_selector
       
       # sc:identifier
       if fragment1.sc::identifier.first
-        selector = generalize(fragment1.sc::identifier + fragment2.sc::identifier)
-        new_fragment.graph         << selector
-        new_fragment.sc::identifier = selector
+        new_identifier = merge(*(fragment1.sc::identifier + fragment2.sc::identifier))
+        return unless new_identifier
+        new_fragment.sc::identifier = new_identifier
+        new_fragment.graph << new_identifier
       end
+      
+      subfragments = mix(fragment1.sc::subfragment, fragment2.sc::subfragment)
+      return unless subfragments
+
+      subfragments.each { |f| return if !f; new_fragment.graph << f }
+      new_fragment.sc::subfragment = subfragments
+      
+      puts "  new fragment #{new_fragment} (#{short_name(new_fragment)}) out of #{fragment1} and #{fragment2}"
 
-      # All new nodes are expected to be inconsistent after performing
-      # subfragments' extractions. Otherwise, if new nodes are consistent, it means
-      # the output from the mixed fragment is different from the separate fragments
-      # and therefore the generalization has failed, so no mixed fragment is returned
       new_fragment
     end
 
-    # Generalize a set of selectors
-    def generalize selectors
+    # Mixes and aligns two set of fragments
+    def mix fragments1, fragments2
+      return unless fragments1.size == fragments2.size
+
+      # Build new fragments
+      used_fragments = []
+      fragments1.map do |fragment1|
+        fragment2 = fragments2.select { |fragment2| signature(fragment1) == signature(fragment2) }.first
+        return unless fragment2
+        return if used_fragments.include?(fragment2)
+
+        used_fragments << fragment2
+        
+        group fragment1, fragment2, false
+      end
+    end
+    
+    def signature fragment
+      [ fragment.sc::type.map(&:to_sym).to_set,
+        fragment.sc::relation.map(&:to_sym).to_set,
+        fragment.sc::superclass.map(&:to_sym).to_set,
+        fragment.sc::sameas.map(&:to_sym).to_set,
+        fragment.sc::identifier.first.nil?,
+        fragment.sc::subfragment.map { |sf| signature(sf) }.to_set ]
+    end
+    
+    # Merges a set of selectors, returning a new more general one
+    def merge *selectors
       selector = Node(nil)
-      selector.rdf::type           = Node('sc:VisualSelector')
-      selector.sc::min_relative_x  = selectors.map { |s| s.sc::min_relative_x.map(&:to_i) }.flatten.min.to_s
-      selector.sc::max_relative_x  = selectors.map { |s| s.sc::max_relative_x.map(&:to_i) }.flatten.max.to_s
-      selector.sc::min_relative_y  = selectors.map { |s| s.sc::min_relative_y.map(&:to_i) }.flatten.min.to_s
-      selector.sc::max_relative_y  = selectors.map { |s| s.sc::max_relative_y.map(&:to_i) }.flatten.max.to_s
-      selector.sc::min_x           = selectors.map { |s| s.sc::min_x.map(&:to_i) }.flatten.min.to_s
-      selector.sc::max_x           = selectors.map { |s| s.sc::max_x.map(&:to_i) }.flatten.max.to_s
-      selector.sc::min_y           = selectors.map { |s| s.sc::min_y.map(&:to_i) }.flatten.min.to_s
-      selector.sc::max_y           = selectors.map { |s| s.sc::max_y.map(&:to_i) }.flatten.max.to_s
-      selector.sc::min_width       = selectors.map { |s| s.sc::min_width.map(&:to_i) }.flatten.min.to_s
-      selector.sc::max_width       = selectors.map { |s| s.sc::max_width.map(&:to_i) }.flatten.max.to_s
-      selector.sc::min_height      = selectors.map { |s| s.sc::min_height.map(&:to_i) }.flatten.min.to_s
-      selector.sc::max_height      = selectors.map { |s| s.sc::max_height.map(&:to_i) }.flatten.max.to_s
-      selector.sc::min_font_size   = selectors.map { |s| s.sc::min_font_size.map(&:to_i) }.flatten.min.to_s if selectors.all? { |s| s.sc::min_font_size.first }
-      selector.sc::max_font_size   = selectors.map { |s| s.sc::max_font_size.map(&:to_i) }.flatten.max.to_s if selectors.all? { |s| s.sc::max_font_size.first }
-      selector.sc::min_font_weight = selectors.map { |s| s.sc::min_font_weight.map(&:to_i) }.flatten.min.to_s if selectors.all? { |s| s.sc::min_font_weight.first }
-      selector.sc::max_font_weight = selectors.map { |s| s.sc::max_font_weight.map(&:to_i) }.flatten.max.to_s if selectors.all? { |s| s.sc::max_font_weight.first }
-      selector.sc::font_family     = selectors.first.sc::font_family if selectors.map { |s| s.sc::font_family }.flatten.uniq.size == 1
-      selector.sc::tag             = selectors.first.sc::tag if selectors.map { |s| s.sc::tag }.flatten.uniq.size == 1
-      selector.sc::attribute       = selectors.first.sc::attribute if selectors.map { |s| s.sc::attribute }.flatten.uniq.size == 1
+      if selectors.first.rdf::type.first == Node('sc:XPathSelector')
+        selector.rdf::type  = Node('sc:XPathSelector')
+        selector.sc::attribute = selectors.first.sc::attribute
+        selector.sc::text = selectors.map { |s| s.sc::text }.flatten
+        
+        xpaths = selectors.map { |s| s.rdf::value }.flatten.map { |s| xpath_for(s) }
+        selector.rdf::value = if selectors.map { |s| s.rdf::value }.uniq.size == 1
+          # All in common
+          selectors.first.rdf::value
+        elsif xpaths.map(&:size).uniq.size == 1
+          # Possible siblings
+          new_xpath = []
+          (0...xpaths.first.size).each do |i|
+            terms      = xpaths.map     { |xp|   xp[i] }
+            tags       = terms.map { |term| term[:tag] }.uniq
+            indexes    = terms.map { |term| term[:index] }.uniq
+            conditions = terms.map { |term| term[:conditions] }
+            
+            tag        = tags.size > 1 ? '*' : tags.first
+            index      = indexes.first if indexes.size == 1
+            conditions = conditions.inject { |acc, n| acc & n }
+          
+            new_xpath << {:tag => tag, :conditions => conditions, :index => index}
+          end
+          xpath_expression_for(new_xpath)
+        else
+          # Nothing in common
+          return
+          nil
+        end
+      elsif selectors.first.rdf::type.first == Node('sc:VisualSelector')
+        selector.rdf::type           = Node('sc:VisualSelector')
+        selector.sc::min_relative_x  = selectors.map { |s| s.sc::min_relative_x.map(&:to_i) }.flatten.min.to_s
+        selector.sc::max_relative_x  = selectors.map { |s| s.sc::max_relative_x.map(&:to_i) }.flatten.max.to_s
+        selector.sc::min_relative_y  = selectors.map { |s| s.sc::min_relative_y.map(&:to_i) }.flatten.min.to_s
+        selector.sc::max_relative_y  = selectors.map { |s| s.sc::max_relative_y.map(&:to_i) }.flatten.max.to_s
+        selector.sc::min_x           = selectors.map { |s| s.sc::min_x.map(&:to_i) }.flatten.min.to_s
+        selector.sc::max_x           = selectors.map { |s| s.sc::max_x.map(&:to_i) }.flatten.max.to_s
+        selector.sc::min_y           = selectors.map { |s| s.sc::min_y.map(&:to_i) }.flatten.min.to_s
+        selector.sc::max_y           = selectors.map { |s| s.sc::max_y.map(&:to_i) }.flatten.max.to_s
+        selector.sc::min_width       = selectors.map { |s| s.sc::min_width.map(&:to_i) }.flatten.min.to_s
+        selector.sc::max_width       = selectors.map { |s| s.sc::max_width.map(&:to_i) }.flatten.max.to_s
+        selector.sc::min_height      = selectors.map { |s| s.sc::min_height.map(&:to_i) }.flatten.min.to_s
+        selector.sc::max_height      = selectors.map { |s| s.sc::max_height.map(&:to_i) }.flatten.max.to_s
+        selector.sc::min_font_size   = selectors.map { |s| s.sc::min_font_size.map(&:to_i) }.flatten.min.to_s if selectors.all? { |s| s.sc::min_font_size.first }
+        selector.sc::max_font_size   = selectors.map { |s| s.sc::max_font_size.map(&:to_i) }.flatten.max.to_s if selectors.all? { |s| s.sc::max_font_size.first }
+        selector.sc::min_font_weight = selectors.map { |s| s.sc::min_font_weight.map(&:to_i) }.flatten.min.to_s if selectors.all? { |s| s.sc::min_font_weight.first }
+        selector.sc::max_font_weight = selectors.map { |s| s.sc::max_font_weight.map(&:to_i) }.flatten.max.to_s if selectors.all? { |s| s.sc::max_font_weight.first }
+        selector.sc::font_family     = selectors.first.sc::font_family if selectors.map { |s| s.sc::font_family.sort }.uniq.size == 1
+        selector.sc::tag             = selectors.first.sc::tag if selectors.map { |s| s.sc::tag.sort }.uniq.size == 1
+        selector.sc::attribute       = selectors.first.sc::attribute if selectors.map { |s| s.sc::attribute.sort }.uniq.size == 1
+      end
 
       selector
     end
+    
+    def distance fragment1, fragment2
+      return @distances[[fragment1.id, fragment2.id]] if @distances[[fragment1.id, fragment2.id]]
+      return 1/0.0 if signature(fragment1) != signature(fragment2)
+
+      # Calculate distances
+      distance  = selector_distance(fragment1.sc::selector.first, fragment2.sc::selector.first)
+      distance += selector_distance(fragment1.sc::identifier.first, fragment2.sc::identifier.first) if fragment1.sc::identifier.first
+      
+      # Calculate subfragments' distances
+      subfragments2 = fragment2.sc::subfragment
+      subdistances  = fragment1.sc::subfragment.map do |subfragment1|
+        subfragment2 = subfragments2.select { |f| signature(subfragment1) == signature(f) }.first
+        subfragments2.delete subfragment2
+        
+        subfragment2.nil? ? 500.0 : distance(subfragment1, subfragment2)
+      end
+      
+      final_distance = distance + subdistances.inject(0.0) {|sum,d| sum+d} + subfragments2.size*500.0
+      @distances[[fragment1.id, fragment2.id]] = final_distance
+      @distances[[fragment2.id, fragment1.id]] = final_distance
+    end
+    
+    def selector_distance selector1, selector2
+      distance  = 0.0
+      distance += (selector1.sc::min_relative_x.first.to_i - selector2.sc::min_relative_x.first.to_i).abs
+      distance += (selector1.sc::max_relative_x.first.to_i - selector2.sc::max_relative_x.first.to_i).abs
+      distance += (selector1.sc::min_relative_y.first.to_i - selector2.sc::min_relative_y.first.to_i).abs
+      distance += (selector1.sc::max_relative_y.first.to_i - selector2.sc::max_relative_y.first.to_i).abs
+      distance += (selector1.sc::min_x.first.to_i - selector2.sc::min_x.first.to_i).abs
+      distance += (selector1.sc::max_x.first.to_i - selector2.sc::max_x.first.to_i).abs
+      distance += (selector1.sc::min_y.first.to_i - selector2.sc::min_y.first.to_i).abs
+      distance += (selector1.sc::max_y.first.to_i - selector2.sc::max_y.first.to_i).abs
+      distance += (selector1.sc::min_width.first.to_i - selector2.sc::min_width.first.to_i).abs
+      distance += (selector1.sc::max_width.first.to_i - selector2.sc::max_width.first.to_i).abs
+      distance += (selector1.sc::min_height.first.to_i - selector2.sc::min_height.first.to_i).abs
+      distance += (selector1.sc::max_height.first.to_i - selector2.sc::max_height.first.to_i).abs
+      distance += (selector1.sc::min_font_size.first.to_i - selector2.sc::min_font_size.first.to_i).abs * 100
+      distance += (selector1.sc::max_font_size.first.to_i - selector2.sc::max_font_size.first.to_i).abs * 100
+      distance += (selector1.sc::min_font_weight.first.to_i - selector2.sc::min_font_weight.first.to_i).abs
+      distance += (selector1.sc::max_font_weight.first.to_i - selector2.sc::max_font_weight.first.to_i).abs
+      distance += 100 if selector1.sc::font_family != selector2.sc::font_family
+      distance += 500 if selector1.sc::tag != selector2.sc::tag
+      distance
+    end
+    
+    def score fragments, docs, kb_type
+      return 0.0 unless fragments
+      docs.inject(0.0) { |sum,doc| doc_score(fragments, doc, kb_type)+sum } / docs.size.to_f
+    end
+    
+    def doc_score fragments, doc, kb_type
+      count = RDF::ID.count
+      extraction = extract_graph(fragments.map(&:proxy), :doc=>doc).triples
+      RDF::ID.count = count # Hack to reduce symbol creation
+
+      correct    = doc[:output]
+      precision, recall, fscore = metrics(correct, extraction, true)
+      
+      kb_type == :patterns ? fscore : recall
+    end
+    
+    def metrics correct, extraction, debug=false
+      right      = correct.size - (correct - extraction).size
+      
+      if debug
+        puts "  Wrong triples: \n"   + RDF::Graph.new(extraction - correct).to_ntriples
+        puts "  Missing triples: \n" + RDF::Graph.new(correct - extraction).to_ntriples
+      end
+      
+      precision  = extraction.size != 0 ? right/extraction.size.to_f : 1.0
+      recall     = correct.size != 0 ? right/correct.size.to_f : 1.0
+      
+      # Calculate fscore
+      fscore = 2.0*(recall*precision)/(precision+recall)
+      
+      puts "  Fscore: #{fscore}" if debug
+      
+      [ precision, recall, fscore ]
+    end
+    
+    private
+    def short_name fragment
+      [ fragment.sc::type.first, fragment.sc::relation.first ].
+        compact.
+        map { |id| RDF::ID.compress(id) } * ", "
+    end
+    
+    def uri_selector_for uris
+      selector = Node(nil)
+      if uris.uniq.size == 1
+        selector.rdf::type    = Node('sc:UriSelector')
+        selector.rdf::value   = uris.first
+        selector
+      else
+        min_length = uris.map(&:length).min
+        pattern = ""
+        (0..min_length).map.reverse.each do |length|
+          pattern = uris.first[0..length]
+          break if uris.all? { |uri| uri.index(pattern) == 0 and uri.length > pattern.length }
+        end
+        selector.rdf::type    = Node('sc:UriPatternSelector')
+        selector.rdf::value   = pattern + "*"
+        selector
+      end
+    end
+    
+    # Parses an xpath expression into an array
+    def xpath_for expression
+      start = expression[0..0]=='/' ? 1 : 0
+      expression.split('/')[start..-1].map do |term|
+        chunks = term.split('[')
+        tag = chunks[0]        
+        conditions = chunks[1]
+        if conditions.to_i.to_s == conditions
+          # It's the index in fact
+          index = chunks[1]
+        else
+          conditions = conditions.chop.split(" and ") if conditions
+          index = chunks[2]
+        end
+        index = index.to_i if index
+        { :tag=>tag, :conditions=>conditions||[], :index=>index }
+      end
+    end
+
+    # Serializes an xpath expression
+    def xpath_expression_for xpath
+      (xpath.first[:tag]=='.' ? "" : "/" ) + xpath.map do |term|
+        term[:tag] +
+        ("[" + (term[:conditions]*' and ') + "]" if term[:conditions].size > 0).to_s +
+        ("[" + term[:index].to_s + "]" if term[:index]).to_s
+      end * '/'
+    end
   end
 end