scrappy 0.3.0 → 0.3.1

data/History.txt

@@ -1,3 +1,9 @@
+=== 0.3.1 2011-03-18
+
+* Support to visual selectors
+* Automatic generation of (overfitting) patterns
+* Added management of patterns to web admin
+
 === 0.3.0 2011-03-11
 
 * Removed unused features: shell, browser, scripting

data/Manifest

@@ -9,32 +9,39 @@ lib/scrappy/agent/agent.rb
 lib/scrappy/agent/blind_agent.rb
 lib/scrappy/agent/cache.rb
 lib/scrappy/agent/dumper.rb
-lib/scrappy/agent/extractor.rb
-lib/scrappy/agent/formats.rb
 lib/scrappy/agent/map_reduce.rb
+lib/scrappy/extractor/extractor.rb
+lib/scrappy/extractor/formats.rb
+lib/scrappy/extractor/fragment.rb
+lib/scrappy/extractor/selector.rb
+lib/scrappy/extractor/selectors/base_uri.rb
+lib/scrappy/extractor/selectors/css.rb
+lib/scrappy/extractor/selectors/new_uri.rb
+lib/scrappy/extractor/selectors/root.rb
+lib/scrappy/extractor/selectors/section.rb
+lib/scrappy/extractor/selectors/slice.rb
+lib/scrappy/extractor/selectors/uri.rb
+lib/scrappy/extractor/selectors/uri_pattern.rb
+lib/scrappy/extractor/selectors/visual.rb
+lib/scrappy/extractor/selectors/xpath.rb
 lib/scrappy/repository.rb
-lib/scrappy/selectors/base_uri.rb
-lib/scrappy/selectors/css.rb
-lib/scrappy/selectors/new_uri.rb
-lib/scrappy/selectors/root.rb
-lib/scrappy/selectors/section.rb
-lib/scrappy/selectors/slice.rb
-lib/scrappy/selectors/uri.rb
-lib/scrappy/selectors/uri_pattern.rb
-lib/scrappy/selectors/xpath.rb
 lib/scrappy/server/admin.rb
 lib/scrappy/server/errors.rb
 lib/scrappy/server/helpers.rb
 lib/scrappy/server/server.rb
 lib/scrappy/support.rb
+lib/scrappy/trainer/trainer.rb
 public/favicon.ico
 public/images/logo.png
 public/images/logo_tiny.png
-public/javascripts/scrappy.js
+public/javascripts/annotator.js
+public/javascripts/remote.js
 public/stylesheets/application.css
 test/test_helper.rb
 test/test_scrappy.rb
+views/extractors.haml
 views/help.haml
 views/home.haml
-views/kb.haml
-views/layout.haml
+views/layout.haml
+views/patterns.haml
+views/samples.haml

data/README.rdoc

@@ -4,7 +4,7 @@
 
 == DESCRIPTION:
 
-scrappy is a tool that allows extracting information from web pages and producing RDF data.
+Scrappy is a tool that allows extracting information from web pages and producing RDF data.
 It uses the scraping ontology to define the mappings between HTML contents and RDF data.
 
 An example of mapping is shown next, which allows extracting all titles from http://www.elmundo.es:
@@ -48,13 +48,13 @@ RDFXML, JSON, NTriples formats, which can also be used to define the mappings).
 
 == SYNOPSIS:
 
-A knowledge base of mappings can be defined by storing RDF files inside ~/.scrappy/kb folder.
+A knowledge base of mappings can be defined by storing RDF files inside ~/.scrappy/extractors folder.
 Then, the command-line tool can be used to get RDF data from web sites. You can get help on this
 tool by typing:
 
   $ scrappy --help
 
-scrappy offers many different interfaces to get RDF data from a web page:
+Scrappy offers many different interfaces to get RDF data from a web page:
 
 * Command-line interface:
 
@@ -78,7 +78,7 @@ scrappy offers many different interfaces to get RDF data from a web page:
 
 * Ruby interface:
 
-  You can use scrappy in a Ruby program by requiring the gem:
+  You can use Scrappy in a Ruby program by requiring the gem:
 
     require 'rubygems'
     require 'scrappy'
@@ -143,11 +143,7 @@ Install it as any other gem:
 The gem also requires raptor library (in Debian systems: sudo aptitude install raptor-utils), which is used
 for outputting different RDF serialization formats.
 
-Additionally, some extra libraries are needed for certain features:
-
-* Visual parsing requires rbwebkitgtk: http://github.com/danlucraft/rbwebkitgtk
-
-* PNG output of RDF graphs requires Graphviz (in Debian systems: sudo aptitude install graphviz).
+PNG output of RDF graphs requires Graphviz (in Debian systems: sudo aptitude install graphviz).
 
 In order to use Sesame, you will need to install it. Further instructions can be found
 in the openRDF website, more precisely, in http://www.openrdf.org/doc/sesame2/users/ch06.html .

data/Rakefile

@@ -9,9 +9,8 @@ Echoe.new('scrappy', Scrappy::VERSION) do |p|
   p.url            = "http://github.com/josei/scrappy"
   p.author         = "Jose Ignacio"
   p.email          = "joseignacio.fernandez@gmail.com"
-  p.install_message = '**(Optional) Remember to install rbwebkitgtk for visual parsing features**'
   p.ignore_pattern = ["pkg/*"]
-  p.dependencies = [['activesupport','>= 2.3.5'], ['sinatra', '>= 1.1.2'], ['thin', '>= 1.2.7'], ['nokogiri', '>= 1.4.1'], ['mechanize','>= 1.0.0'], ['lightrdf','>= 0.2.1'], ['i18n', '>= 0.4.2'], ['rest-client', '>=1.6.1'], ['haml', '>= 3.0.24']]
+  p.dependencies = [['activesupport','>= 2.3.5'], ['sinatra', '>= 1.1.2'], ['thin', '>= 1.2.7'], ['nokogiri', '>= 1.4.1'], ['mechanize','>= 1.0.0'], ['lightrdf','>= 0.3.0'], ['i18n', '>= 0.4.2'], ['rest-client', '>=1.6.1'], ['haml', '>= 3.0.24'], ['rack-flash', '>= 0.1.1']]
 end
 
 Rake::RDocTask.new(:rdoc) do |rdoc|

data/bin/scrappy

@@ -18,14 +18,10 @@ module Scrappy
   
   SESSION_TOKEN = rand(100000000)
   Options = OpenStruct.new
+  Kb      = OpenStruct.new
 
   class App
-    def self.quit
-      puts "\"#{Quotes.sort_by{rand}.first}\"" unless Options.quiet
-      exit
-    end
-    
-    def initialize
+    def self.run
       Options.port = 3434
       Agent::Options.workers = 10
       Agent::Options.depth = -1
@@ -51,28 +47,22 @@ module Scrappy
         opts.on('-o URIs', '--observe URIs')    { |uris| Options.observe = uris.split(',') }
       end.parse!(args)
       @file = args.shift
-    end
 
-    def run
-      onload
       if Options.uri
         Options.quiet = true
         puts Agent.new.proxy(:http_method=>:get, :uri=>Options.uri).output
       elsif Options.observe
         Agent.new.observe(Options.observe)
-      elsif Options.admin
-        puts "Launching Scrappy Web Admin (browse http://localhost:#{Options.port})..."
+      elsif Options.admin or Options.server
+        if Options.admin
+          puts "Launching Scrappy Web Admin (browse http://localhost:#{Options.port})..."
+        else
+          puts "Launching Scrappy Web Server..."
+        end
         require 'scrappy/server/server'
         Thin::Logging.silent = true
-        Scrappy::Server.register Scrappy::Admin
         Scrappy::Server.run! :host => 'localhost', :port => Options.port, :environment => :production,
                              :base_uri=>Options.base_uri
-      elsif Options.server
-        puts "Launching Scrappy Web Server..."
-        require 'scrappy/server/server'
-        Thin::Logging.silent = true
-        Scrappy::Server.run! :host => 'localhost', :port => Options.port, :environment => :production,
-                             :base_uri => Options.base_uri
       else
         output_version
         puts 'To get help use: scrappy -h'
@@ -81,8 +71,75 @@ module Scrappy
       Scrappy::App.quit
     end
 
+    def self.quit
+      puts "\"#{Quotes.sort_by{rand}.first}\"" unless Options.quiet
+      exit
+    end
+    
+    def self.data_folder
+      @data_folder
+    end
+    def self.cache_folder
+      @cache_folder
+    end
+    def self.samples_file
+      @samples_file
+    end
+    def self.samples
+      @samples ||= []
+    end
+    
+    def self.add_sample sample
+      self.samples.unshift sample
+      save_samples
+      sample
+    end
+    def self.delete_sample id
+      @samples.delete @samples[id]
+      save_samples
+    end
+    def self.save_samples
+      open(@samples_file, "w") { |f| Marshal.dump(@samples, f) }
+    end
+    def self.editable_kb?
+      @editable_kb
+    end
+    def self.add_pattern graph
+      new_patterns = Scrappy::Kb.patterns.merge graph
+      open(@patterns_file, "w") { |f| f.write new_patterns.serialize(:yarf) }
+      onload
+    end
+    def self.delete_pattern uri
+      graph = Scrappy::Kb.patterns
+      fragments = graph.find(nil, Node('rdf:type'), Node('sc:Fragment')).
+                        select { |fragment| fragment.sc::type.include?(Node(uri)) }
+      fragments.each { |fragment| graph.triples -= fragment.all_triples }
+      open(@patterns_file, "w") { |f| f.write graph.serialize(:yarf) }
+      onload
+    end
+    def self.add_extractor graph
+      open(File.join(@extractors_folder,"extractor_#{Dir[File.join(@extractors_folder,'*')].size}.yarf"), "w") { |f| f.write graph.serialize(:yarf) }
+      onload
+    end
+    def self.delete_extractor uri
+      Dir[File.join(@extractors_folder, '*')].each do |file|
+        format = file.split('.').last.to_sym
+        next if format==:ignore
+        graph = RDF::Parser.parse format, open(file).read
+        uri_selectors  = (graph.find(nil, Node('rdf:type'), Node('sc:UriSelector')) +
+                          graph.find(nil, Node('rdf:type'), Node('sc:UriPatternSelector'))).
+                          flatten.select do |uri_selector|
+                            uri_selector.rdf::value.include?(uri)
+        end
+        fragments = uri_selectors.map { |uri_selector| graph.find(nil, Node('sc:selector'), uri_selector) }.flatten
+        fragments.each { |fragment| graph.triples -= fragment.all_triples }
+        open(file, "w") { |f| f.write graph.serialize(format) } if fragments.any?
+      end
+      onload
+    end
+
     protected
-    def output_help
+    def self.output_help
       output_version
       puts """Synopsis 
   Scrappy is a tool to scrape semantic data out of the unstructured web
@@ -123,59 +180,92 @@ Copyright
   http://www.opensource.org/licenses/mit-license.php"""
     end
    
-    def output_version
+    def self.output_version
       puts "Scrappy v#{Scrappy::VERSION}"
     end
 
-    def onload
+    def self.define_paths
       # Check local or global knowledge base
-      home = WINDOWS_PLATFORM ? "#{ENV['HOME']}/scrappy" : "#{ENV['HOME']}/.scrappy"
+      @home = WINDOWS_PLATFORM ? "#{ENV['HOME']}/scrappy" : "#{ENV['HOME']}/.scrappy"
       
-      data_dirname    = "kb"
-      cache_dirname   = "cache"
-      cache_filename  = "scrappy-#{Scrappy::VERSION}.kb"
-      config_filename = "config.yml"
+      extractors_dirname = "extractors"
+      cache_dirname      = "cache"
+      cache_filename     = "scrappy-#{Scrappy::VERSION}.kb"
+      config_filename    = "config.yml"
+      samples_filename   = "samples"
+      patterns_filename  = "patterns.yarf"
       
-      if File.exists?(File.join(home, data_dirname))
-        data_folder  = File.join home, data_dirname
-        cache_folder = File.join home, cache_dirname
+      if File.exists?(File.join(@home, extractors_dirname))
+        @editable_kb       = true
+        @extractors_folder = File.join @home, extractors_dirname
+        @cache_folder      = File.join @home, cache_dirname
       else
-        data_folder  = File.join Scrappy::Root, data_dirname
-        cache_folder = Dir.tmpdir
+        @editable_kb       = false
+        @extractors_folder = File.join Scrappy::Root, extractors_dirname
+        @cache_folder      = Dir.tmpdir
       end
-      Dir.mkdir cache_folder if Dir[cache_folder].empty?
-      cache_file  = File.join cache_folder, cache_filename
-      config_file = File.join home, config_filename
-
+      @cache_file    = File.join @cache_folder, cache_filename
+      @config_file   = File.join @home, config_filename
+      @samples_file  = File.join @home, samples_filename
+      @patterns_file = File.join @home, patterns_filename
+      Dir.mkdir @home            if Dir[@home].empty?
+      Dir.mkdir cache_folder     if Dir[cache_folder].empty?
+    end
+    
+    def self.load_files_from folder
+      kb = RDF::Graph.new
+      kb.triples = ( Dir[File.join(folder, "*")].inject([]) do |triples, file|
+        extension = file.split('.').last.to_sym
+        triples + if extension==:ignore or File.directory?(file)
+          []
+        else
+          graph = RDF::Parser.parse(extension, open(file).read)
+          kb.ns.merge! graph.ns
+          graph.triples
+        end
+      end )
+      kb
+    end
+    
+    def self.onload
+      define_paths
+      
       # Load knowledge base
-      Agent::Options.kb = if File.exists?(cache_file) and File.mtime(cache_file) >= Dir["#{data_folder}/*", data_folder].map{ |f| File.mtime(f) }.max
+      Agent::Options.kb ||= RDF::Graph.new
+      
+      Kb.extractors, Kb.patterns = if File.exists?(@cache_file) and File.mtime(@cache_file) >= Dir["#{@extractors_folder}/*",@extractors_folder,@patterns_file].map{ |f| File.mtime(f) }.max
         # Just load kb from cache
-        open(cache_file) { |f| Marshal.load(f) }
+        open(@cache_file) { |f| Marshal.load(f) }
       else
         # Load YARF files and cache kb
-        data = Dir[File.join(data_folder, "*")].inject(RDF::Graph.new) do |kb, file|
-          extension = file.split('.').last.to_sym
-          graph = RDF::Parser.parse(extension, open(file).read)
-          kb.ns.merge! graph.ns
-          kb.merge!(extension==:ignore ? RDF::Graph.new : graph)
-          kb
-        end
-        open(cache_file, "w") { |f| Marshal.dump(data, f) }
-        data
+        extractors = load_files_from(@extractors_folder)
+        patterns   = File.exists?(@patterns_file) ? RDF::Parser.parse(:yarf, open(@patterns_file).read) : RDF::Graph.new
+        open(@cache_file, "w") { |f| Marshal.dump([extractors, patterns], f) }
+        [extractors, patterns]
       end
 
-      # Looks for a configuration file. If it does not exist, Scrappy does not uses Sesame
+      # Sets new kb
+      Agent::Options.kb.replace Kb.extractors
+      Agent::Options.kb.ns = Kb.extractors.ns
+      # Adds defined prefixes to namespace
+      RDF::ID.ns.merge! Agent::Options.kb.ns
+
+      # Looks for a configuration file. If it does not exist, Scrappy does not use Sesame
       # It looks for it in the home .scrappy dir
-      if File.exist?(config_file)
-        config = YAML::load_file(config_file)["repository"]
+      if File.exist?(@config_file)
+        config = YAML::load_file(@config_file)["repository"]
         # Convert the strings from the YAML file into symbols
         repository_options = {}
         config.each { |k,v| repository_options[k.to_sym] = v }
         Agent::Options.repository = Repository.new repository_options
       end
 
-      RDF::ID.ns.merge! Agent::Options.kb.ns
+      if File.exist?(@samples_file)
+        @samples = open(@samples_file) { |f| Marshal.load(f) }
+      end
     end
+
+    self.onload
   end
 
   Quotes = """Knowledge talks, wisdom listens
@@ -259,4 +349,4 @@ The reward of a thing well done is to have done it
 Don’t argue with idiots. They will bring you down to their level and beat you with experience""".split("\n")
 end
 
-Scrappy::App.new.run
+Scrappy::App.run

data/lib/scrappy.rb

@@ -9,22 +9,19 @@ require 'active_support'
 require 'tmpdir'
 require 'lightrdf'
 
+Namespace :sc, 'http://lab.gsi.dit.upm.es/scraping.rdf#'
+
 require 'scrappy/support'
 require 'scrappy/repository'
 
-require 'scrappy/agent/extractor'
+require 'scrappy/extractor/extractor'
+require 'scrappy/trainer/trainer'
 require 'scrappy/agent/map_reduce'
 require 'scrappy/agent/cache'
 require 'scrappy/agent/dumper'
-require 'scrappy/agent/formats'
 require 'scrappy/agent/blind_agent'
 require 'scrappy/agent/agent'
 
-Namespace :sc, 'http://lab.gsi.dit.upm.es/scraping.rdf#'
-
 module Scrappy
-  VERSION = '0.3.0'
-end
-
-# Require selectors
-Dir["#{File.expand_path(File.dirname(__FILE__))}/scrappy/selectors/*.rb"].each { |f| require f }
+  VERSION = '0.3.1'
+end

data/lib/scrappy/agent/agent.rb

@@ -2,6 +2,7 @@ module Scrappy
   class Agent
     include MonitorMixin
     include Extractor
+    include Trainer
     include MapReduce
     include Cached
     include BlindAgent
@@ -151,8 +152,7 @@ module Scrappy
     end
 
     def clean triples
-      triples.uniq.select { |s,p,o| p!=ID('rdf:type') or ![ID('sc:Index'), ID('sc:Page')].include?(o) }.
-                   select { |s,p,o| p!=Node('rdf:type') or ![Node('sc:Index'), Node('sc:Page')].include?(o) } 
+      triples.uniq.select { |s,p,o| p!=ID('rdf:type') or ![ID('sc:Index'), ID('sc:Page')].include?(o) }
     end
     
     # Do the extraction using RDF repository
@@ -223,7 +223,7 @@ module Scrappy
       puts 'done!' if options.debug
       
       if self.html_data?
-        triples = extract(self.uri, html, options.referenceable) # Extract data
+        triples = extract(self.uri, html, self.kb, options.referenceable) # Extract data
         Dumper.dump self.uri, clean(triples), options.format if options.dump # Dump results to disk
         triples
       else

data/lib/scrappy/extractor/extractor.rb

@@ -0,0 +1,108 @@
+require 'digest/md5'
+require 'scrappy/extractor/fragment'
+require 'scrappy/extractor/formats'
+require 'scrappy/extractor/selector'
+# Require selectors
+Dir["#{File.expand_path(File.dirname(__FILE__))}/selectors/*.rb"].each { |f| require f }
+
+module Scrappy
+  module Extractor
+    def extract uri, html, kb, referenceable=nil
+      synchronize do
+        if options.debug
+          print "Extracting #{uri}..."; $stdout.flush
+        end
+        
+        # Restart stateful selectors
+        kb = RDF::Graph.new(kb.triples)
+        
+        # Parse document
+        content = Nokogiri::HTML(html, nil, 'utf-8')
+        
+        # Extract each fragment
+        options = { :doc => { :uri=>uri, :content=>content }, :referenceable=>referenceable }
+        triples = []
+        fragments_for(kb, uri).each do |fragment|
+          kb.node(fragment).extract(options).each do |node|
+            triples += node.graph.triples
+          end
+        end
+
+        # Add references to sources if requested
+        triples += add_referenceable_data uri, content, triples, referenceable if referenceable
+
+        puts "done!" if self.options.debug
+
+        triples
+      end
+    end
+    
+    def fragments_for kb, uri
+      uri_selectors = ( kb.find(nil, Node('rdf:type'), Node('sc:UriSelector')) +
+                        kb.find(nil, Node('rdf:type'), Node('sc:UriPatternSelector')) ).
+                        flatten.select do |uri_selector|
+        !kb.node(uri_selector).filter(:uri=>uri).empty?
+      end
+
+      visual_selectors = kb.find(nil, Node('rdf:type'), Node('sc:VisualSelector'))
+      
+      selectors = uri_selectors + visual_selectors
+
+      selectors.map { |selector| kb.find(nil, Node('sc:selector'), selector) }.
+                flatten.
+                select { |selector| selector.rdf::type.include?(Node('sc:Fragment')) }
+    end
+    
+    private
+    def add_referenceable_data uri, content, given_triples, referenceable
+      triples = []
+      resources = {}; given_triples.each { |s,p,o| resources[s] = resources[o] = true }
+      
+      fragment = Node(Extractor.node_hash(uri, '/'))
+      selector = Node(nil)
+      presentation = Node(nil)
+
+      selector.rdf::type = Node('sc:UnivocalSelector')
+      selector.sc::path = '/'
+      selector.sc::document = uri
+
+      fragment.sc::selector = selector
+
+      triples.push(*fragment.graph.merge(presentation.graph).merge(selector.graph).triples) if referenceable==:dump or resources[fragment.id]
+
+      content.search('*').each do |node|
+        next if node.text?
+
+        fragment = Extractor.node_hash(uri, node.path)
+        
+        if referenceable == :dump or resources[fragment]
+          selector     = ID(nil)
+          presentation = ID(nil)
+          
+          triples << [selector, ID('rdf:type'), ID('sc:UnivocalSelector')]
+          triples << [selector, ID('sc:path'), node.path.to_s]
+          triples << [selector, ID('sc:tag'), node.name.to_s]
+          triples << [selector, ID('sc:document'), uri]
+          
+          triples << [presentation, ID('sc:x'), node[:vx].to_s] if node[:vx]
+          triples << [presentation, ID('sc:y'), node[:vy].to_s] if node[:vy]
+          triples << [presentation, ID('sc:width'), node[:vw].to_s] if node[:vw]
+          triples << [presentation, ID('sc:height'), node[:vh].to_s] if node[:vh]
+          triples << [presentation, ID('sc:font_size'), node[:vsize].gsub("px","").to_s] if node[:vsize]
+          triples << [presentation, ID('sc:font_family'), node[:vfont]] if node[:vfont]
+          triples << [presentation, ID('sc:font_weight'), node[:vweight].to_s] if node[:vweight]
+          triples << [presentation, ID('sc:text'), node.text.strip]
+
+          triples << [fragment, ID('sc:selector'), selector]
+          triples << [fragment, ID('sc:presentation'), presentation]
+        end
+      end
+      triples
+    end
+
+    def self.node_hash uri, path
+      digest = Digest::MD5.hexdigest("#{uri} #{path}")
+      :"_:bnode#{digest}"
+    end
+  end
+end