scrappy 0.3.0 → 0.3.1

data/lib/scrappy/extractor/fragment.rb

@@ -0,0 +1,111 @@
+module Sc
+  class Fragment
+    include RDF::NodeProxy
+
+    def extract options={}
+      uri    = options[:doc][:uri]
+
+      # Identify the fragment's mappings
+      docs = sc::selector.map { |s| graph.node(s).select options[:doc] }.flatten
+
+      # Generate nodes for each page mapping
+      docs.map do |doc|
+        # Build RDF nodes from identifier selectors (if present)
+        nodes = self.nodes(uri, doc, options[:referenceable])
+        
+        # Add info to each node
+        nodes.map do |node|
+          # Build the object -- it can be a node or a literal
+          object = if sc::type.include?(Node('rdf:Literal'))
+            value = doc[:value].to_s.strip
+            if options[:referenceable]
+              node.rdf::value = value
+              node.rdf::type  = Node('rdf:Literal')
+              node
+            else
+              value
+            end
+          else
+            # Add statements about the node
+            sc::type.each       { |type|       node.rdf::type += [type] if type != Node('rdf:Resource') }
+            sc::superclass.each { |superclass| node.rdfs::subClassOf += [superclass] }
+            sc::sameas.each     { |samenode|   node.owl::sameAs += [samenode] }
+
+            node
+          end
+
+          # Process subfragments
+          consistent = true
+          sc::subfragment.each do |subfragment|
+            # Get subfragment object
+            subfragment = graph.node(subfragment, Node('sc:Fragment'))
+            # Extract data from the subfragment
+            subnodes    = subfragment.extract(options.merge(:doc=>doc))
+            
+            # Add relations
+            subnodes.each do |subnode|
+              node.graph << subnode if subnode.is_a?(RDF::Node)
+              subfragment.sc::relation.each { |relation| node[relation] += [subnode] }
+            end
+            
+            # Check consistency
+            consistent = false if subfragment.sc::min_cardinality.first and subnodes.size < subfragment.sc::min_cardinality.first.to_i
+            consistent = false if subfragment.sc::max_cardinality.first and subnodes.size > subfragment.sc::max_cardinality.first.to_i
+          end
+
+          # Skip the node if it has inconsistent relations
+          # For example: extracting a sioc:Post with no dc:title would
+          # violate the constraint sc:min_cardinality = 1
+          next if !consistent
+          
+          # Add referenceable data if requested
+          if options[:referenceable]
+            sources = [doc[:content]].flatten.map { |n| Node(Scrappy::Extractor.node_hash(doc[:uri], n.path)) }
+            sources.each do |source|
+              sc::type.each     { |type|     source.sc::type     += [type] }
+              sc::relation.each { |relation| source.sc::relation += [relation] }
+              node.graph << source
+              node.sc::source += [source]
+            end
+          end
+          
+          # Object points to either the node or the literal
+          object
+        end
+      end.flatten.compact
+    end
+    
+    def nodes uri, doc, referenceable
+      nodes = sc::identifier.map { |s| graph.node(s).select doc }.flatten.map do |d|
+        node = Node(parse_uri(uri, d[:value]))
+        
+        if referenceable
+          # Include the fragment where the URI was built from
+          uri_node = Node(nil, node.graph)
+          hash     = Scrappy::Extractor.node_hash(d[:uri], d[:content].path)
+          
+          node.sc::uri        = uri_node
+          uri_node.rdf::value = node.to_s
+          uri_node.sc::source = Node(hash)
+        end
+        
+        node
+      end
+      nodes << Node(nil) if nodes.empty?
+      
+      nodes
+    end
+    
+    private
+    # Parses a URI by resolving relative paths
+    def parse_uri(uri, rel_uri)
+      return ID('*') if rel_uri.nil?
+      begin
+        ID(URI::parse(uri.split('/')[0..3]*'/').merge(rel_uri).to_s)
+      rescue
+        ID('*')
+      end
+    end
+
+  end
+end

data/lib/scrappy/extractor/selector.rb

@@ -0,0 +1,41 @@
+module Sc
+  class Selector
+    include RDF::NodeProxy
+    include Scrappy::Formats
+    
+    def select doc
+      if sc::debug.first=="true" and Scrappy::Agent::Options.debug
+        puts '== DEBUG'
+        puts '== Selector:'
+        puts node.serialize(:yarf, false)
+        puts '== On fragment:'
+        puts "URI: #{doc[:uri]}"
+        puts "Content: #{doc[:content]}"
+        puts "Value: #{doc[:value]}"
+      end
+
+      # Process selector
+      # Filter method is defined in each subclass
+      results = filter doc
+
+      if sc::debug.first=="true" and Scrappy::Agent::Options.debug
+        puts "== No results" if results.empty?
+        results.each_with_index do |result, i|
+          puts "== Result ##{i}:"
+          puts "URI: #{result[:uri]}"
+          puts "Content: #{result[:content]}"
+          puts "Value: #{result[:value].inspect}"
+        end
+        puts
+      end
+      
+      # Return results if no nested selectors
+      return results if sc::selector.empty?
+
+      # Process nested selectors
+      results.map do |result|
+        sc::selector.map { |s| graph.node(s).select result }
+      end.flatten
+    end
+  end
+end

data/lib/scrappy/{selectors → extractor/selectors}/base_uri.rb

@@ -1,7 +1,5 @@
 module Sc
-  class BaseUriSelector
-    include RDF::NodeProxy
-
+  class BaseUriSelector < Selector
     def filter doc
       [ { :uri=>doc[:uri], :content=>doc[:content], :value=>doc[:uri] } ]
     end

data/lib/scrappy/extractor/selectors/css.rb

@@ -0,0 +1,5 @@
+module Sc
+  class CssSelector < XPathSelector
+    # By using Nokogiri, CSS and XPath use the same search method
+  end
+end

data/lib/scrappy/{selectors → extractor/selectors}/new_uri.rb

@@ -1,7 +1,5 @@
 module Sc
-  class NewUriSelector
-    include RDF::NodeProxy
-    
+  class NewUriSelector < Selector
     def filter doc
       contents = if sc::attribute.first
         # Select node's attribute if given

data/lib/scrappy/{selectors → extractor/selectors}/root.rb

@@ -1,8 +1,5 @@
 module Sc
-  class RootSelector
-    include RDF::NodeProxy
-    include Scrappy::Formats
-
+  class RootSelector < Selector
     def filter doc
       if sc::attribute.first
         # Select node's attribute if given

data/lib/scrappy/{selectors → extractor/selectors}/section.rb

@@ -1,8 +1,5 @@
 module Sc
-  class SectionSelector
-    include RDF::NodeProxy
-    include Scrappy::Formats
-    
+  class SectionSelector < Selector
     def filter doc
       rdf::value.map do |pattern|
         doc[:content].search('h1, h2, h3, h4, h5, h6, h7, h8, h9, h10').select { |n| n.parent.name!='script' and n.text.downcase.strip == pattern }.map do |node|

data/lib/scrappy/{selectors → extractor/selectors}/slice.rb

@@ -1,7 +1,5 @@
 module Sc
-  class SliceSelector
-    include RDF::NodeProxy
-
+  class SliceSelector < Selector
     def filter doc
       rdf::value.map do |separator|
         slices = doc[:value].split(separator)

data/lib/scrappy/{selectors → extractor/selectors}/uri.rb

@@ -1,11 +1,9 @@
 module Sc
-  class UriSelector
-    include RDF::NodeProxy
-    
+  class UriSelector < Selector
     def filter doc
       # Check if the UriSelector has this URI as value (without params: ?param1=value1&param2=value2)
       if rdf::value.include?(doc[:uri].match(/\A([^\?]*)(\?.*\Z)?/).captures.first)
-        [ { :uri=>doc[:uri], :content=>doc[:content], :value=>doc[:content].text } ]
+        [ { :uri=>doc[:uri], :content=>doc[:content], :value=>format(doc[:value], sc::format, doc[:uri]) } ]
       else
         []
       end

data/lib/scrappy/{selectors → extractor/selectors}/uri_pattern.rb

@@ -1,11 +1,9 @@
 module Sc
-  class UriPatternSelector
-    include RDF::NodeProxy
-
+  class UriPatternSelector < Selector
     def filter doc
       # Check if the uri fits the pattern
       if rdf::value.any? { |v| doc[:uri] =~ /\A#{v.gsub('.','\.').gsub('*', '.+')}\Z/ }
-        [ { :uri=>doc[:uri], :content=>doc[:content], :value=>doc[:content].text } ]
+        [ { :uri=>doc[:uri], :content=>doc[:content], :value=>format(doc[:value], sc::format, doc[:uri]) } ]
       else
         []
       end

data/lib/scrappy/extractor/selectors/visual.rb

@@ -0,0 +1,39 @@
+module Sc
+  class VisualSelector < Selector
+    def filter doc
+      doc[:content].search(sc::tag.first || "*").select do |node|
+        relative_x = node['vx'].to_i - doc[:content]['vx'].to_i
+        relative_y = node['vy'].to_i - doc[:content]['vy'].to_i
+        
+        !node.text? and
+        ( !sc::min_relative_x.first  or relative_x          >= sc::min_relative_x.first.to_i) and
+        ( !sc::max_relative_x.first  or relative_x          <= sc::max_relative_x.first.to_i) and
+        ( !sc::min_relative_y.first  or relative_y          >= sc::min_relative_y.first.to_i) and
+        ( !sc::max_relative_y.first  or relative_y          <= sc::max_relative_y.first.to_i) and
+        
+        ( !sc::min_x.first           or node['vx'].to_i      >= sc::min_x.first.to_i) and
+        ( !sc::max_x.first           or node['vx'].to_i      <= sc::max_x.first.to_i) and
+        ( !sc::min_y.first           or node['vy'].to_i      >= sc::min_y.first.to_i) and
+        ( !sc::max_y.first           or node['vy'].to_i      <= sc::max_y.first.to_i) and
+        
+        ( !sc::min_width.first       or node['vw'].to_i      >= sc::min_width.first.to_i) and
+        ( !sc::max_width.first       or node['vw'].to_i      <= sc::max_width.first.to_i) and
+        ( !sc::min_height.first      or node['vh'].to_i      >= sc::min_height.first.to_i) and
+        ( !sc::max_height.first      or node['vh'].to_i      <= sc::max_height.first.to_i) and
+        
+        ( !sc::min_font_size.first   or node['vsize'].to_i   >= sc::min_font_size.first.to_i) and
+        ( !sc::max_font_size.first   or node['vsize'].to_i   <= sc::max_font_size.first.to_i) and
+        ( !sc::min_font_weight.first or node['vweight'].to_i >= sc::min_font_weight.first.to_i) and
+        ( !sc::max_font_weight.first or node['vweight'].to_i <= sc::max_font_weight.first.to_i) and
+        ( !sc::font_family.first     or node['vfont']        == sc::font_family.first)
+      end.map do |content|
+        if sc::attribute.first
+          # Select node's attribute if given
+          sc::attribute.map { |attribute| { :uri=>doc[:uri], :content=>content, :value=>content[attribute] } }
+        else
+          [ { :uri=>doc[:uri], :content=>content, :value=>format(content, sc::format, doc[:uri]) } ]
+        end
+      end.flatten
+    end
+  end
+end

data/lib/scrappy/{selectors → extractor/selectors}/xpath.rb

@@ -1,8 +1,5 @@
 module Sc
-  class XPathSelector
-    include RDF::NodeProxy
-    include Scrappy::Formats
-    
+  class XPathSelector < Selector
     def filter doc
       rdf::value.map do |pattern|
         interval = if sc::index.first

data/lib/scrappy/server/admin.rb

@@ -1,6 +1,12 @@
+require 'iconv'
+require 'rack-flash'
+
 module Scrappy
   module Admin
     def self.registered app
+      app.set :method_override, true
+      app.use Rack::Flash
+      
       app.get '/' do
         if params[:format] and params[:uri]
           redirect "#{settings.base_uri}/#{params[:format]}/#{simplify_uri(params[:uri])}"
@@ -9,15 +15,96 @@ module Scrappy
         end
       end
       
+      app.get '/javascript' do
+        fragments = agent.fragments_for(Scrappy::Kb.extractors, params[:uri])
+        content_type 'application/javascript'
+        "window.scrappy_extractor=#{fragments.any?};" + open("#{settings.public}/javascripts/annotator.js").read
+      end
+      
       app.get '/help' do
         haml :help
       end
       
-      app.get '/kb' do
+      # Extractors 
+      
+      app.get '/extractors' do
         @uris = ( Agent::Options.kb.find(nil, Node('rdf:type'), Node('sc:UriSelector')) +
                   Agent::Options.kb.find(nil, Node('rdf:type'), Node('sc:UriPatternSelector')) ).
                   map { |node| node.rdf::value }.flatten.sort.map(&:to_s)
-        haml :kb
+        haml :extractors
+      end
+      
+      app.post '/extractors' do
+        if params[:html]
+          # Generate extractor automatically
+          iconv = Iconv.new(params[:encoding], 'UTF-8')
+          html  = iconv.iconv(params[:html]) 
+          puts params[:html]
+          puts params[:uri]
+          raise Exception, "Automatic generation of extractors is not supported yet"
+        else
+          # Store the given extractor
+          Scrappy::App.add_extractor RDF::Parser.parse(:yarf,params[:rdf])
+        end
+        flash[:notice] = "Extractor stored"
+        redirect "#{settings.base_uri}/extractors"
+      end      
+      app.delete '/extractors/*' do |uri|
+        Scrappy::App.delete_extractor uri
+        flash[:notice] = "Extractor deleted"
+        redirect "#{settings.base_uri}/extractors"
+      end
+
+      # Patterns
+      
+      app.get '/patterns' do
+        @uris = Scrappy::Kb.patterns.find(nil, Node('rdf:type'), Node('sc:Fragment')).
+                map { |node| node.sc::type }.flatten.map(&:to_s).sort
+        haml :patterns
+      end
+      
+      app.delete '/patterns/*' do |uri|
+        Scrappy::App.delete_pattern uri
+        flash[:notice] = "Pattern deleted"
+        redirect "#{settings.base_uri}/patterns"
+      end
+      
+      # Samples
+      
+      app.get '/samples' do
+        @samples = Scrappy::App.samples
+        haml :samples
+      end
+      
+      app.get '/samples/:id' do |id|
+        Scrappy::App.samples[id.to_i][:html]
+      end
+      
+      app.get '/samples/:id/:kb_type' do |id,kb_type|
+        kb = (kb_type == "patterns" ? Scrappy::Kb.patterns : Scrappy::Kb.extractors)
+        sample = Scrappy::App.samples[id.to_i]
+        headers 'Content-Type' => 'text/plain'
+        RDF::Graph.new(agent.extract(sample[:uri], sample[:html], kb, Agent::Options.referenceable)).serialize(:yarf)
+      end
+      
+      app.post '/samples/:id/train' do |id|
+        new_extractor = agent.train Scrappy::App.samples[id.to_i]
+        Scrappy::App.add_pattern new_extractor
+        flash[:notice] = "Training completed"
+        redirect "#{settings.base_uri}/samples"
+      end
+      
+      app.post '/samples' do
+        html   = Iconv.iconv('UTF-8', params[:encoding], params[:html]).first
+        sample = Scrappy::App.add_sample(:html=>html, :uri=>params[:uri], :date=>Time.now)
+        flash[:notice] = "Sample stored"
+        redirect "#{settings.base_uri}/samples"
+      end
+      
+      app.delete '/samples/:id' do |id|
+        Scrappy::App.delete_sample id.to_i
+        flash[:notice] = "Sample deleted"
+        redirect "#{settings.base_uri}/samples"
       end
     end
   end

data/lib/scrappy/server/helpers.rb

@@ -6,10 +6,19 @@ module Scrappy
         "var e=document.createElement('script');" +
         "e.src='https://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js';" +
         "e.id='scrappy';" +
-        "document.getElementsByTagName('head')[0].appendChild(e);};" +
+        "document.getElementsByTagName('head')[0].appendChild(e);" +
+        "e=document.createElement('script');" +
+        "e.src='https://ajax.googleapis.com/ajax/libs/jqueryui/1.8.10/jquery-ui.min.js';" +
+        "document.getElementsByTagName('head')[0].appendChild(e);" +
+        "e=document.createElement('link');" +
+        "e.href='http://ajax.googleapis.com/ajax/libs/jqueryui/1.8.10/themes/ui-lightness/jquery-ui.css';" +
+        "e.rel='stylesheet';" +
+        "e.type='text/css';" +
+        "document.getElementsByTagName('head')[0].appendChild(e);" +
+        "};" +
       "if(!window.scrappy_loaded){" +
         "e=document.createElement('script');" +
-        "e.src='http://localhost:3434/javascripts/scrappy.js?_=#{Time.now.to_i}';" +
+        "e.src='http://localhost:3434/javascript?#{Time.now.to_i}&uri='+escape(window.location);" +
         "e.onerror=function(){alert('Error: Please start Scrappy Server at http://localhost:3434');};" +
         "document.getElementsByTagName('head')[0].appendChild(e);" +
       "}"+

data/lib/scrappy/server/server.rb

@@ -9,6 +9,7 @@ module Scrappy
   class Server < Sinatra::Base
     helpers JavaScriptHelpers
     register Errors
+    register Admin if Scrappy::Options.admin
     
     enable :sessions
     set    :root,   File.join(File.dirname(__FILE__), '..', '..', '..')

data/lib/scrappy/trainer/trainer.rb

@@ -0,0 +1,101 @@
+module Scrappy
+  module Trainer
+    # Generates visual patterns
+    def train *samples
+      RDF::Graph.new( samples.inject([]) do |triples, sample|
+        triples + train_sample(sample).triples
+      end )
+    end
+
+    # Optimizes the knowledge base by generalizing patterns
+    def optimize
+    end
+    
+    private
+    def train_sample sample
+      results = RDF::Graph.new extract(sample[:uri], sample[:html], Scrappy::Kb.extractors, :minimum)
+
+      typed_nodes     = results.find(nil, Node("rdf:type"), [])
+      non_root_nodes  = results.find([], [], nil)
+
+      nodes = typed_nodes - non_root_nodes
+      
+      RDF::Graph.new( nodes.inject([]) do |triples, node|
+        triples + fragment_for(node).graph.triples
+      end )
+    end
+    
+    def fragment_for node, parent=nil
+      fragment = Node(nil)
+      node.keys.each do |predicate|
+        case predicate
+        when ID("sc:source") then
+          selector = selector_for(node.sc::source.first, parent)
+          fragment.graph << selector
+          fragment.sc::selector = selector
+        when ID("sc:uri") then
+          # Assumption: URIs are extracted from a link
+          selector = selector_for(node.sc::uri.first.sc::source.first, node)
+          selector.sc::tag = "a"
+          selector.sc::attribute = "href"
+
+          fragment.graph << selector
+          fragment.sc::identifier = selector
+        when ID("rdf:type") then
+          fragment.sc::type = node.rdf::type
+        else
+          if node[predicate].map(&:class).uniq.first != String
+            subfragments = node[predicate].map { |subnode| fragment_for(subnode, node) }
+            # Mix the subfragments
+            id = subfragments.first
+            graph = RDF::Graph.new( subfragments.inject([]) do |triples, subfragment|
+              triples + subfragment.graph.triples.map { |s,p,o| [s==subfragment.id ? id : s,p,o] }
+            end )
+            subfragment = graph[id]
+            subfragment.sc::relation = Node(predicate)
+            subfragment.sc::min_cardinality = "1"
+          
+            fragment.graph << subfragment
+            fragment.sc::subfragment += [subfragment]
+          end
+        end
+      end
+      fragment.rdf::type = Node("sc:Fragment") if parent.nil?
+      fragment
+    end
+    
+    def selector_for fragment, parent=nil
+      presentation = fragment.sc::presentation.first
+      
+      selector = Node(nil)
+      selector.rdf::type = Node("sc:VisualSelector")
+
+      origin_x = parent ? parent.sc::source.first.sc::presentation.first.sc::x.first.to_i : 0
+      origin_y = parent ? parent.sc::source.first.sc::presentation.first.sc::y.first.to_i : 0
+      
+      relative_x = presentation.sc::x.first.to_i - origin_x
+      relative_y = presentation.sc::y.first.to_i - origin_y
+      
+      selector.sc::min_relative_x  = relative_x.to_s
+      selector.sc::max_relative_x  = relative_x.to_s
+      selector.sc::min_relative_y  = relative_y.to_s
+      selector.sc::max_relative_y  = relative_y.to_s
+      selector.sc::min_x           = presentation.sc::x
+      selector.sc::max_x           = presentation.sc::x
+      selector.sc::min_y           = presentation.sc::y
+      selector.sc::max_y           = presentation.sc::y
+
+      selector.sc::min_width       = presentation.sc::width
+      selector.sc::max_width       = presentation.sc::width
+      selector.sc::min_height      = presentation.sc::height
+      selector.sc::max_height      = presentation.sc::height
+      selector.sc::min_font_size   = presentation.sc::font_size
+      selector.sc::max_font_size   = presentation.sc::font_size
+      selector.sc::min_font_weight = presentation.sc::font_weight
+      selector.sc::max_font_weight = presentation.sc::font_weight
+      selector.sc::font_family     = presentation.sc::font_family
+      
+      selector
+    end
+  end
+end