scrappy 0.1

data/kb/elmundo.yarf

@@ -0,0 +1,92 @@
+sioc: http://rdfs.org/sioc/ns#
+sc: http://lab.gsi.dit.upm.es/scraping.rdf#
+loc: http://www.daml.org/experiment/ontology/location-ont#
+
+_:elmundoindice:
+  rdf:type: sc:Fragment
+  sc:selector:
+    *:
+      rdf:type: sc:UriSelector
+      rdf:value: "http://www.elmundo.es/"
+  sc:identifier:
+    *:
+      rdf:type: sc:BaseUriSelector
+  sc:subfragment:
+    *:
+      sc:type: sioc:Post
+      sc:selector:
+        *:
+          rdf:type: sc:CssSelector
+          rdf:value: ".noticia h2, .noticia h3, .noticia h4"
+      sc:identifier:
+        *:
+          rdf:type: sc:CssSelector
+          rdf:value: "a"
+          sc:attribute: "href"
+      sc:subfragment:
+        *:
+          sc:type:     rdf:Literal
+          sc:relation: dc:title
+          sc:selector:
+            *:
+              rdf:type:  sc:CssSelector
+              rdf:value: "a"
+
+_:elmundonoticia:
+  rdf:type: sc:Fragment
+  sc:type: sioc:Post
+  sc:selector:
+    *:
+      rdf:type: sc:UriPatternSelector
+      rdf:value: "http://www.elmundo.es/*"
+  sc:identifier:
+    *:
+      rdf:type: sc:BaseUriSelector
+  sc:subfragment:
+    *:
+      sc:type:     rdf:Literal
+      sc:relation: dc:creator
+      sc:selector:
+        *:
+          rdf:type: sc:CssSelector
+          rdf:value: ".noticia .firma em"
+    *:
+      sc:type:     rdf:Literal
+      sc:relation: dc:title
+      sc:selector:
+        *:
+          rdf:type:  sc:CssSelector
+          rdf:value: ".noticia h2"
+    *:
+      sc:type:     loc:Location
+      sc:relation: loc:location
+      sc:selector:
+        *:
+          rdf:type:  sc:CssSelector
+          rdf:value: ".noticia .firma .localizacion"
+      sc:subfragment:
+        *:
+          sc:type:      rdf:Literal
+          sc:relation: rdf:label
+          sc:selector:
+            *:
+              rdf:type:  sc:RootSelector
+    *:
+      sc:type:     rdf:Literal
+      sc:relation: dc:date
+      sc:selector:
+        *:
+          rdf:type:  sc:CssSelector
+          rdf:value: ".metadata_noticia .fecha"
+    *:
+      sc:type:     rdf:Literal
+      sc:relation: dc:description
+      sc:selector:
+        *:
+          rdf:type: sc:CssSelector
+          rdf:value: ".contenido_noticia_01 .antetitulo"
+          sc:selector:
+            *:
+              rdf:type: sc:SliceSelector
+              rdf:value: "|"
+              sc:index:  "1"

data/lib/scrappy.rb

@@ -0,0 +1,22 @@
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+require 'nokogiri'
+require 'thread'
+require 'monitor'
+require 'mechanize'
+require 'ostruct'
+require 'active_support'
+require 'tmpdir'
+require 'lightrdf'
+
+require 'scrappy/support'
+require 'scrappy/agent/extractor'
+require 'scrappy/agent/cluster'
+require 'scrappy/agent/agent'
+
+Namespace :sc, 'http://lab.gsi.dit.upm.es/scraping.rdf#'
+
+module Scrappy
+  VERSION = '0.1'
+end

data/lib/scrappy/agent/agent.rb

@@ -0,0 +1,90 @@
+module Scrappy
+  class Agent
+    include Extractor
+    include MonitorMixin
+    include Cluster
+
+    Options = OpenStruct.new :format=>:yarf, :depth=>0, :agent=>:blind, :delay=>0
+    ContentTypes = { :png => 'image/png', :rdfxml => 'application/rdf+xml',
+                     :rdf => 'application/rdf+xml' }
+
+    def self.pool
+      @pool ||= {}
+    end
+    def self.[] id
+      pool[id] || Agent.create(:id=>id)
+    end
+
+    def self.create args={}
+      if (args[:agent] || Options.agent) == :visual
+        require 'scrappy/agent/visual_agent'
+        VisualAgent.new args
+      else
+        require 'scrappy/agent/blind_agent'
+        BlindAgent.new args
+      end
+    end
+
+    attr_accessor :id, :output, :content_type, :status, :options, :kb
+
+    def initialize args={}
+      super()
+      @id = args[:id] || Agent.pool.keys.size
+      Agent.pool[@id] = self
+      @kb = args[:kb] || Options.kb
+      @options = Options.clone
+    end
+
+    def request http_method, uri, inputs={}, depth=options.depth
+      synchronize do
+        uri = "#{uri}.com" if uri =~ /\A\w+\Z/
+        uri = "http://#{uri}" if uri.index(/\A\w*:/) != 0
+
+        # Perform the request
+        if http_method == :get
+          self.uri = uri
+          return RDF::Graph.new unless self.html_data?
+        else
+          raise Exception, 'POST requests not supported yet'
+        end
+
+        # Adds tags including visual information
+        add_visual_data! if options.referenceable
+
+        # Extract data
+        triples = extract self.uri, html, options.referenceable
+
+        # Iterate through subresources
+        if depth > 0
+          uris = (triples.map{|t| [t[0],t[2]]}.flatten-[Node(self.uri)]).uniq.select{|n| n.is_a?(RDF::Node) and n.id.is_a?(URI)}.map(&:to_s)
+          Agent.process(uris, :depth=>depth-1).each { |result| triples += result }
+        end
+        RDF::Graph.new(triples.uniq)
+      end
+    end
+
+    def proxy http_method, uri, inputs={}, format=options.format, depth=options.depth
+      synchronize do
+        if @status == :redirect and uri == self.uri
+          @status = :ok
+        else
+          @output = request(http_method, uri, inputs, depth).serialize(format)
+          @content_type = ContentTypes[format] || 'text/plain'
+          @status = if self.html_data?
+            self.uri == uri ? :ok : :redirect
+          else
+            :error
+          end
+        end
+
+        @output
+      end
+    end
+
+    # Method used when consuming a list of uris
+    def process uri, args={}
+      sleep 0.001 * options.delay.to_f
+      request(:get, uri, {}, args[:depth]).triples
+    end
+  end
+end

data/lib/scrappy/agent/blind_agent.rb

@@ -0,0 +1,34 @@
+module Scrappy
+  class BlindAgent < Agent
+    def initialize args={}
+      super
+      @mechanize = Mechanize.new
+    end
+
+    def uri
+      @loaded ? @mechanize.current_page.uri.to_s : nil
+    end
+
+    def uri= uri
+      synchronize do
+        begin
+          @mechanize.get uri
+          @loaded = true
+        rescue
+          @loaded = false
+        end
+      end
+    end
+
+    def html_data?
+      !uri.nil? and @mechanize.current_page.is_a?(Mechanize::Page)
+    end
+
+    def html
+      @mechanize.current_page.root.to_html :encoding=>'UTF-8'
+    end
+
+    def add_visual_data!
+    end
+  end
+end

data/lib/scrappy/agent/cluster.rb

@@ -0,0 +1,35 @@
+module Cluster
+
+  def self.included(klass)
+    klass.extend ClassMethods
+    klass.extend MonitorMixin
+  end
+
+  def consume(list, results, args={})
+    begin
+      element = list.synchronize { list.pop }
+      unless element.nil?
+        result = process(element, args)
+        results.synchronize { results << result }
+      end
+    end until element.nil?
+  end
+
+  module ClassMethods
+    def cluster; @cluster; end
+    def cluster= value; @cluster=value; end
+
+    def create_cluster count, *args
+      self.cluster = (1..count).map { args.nil? ? create : create(*args) }
+    end
+
+    def process(list=[], args={})
+      results = []
+      list.extend MonitorMixin
+      results.extend MonitorMixin
+      cluster.map { |o| Thread.new { o.consume(list, results, args) } }.each { |t| t.join }
+      results
+    end
+  end
+
+end

data/lib/scrappy/agent/extractor.rb

@@ -0,0 +1,159 @@
+module Scrappy
+  module Extractor
+    def extract uri, html, referenceable=nil
+      triples = []
+      content = Nokogiri::HTML(html, nil, 'utf-8')
+      uri_selectors  = kb.find(nil, Node('rdf:type'), Node('sc:UriSelector')).select{ |n| n.rdf::value.include?(uri.match(/\A([^\?]*)(\?.*\Z)?/).captures.first) }
+      uri_selectors += kb.find(nil, Node('rdf:type'), Node('sc:UriPatternSelector')).select{|n| n.rdf::value.any?{|v| /\A#{v.gsub('.','\.').gsub('*', '.+')}\Z/ =~ uri} }
+
+      fragments = uri_selectors.map { |uri_selector| kb.find(nil, Node('sc:selector'), uri_selector) }.flatten
+      fragments.each do |fragment|
+        extract_fragment fragment, :doc=>{:uri=>uri, :content=>content },
+                                   :parent=>uri, :triples=>triples, :referenceable=>!referenceable.nil?
+      end
+
+      add_referenceable_data content, triples, referenceable if referenceable
+
+      triples
+    end
+    
+    private
+    def extract_fragment fragment, options={}
+      node = Node(options[:parent])
+      uri = options[:doc][:uri]
+
+      # Select nodes
+      docs = fragment.sc::selector.map { |s| filter s, options[:doc] }.flatten
+
+      # Generate triples
+      docs.each do |doc|
+        # Build URIs if identifier present
+        nodes = fragment.sc::identifier.map { |s| filter s, doc }.flatten.map{ |d| Node(parse_uri(uri, d[:value])) }
+        nodes << Node(nil) if nodes.empty?
+
+        nodes.each do |node|
+          # Build the object
+          object = if fragment.sc::type.first == Node('rdf:Literal')
+            value = doc[:value].strip
+            if options[:referenceable]
+              bnode = Node(nil)
+              bnode.rdf::value = value
+              bnode.rdf::type = Node('rdf:Literal')
+              bnode
+            else
+              value
+            end
+          elsif fragment.sc::type.first
+            options[:triples] << [node, Node('rdf:type'), fragment.sc::type.first]
+            node
+          else
+            node
+          end
+          fragment.sc::relation.each { |relation| options[:triples] << [options[:parent], relation, object] }
+
+          # Add referenceable data if requested
+          if options[:referenceable]
+            source = Node("_:#{doc[:uri]}|#{doc[:content].path}")
+            options[:triples] << [ object, Node("sc:source"), source ]
+            fragment.sc::type.each { |t| options[:triples] << [ source, Node("sc:type"), t ] }
+            fragment.sc::relation.each { |relation| options[:triples] << [ source, Node("sc:relation"), relation ] }
+          end
+
+          # Process subfragments
+          fragment.sc::subfragment.each { |subfragment| extract_fragment subfragment, options.merge(:doc=>doc, :parent=>object) }
+        end
+      end
+    end
+
+    def filter selector, doc
+      content = doc[:content]
+      uri = doc[:uri]
+      results = if selector.rdf::type.include?(Node('sc:CssSelector')) or
+         selector.rdf::type.include?(Node('sc:XPathSelector'))
+        selector.rdf::value.map do |pattern|
+          content.search(pattern).map do |result|
+            if selector.sc::attribute.first
+              # Select node's attribute if given
+              selector.sc::attribute.map { |attribute| { :uri=>uri, :content=>result, :value=>result[attribute] } }
+            else
+              # Select node
+              [ { :uri=>uri, :content=>result, :value=>result.text } ]
+            end
+          end
+        end.flatten
+
+      elsif selector.rdf::type.include?(Node('sc:SliceSelector'))
+        text = content.text
+        selector.rdf::value.map do |separator|
+          slices = text.split(separator)
+          selector.sc::index.map { |index| { :uri=>uri, :content=>content, :value=>slices[index.to_i].to_s.strip} }
+        end.flatten
+
+      elsif selector.rdf::type.include?(Node('sc:BaseUriSelector'))
+        [ { :uri=>uri, :content=>content, :value=>uri } ]
+
+      else
+        [ { :uri=>uri, :content=>content, :value=>content.text } ]
+      end
+
+      # Process nested selectors, if any
+      return results if selector.sc::selector.empty?
+      results.map do |result|
+        selector.sc::selector.map { |s| filter s, result }
+      end.flatten
+    end
+    
+    def parse_uri(uri, rel_uri)
+      return ID('*') if rel_uri.nil?
+      begin
+        ID(URI::parse(uri.split('/')[0..3]*'/').merge(rel_uri))
+      rescue
+        ID('*')
+      end
+    end
+
+    def add_referenceable_data content, triples, referenceable
+      resources = triples.map{|s,p,o| [[s],[o]]}.flatten
+
+      fragment = Node("_:#{uri}|/")
+      selector = Node(nil)
+      presentation = Node(nil)
+
+      selector.rdf::type = Node('sc:UnivocalSelector')
+      selector.sc::path = '/'
+      selector.sc::uri = uri
+
+      fragment.sc::selector = selector
+
+      triples.push(*fragment.graph.merge(presentation.graph).merge(selector.graph).triples) if referenceable==:dump or resources.include?(fragment)
+
+      content.search('*').each do |node|
+        fragment = Node("_:#{uri}|#{node.path}")
+
+        if referenceable == :dump or resources.include?(fragment)
+          selector = Node(nil)
+          presentation = Node(nil)
+
+          selector.rdf::type = Node('sc:UnivocalSelector')
+          selector.sc::path = node.path.to_s
+          selector.sc::tag = node.name.to_s
+          selector.sc::uri = uri
+
+          presentation.sc::x = node[:vx].to_s if node[:vx]
+          presentation.sc::y = node[:vy].to_s if node[:vy]
+          presentation.sc::width = node[:vw].to_s if node[:vw]
+          presentation.sc::height = node[:vh].to_s if node[:vh]
+          presentation.sc::font_size = node[:vsize].gsub("px","").to_s if node[:vsize]
+          presentation.sc::font_weight = node[:vweight].to_s if node[:vweight]
+          presentation.sc::color = node[:vcolor].to_s if node[:vcolor]
+          presentation.sc::background_color = node[:vbcolor].to_s if node[:vbcolor]
+
+          fragment.sc::selector = selector
+          fragment.sc::presentation = presentation unless presentation.empty?
+
+          triples.push(*fragment.graph.merge(presentation.graph).merge(selector.graph).triples)
+        end
+      end
+    end
+  end
+end

data/lib/scrappy/agent/visual_agent.rb

@@ -0,0 +1,72 @@
+# Hack to hide annoying gtk debug messages
+old_stderr = $stderr.clone
+$stderr.reopen '/dev/null'
+require 'scrappy/webkit/webkit'
+$stderr = old_stderr
+
+module Scrappy
+  class VisualAgent < Agent
+    def initialize args={}
+      super
+
+      @cv = new_cond
+
+      @webview = Gtk::WebKit::WebView.new
+      @webview.signal_connect("load_finished") { synchronize { @cv.signal } }
+
+      @window = Gtk::Window.new
+      @window.signal_connect("destroy") { Gtk.main_quit }
+      @window.add(@webview)
+      @window.set_size_request(1024, 600)
+      @window.show_all if args[:window] or (args[:window].nil? and Agent::Options.window)
+    end
+
+    def uri
+      @webview.uri
+    end
+
+    def uri= uri
+      synchronize do
+        @webview.open uri.to_s
+        @cv.wait(60) # 1 minute to open the page
+        sleep(1) while !Nokogiri::HTML(html).search("head").empty? and Nokogiri::HTML(html).search("body").empty?
+      end
+    end
+
+    def html_data?
+      uri.to_s != ""
+    end
+
+    def html
+      js "document.documentElement.outerHTML"
+    end
+    
+    def add_visual_data!
+      js """var items = document.documentElement.getElementsByTagName('*');
+            var i=0;
+            for(var i=0; i<items.length; i++) {
+              var item = items[i];
+              item.setAttribute('vx', item.offsetLeft)
+              item.setAttribute('vy', item.offsetTop)
+              item.setAttribute('vw', item.offsetWidth)
+              item.setAttribute('vh', item.offsetHeight)
+              item.setAttribute('vsize', document.defaultView.getComputedStyle(item, null).getPropertyValue('font-size'))
+              item.setAttribute('vweight', document.defaultView.getComputedStyle(item, null).getPropertyValue('font-weight'))
+              item.setAttribute('vcolor', document.defaultView.getComputedStyle(item, null).getPropertyValue('color'))
+              item.setAttribute('vbcolor', document.defaultView.getComputedStyle(item, null).getPropertyValue('background-color'))
+            }"""
+    end
+
+
+    private
+    def js code
+      old_title = @webview.title
+      @webview.execute_script("document.title = JSON.stringify(eval(#{ActiveSupport::JSON.encode(code)}))")
+      title = ActiveSupport::JSON.decode(@webview.title)
+      @webview.execute_script("document.title = #{ActiveSupport::JSON.encode(old_title)}")
+      title
+    end
+  end
+end
+
+Thread.new { Gtk.main }