scrappy 0.1.4 → 0.1.5

data/History.txt

@@ -1,3 +1,7 @@
+=== 0.1.5 2010-11-29
+
+* Better map-reduce-based design
+
 === 0.1.4 2010-11-24
 
 * Support for node sets in extractions

data/Manifest

@@ -8,7 +8,8 @@ lib/js/annotator.js
 lib/scrappy.rb
 lib/scrappy/agent/agent.rb
 lib/scrappy/agent/blind_agent.rb
-lib/scrappy/agent/cluster.rb
+lib/scrappy/agent/cache.rb
+lib/scrappy/agent/map_reduce.rb
 lib/scrappy/agent/extractor.rb
 lib/scrappy/agent/visual_agent.rb
 lib/scrappy/proxy.rb

data/bin/scrappy

@@ -28,7 +28,7 @@ module Scrappy
     
     def initialize
       Options.port = 3434
-      Options.concurrence = 10
+      Agent::Options.workers = 10
       Agent::Options.depth = 1
       args = ARGV.map { |arg| arg.split(" ") }.flatten
 
@@ -41,10 +41,10 @@ module Scrappy
         opts.on('-s', '--server')               { Options.server = true }
         opts.on('-S', '--proxy-server')         { Options.proxy = true }
         opts.on('-P P', '--port P')             { |p| Options.port = p }
-        opts.on('-c C', '--concurrence C')      { |c| Options.concurrence = c.to_i }
-        opts.on('-d D', '--delay D')            { |d| Agent::Options.delay = d; Options.concurrence = 1 }
+        opts.on('-c C', '--concurrence C')      { |c| Agent::Options.workers = c.to_i }
+        opts.on('-d D', '--delay D')            { |d| Agent::Options.delay = d; Agent::Options.workers = 1 }
         opts.on('-l L', '--levels L')           { |l| Agent::Options.depth = l.to_i }
-        opts.on('-v', '--visual')               { Agent::Options.agent = :visual; Options.concurrence = 1 }
+        opts.on('-v', '--visual')               { Agent::Options.agent = :visual; Agent::Options.workers = 1 }
         opts.on('-r', '--reference')            { Agent::Options.referenceable = :minimum }
         opts.on('-R', '--reference-all')        { Agent::Options.referenceable = :dump }
         opts.on('-w', '--window')               { Agent::Options.window = true }
@@ -139,10 +139,6 @@ Copyright
         open(cache_file, "w") { |f| Marshal.dump(data, f) }
         data
       end
-
-      # Create cluster of agents
-      Agent.create_cluster Options.concurrence, :referenceable=>Agent::Options.referenceable,
-                                                :agent=>Agent::Options.agent, :window=>false
     end
   end
 

data/lib/scrappy.rb

@@ -3,7 +3,6 @@ $:.unshift(File.dirname(__FILE__)) unless
 
 require 'nokogiri'
 require 'thread'
-require 'monitor'
 require 'mechanize'
 require 'ostruct'
 require 'active_support'
@@ -13,13 +12,14 @@ require 'lightrdf'
 require 'scrappy/support'
 
 require 'scrappy/agent/extractor'
-require 'scrappy/agent/cluster'
+require 'scrappy/agent/map_reduce'
+require 'scrappy/agent/cache'
 require 'scrappy/agent/agent'
 
 Namespace :sc, 'http://lab.gsi.dit.upm.es/scraping.rdf#'
 
 module Scrappy
-  VERSION = '0.1.4'
+  VERSION = '0.1.5'
 end
 
 # Require selectors

data/lib/scrappy/agent/agent.rb

@@ -1,10 +1,11 @@
 module Scrappy
   class Agent
-    include Extractor
     include MonitorMixin
-    include Cluster
+    include Extractor
+    include MapReduce
+    include Cached
 
-    Options = OpenStruct.new :format=>:yarf, :depth=>0, :agent=>:blind, :delay=>0
+    Options = OpenStruct.new :format=>:yarf, :depth=>0, :agent=>:blind, :delay=>0, :workers=>10
     ContentTypes = { :png => 'image/png', :rdfxml => 'application/rdf+xml',
                      :rdf => 'application/rdf+xml' }
 
@@ -14,9 +15,6 @@ module Scrappy
     def self.[] id
       pool[id] || Agent.create(:id=>id)
     end
-    def self.cache
-      @cache ||= {}
-    end
 
     def self.create args={}
       if (args[:agent] || Options.agent) == :visual
@@ -32,72 +30,82 @@ module Scrappy
 
     def initialize args={}
       super()
+      @cluster_count   = args[:workers] || Options.workers
+      @cluster_options = [ { :referenceable=>Options.referenceable, :agent=>Options.agent,
+                             :workers=>1, :window=>false } ]
+      @cluster = args[:parent]
       @id = args[:id] || Agent.pool.keys.size
       Agent.pool[@id] = self
       @kb = args[:kb] || Options.kb
       @options = Options.clone
     end
 
-    def request args={}
-      synchronize do
-        depth = args[:depth]
-        request = { :method=>:get, :inputs=>{} }.merge :method=>args[:method], :uri=>complete_uri(args[:uri]), :inputs=>args[:inputs]||{}
+    def map args, queue=nil
+      depth = args[:depth]
+      request = { :method=>args[:method]||:get, :uri=>complete_uri(args[:uri]), :inputs=>args[:inputs]||{} }
 
-        # Expire cache
-        Agent::cache.keys.each { |req| Agent::cache.delete(req) if Time.now.to_i - Agent::cache[req][:time].to_i > 300 }
+      # Expire cache
+      cache.expire! 300 # 5 minutes
 
-        # Lookup in cache
-        triples = if Agent::cache[request]
-          Agent::cache[request][:response]
+      # Lookup in cache
+      triples = if cache[request]
+        cache[request][:response]
+      else
+        # Perform the request
+        
+        sleep 0.001 * options.delay.to_f # Sleep if requested
+        
+        if request[:method] == :get
+          self.uri = request[:uri]
+        else
+          raise Exception, 'POST requests not supported yet'
+        end
+        
+        response = if self.html_data?
+          add_visual_data! if options.referenceable     # Adds tags including visual information
+          extract self.uri, html, options.referenceable # Extract data
         else
-          # Perform the request
-          if request[:method] == :get
-            self.uri = request[:uri]
-          else
-            raise Exception, 'POST requests not supported yet'
-          end
-          
-          response = if self.html_data?
-            add_visual_data! if options.referenceable     # Adds tags including visual information
-            extract self.uri, html, options.referenceable # Extract data
-          else
-            []
-          end
+          []
+        end
 
-          # Cache the request
-          Agent::cache[request]                       = { :time=>Time.now, :response=>response }
-          Agent::cache[request.merge(:uri=>self.uri)] = { :time=>Time.now, :response=>response } unless self.uri.nil?
+        # Cache the request
+        cache[request]                       = { :time=>Time.now, :response=>response }
+        cache[request.merge(:uri=>self.uri)] = { :time=>Time.now, :response=>response } unless self.uri.nil?
 
-          response
-        end
+        response
+      end
 
-        # Iterate through subresources
-        if depth > 0
-          uris = (triples.map{|t| [t[0],t[2]]}.flatten-[Node(self.uri)]).uniq.select{|n| n.is_a?(RDF::Node) and n.id.is_a?(URI)}.map(&:to_s)
-          Agent.process(uris, :depth=>depth-1).each { |result| triples += result }
+      # Enqueue subresources
+      if depth > 0
+        items = (triples.map{|t| [t[0],t[2]]}.flatten-[Node(self.uri)]).uniq.select{|n| n.is_a?(RDF::Node) and n.id.is_a?(URI)}.map { |uri| {:uri=>uri.to_s, :depth=>depth-1} }
+        if queue.nil?
+          triples += process items
+        else
+          items.each { |item| queue << item }
         end
-        
-        RDF::Graph.new(triples.uniq)
       end
+      
+      triples
+    end
+    
+    def reduce results
+      triples = []; results.each { |result| triples += result }
+      triples
+    end
+
+    def request args={}
+      RDF::Graph.new map(args).uniq
     end
 
     def proxy args={}
-      synchronize do
-        request  = { :method=>:get, :inputs=>{}, :format=>options.format, :depth=>options.depth }.merge(args)
+      request  = { :method=>:get, :inputs=>{}, :format=>options.format, :depth=>options.depth }.merge(args)
 
-        OpenStruct.new :output => self.request(request).serialize(request[:format]),
-                       :content_type => ContentTypes[request[:format]] || 'text/plain',
-                       :uri => self.uri,
-                       :status => self.html_data? ? (self.uri == request[:uri] ? :ok : :redirect) : :error
-      end
+      OpenStruct.new :output => self.request(request).serialize(request[:format]),
+                     :content_type => ContentTypes[request[:format]] || 'text/plain',
+                     :uri => self.uri,
+                     :status => self.html_data? ? (self.uri == request[:uri] ? :ok : :redirect) : :error
     end
 
-    # Method used when consuming a list of uris
-    def process uri, args={}
-      sleep 0.001 * options.delay.to_f
-      request(:method=>:get, :uri=>uri, :depth=>args[:depth]).triples
-    end
-    
     def complete_uri uri
       uri = "#{uri}.com" if uri =~ /\A\w+\Z/
       uri = "http://#{uri}" if uri.index(/\A\w*:/) != 0

data/lib/scrappy/agent/cache.rb

@@ -0,0 +1,37 @@
+require 'monitor'
+
+module Scrappy
+  module Cached
+    def self.included base
+      base.extend Cached::ClassMethods
+    end
+
+    module ClassMethods
+      def cache
+        @cache ||= Cache.new
+      end
+    end
+    
+    def cache
+      self.class.cache
+    end
+  end
+
+  class Cache < Hash
+    include MonitorMixin
+    
+    def expire! timeout
+      synchronize do
+        keys.each { |req| delete(req) if Time.now.to_i - self[req][:time].to_i > timeout }
+      end
+    end
+
+    def []= key, value
+      synchronize { super }
+    end
+
+    def [] key
+      synchronize { super }
+    end
+  end
+end

data/lib/scrappy/agent/extractor.rb

@@ -79,17 +79,21 @@ module Scrappy
       # From "BaseUriSelector" to "base_uri"
       class_name = selector.rdf::type.first.to_s.split('#').last
 
-      # Process selector
-      results = Kernel.const_get(class_name).filter selector, doc
-
       if !selector.sc::debug.empty?
         puts '== DEBUG'
         puts '== Selector:'
         puts selector.serialize(:yarf, false)
-        puts '== Applied on fragment:'
+        puts '== On fragment:'
         puts "URI: #{doc[:uri]}"
         puts "Content: #{doc[:content]}"
         puts "Value: #{doc[:value]}"
+      end
+
+      # Process selector
+      results = Kernel.const_get(class_name).filter selector, doc
+
+      if !selector.sc::debug.empty?
+        puts "== No results" if results.empty?
         results.each_with_index do |result, i|
           puts "== Result ##{i}:"
           puts "URI: #{result[:uri]}"

data/lib/scrappy/agent/map_reduce.rb

@@ -0,0 +1,66 @@
+require 'thread'
+require 'monitor'
+
+module MapReduce
+
+  class Queue
+    def initialize
+      @items = []
+      @items.extend MonitorMixin
+    end
+    
+    def pop
+      yielded = false
+      item = nil
+      @items.synchronize do
+        item = @items.shift
+        if @items.empty?
+          yield item if (block_given? and item)
+          yielded = true
+        end
+      end
+      yield item if (block_given? and not yielded)
+      item
+    end
+    
+    def << value
+      @items << value
+    end
+    
+    def push value
+      self << value
+    end
+
+    def empty?
+      @items.synchronize { @items.empty? }
+    end
+  end
+
+  
+  def cluster
+    @cluster ||= (1..@cluster_count || 1).map { self.class.new(*(@cluster_options || [])) }
+  end
+
+  def process list
+    results = []
+    results.extend MonitorMixin
+    
+    queue = Queue.new
+    list.each { |element| queue << element }
+    
+    cluster.map { |obj| Thread.new { obj.work queue, results } }.each { |t| t.join }
+    
+    reduce results
+  end
+
+
+  def work queue, results
+    begin
+      queue.pop do |item|
+        result = map item, queue
+        results.synchronize { results << result }
+      end
+    end until queue.empty?
+  end
+
+end

data/lib/scrappy/selectors/xpath.rb

@@ -6,7 +6,8 @@ module XPathSelector
       else
         (0..-1)
       end
-      (doc[:content].search(pattern)[interval] || []).map do |result|
+      patterns = selector.sc::keyword
+      (doc[:content].search(pattern)[interval] || []).select { |node| patterns.any? ? patterns.include?(node.text.downcase.strip) : true }.map do |result|
         if selector.sc::attribute.first
           # Select node's attribute if given
           selector.sc::attribute.map { |attribute| { :uri=>doc[:uri], :content=>result, :value=>result[attribute] } }

data/scrappy.gemspec

@@ -2,17 +2,17 @@
 
 Gem::Specification.new do |s|
   s.name = %q{scrappy}
-  s.version = "0.1.4"
+  s.version = "0.1.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jose Ignacio"]
-  s.date = %q{2010-11-24}
+  s.date = %q{2010-11-29}
   s.default_executable = %q{scrappy}
   s.description = %q{RDF web scraper}
   s.email = %q{joseignacio.fernandez@gmail.com}
   s.executables = ["scrappy"]
-  s.extra_rdoc_files = ["README.rdoc", "bin/scrappy", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cluster.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/proxy.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/webkit/webkit.rb"]
-  s.files = ["History.txt", "Manifest", "README.rdoc", "Rakefile", "bin/scrappy", "kb/elmundo.yarf", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cluster.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/proxy.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/webkit/webkit.rb", "test/test_helper.rb", "test/test_scrappy.rb", "scrappy.gemspec"]
+  s.extra_rdoc_files = ["README.rdoc", "bin/scrappy", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/proxy.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/webkit/webkit.rb"]
+  s.files = ["History.txt", "Manifest", "README.rdoc", "Rakefile", "bin/scrappy", "kb/elmundo.yarf", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/proxy.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/webkit/webkit.rb", "test/test_helper.rb", "test/test_scrappy.rb", "scrappy.gemspec"]
   s.homepage = %q{http://github.com/josei/scrappy}
   s.post_install_message = %q{**(Optional) Remember to install rbwebkitgtk for visual parsing features**}
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Scrappy", "--main", "README.rdoc"]

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 4
-  version: 0.1.4
+  - 5
+  version: 0.1.5
 platform: ruby
 authors: 
 - Jose Ignacio
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-24 00:00:00 +01:00
+date: 2010-11-29 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -126,7 +126,8 @@ extra_rdoc_files:
 - lib/scrappy.rb
 - lib/scrappy/agent/agent.rb
 - lib/scrappy/agent/blind_agent.rb
-- lib/scrappy/agent/cluster.rb
+- lib/scrappy/agent/cache.rb
+- lib/scrappy/agent/map_reduce.rb
 - lib/scrappy/agent/extractor.rb
 - lib/scrappy/agent/visual_agent.rb
 - lib/scrappy/proxy.rb
@@ -152,7 +153,8 @@ files:
 - lib/scrappy.rb
 - lib/scrappy/agent/agent.rb
 - lib/scrappy/agent/blind_agent.rb
-- lib/scrappy/agent/cluster.rb
+- lib/scrappy/agent/cache.rb
+- lib/scrappy/agent/map_reduce.rb
 - lib/scrappy/agent/extractor.rb
 - lib/scrappy/agent/visual_agent.rb
 - lib/scrappy/proxy.rb

data/lib/scrappy/agent/cluster.rb

@@ -1,35 +0,0 @@
-module Cluster
-
-  def self.included(klass)
-    klass.extend ClassMethods
-    klass.extend MonitorMixin
-  end
-
-  def consume(list, results, args={})
-    begin
-      element = list.synchronize { list.pop }
-      unless element.nil?
-        result = process(element, args)
-        results.synchronize { results << result }
-      end
-    end until element.nil?
-  end
-
-  module ClassMethods
-    def cluster; @cluster; end
-    def cluster= value; @cluster=value; end
-
-    def create_cluster count, *args
-      self.cluster = (1..count).map { args.nil? ? create : create(*args) }
-    end
-
-    def process(list=[], args={})
-      results = []
-      list.extend MonitorMixin
-      results.extend MonitorMixin
-      cluster.map { |o| Thread.new { o.consume(list, results, args) } }.each { |t| t.join }
-      results
-    end
-  end
-
-end