scrappy 0.1.14 → 0.1.15

data/History.txt

@@ -1,3 +1,8 @@
+=== 0.1.15 2011-02-04
+
+* Added literal output formats such as HTML or Wikitext
+* Added support to infinite crawling
+
 === 0.1.14 2011-01-30
 
 * Added missing file to Manifest

data/Manifest

@@ -10,6 +10,7 @@ lib/scrappy/agent/agent.rb
 lib/scrappy/agent/blind_agent.rb
 lib/scrappy/agent/cache.rb
 lib/scrappy/agent/dumper.rb
+lib/scrappy/agent/formats.rb
 lib/scrappy/agent/map_reduce.rb
 lib/scrappy/agent/extractor.rb
 lib/scrappy/agent/visual_agent.rb

data/bin/scrappy

@@ -31,7 +31,7 @@ module Scrappy
     def initialize
       Options.port = 3434
       Agent::Options.workers = 10
-      Agent::Options.depth = 1
+      Agent::Options.depth = -1
       args = ARGV.map { |arg| arg.split(" ") }.flatten
 
       OptionParser.new do |opts|
@@ -100,7 +100,7 @@ Options
   -g, --get URL            Gets requested URL
   -p, --post URL           Posts requested URL
   -c, --concurrence VALUE  Sets number of concurrent connections for crawling (default is 10)
-  -l, --levels VALUE       Sets recursion levels for resource crawling (default is 1)
+  -l, --levels VALUE       Sets recursion levels for resource crawling (default is infinite crawling)
   -d, --delay VALUE        Sets delay (in ms) between requests (default is 0)
   -D, --dump               Dumps RDF data to disk
   -u, --debug              Shows debugging traces

data/lib/scrappy.rb

@@ -15,12 +15,13 @@ require 'scrappy/agent/extractor'
 require 'scrappy/agent/map_reduce'
 require 'scrappy/agent/cache'
 require 'scrappy/agent/dumper'
+require 'scrappy/agent/formats'
 require 'scrappy/agent/agent'
 
 Namespace :sc, 'http://lab.gsi.dit.upm.es/scraping.rdf#'
 
 module Scrappy
-  VERSION = '0.1.14'
+  VERSION = '0.1.15'
 end
 
 # Require selectors

data/lib/scrappy/agent/agent.rb

@@ -86,26 +86,24 @@ module Scrappy
       end
 
       # Enqueue subresources
-      if depth >= 0
-        # Pages are enqueued without reducing depth
-        pages = triples.select { |s,p,o| p==Node("rdf:type") and o==Node("sc:Page") }.map{|s,p,o| s}.select{|n| n.is_a?(RDF::Node) and n.id.is_a?(URI)}
+      # Pages are enqueued without reducing depth
+      pages = triples.select { |s,p,o| p==Node("rdf:type") and o==Node("sc:Page") }.map{|s,p,o| s}.select{|n| n.is_a?(RDF::Node) and n.id.is_a?(URI)}
 
-        # All other URIS are enqueued with depth reduced
-        uris = if depth > 0
-          (triples.map { |s, p, o| [s,o] }.flatten - [Node(self.uri)] - pages).select{|n| n.is_a?(RDF::Node) and n.id.is_a?(URI)}
-        else
-          []
-        end
-        
-        items = (pages.map { |uri| {:uri=>uri.to_s, :depth=>depth} } + uris.map { |uri| {:uri=>uri.to_s, :depth=>depth-1} }).uniq
-        
-        items.each { |item| puts "Enqueuing (depth = #{item[:depth]}): #{item[:uri]}" } if options.debug
-        
-        if queue.nil?
-          triples += process items
-        else
-          items.each { |item| queue << item }
-        end
+      # All other URIS are enqueued with depth reduced
+      uris = if depth != 0
+        (triples.map { |s, p, o| [s,o] }.flatten - [Node(self.uri)] - pages).select{|n| n.is_a?(RDF::Node) and n.id.is_a?(URI)}
+      else
+        []
+      end
+      
+      items = (pages.map { |uri| {:uri=>uri.to_s, :depth=>[-1, depth].max} } + uris.map { |uri| {:uri=>uri.to_s, :depth=>[-1, depth-1].max} }).uniq
+      
+      items.each { |item| puts "Enqueuing (depth = #{item[:depth]}): #{item[:uri]}" } if options.debug
+      
+      if queue.nil?
+        triples += process items
+      else
+        items.each { |item| queue.push_unless_done item }
       end
 
       triples unless options.dump

data/lib/scrappy/agent/extractor.rb

@@ -136,7 +136,6 @@ module Scrappy
 
       selector.rdf::type = Node('sc:UnivocalSelector')
       selector.sc::path = '/'
-      selector.sc::children = content.search('*').size.to_s
       selector.sc::uri = uri
 
       fragment.sc::selector = selector

data/lib/scrappy/agent/formats.rb

@@ -0,0 +1,27 @@
+module Scrappy
+  module Formats
+
+    def format node, formats
+      case formats.first
+      when Node('sc:WikiText') then
+        doc = Nokogiri::XML(node.to_html)
+        doc.search("h1").each {|n| n.replace(Nokogiri::XML::Text.new("= #{n.text.strip} =", n.document)) }
+        doc.search("h2").each {|n| n.replace(Nokogiri::XML::Text.new("== #{n.text.strip} ==", n.document)) }
+        doc.search("h3").each {|n| n.replace(Nokogiri::XML::Text.new("=== #{n.text.strip} ===", n.document)) }
+        doc.search("h4").each {|n| n.replace(Nokogiri::XML::Text.new("==== #{n.text.strip} ====", n.document)) }
+        doc.search("h5").each {|n| n.replace(Nokogiri::XML::Text.new("===== #{n.text.strip} =====", n.document)) }
+        doc.search("b").each  {|n| n.replace(Nokogiri::XML::Text.new("'''#{n.text.strip}'''", n.document)) }
+        doc.search("li").each {|n| n.replace(Nokogiri::XML::Text.new("* #{n.text.strip}", n.document)) }
+        doc.search("ul").each {|n| n.replace(Nokogiri::XML::Text.new(n.text.strip, n.document)) }
+        doc.search("pre, code").each {|n| n.replace(Nokogiri::XML::Text.new("<pre>\n#{n.text.strip}\n</pre>", n.document)) }
+        doc.search("p").each {|n| n.replace(Nokogiri::XML::Text.new("#{n.text.strip}\n", n.document)) }
+        doc.text.strip
+      when Node('sc:Html') then
+        node.to_html
+      else
+        node.text
+      end
+    end
+
+  end
+end

data/lib/scrappy/agent/map_reduce.rb

@@ -4,16 +4,20 @@ require 'monitor'
 module MapReduce
 
   class Queue
+    include MonitorMixin
+    
     def initialize
-      @items = []
-      @items.extend MonitorMixin
+      super
+      @items   = []
+      @history = []
     end
     
     def pop
       yielded = false
       item = nil
-      @items.synchronize do
+      synchronize do
         item = @items.shift
+        @history << item
         if @items.empty?
           yield item if (block_given? and item)
           yielded = true
@@ -24,15 +28,19 @@ module MapReduce
     end
     
     def << value
-      @items << value
+      push value
     end
     
     def push value
-      self << value
+      synchronize { @items << value }
     end
 
+    def push_unless_done value
+      synchronize { @items << value unless @history.include?(value) }
+    end
+    
     def empty?
-      @items.synchronize { @items.empty? }
+      synchronize { @items.empty? }
     end
   end
 

data/lib/scrappy/selectors/section.rb

@@ -1,10 +1,13 @@
 module SectionSelector
+  extend Scrappy::Formats
+
   def self.filter selector, doc
     selector.rdf::value.map do |pattern|
       doc[:content].search('h1, h2, h3, h4, h5, h6, h7, h8, h9, h10').select { |n| n.parent.name!='script' and n.text.downcase.strip == pattern }.map do |node|
         found = false
         content = node.parent.children[node.parent.children.index(node)+1..-1].select { |n| found ||= (n.name==node.name); !found }
-        [ { :uri=>doc[:uri], :content=>content, :value=>content.map{|t| t.text.strip}.select{|t| t.to_s.strip!=''}*"\n" } ]
+
+        [ { :uri=>doc[:uri], :content=>content, :value=>content.map{|t| format(t, selector.sc::format)}.select{|t| t.to_s.strip!=''}*"\n\n" } ]
       end
     end.flatten
   end

data/lib/scrappy/selectors/xpath.rb

@@ -1,4 +1,6 @@
 module XPathSelector
+  extend Scrappy::Formats
+  
   def self.filter selector, doc
     selector.rdf::value.map do |pattern|
       interval = if selector.sc::index.first
@@ -13,7 +15,7 @@ module XPathSelector
           selector.sc::attribute.map { |attribute| { :uri=>doc[:uri], :content=>result, :value=>result[attribute] } }
         else
           # Select node
-          [ { :uri=>doc[:uri], :content=>result, :value=>result.text } ]
+          [ { :uri=>doc[:uri], :content=>result, :value=>format(result, selector.sc::format) } ]
         end
       end
     end.flatten

data/scrappy.gemspec

@@ -2,17 +2,17 @@
 
 Gem::Specification.new do |s|
   s.name = %q{scrappy}
-  s.version = "0.1.14"
+  s.version = "0.1.15"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jose Ignacio"]
-  s.date = %q{2011-01-30}
+  s.date = %q{2011-02-04}
   s.default_executable = %q{scrappy}
   s.description = %q{RDF web scraper}
   s.email = %q{joseignacio.fernandez@gmail.com}
   s.executables = ["scrappy"]
-  s.extra_rdoc_files = ["README.rdoc", "bin/scrappy", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/dumper.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/proxy.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/new_uri.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/section.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/webkit/webkit.rb"]
-  s.files = ["History.txt", "Manifest", "README.rdoc", "Rakefile", "bin/scrappy", "kb/elmundo.yarf", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/dumper.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/proxy.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/new_uri.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/section.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/webkit/webkit.rb", "test/test_helper.rb", "test/test_scrappy.rb", "scrappy.gemspec"]
+  s.extra_rdoc_files = ["README.rdoc", "bin/scrappy", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/dumper.rb", "lib/scrappy/agent/formats.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/proxy.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/new_uri.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/section.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/webkit/webkit.rb"]
+  s.files = ["History.txt", "Manifest", "README.rdoc", "Rakefile", "bin/scrappy", "kb/elmundo.yarf", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/dumper.rb", "lib/scrappy/agent/formats.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/proxy.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/new_uri.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/section.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/webkit/webkit.rb", "test/test_helper.rb", "test/test_scrappy.rb", "scrappy.gemspec"]
   s.homepage = %q{http://github.com/josei/scrappy}
   s.post_install_message = %q{**(Optional) Remember to install rbwebkitgtk for visual parsing features**}
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Scrappy", "--main", "README.rdoc"]

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 14
-  version: 0.1.14
+  - 15
+  version: 0.1.15
 platform: ruby
 authors: 
 - Jose Ignacio
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-30 00:00:00 +01:00
+date: 2011-02-04 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -142,6 +142,7 @@ extra_rdoc_files:
 - lib/scrappy/agent/blind_agent.rb
 - lib/scrappy/agent/cache.rb
 - lib/scrappy/agent/dumper.rb
+- lib/scrappy/agent/formats.rb
 - lib/scrappy/agent/map_reduce.rb
 - lib/scrappy/agent/extractor.rb
 - lib/scrappy/agent/visual_agent.rb
@@ -172,6 +173,7 @@ files:
 - lib/scrappy/agent/blind_agent.rb
 - lib/scrappy/agent/cache.rb
 - lib/scrappy/agent/dumper.rb
+- lib/scrappy/agent/formats.rb
 - lib/scrappy/agent/map_reduce.rb
 - lib/scrappy/agent/extractor.rb
 - lib/scrappy/agent/visual_agent.rb