scrappy 0.1.10 → 0.1.11

data/History.txt

@@ -1,3 +1,10 @@
+=== 0.1.11 2010-12-23
+
+* Handling of timeout errors
+* Support for paged resources with sc:Page
+* Added missing gem dependecy (i18n)
+* Added debug mode
+
 === 0.1.10 2010-12-20
 
 * Fixed gem dependencies

data/Rakefile

@@ -11,7 +11,7 @@ Echoe.new('scrappy', Scrappy::VERSION) do |p|
   p.email          = "joseignacio.fernandez@gmail.com"
   p.install_message = '**(Optional) Remember to install rbwebkitgtk for visual parsing features**'
   p.ignore_pattern = ["pkg/*"]
-  p.dependencies = [['activesupport','>= 2.3.5'], ['markaby', '>= 0.7.1'], ['camping', '= 2.0'], ['nokogiri', '>= 1.4.1'], ['mechanize','>= 1.0.0'], ['lightrdf','>= 0.1'], ['mongrel', '>= 1.1.5']]
+  p.dependencies = [['activesupport','>= 2.3.5'], ['markaby', '>= 0.7.1'], ['camping', '= 2.0'], ['nokogiri', '>= 1.4.1'], ['mechanize','>= 1.0.0'], ['lightrdf','>= 0.1'], ['mongrel', '>= 1.1.5'], ['i18n', '>= 0.4.2']]
 end
 
 Rake::RDocTask.new(:rdoc) do |rdoc|

data/bin/scrappy

@@ -37,6 +37,7 @@ module Scrappy
         opts.on('-h', '--help')                 { output_help; exit 0 }
         opts.on('-g URL', '--get URL')          { |url| Options.url = url; Options.http_method=:get }
         opts.on('-p URL', '--post URL')         { |url| Options.url = url; Options.http_method=:post }
+        opts.on('-u', '--debug')                { Agent::Options.debug = true }
         opts.on('-i', '--interactive')          { Options.shell = true }
         opts.on('-s', '--server')               { Options.server = true }
         opts.on('-S', '--proxy-server')         { Options.proxy = true }
@@ -98,6 +99,7 @@ Options
   -c, --concurrence VALUE  Sets number of concurrent connections for crawling (default is 10)
   -l, --levels VALUE       Sets recursion levels for resource crawling (default is 1)
   -d, --delay VALUE        Sets delay (in ms) between requests (default is 0)
+  -u, --debug              Shows debugging traces
   -i, --interactive        Runs interactive shell
   -s, --server             Runs web server
   -S, --proxy-server       Runs web proxy

data/lib/scrappy.rb

@@ -19,7 +19,7 @@ require 'scrappy/agent/agent'
 Namespace :sc, 'http://lab.gsi.dit.upm.es/scraping.rdf#'
 
 module Scrappy
-  VERSION = '0.1.10'
+  VERSION = '0.1.11'
 end
 
 # Require selectors

data/lib/scrappy/agent/agent.rb

@@ -44,23 +44,28 @@ module Scrappy
       depth = args[:depth]
       request = { :method=>args[:method]||:get, :uri=>complete_uri(args[:uri]), :inputs=>args[:inputs]||{} }
 
-      # Expire cache
-      cache.expire! 300 # 5 minutes
-
       # Lookup in cache
       triples = if cache[request]
+        puts "Retrieving cached #{request[:uri]}...done!" if options.debug
+        
         cache[request][:response]
       else
         # Perform the request
         
         sleep 0.001 * options.delay.to_f # Sleep if requested
-        
+
+        if options.debug
+          print "Opening #{request[:uri]}..."; $stdout.flush
+        end
+
         if request[:method] == :get
           self.uri = request[:uri]
         else
           raise Exception, 'POST requests not supported yet'
         end
         
+        puts 'done!' if options.debug
+        
         response = if self.html_data?
           add_visual_data! if options.referenceable     # Adds tags including visual information
           extract self.uri, html, options.referenceable # Extract data
@@ -76,8 +81,21 @@ module Scrappy
       end
 
       # Enqueue subresources
-      if depth > 0
-        items = (triples.map{|t| [t[0],t[2]]}.flatten-[Node(self.uri)]).uniq.select{|n| n.is_a?(RDF::Node) and n.id.is_a?(URI)}.map { |uri| {:uri=>uri.to_s, :depth=>depth-1} }
+      if depth >= 0
+        # Pages are enqueued without reducing depth
+        pages = triples.select { |s,p,o| p==Node("rdf:type") and o==Node("sc:Page") }.map{|s,p,o| s}.select{|n| n.is_a?(RDF::Node) and n.id.is_a?(URI)}
+
+        # All other URIS are enqueued with depth reduced
+        uris = if depth > 0
+          (triples.map { |s, p, o| [s,o] }.flatten - [Node(self.uri)] - pages).select{|n| n.is_a?(RDF::Node) and n.id.is_a?(URI)}
+        else
+          []
+        end
+        
+        items = (pages.map { |uri| {:uri=>uri.to_s, :depth=>depth} } + uris.map { |uri| {:uri=>uri.to_s, :depth=>depth-1} }).uniq
+        
+        items.each { |item| puts "Enqueuing (depth = #{item[:depth]}): #{item[:uri]}" } if options.debug
+        
         if queue.nil?
           triples += process items
         else
@@ -89,18 +107,38 @@ module Scrappy
     end
     
     def reduce results
+      if options.debug
+        print "Merging results..."; $stdout.flush
+      end
+      
       triples = []; results.each { |result| triples += result }
+      
+      puts 'done!'if options.debug
+      
       triples
     end
 
     def request args={}
-      RDF::Graph.new(map(args).uniq.select { |s,p,o| p!=Node('rdf:type') or o!=Node('sc:Index') })
+      # Expire cache
+      cache.expire! 300 # 5 minutes
+
+      RDF::Graph.new(map(args).uniq.select { |s,p,o| p!=Node('rdf:type') or ![Node('sc:Index'), Node('sc:Page')].include?(o) })
     end
 
     def proxy args={}
       request  = { :method=>:get, :inputs=>{}, :format=>options.format, :depth=>options.depth }.merge(args)
+      
+      response = self.request(request)
+      
+      if options.debug
+        print "Serializing..."; $stdout.flush
+      end
+      
+      output = response.serialize(request[:format])
+      
+      puts 'done!'if options.debug
 
-      OpenStruct.new :output => self.request(request).serialize(request[:format]),
+      OpenStruct.new :output => output,
                      :content_type => ContentTypes[request[:format]] || 'text/plain',
                      :uri => self.uri,
                      :status => self.html_data? ? (self.uri == request[:uri] ? :ok : :redirect) : :error

data/lib/scrappy/agent/blind_agent.rb

@@ -14,6 +14,8 @@ module Scrappy
         begin
           @mechanize.get uri
           @loaded = true
+        rescue Timeout::Error
+          @loaded = false
         rescue
           @loaded = false
         end

data/lib/scrappy/agent/extractor.rb

@@ -3,6 +3,10 @@ require 'digest/md5'
 module Scrappy
   module Extractor
     def extract uri, html, referenceable=nil
+      if options.debug
+        print "Extracting #{uri}..."; $stdout.flush
+      end
+      
       triples = []
       content = Nokogiri::HTML(html, nil, 'utf-8')
 
@@ -21,6 +25,8 @@ module Scrappy
 
       add_referenceable_data content, triples, referenceable if referenceable
 
+      puts "done!" if options.debug
+      
       triples
     end
     
@@ -40,7 +46,7 @@ module Scrappy
 
         nodes.each do |node|
           # Build the object
-          object = if fragment.sc::type.first == Node('rdf:Literal')
+          object = if fragment.sc::type.include?(Node('rdf:Literal'))
             value = doc[:value].to_s.strip
             if options[:referenceable]
               bnode = Node(nil)
@@ -52,15 +58,13 @@ module Scrappy
               value
             end
           else
-            if fragment.sc::type.first and fragment.sc::type.first != Node('rdf:Resource')
-              options[:triples] << [node, Node('rdf:type'), fragment.sc::type.first]
-            end
+            fragment.sc::type.each { |type| options[:triples] << [node, Node('rdf:type'), type] if type != Node('rdf:Resource') }
             fragment.sc::superclass.each { |superclass| options[:triples] << [node, Node('rdfs:subClassOf'), superclass] }
             fragment.sc::sameas.each { |samenode| options[:triples] << [node, Node('owl:sameAs'), samenode] }
             node
           end
           fragment.sc::relation.each { |relation| options[:triples] << [options[:parent], relation, object] }
-
+          
           # Add referenceable data if requested
           if options[:referenceable]
             sources = [doc[:content]].flatten.map { |node| Node(node_hash(doc[:uri], node.path)) }
@@ -81,7 +85,7 @@ module Scrappy
       # From "BaseUriSelector" to "base_uri"
       class_name = selector.rdf::type.first.to_s.split('#').last
 
-      if !selector.sc::debug.empty?
+      if !selector.sc::debug.empty? and options.debug
         puts '== DEBUG'
         puts '== Selector:'
         puts selector.serialize(:yarf, false)
@@ -94,7 +98,7 @@ module Scrappy
       # Process selector
       results = Kernel.const_get(class_name).filter selector, doc
 
-      if !selector.sc::debug.empty?
+      if !selector.sc::debug.empty? and options.debug
         puts "== No results" if results.empty?
         results.each_with_index do |result, i|
           puts "== Result ##{i}:"

data/scrappy.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{scrappy}
-  s.version = "0.1.10"
+  s.version = "0.1.11"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jose Ignacio"]
-  s.date = %q{2010-12-20}
+  s.date = %q{2010-12-23}
   s.default_executable = %q{scrappy}
   s.description = %q{RDF web scraper}
   s.email = %q{joseignacio.fernandez@gmail.com}
@@ -34,6 +34,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<mechanize>, [">= 1.0.0"])
       s.add_runtime_dependency(%q<lightrdf>, [">= 0.1"])
       s.add_runtime_dependency(%q<mongrel>, [">= 1.1.5"])
+      s.add_runtime_dependency(%q<i18n>, [">= 0.4.2"])
     else
       s.add_dependency(%q<activesupport>, [">= 2.3.5"])
       s.add_dependency(%q<markaby>, [">= 0.7.1"])
@@ -42,6 +43,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<mechanize>, [">= 1.0.0"])
       s.add_dependency(%q<lightrdf>, [">= 0.1"])
       s.add_dependency(%q<mongrel>, [">= 1.1.5"])
+      s.add_dependency(%q<i18n>, [">= 0.4.2"])
     end
   else
     s.add_dependency(%q<activesupport>, [">= 2.3.5"])
@@ -51,5 +53,6 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<mechanize>, [">= 1.0.0"])
     s.add_dependency(%q<lightrdf>, [">= 0.1"])
     s.add_dependency(%q<mongrel>, [">= 1.1.5"])
+    s.add_dependency(%q<i18n>, [">= 0.4.2"])
   end
 end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 10
-  version: 0.1.10
+  - 11
+  version: 0.1.11
 platform: ruby
 authors: 
 - Jose Ignacio
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-20 00:00:00 +01:00
+date: 2010-12-23 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -113,6 +113,20 @@ dependencies:
         version: 1.1.5
   type: :runtime
   version_requirements: *id007
+- !ruby/object:Gem::Dependency 
+  name: i18n
+  prerelease: false
+  requirement: &id008 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 4
+        - 2
+        version: 0.4.2
+  type: :runtime
+  version_requirements: *id008
 description: RDF web scraper
 email: joseignacio.fernandez@gmail.com
 executables: