scrappy 0.1.12 → 0.1.14

data/History.txt

@@ -1,3 +1,13 @@
+=== 0.1.14 2011-01-30
+
+* Added missing file to Manifest
+
+=== 0.1.13 2011-01-27
+
+* Improvements on memory consumption
+* Correction in redirections in the web interface
+* Dump to disk feature
+
 === 0.1.12 2011-01-10
 
 * Correction for Windows compatibility

data/Manifest

@@ -9,6 +9,7 @@ lib/scrappy.rb
 lib/scrappy/agent/agent.rb
 lib/scrappy/agent/blind_agent.rb
 lib/scrappy/agent/cache.rb
+lib/scrappy/agent/dumper.rb
 lib/scrappy/agent/map_reduce.rb
 lib/scrappy/agent/extractor.rb
 lib/scrappy/agent/visual_agent.rb

data/bin/scrappy

@@ -39,6 +39,7 @@ module Scrappy
         opts.on('-h', '--help')                 { output_help; exit 0 }
         opts.on('-g URL', '--get URL')          { |url| Options.url = url; Options.http_method=:get }
         opts.on('-p URL', '--post URL')         { |url| Options.url = url; Options.http_method=:post }
+        opts.on('-D', '--dump')                 { Agent::Options.dump = true; Agent::Options.format = :rdf }
         opts.on('-u', '--debug')                { Agent::Options.debug = true }
         opts.on('-i', '--interactive')          { Options.shell = true }
         opts.on('-s', '--server')               { Options.server = true }
@@ -101,6 +102,7 @@ Options
   -c, --concurrence VALUE  Sets number of concurrent connections for crawling (default is 10)
   -l, --levels VALUE       Sets recursion levels for resource crawling (default is 1)
   -d, --delay VALUE        Sets delay (in ms) between requests (default is 0)
+  -D, --dump               Dumps RDF data to disk
   -u, --debug              Shows debugging traces
   -i, --interactive        Runs interactive shell
   -s, --server             Runs web server

data/lib/scrappy.rb

@@ -14,12 +14,13 @@ require 'scrappy/support'
 require 'scrappy/agent/extractor'
 require 'scrappy/agent/map_reduce'
 require 'scrappy/agent/cache'
+require 'scrappy/agent/dumper'
 require 'scrappy/agent/agent'
 
 Namespace :sc, 'http://lab.gsi.dit.upm.es/scraping.rdf#'
 
 module Scrappy
-  VERSION = '0.1.12'
+  VERSION = '0.1.14'
 end
 
 # Require selectors

data/lib/scrappy/agent/agent.rb

@@ -44,6 +44,9 @@ module Scrappy
       depth = args[:depth]
       request = { :method=>args[:method]||:get, :uri=>complete_uri(args[:uri]), :inputs=>args[:inputs]||{} }
 
+      # Expire cache
+      cache.expire! 300 # 5 minutes
+
       # Lookup in cache
       triples = if cache[request]
         puts "Retrieving cached #{request[:uri]}...done!" if options.debug
@@ -67,15 +70,17 @@ module Scrappy
         puts 'done!' if options.debug
         
         response = if self.html_data?
-          add_visual_data! if options.referenceable     # Adds tags including visual information
-          extract self.uri, html, options.referenceable # Extract data
+          add_visual_data! if options.referenceable                     # Adds tags including visual information
+          extraction = extract self.uri, html, options.referenceable       # Extract data
+          Dumper.dump self.uri, clean(extraction), options.format if options.dump # Dump results to disk
+          extraction
         else
           []
         end
 
         # Cache the request
-#        cache[request]                       = { :time=>Time.now, :response=>response }
-#        cache[request.merge(:uri=>self.uri)] = { :time=>Time.now, :response=>response } unless self.uri.nil?
+        cache[request]                       = { :time=>Time.now, :response=>response }
+        cache[request.merge(:uri=>self.uri)] = { :time=>Time.now, :response=>response } unless self.uri.nil?
 
         response
       end
@@ -102,11 +107,13 @@ module Scrappy
           items.each { |item| queue << item }
         end
       end
-      
-      triples
+
+      triples unless options.dump
     end
     
     def reduce results
+      return [] if options.dump
+      
       if options.debug
         print "Merging results..."; $stdout.flush
       end
@@ -119,10 +126,7 @@ module Scrappy
     end
 
     def request args={}
-      # Expire cache
-      cache.expire! 300 # 5 minutes
-
-      RDF::Graph.new(map(args).uniq.select { |s,p,o| p!=Node('rdf:type') or ![Node('sc:Index'), Node('sc:Page')].include?(o) })
+      RDF::Graph.new clean(map(args) || [])
     end
 
     def proxy args={}
@@ -130,13 +134,19 @@ module Scrappy
       
       response = self.request(request)
       
-      if options.debug
-        print "Serializing..."; $stdout.flush
-      end
-      
-      output = response.serialize(request[:format])
+      output = if options.dump
+        ""
+      else
+        if options.debug
+          print "Serializing..."; $stdout.flush
+        end
+        
+        output = response.serialize request[:format]
       
-      puts 'done!'if options.debug
+        puts 'done!'if options.debug
+        
+        output
+      end
 
       OpenStruct.new :output => output,
                      :content_type => ContentTypes[request[:format]] || 'text/plain',
@@ -149,5 +159,9 @@ module Scrappy
       uri = "http://#{uri}" if uri.index(/\A\w*:/) != 0
       uri
     end
+    
+    def clean triples
+      triples.uniq.select { |s,p,o| p!=Node('rdf:type') or ![Node('sc:Index'), Node('sc:Page')].include?(o) }
+    end
   end
 end

data/lib/scrappy/agent/blind_agent.rb

@@ -3,6 +3,7 @@ module Scrappy
     def initialize args={}
       super
       @mechanize = Mechanize.new
+      @mechanize.max_history = 20
     end
 
     def uri

data/lib/scrappy/agent/cache.rb

@@ -19,10 +19,13 @@ module Scrappy
 
   class Cache < Hash
     include MonitorMixin
+    MAX_ELEMENTS = 100
     
     def expire! timeout
       synchronize do
-        keys.each { |req| delete(req) if Time.now.to_i - self[req][:time].to_i > timeout }
+        keys.each { |key| delete(key) if Time.now.to_i - self[key][:time].to_i > timeout }
+        sort_by { |key, value| value[:time].to_i }[0...size-MAX_ELEMENTS].each { |key, value| delete key } if size > MAX_ELEMENTS
+        self
       end
     end
 

data/lib/scrappy/agent/dumper.rb

@@ -0,0 +1,13 @@
+module Scrappy
+  class Dumper
+    Mux = Mutex.new
+
+    def self.dump uri, triples, format
+      Mux.synchronize do
+        filename = uri.gsub("http://", "").gsub("https://", "").gsub("/", "-").gsub(".", "_").gsub("?", "+").gsub("&", "+") + ".#{format}"        
+        data = RDF::Graph.new(triples).serialize(format)
+        File.open(filename, "w") { |f| f.write data }
+      end
+    end
+  end
+end

data/lib/scrappy/selectors/slice.rb

@@ -1,7 +1,7 @@
 module SliceSelector
   def self.filter selector, doc
     selector.rdf::value.map do |separator|
-      slices = doc[:content].text.split(separator)
+      slices = doc[:value].split(separator)
       selector.sc::index.map { |index| { :uri=>doc[:uri], :content=>doc[:content], :value=>slices[index.to_i].to_s.strip} }
     end.flatten
   end

data/lib/scrappy/server.rb

@@ -51,10 +51,9 @@ module Scrappy
       def process_request method, format, url
         callback = @input['callback']
         response = agent.proxy :method=>method, :uri=>url, :inputs=>@input.reject{|k,v| k=='callback'}, :format=>format.to_sym
-
         case response.status
         when :redirect
-          redirect "/#{format}/#{response.uri}#{inputs}"
+          redirect "/#{format}/#{CGI::escape(response.uri).gsub('%2F','/').gsub('%3A',':')}#{inputs}"
         when :ok
           @headers['Content-Type'] = response.content_type
           callback ? "#{callback}(#{response.output})" : response.output

data/scrappy.gemspec

@@ -2,17 +2,17 @@
 
 Gem::Specification.new do |s|
   s.name = %q{scrappy}
-  s.version = "0.1.12"
+  s.version = "0.1.14"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jose Ignacio"]
-  s.date = %q{2011-01-10}
+  s.date = %q{2011-01-30}
   s.default_executable = %q{scrappy}
   s.description = %q{RDF web scraper}
   s.email = %q{joseignacio.fernandez@gmail.com}
   s.executables = ["scrappy"]
-  s.extra_rdoc_files = ["README.rdoc", "bin/scrappy", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/proxy.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/new_uri.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/section.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/webkit/webkit.rb"]
-  s.files = ["History.txt", "Manifest", "README.rdoc", "Rakefile", "bin/scrappy", "kb/elmundo.yarf", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/proxy.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/new_uri.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/section.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/webkit/webkit.rb", "test/test_helper.rb", "test/test_scrappy.rb", "scrappy.gemspec"]
+  s.extra_rdoc_files = ["README.rdoc", "bin/scrappy", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/dumper.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/proxy.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/new_uri.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/section.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/webkit/webkit.rb"]
+  s.files = ["History.txt", "Manifest", "README.rdoc", "Rakefile", "bin/scrappy", "kb/elmundo.yarf", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/dumper.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/proxy.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/new_uri.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/section.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/webkit/webkit.rb", "test/test_helper.rb", "test/test_scrappy.rb", "scrappy.gemspec"]
   s.homepage = %q{http://github.com/josei/scrappy}
   s.post_install_message = %q{**(Optional) Remember to install rbwebkitgtk for visual parsing features**}
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Scrappy", "--main", "README.rdoc"]

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 12
-  version: 0.1.12
+  - 14
+  version: 0.1.14
 platform: ruby
 authors: 
 - Jose Ignacio
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-10 00:00:00 +01:00
+date: 2011-01-30 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -141,6 +141,7 @@ extra_rdoc_files:
 - lib/scrappy/agent/agent.rb
 - lib/scrappy/agent/blind_agent.rb
 - lib/scrappy/agent/cache.rb
+- lib/scrappy/agent/dumper.rb
 - lib/scrappy/agent/map_reduce.rb
 - lib/scrappy/agent/extractor.rb
 - lib/scrappy/agent/visual_agent.rb
@@ -170,6 +171,7 @@ files:
 - lib/scrappy/agent/agent.rb
 - lib/scrappy/agent/blind_agent.rb
 - lib/scrappy/agent/cache.rb
+- lib/scrappy/agent/dumper.rb
 - lib/scrappy/agent/map_reduce.rb
 - lib/scrappy/agent/extractor.rb
 - lib/scrappy/agent/visual_agent.rb