scrappy 0.1.17 → 0.1.18

data/History.txt

@@ -1,3 +1,8 @@
+=== 0.1.18 2011-02-18
+
+* Retry support when opening URLs
+* Update for latest lightRDF version
+
 === 0.1.17 2011-02-15
 
 * Enabling headless yarf serialization in shell mode

data/README.rdoc

@@ -125,8 +125,11 @@ scrappy offers many different interfaces to get RDF data from a web page:
     # Parse a knowledge base
     kb = RDF::Parser.parse :yarf, open("https://github.com/josei/scrappy/raw/master/kb/elmundo.yarf").read
 
+    # Set kb as default knowledge base
+    Scrappy::Agent::Options.kb = kb
+
     # Create an agent
-    agent = Scrappy::Agent.create :kb=>kb
+    agent = Scrappy::Agent.create
 
     # Get RDF output
     output = agent.request :method=>:get, :uri=>'http://www.elmundo.es'

data/Rakefile

@@ -11,7 +11,7 @@ Echoe.new('scrappy', Scrappy::VERSION) do |p|
   p.email          = "joseignacio.fernandez@gmail.com"
   p.install_message = '**(Optional) Remember to install rbwebkitgtk for visual parsing features**'
   p.ignore_pattern = ["pkg/*"]
-  p.dependencies = [['activesupport','>= 2.3.5'], ['markaby', '>= 0.7.1'], ['camping', '= 2.0'], ['nokogiri', '>= 1.4.1'], ['mechanize','>= 1.0.0'], ['lightrdf','>= 0.1'], ['mongrel', '>= 1.1.5'], ['i18n', '>= 0.4.2']]
+  p.dependencies = [['activesupport','>= 2.3.5'], ['markaby', '>= 0.7.1'], ['camping', '= 2.0'], ['nokogiri', '>= 1.4.1'], ['mechanize','>= 1.0.0'], ['lightrdf','>= 0.1.9'], ['mongrel', '>= 1.1.5'], ['i18n', '>= 0.4.2']]
 end
 
 Rake::RDocTask.new(:rdoc) do |rdoc|

data/bin/scrappy

@@ -153,7 +153,7 @@ Copyright
         open(cache_file, "w") { |f| Marshal.dump(data, f) }
         data
       end
-      RDF::QURI.ns.merge! Agent::Options.kb.ns
+      RDF::ID.ns.merge! Agent::Options.kb.ns
     end
   end
 

data/lib/scrappy.rb

@@ -21,7 +21,7 @@ require 'scrappy/agent/agent'
 Namespace :sc, 'http://lab.gsi.dit.upm.es/scraping.rdf#'
 
 module Scrappy
-  VERSION = '0.1.17'
+  VERSION = '0.1.18'
 end
 
 # Require selectors

data/lib/scrappy/agent/agent.rb

@@ -87,11 +87,11 @@ module Scrappy
 
       # Enqueue subresources
       # Pages are enqueued without reducing depth
-      pages = triples.select { |s,p,o| p==Node("rdf:type") and o==Node("sc:Page") }.map{|s,p,o| s}.select{|n| n.is_a?(RDF::Node) and n.id.is_a?(URI)}
+      pages = triples.select { |s,p,o| p==Node("rdf:type") and o==Node("sc:Page") }.map{|s,p,o| s}.select{|n| n.is_a?(RDF::Node) and n.id.is_a?(Symbol)}
 
       # All other URIS are enqueued with depth reduced
       uris = if depth != 0
-        (triples.map { |s, p, o| [s,o] }.flatten - [Node(self.uri)] - pages).select{|n| n.is_a?(RDF::Node) and n.id.is_a?(URI)}
+        (triples.map { |s, p, o| [s,o] }.flatten - [Node(self.uri)] - pages).select{|n| n.is_a?(RDF::Node) and n.id.is_a?(Symbol)}
       else
         []
       end

data/lib/scrappy/agent/blind_agent.rb

@@ -12,14 +12,20 @@ module Scrappy
 
     def uri= uri
       synchronize do
-        begin
-          @mechanize.get uri
-          @loaded = true
-        rescue Timeout::Error
-          @loaded = false
-        rescue
-          @loaded = false
+        retries = 10
+        @loaded = false
+        while retries > 0 and !@loaded
+          begin
+            @mechanize.get uri
+            @loaded = true
+          rescue Timeout::Error
+            @loaded = false
+          rescue
+            @loaded = false
+          end
+          retries -= 1 unless @loaded
         end
+        @loaded
       end
     end
 

data/lib/scrappy/agent/extractor.rb

@@ -121,7 +121,7 @@ module Scrappy
     def parse_uri(uri, rel_uri)
       return ID('*') if rel_uri.nil?
       begin
-        ID(URI::parse(uri.split('/')[0..3]*'/').merge(rel_uri))
+        ID(URI::parse(uri.split('/')[0..3]*'/').merge(rel_uri).to_s)
       rescue
         ID('*')
       end

data/scrappy.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{scrappy}
-  s.version = "0.1.17"
+  s.version = "0.1.18"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jose Ignacio"]
-  s.date = %q{2011-02-15}
+  s.date = %q{2011-02-18}
   s.default_executable = %q{scrappy}
   s.description = %q{RDF web scraper}
   s.email = %q{joseignacio.fernandez@gmail.com}
@@ -32,7 +32,7 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<camping>, ["= 2.0"])
       s.add_runtime_dependency(%q<nokogiri>, [">= 1.4.1"])
       s.add_runtime_dependency(%q<mechanize>, [">= 1.0.0"])
-      s.add_runtime_dependency(%q<lightrdf>, [">= 0.1"])
+      s.add_runtime_dependency(%q<lightrdf>, [">= 0.1.9"])
       s.add_runtime_dependency(%q<mongrel>, [">= 1.1.5"])
       s.add_runtime_dependency(%q<i18n>, [">= 0.4.2"])
     else
@@ -41,7 +41,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<camping>, ["= 2.0"])
       s.add_dependency(%q<nokogiri>, [">= 1.4.1"])
       s.add_dependency(%q<mechanize>, [">= 1.0.0"])
-      s.add_dependency(%q<lightrdf>, [">= 0.1"])
+      s.add_dependency(%q<lightrdf>, [">= 0.1.9"])
       s.add_dependency(%q<mongrel>, [">= 1.1.5"])
       s.add_dependency(%q<i18n>, [">= 0.4.2"])
     end
@@ -51,7 +51,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<camping>, ["= 2.0"])
     s.add_dependency(%q<nokogiri>, [">= 1.4.1"])
     s.add_dependency(%q<mechanize>, [">= 1.0.0"])
-    s.add_dependency(%q<lightrdf>, [">= 0.1"])
+    s.add_dependency(%q<lightrdf>, [">= 0.1.9"])
     s.add_dependency(%q<mongrel>, [">= 1.1.5"])
     s.add_dependency(%q<i18n>, [">= 0.4.2"])
   end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 17
-  version: 0.1.17
+  - 18
+  version: 0.1.18
 platform: ruby
 authors: 
 - Jose Ignacio
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-02-15 00:00:00 +01:00
+date: 2011-02-18 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -96,7 +96,8 @@ dependencies:
         segments: 
         - 0
         - 1
-        version: "0.1"
+        - 9
+        version: 0.1.9
   type: :runtime
   version_requirements: *id006
 - !ruby/object:Gem::Dependency