scrappy 0.1.7 → 0.1.8

data/History.txt

@@ -1,3 +1,13 @@
+=== 0.1.8 2010-12-14
+
+* Added sc:sameas
+* Added sc:attribute support to sc:RootSelector
+* Added sc:NewUriSelector for constructing new uris
+
+=== 0.1.7 2010-12-09
+
+* Added section selector
+
 === 0.1.6 2010-12-09
 
 * Added sc:superclass to sc:Fragments

data/Manifest

@@ -15,6 +15,7 @@ lib/scrappy/agent/visual_agent.rb
 lib/scrappy/proxy.rb
 lib/scrappy/selectors/base_uri.rb
 lib/scrappy/selectors/css.rb
+lib/scrappy/selectors/new_uri.rb
 lib/scrappy/selectors/root.rb
 lib/scrappy/selectors/section.rb
 lib/scrappy/selectors/slice.rb

data/lib/scrappy.rb

@@ -19,7 +19,7 @@ require 'scrappy/agent/agent'
 Namespace :sc, 'http://lab.gsi.dit.upm.es/scraping.rdf#'
 
 module Scrappy
-  VERSION = '0.1.7'
+  VERSION = '0.1.8'
 end
 
 # Require selectors

data/lib/scrappy/agent/agent.rb

@@ -94,7 +94,7 @@ module Scrappy
     end
 
     def request args={}
-      RDF::Graph.new map(args).uniq
+      RDF::Graph.new(map(args).uniq.select { |s,p,o| p!=Node('rdf:type') or o!=Node('sc:Index') })
     end
 
     def proxy args={}

data/lib/scrappy/agent/extractor.rb

@@ -56,6 +56,7 @@ module Scrappy
               options[:triples] << [node, Node('rdf:type'), fragment.sc::type.first]
             end
             fragment.sc::superclass.each { |superclass| options[:triples] << [node, Node('rdfs:subClassOf'), superclass] }
+            fragment.sc::sameas.each { |samenode| options[:triples] << [node, Node('owl:sameAs'), samenode] }
             node
           end
           fragment.sc::relation.each { |relation| options[:triples] << [options[:parent], relation, object] }

data/lib/scrappy/selectors/new_uri.rb

@@ -0,0 +1,15 @@
+module NewUriSelector
+  def self.filter selector, doc
+    contents = if selector.sc::attribute.first
+      # Select node's attribute if given
+      selector.sc::attribute.map { |attribute| doc[:content][attribute] }
+    else
+      [ doc[:content].text ]
+    end
+
+    contents.map do |content|
+      new_uri = selector.sc::prefix.to_s + content.wikify
+      { :uri=>new_uri, :content=>doc[:content], :value=>new_uri }
+    end
+  end
+end

data/lib/scrappy/selectors/root.rb

@@ -1,5 +1,10 @@
 module RootSelector
   def self.filter selector, doc
-    [ { :uri=>doc[:uri], :content=>doc[:content], :value=>doc[:content].text } ]
+    if selector.sc::attribute.first
+      # Select node's attribute if given
+      selector.sc::attribute.map { |attribute| { :uri=>doc[:uri], :content=>doc[:content], :value=>doc[:content][attribute] } }
+    else
+      [ { :uri=>doc[:uri], :content=>doc[:content], :value=>doc[:content].text } ]
+    end
   end
 end

data/lib/scrappy/support.rb

@@ -25,4 +25,10 @@ module Nokogiri
       end
     end
   end
-end
+end
+
+class String
+  def wikify
+    gsub(/^[a-z]|\s+[a-z]/) { |a| a.upcase }.gsub(/\s/, '')
+  end
+end

data/scrappy.gemspec

@@ -2,17 +2,17 @@
 
 Gem::Specification.new do |s|
   s.name = %q{scrappy}
-  s.version = "0.1.7"
+  s.version = "0.1.8"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jose Ignacio"]
-  s.date = %q{2010-12-09}
+  s.date = %q{2010-12-14}
   s.default_executable = %q{scrappy}
   s.description = %q{RDF web scraper}
   s.email = %q{joseignacio.fernandez@gmail.com}
   s.executables = ["scrappy"]
-  s.extra_rdoc_files = ["README.rdoc", "bin/scrappy", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/proxy.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/section.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/webkit/webkit.rb"]
-  s.files = ["History.txt", "Manifest", "README.rdoc", "Rakefile", "bin/scrappy", "kb/elmundo.yarf", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/proxy.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/section.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/webkit/webkit.rb", "test/test_helper.rb", "test/test_scrappy.rb", "scrappy.gemspec"]
+  s.extra_rdoc_files = ["README.rdoc", "bin/scrappy", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/proxy.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/new_uri.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/section.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/webkit/webkit.rb"]
+  s.files = ["History.txt", "Manifest", "README.rdoc", "Rakefile", "bin/scrappy", "kb/elmundo.yarf", "lib/js/annotator.js", "lib/scrappy.rb", "lib/scrappy/agent/agent.rb", "lib/scrappy/agent/blind_agent.rb", "lib/scrappy/agent/cache.rb", "lib/scrappy/agent/map_reduce.rb", "lib/scrappy/agent/extractor.rb", "lib/scrappy/agent/visual_agent.rb", "lib/scrappy/proxy.rb", "lib/scrappy/selectors/base_uri.rb", "lib/scrappy/selectors/css.rb", "lib/scrappy/selectors/new_uri.rb", "lib/scrappy/selectors/root.rb", "lib/scrappy/selectors/section.rb", "lib/scrappy/selectors/slice.rb", "lib/scrappy/selectors/uri.rb", "lib/scrappy/selectors/uri_pattern.rb", "lib/scrappy/selectors/xpath.rb", "lib/scrappy/server.rb", "lib/scrappy/shell.rb", "lib/scrappy/support.rb", "lib/scrappy/webkit/webkit.rb", "test/test_helper.rb", "test/test_scrappy.rb", "scrappy.gemspec"]
   s.homepage = %q{http://github.com/josei/scrappy}
   s.post_install_message = %q{**(Optional) Remember to install rbwebkitgtk for visual parsing features**}
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Scrappy", "--main", "README.rdoc"]

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 7
-  version: 0.1.7
+  - 8
+  version: 0.1.8
 platform: ruby
 authors: 
 - Jose Ignacio
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-09 00:00:00 +01:00
+date: 2010-12-14 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -133,6 +133,7 @@ extra_rdoc_files:
 - lib/scrappy/proxy.rb
 - lib/scrappy/selectors/base_uri.rb
 - lib/scrappy/selectors/css.rb
+- lib/scrappy/selectors/new_uri.rb
 - lib/scrappy/selectors/root.rb
 - lib/scrappy/selectors/section.rb
 - lib/scrappy/selectors/slice.rb
@@ -161,6 +162,7 @@ files:
 - lib/scrappy/proxy.rb
 - lib/scrappy/selectors/base_uri.rb
 - lib/scrappy/selectors/css.rb
+- lib/scrappy/selectors/new_uri.rb
 - lib/scrappy/selectors/root.rb
 - lib/scrappy/selectors/section.rb
 - lib/scrappy/selectors/slice.rb