scrappy 0.1.3 → 0.1.4

data/History.txt

@@ -1,3 +1,9 @@
+=== 0.1.4 2010-11-24
+
+* Support for node sets in extractions
+* Support for index selection in CSS and XPath selectors
+* Debugging mode
+
 === 0.1.3 2010-11-18
 
 * RDF node caching

data/lib/scrappy/agent/extractor.rb

@@ -61,10 +61,12 @@ module Scrappy
 
           # Add referenceable data if requested
           if options[:referenceable]
-            source = Node(node_hash(doc[:uri], doc[:content].path))
-            options[:triples] << [ object, Node("sc:source"), source ]
-            fragment.sc::type.each { |t| options[:triples] << [ source, Node("sc:type"), t ] }
-            fragment.sc::relation.each { |relation| options[:triples] << [ source, Node("sc:relation"), relation ] }
+            sources = [doc[:content]].flatten.map { |node| Node(node_hash(doc[:uri], node.path)) }
+            sources.each do |source|
+              options[:triples] << [ object, Node("sc:source"), source ]
+              fragment.sc::type.each { |t| options[:triples] << [ source, Node("sc:type"), t ] }
+              fragment.sc::relation.each { |relation| options[:triples] << [ source, Node("sc:relation"), relation ] }
+            end
           end
 
           # Process subfragments
@@ -80,6 +82,23 @@ module Scrappy
       # Process selector
       results = Kernel.const_get(class_name).filter selector, doc
 
+      if !selector.sc::debug.empty?
+        puts '== DEBUG'
+        puts '== Selector:'
+        puts selector.serialize(:yarf, false)
+        puts '== Applied on fragment:'
+        puts "URI: #{doc[:uri]}"
+        puts "Content: #{doc[:content]}"
+        puts "Value: #{doc[:value]}"
+        results.each_with_index do |result, i|
+          puts "== Result ##{i}:"
+          puts "URI: #{result[:uri]}"
+          puts "Content: #{result[:content]}"
+          puts "Value: #{result[:value].inspect}"
+        end
+        puts
+      end
+      
       # Return results if no nested selectors
       return results if selector.sc::selector.empty?
 

data/lib/scrappy/selectors/xpath.rb

@@ -1,7 +1,12 @@
 module XPathSelector
   def self.filter selector, doc
     selector.rdf::value.map do |pattern|
-      doc[:content].search(pattern).map do |result|
+      interval = if selector.sc::index.first
+        (selector.sc::index.first.to_i..selector.sc::index.first.to_i)
+      else
+        (0..-1)
+      end
+      (doc[:content].search(pattern)[interval] || []).map do |result|
         if selector.sc::attribute.first
           # Select node's attribute if given
           selector.sc::attribute.map { |attribute| { :uri=>doc[:uri], :content=>result, :value=>result[attribute] } }
@@ -12,4 +17,4 @@ module XPathSelector
       end
     end.flatten
   end
-end
+end

data/lib/scrappy/support.rb

@@ -16,3 +16,13 @@ module Scrappy
     end
   end
 end
+
+module Nokogiri
+  module XML
+    class NodeSet
+      def select &block
+        NodeSet.new(document, super(&block))
+      end
+    end
+  end
+end

data/lib/scrappy.rb

@@ -19,7 +19,7 @@ require 'scrappy/agent/agent'
 Namespace :sc, 'http://lab.gsi.dit.upm.es/scraping.rdf#'
 
 module Scrappy
-  VERSION = '0.1.3'
+  VERSION = '0.1.4'
 end
 
 # Require selectors

data/scrappy.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{scrappy}
-  s.version = "0.1.3"
+  s.version = "0.1.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jose Ignacio"]
-  s.date = %q{2010-11-18}
+  s.date = %q{2010-11-24}
   s.default_executable = %q{scrappy}
   s.description = %q{RDF web scraper}
   s.email = %q{joseignacio.fernandez@gmail.com}

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 3
-  version: 0.1.3
+  - 4
+  version: 0.1.4
 platform: ruby
 authors: 
 - Jose Ignacio
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-18 00:00:00 +01:00
+date: 2010-11-24 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency