scrappy 0.4.1 → 0.4.2

data/History.txt

@@ -1,3 +1,8 @@
+=== 0.4.2 2011-07-06
+
+* Support to sc:nonempty and sc:boolean in selectors
+* Bugfix in NewUriSelector
+
 === 0.4.1 2011-07-05
 
 * Bug correction in NewUriSelector

data/lib/scrappy.rb

@@ -24,5 +24,5 @@ require 'scrappy/agent/blind_agent'
 require 'scrappy/agent/agent'
 
 module Scrappy
-  VERSION = '0.4.1'
+  VERSION = '0.4.2'
 end

data/lib/scrappy/extractor/fragment.rb

@@ -64,7 +64,7 @@ module Sc
 
         # Build the object -- it can be a node or a literal
         object = if sc::type.include?(Node('rdf:Literal'))
-          value = doc[:value].to_s.strip
+          value = doc[:value].to_s.gsub("\302\240"," ").strip
           if options[:referenceable]
             node.rdf::value = value
             node.rdf::type += [Node('rdf:Literal')]

data/lib/scrappy/extractor/selector.rb

@@ -20,6 +20,27 @@ module Sc
       # Filter method is defined in each subclass
       results = filter doc
 
+      if sc::boolean.first=="true"
+        results = results.map do |r|
+          affirmations = ["yes", "true"]
+          negations = ["no", "none", "false", "-", "--"]
+          no  = negations.include?(r[:value].gsub("\302\240"," ").strip.downcase)
+          yes = affirmations.include?(r[:value].gsub("\302\240"," ").strip.downcase)
+          if no
+            value = "false" 
+          elsif yes
+            value = "true"
+          else
+            value = :remove
+          end
+          r.merge :value=>value
+        end
+        results = results.select{ |r| r[:value] != :remove }
+      end
+      if sc::nonempty.first=="true"
+        results = results.select{ |r| r[:value].gsub("\302\240"," ").strip!=""}
+      end
+      
       if sc::debug.first=="true" and Scrappy::Agent::Options.debug and
         (Scrappy::Agent::Options.debug_key.nil? or doc[:value].downcase.include?(Scrappy::Agent::Options.debug_key) )
         

data/lib/scrappy/extractor/selectors/new_uri.rb

@@ -10,7 +10,9 @@ module Sc
       
       @indexes ||= Hash.new(0)
       prefix = sc::prefix.first.to_s
-      prefix = (prefix =~ /\Ahttp\:/ or prefix =~ /\Ahttps\:/) ? URI::parse(doc[:uri]).merge(prefix).to_s : "#{doc[:uri]}#{prefix}"
+      if !["http://", "https://"].include?(prefix)
+        prefix = (prefix =~ /\Ahttp\:/ or prefix =~ /\Ahttps\:/) ? URI::parse(doc[:uri]).merge(prefix).to_s : "#{doc[:uri]}#{prefix}"
+      end
       suffix = sc::suffix.first.to_s
       
       nofollow = (sc::follow.first != "true")

data/scrappy.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{scrappy}
-  s.version = "0.4.1"
+  s.version = "0.4.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jose Ignacio"]
-  s.date = %q{2011-07-05}
+  s.date = %q{2011-07-06}
   s.default_executable = %q{scrappy}
   s.description = %q{RDF web scraper}
   s.email = %q{joseignacio.fernandez@gmail.com}

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 4
-  - 1
-  version: 0.4.1
+  - 2
+  version: 0.4.2
 platform: ruby
 authors: 
 - Jose Ignacio
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-07-05 00:00:00 +02:00
+date: 2011-07-06 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency