scrappy 0.4.8 → 0.4.9

data/History.txt

@@ -1,3 +1,7 @@
+=== 0.4.9 2011-11-28
+
+* Refactorization of how encoding issues are managed
+
 === 0.4.8 2011-11-24
 
 * Encoding fixes

data/lib/scrappy.rb

@@ -8,7 +8,6 @@ require 'ostruct'
 require 'active_support'
 require 'tmpdir'
 require 'lightrdf'
-require 'iconv'
 
 Namespace :sc, 'http://lab.gsi.dit.upm.es/scraping.rdf#'
 
@@ -25,5 +24,5 @@ require 'scrappy/agent/blind_agent'
 require 'scrappy/agent/agent'
 
 module Scrappy
-  VERSION = '0.4.8'
+  VERSION = '0.4.9'
 end

data/lib/scrappy/extractor/formats.rb

@@ -4,7 +4,7 @@ module Scrappy
     def format node, formats, uri
       case formats.first
       when Node('sc:WikiText') then
-        doc = Nokogiri::XML(node.to_html)
+        doc = Nokogiri::XML(node.to_html.clean)
         doc.search("a").each {|n| n.replace(Nokogiri::XML::Text.new(URI.parse(uri).merge(n["href"]).to_s, n.document)) }
         doc.search("h1").each {|n| n.replace(Nokogiri::XML::Text.new("= #{n.text.strip} =", n.document)) }
         doc.search("h2").each {|n| n.replace(Nokogiri::XML::Text.new("== #{n.text.strip} ==", n.document)) }
@@ -26,15 +26,15 @@ module Scrappy
         doc.text.strip
       when Node('sc:Html') then
         if node.respond_to? :to_html
-          node.to_html
+          node.to_html.clean
         else
-          node.to_s
+          node.to_s.clean
         end
       else
         if node.respond_to? :text
-          node.text
+          node.text.clean
         else
-          node.to_s
+          node.to_s.clean
         end
       end
     end

data/lib/scrappy/extractor/fragment.rb

@@ -65,14 +65,12 @@ module Sc
 
         # Build the object -- it can be a node or a literal
         object = if sc::type.include?(Node('rdf:Literal'))
-          ic = Iconv.new('UTF-8//IGNORE', 'UTF-8')
-          value = ic.iconv(doc[:value].to_s + ' ')[0..-2].gsub("\302\240"," ").strip
           if options[:referenceable]
-            node.rdf::value = value
+            node.rdf::value = doc[:value]
             node.rdf::type += [Node('rdf:Literal')]
             node
           else
-            value
+            doc[:value]
           end
         else
           # Add statements about the node

data/lib/scrappy/extractor/selector.rb

@@ -20,13 +20,13 @@ module Sc
       # Process selector
       # Filter method is defined in each subclass
       results = filter doc
-
+      
       if sc::boolean.first=="true"
         results = results.map do |r|
           affirmations = ["yes", "true"]
           negations = ["no", "none", "false", "-", "--"]
-          no  = negations.include?(r[:value].gsub("\302\240"," ").strip.downcase)
-          yes = affirmations.include?(r[:value].gsub("\302\240"," ").strip.downcase)
+          no  = negations.include?(r[:value].downcase)
+          yes = affirmations.include?(r[:value].downcase)
           if no
             value = "false" 
           elsif yes
@@ -41,10 +41,16 @@ module Sc
       if sc::normalize_max.first
         max = sc::normalize_max.first.to_f
         min = sc::normalize_min.first.to_f
-        results.each { |r| r[:value] = ((r[:value].to_f-min) / (max-min)).to_s }
+        in_range = sc::normalize_in_range.first == "true"
+        results.each do |r|
+          r[:value] = ((r[:value].to_f-min) / (max-min)).to_s
+        end
+        if in_range
+          results = results.select { |r| r[:value].to_f <= 1.0 and r[:value].to_f >= 0.0 }
+        end
       end
       if sc::nonempty.first=="true"
-        results = results.select{ |r| r[:value].gsub("\302\240"," ").strip!=""}
+        results = results.select{ |r| r[:value] != ""}
       end
       
       if sc::debug.first=="true" and Scrappy::Agent::Options.debug and

data/lib/scrappy/extractor/selectors/visual.rb

@@ -62,7 +62,7 @@ module Sc
         end.map do |content|
           if attributes.first
             # Select node's attribute if given
-            attributes.map { |attribute| { :uri=>doc[:uri], :content=>content, :value=>content[attribute], :attribute=>attribute } }
+            attributes.map { |attribute| { :uri=>doc[:uri], :content=>content, :value=>content[attribute].clean, :attribute=>attribute } }
           else
             [ { :uri=>doc[:uri], :content=>content, :value=>format(content, formats, doc[:uri]) } ]
           end

data/lib/scrappy/extractor/selectors/xpath.rb

@@ -8,10 +8,12 @@ module Sc
           (0..-1)
         end
         patterns = sc::keyword
-        (doc[:content].search(pattern)[interval] || []).select { |node| patterns.any? ? patterns.include?(node.text.downcase.strip) : true }.map do |result|
+        (doc[:content].search(pattern)[interval] || []).select do |node|
+          patterns.any? ? patterns.include?(node.text.clean.downcase) : true
+        end.map do |result|
           if sc::attribute.first
             # Select node's attribute if given
-            sc::attribute.map { |attribute| { :uri=>doc[:uri], :content=>result, :value=>result[attribute], :attribute=>attribute } }
+            sc::attribute.map { |attribute| { :uri=>doc[:uri], :content=>result, :value=>result[attribute].clean, :attribute=>attribute } }
           else
             # Select node
             [ { :uri=>doc[:uri], :content=>result, :value=>format(result, sc::format, doc[:uri]) } ]

data/lib/scrappy/support.rb

@@ -1,3 +1,5 @@
+# -*- encoding: utf-8 -*-
+require 'iconv'
 require 'open-uri'
 require 'net/http'
 require 'net/https'
@@ -19,6 +21,8 @@ module Nokogiri
 end
 
 class String
+  Utf8Iconv = Iconv.new('UTF-8//IGNORE', 'UTF-8')
+  
   def wikify
     gsub(/^[a-z]|\s+[a-z]/) { |a| a.upcase }.gsub(/\s/, '')
   end
@@ -30,4 +34,7 @@ class String
     gsub(/\s+/,"_").
     downcase
   end
+  def clean
+    Utf8Iconv.iconv(self + '  ')[0..-3].gsub("\302\240"," ").strip
+  end
 end

data/scrappy.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = "scrappy"
-  s.version = "0.4.8"
+  s.version = "0.4.9"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jose Ignacio"]
-  s.date = "2011-11-24"
+  s.date = "2011-11-28"
   s.description = "RDF web scraper"
   s.email = "joseignacio.fernandez@gmail.com"
   s.executables = ["scrappy"]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scrappy
 version: !ruby/object:Gem::Version
-  version: 0.4.8
+  version: 0.4.9
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-11-24 00:00:00.000000000Z
+date: 2011-11-28 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &82230020 !ruby/object:Gem::Requirement
+  requirement: &85217640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: 2.3.5
   type: :runtime
   prerelease: false
-  version_requirements: *82230020
+  version_requirements: *85217640
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &82229760 !ruby/object:Gem::Requirement
+  requirement: &85217340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 1.1.2
   type: :runtime
   prerelease: false
-  version_requirements: *82229760
+  version_requirements: *85217340
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &82229510 !ruby/object:Gem::Requirement
+  requirement: &85217090 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: 1.2.7
   type: :runtime
   prerelease: false
-  version_requirements: *82229510
+  version_requirements: *85217090
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &82229270 !ruby/object:Gem::Requirement
+  requirement: &85216790 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: 1.4.1
   type: :runtime
   prerelease: false
-  version_requirements: *82229270
+  version_requirements: *85216790
 - !ruby/object:Gem::Dependency
   name: mechanize
-  requirement: &82229010 !ruby/object:Gem::Requirement
+  requirement: &85216530 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *82229010
+  version_requirements: *85216530
 - !ruby/object:Gem::Dependency
   name: lightrdf
-  requirement: &82228740 !ruby/object:Gem::Requirement
+  requirement: &85216280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: 0.4.1
   type: :runtime
   prerelease: false
-  version_requirements: *82228740
+  version_requirements: *85216280
 - !ruby/object:Gem::Dependency
   name: i18n
-  requirement: &82228500 !ruby/object:Gem::Requirement
+  requirement: &85215970 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: 0.4.2
   type: :runtime
   prerelease: false
-  version_requirements: *82228500
+  version_requirements: *85215970
 - !ruby/object:Gem::Dependency
   name: rest-client
-  requirement: &82228260 !ruby/object:Gem::Requirement
+  requirement: &85215720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: 1.6.1
   type: :runtime
   prerelease: false
-  version_requirements: *82228260
+  version_requirements: *85215720
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &82227980 !ruby/object:Gem::Requirement
+  requirement: &85215430 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: 3.0.24
   type: :runtime
   prerelease: false
-  version_requirements: *82227980
+  version_requirements: *85215430
 - !ruby/object:Gem::Dependency
   name: rack-flash
-  requirement: &82227730 !ruby/object:Gem::Requirement
+  requirement: &85215190 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,7 +120,7 @@ dependencies:
         version: 0.1.1
   type: :runtime
   prerelease: false
-  version_requirements: *82227730
+  version_requirements: *85215190
 description: RDF web scraper
 email: joseignacio.fernandez@gmail.com
 executables: