scrappy 0.4.7 → 0.4.8

data/History.txt

@@ -1,3 +1,8 @@
+=== 0.4.8 2011-11-24
+
+* Encoding fixes
+* Added sc:normalize_max and sc:normalize_min support
+
 === 0.4.7 2011-11-24
 
 * Bug fix in Ruby 1.9

data/lib/scrappy.rb

@@ -8,6 +8,7 @@ require 'ostruct'
 require 'active_support'
 require 'tmpdir'
 require 'lightrdf'
+require 'iconv'
 
 Namespace :sc, 'http://lab.gsi.dit.upm.es/scraping.rdf#'
 
@@ -24,5 +25,5 @@ require 'scrappy/agent/blind_agent'
 require 'scrappy/agent/agent'
 
 module Scrappy
-  VERSION = '0.4.7'
+  VERSION = '0.4.8'
 end

data/lib/scrappy/extractor/fragment.rb

@@ -65,7 +65,8 @@ module Sc
 
         # Build the object -- it can be a node or a literal
         object = if sc::type.include?(Node('rdf:Literal'))
-          value = doc[:value].to_s.gsub("\302\240"," ").strip
+          ic = Iconv.new('UTF-8//IGNORE', 'UTF-8')
+          value = ic.iconv(doc[:value].to_s + ' ')[0..-2].gsub("\302\240"," ").strip
           if options[:referenceable]
             node.rdf::value = value
             node.rdf::type += [Node('rdf:Literal')]

data/lib/scrappy/extractor/selector.rb

@@ -1,3 +1,4 @@
+# -*- encoding: utf-8 -*-
 module Sc
   class Selector
     include RDF::NodeProxy
@@ -37,6 +38,11 @@ module Sc
         end
         results = results.select{ |r| r[:value] != :remove }
       end
+      if sc::normalize_max.first
+        max = sc::normalize_max.first.to_f
+        min = sc::normalize_min.first.to_f
+        results.each { |r| r[:value] = ((r[:value].to_f-min) / (max-min)).to_s }
+      end
       if sc::nonempty.first=="true"
         results = results.select{ |r| r[:value].gsub("\302\240"," ").strip!=""}
       end

data/scrappy.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name = "scrappy"
-  s.version = "0.4.7"
+  s.version = "0.4.8"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Jose Ignacio"]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scrappy
 version: !ruby/object:Gem::Version
-  version: 0.4.7
+  version: 0.4.8
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2011-11-24 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &81718490 !ruby/object:Gem::Requirement
+  requirement: &82230020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: 2.3.5
   type: :runtime
   prerelease: false
-  version_requirements: *81718490
+  version_requirements: *82230020
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &81718210 !ruby/object:Gem::Requirement
+  requirement: &82229760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 1.1.2
   type: :runtime
   prerelease: false
-  version_requirements: *81718210
+  version_requirements: *82229760
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &81717960 !ruby/object:Gem::Requirement
+  requirement: &82229510 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: 1.2.7
   type: :runtime
   prerelease: false
-  version_requirements: *81717960
+  version_requirements: *82229510
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &81717720 !ruby/object:Gem::Requirement
+  requirement: &82229270 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: 1.4.1
   type: :runtime
   prerelease: false
-  version_requirements: *81717720
+  version_requirements: *82229270
 - !ruby/object:Gem::Dependency
   name: mechanize
-  requirement: &81717430 !ruby/object:Gem::Requirement
+  requirement: &82229010 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: 1.0.0
   type: :runtime
   prerelease: false
-  version_requirements: *81717430
+  version_requirements: *82229010
 - !ruby/object:Gem::Dependency
   name: lightrdf
-  requirement: &81717180 !ruby/object:Gem::Requirement
+  requirement: &82228740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: 0.4.1
   type: :runtime
   prerelease: false
-  version_requirements: *81717180
+  version_requirements: *82228740
 - !ruby/object:Gem::Dependency
   name: i18n
-  requirement: &81716890 !ruby/object:Gem::Requirement
+  requirement: &82228500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: 0.4.2
   type: :runtime
   prerelease: false
-  version_requirements: *81716890
+  version_requirements: *82228500
 - !ruby/object:Gem::Dependency
   name: rest-client
-  requirement: &81716630 !ruby/object:Gem::Requirement
+  requirement: &82228260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: 1.6.1
   type: :runtime
   prerelease: false
-  version_requirements: *81716630
+  version_requirements: *82228260
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &81716070 !ruby/object:Gem::Requirement
+  requirement: &82227980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: 3.0.24
   type: :runtime
   prerelease: false
-  version_requirements: *81716070
+  version_requirements: *82227980
 - !ruby/object:Gem::Dependency
   name: rack-flash
-  requirement: &81715820 !ruby/object:Gem::Requirement
+  requirement: &82227730 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,7 +120,7 @@ dependencies:
         version: 0.1.1
   type: :runtime
   prerelease: false
-  version_requirements: *81715820
+  version_requirements: *82227730
 description: RDF web scraper
 email: joseignacio.fernandez@gmail.com
 executables: