kabutops 0.0.12 → 0.0.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0b4f48640fb6de941353f09dee76176c4ce9ff07
-  data.tar.gz: 41173f3f0826601540cdcd783da88c4c4dae9169
+  metadata.gz: 2079759508e14efc73da0711fefa3bafa7e2cffa
+  data.tar.gz: 0034b70dbeca4dafd3471125f47392a4c7e98693
 SHA512:
-  metadata.gz: 72097b1fa2b063c628fe40c1f02557c40ee4eba0c25e020140511e6154b1e3bcf5b05d89d07c1cd10aaeec6b1b54f03fccd9b25415bf7bfa11661c2403d1a995
-  data.tar.gz: c1d42e0efdcc0ba81dfd3fe7c885786c7e639d7a022c00d9999bfa48cd1c75d7d0cc7b8be13c143ce9e84ee017ce3571f24ecc024967c3fd9859af628d296abd
+  metadata.gz: 5315b48bff4207c58ff56242120cf6c9feb5297b29d668607241238ff9f1e0f01df16bf0ed45367b927690f6c55480b212761aec3dda0261a6af3ade1676e6b7
+  data.tar.gz: c00e64003cd9b3bb26685ed74dd16052e9b952cfa5e89baf0cf36f35d4d2cca77e3ee618c50d68f82d3e15aa2d239d45aebadea33b774ec507813ce15abfeb51

data/README.md

@@ -24,7 +24,7 @@ gem install kabutops
 Or you can put it in your Gemfile
 
 ```ruby
-gem 'kabutops', '~> 0.0.12'
+gem 'kabutops', '~> 0.0.13'
 ```
 
 You will also need Redis database installed and running.

data/lib/kabutops/crawler.rb

@@ -75,6 +75,8 @@ module Kabutops
 
       page = crawl(resource)
 
+      return if page.nil?
+
       adapters.each do |adapter|
         adapter.process(resource, page)
       end
@@ -113,6 +115,13 @@ module Kabutops
       page = Nokogiri::HTML(content) if page.nil?
       self.class.notify(:after_crawl, resource, page)
       page
+    rescue Mechanize::ResponseCodeError => e
+      if e.response_code.to_i == 404
+        nil
+      else
+        p e.response_code
+        raise
+      end
     end
 
     def agent

data/lib/kabutops/recipe.rb

@@ -17,8 +17,8 @@ module Kabutops
         @items[name] = RecipeItem.new(:recipe, recipe)
         @nested = true
       else
-        type, value = args[0..1]
-        @items[name] = RecipeItem.new(type, value)
+        type, value, convert_to = args[0..2]
+        @items[name] = RecipeItem.new(type, value, convert_to)
       end
     end
 

data/lib/kabutops/recipe_item.rb

@@ -5,27 +5,45 @@ module Kabutops
   class RecipeItem
     attr_reader :type, :value
 
-    def initialize type, value
+    def initialize type, value, convert_to=nil
       @type = type
       @value = value
+      @convert_to = convert_to
     end
 
     def process resource, page
+      convert(get(resource, page))
+    end
+
+    protected
+
+    def get resource, page
       case @type
       when :var
         resource[@value]
       when :recipe
         @value.process(resource, page)
       when :css
-        page.css(@value).text
+        page.css(@value).text.gsub(/\u00a0/, ' ').strip
       when :xpath
-        page.xpath(@value).text
+        page.xpath(@value).text.gsub(/\u00a0/, ' ').strip
       when :lambda, :proc
         @value.call(resource, page)
       else
         raise "unknown recipe item type '#{item.type}'"
       end
     end
+
+    def convert v
+      return nil if v.nil?
+
+      case @convert_to
+      when nil then v
+      when :int then v[/\d+/].to_i
+      when :float then v[/\d+(\.\d+)?/].to_f
+      end
+    end
   end
 
+
 end

data/lib/kabutops/version.rb

@@ -1,5 +1,5 @@
 # -*- encoding : utf-8 -*-
 
 module Kabutops
-  VERSION = '0.0.12'
+  VERSION = '0.0.13'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kabutops
 version: !ruby/object:Gem::Version
-  version: 0.0.12
+  version: 0.0.13
 platform: ruby
 authors:
 - Rene Klacan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-16 00:00:00.000000000 Z
+date: 2014-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize