kabutops 0.0.11 → 0.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 491dade458c54b02dbcdd36da92010fcaa1a5463
-  data.tar.gz: 9d00eb9b6ad5e61ddb27dfa7137f0734e9eda03f
+  metadata.gz: 0b4f48640fb6de941353f09dee76176c4ce9ff07
+  data.tar.gz: 41173f3f0826601540cdcd783da88c4c4dae9169
 SHA512:
-  metadata.gz: 9d00a7d31dfa94e85c9f2d31c49d56b3df2da5fd1b0580c769724f64bad12b331bc8c87a3432071944bc427543458ffe0257743ba5f6dd1840e7e0a8674e7e88
-  data.tar.gz: aa6a67c2dcbaf8e14cf6d4a38e94b78979ef9f594ad492e5618bd3d1db36fbca305895e1c756cae2f355d21045fd82489ee315eb7150a2cac7f17d6fbc744fdb
+  metadata.gz: 72097b1fa2b063c628fe40c1f02557c40ee4eba0c25e020140511e6154b1e3bcf5b05d89d07c1cd10aaeec6b1b54f03fccd9b25415bf7bfa11661c2403d1a995
+  data.tar.gz: c1d42e0efdcc0ba81dfd3fe7c885786c7e639d7a022c00d9999bfa48cd1c75d7d0cc7b8be13c143ce9e84ee017ce3571f24ecc024967c3fd9859af628d296abd

data/README.md

@@ -24,7 +24,7 @@ gem install kabutops
 Or you can put it in your Gemfile
 
 ```ruby
-gem 'kabutops', '~> 0.0.11'
+gem 'kabutops', '~> 0.0.12'
 ```
 
 You will also need Redis database installed and running.

data/lib/kabutops/crawler.rb

@@ -17,7 +17,7 @@ module Kabutops
 
       params :collection, :proxy, :cache, :wait,
              :skip_existing, :agent
-      callbacks :after_crawl
+      callbacks :after_crawl, :before_cache
 
       def adapters
         @adapters ||= []
@@ -99,13 +99,18 @@ module Kabutops
     end
 
     def crawl resource
-      cache_key = (resource[:id] || resource[:url]).to_s
+      page = nil
+      cache_key = (resource[:id] || Digest::SHA256.hexdigest(resource[:url])).to_s
+
       content = Cachy.cache_if(params.cache, cache_key) do
         sleep params[:wait] || 0 # wait only if value is not from cache
-        agent.get(resource[:url]).body
+        body = agent.get(resource[:url]).body
+        page = Nokogiri::HTML(body)
+        self.class.notify(:before_cache, resource, page)
+        body
       end
 
-      page = Nokogiri::HTML(content)
+      page = Nokogiri::HTML(content) if page.nil?
       self.class.notify(:after_crawl, resource, page)
       page
     end

data/lib/kabutops/spider.rb

@@ -5,7 +5,7 @@ module Kabutops
   class Spider < Crawler
     class << self
       params :url
-      callbacks :after_crawl, :follow_if
+      callbacks :after_crawl, :before_cache, :follow_if
 
       def debug_spider
         enable_debug
@@ -14,6 +14,10 @@ module Kabutops
         })
       end
 
+      def crawl collection=nil
+        super(collection || [{ url: params.url, }])
+      end
+
       def << resource
         if resource_status(resource).nil?
           resource_status(resource, 'new')
@@ -72,7 +76,7 @@ module Kabutops
         follow = self.class.notify(:follow_if, a['href']).any?
         if follow
           self << {
-            url: a['href'],
+            url: URI.join(params.url, URI.escape(a['href'])).to_s
           }
         end
       end

data/lib/kabutops/version.rb

@@ -1,5 +1,5 @@
 # -*- encoding : utf-8 -*-
 
 module Kabutops
-  VERSION = '0.0.11'
+  VERSION = '0.0.12'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kabutops
 version: !ruby/object:Gem::Version
-  version: 0.0.11
+  version: 0.0.12
 platform: ruby
 authors:
 - Rene Klacan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-15 00:00:00.000000000 Z
+date: 2014-07-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize