kabutops 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c9d30b351bb730c7b4b6d44de1dfb4b34630c702
-  data.tar.gz: b39802e0b31ad087cd96259957d16624307fe011
+  metadata.gz: 9f6f70b8bec88cff6caf4d04cb1e24e058e411c0
+  data.tar.gz: c7c77ccc1785e79e2b18b7c00aeae9d8d80954d3
 SHA512:
-  metadata.gz: 6ff995f573eeafac0895a0f8095161bda425f970a6095c637e0cf71028e122348a9b7ea2c65cd62e20d013efacb5178a0aa7c26fa6ed9b8e753b5dd61d9a744e
-  data.tar.gz: f9470dee147282accc9dd9a31debd11706f634e14679f085242febdc874aded7ae9e09439ae3e2a77f55892f6a53e6b887f9832bbeebf5a51bdff694d5fd57f8
+  metadata.gz: 1b85f06cbc5310797e30d7cbf4397a4ae08843e4377f478fffecb4a28db4f80884c3a9bf77542e257480fdb2c6a9ca940d35274d215b6eb0fbbf7004fad4ed39
+  data.tar.gz: f77ef30437460ca717e9e87dcdae84ff5509ff25c3983783bec803d624f7721ca796e3ec222934549b82a68c07821cbb8f6af0f74ee977fe8a91a78557bca4a4

data/lib/kabutops/adapters/database_adapter.rb

@@ -41,7 +41,7 @@ module Kabutops
         raise NotImplementedError
       end
 
-      def nested?
+      def find resource
         raise NotImplementedError
       end
     end

data/lib/kabutops/adapters/elastic_search.rb

@@ -68,10 +68,6 @@ module Kabutops
         result['hits']['hits'].map{ |hit| hit['_source'] }
       end
 
-      def nested?
-        true
-      end
-
       protected
 
       def client

data/lib/kabutops/adapters/mongo.rb

@@ -21,10 +21,6 @@ module Kabutops
         end
       end
 
-      def nested?
-        true
-      end
-
       protected
 
       def client

data/lib/kabutops/adapters/redis.rb

@@ -13,8 +13,8 @@ module Kabutops
         client[result[:id]] = JSON.dump(result.to_hash)
       end
 
-      def nested?
-        true
+      def find resource
+        client[resource[:id] || resource[:url]]
       end
 
       protected

data/lib/kabutops/crawler.rb

@@ -64,22 +64,13 @@ module Kabutops
       resource = Hashie::Mash.new(resource)
 
       adapters = self.class.adapters.select do |adapter|
-        if params.skip_existing && adapter.respond_to?(:find)
-          adapter.find(resource).nil?
-        else
-          true
-        end
+        params.skip_existing ? adapter.find(resource).nil? : true
       end
 
       return if adapters.nil?
-
       page = crawl(resource)
-
       return if page.nil?
-
-      save = (self.class.notify(:store_if, resource, page) || []).all?
-
-      return unless save
+      return unless (self.class.notify(:store_if, resource, page) || []).all?
 
       adapters.each do |adapter|
         adapter.process(resource, page)
@@ -105,19 +96,7 @@ module Kabutops
     end
 
     def crawl resource
-      page = nil
-      cache_key = (resource[:id] || Digest::SHA256.hexdigest(resource[:url])).to_s
-
-      content = Cachy.cache_if(params.cache, cache_key) do
-        sleep params[:wait] || 0 # wait only if value is not from cache
-        body = agent.get(resource[:url]).body
-        body.encode!('utf-8', params[:encoding]) if params[:encoding]
-        page = Nokogiri::HTML(body)
-        self.class.notify(:before_cache, resource, page)
-        body
-      end
-
-      page = Nokogiri::HTML(content) if page.nil?
+      page = get_cache_or_hit(resource)
       self.class.notify(:after_crawl, resource, page)
       page
     rescue Mechanize::ResponseCodeError => e
@@ -129,6 +108,26 @@ module Kabutops
       end
     end
 
+    def get_cache_or_hit resource
+      cache_key = (resource[:id] || Digest::SHA256.hexdigest(resource[:url])).to_s
+      page = nil
+
+      content = Cachy.cache_if(params.cache, cache_key) do
+        sleep params[:wait] || 0 # wait only if value is not from cache
+        page = get_page(resource[:url])
+        self.class.notify(:before_cache, resource, page)
+        page.to_s
+      end
+
+      page ? page : Nokogiri::HTML(content)
+    end
+
+    def get_page url
+      body = agent.get(url).body
+      body.encode!('utf-8', params[:encoding]) if params[:encoding]
+      Nokogiri::HTML(body)
+    end
+
     def agent
       if params[:agent].is_a?(Proc)
         @agent = params[:agent].call

data/lib/kabutops/version.rb

@@ -1,5 +1,5 @@
 # -*- encoding : utf-8 -*-
 
 module Kabutops
-  VERSION = '0.1.6'
+  VERSION = '0.1.7'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kabutops
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - Rene Klacan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-11 00:00:00.000000000 Z
+date: 2014-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: 2.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: 2.6.0
 - !ruby/object:Gem::Dependency
   name: cachy
   requirement: !ruby/object:Gem::Requirement
@@ -249,3 +249,4 @@ specification_version: 4
 summary: Dead simple yet powerful Ruby crawler for easy parallel crawling with support
   for an anonymity.
 test_files: []
+has_rdoc: