kabutops 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 30185d1be8fe797322576ca2ded85d792f26d6ce
-  data.tar.gz: fba6fe7cf866ad20f8c632f9fc5761c3e5920c6a
+  metadata.gz: 321de7144e42cfc9cba6d6e6675eac975a2981a5
+  data.tar.gz: 39e39c701812ff208f6131e5caa31438c3268ed5
 SHA512:
-  metadata.gz: ed6ab102fb990e4e83d6efb7eedbe3a84136979452395f448827b529b772dd410ec3eed39ba4f51cd844a2b780ac6e1a7e84d94479fa30993048b20ba6cc1c0f
-  data.tar.gz: 4ec9f7872df1e79a8f4780bf50233d1ce761ee2b8702efb22058bd52900c551147dae3fe174d05fc75b655e44a3e340c8f5cb369758dcfc11ba1f65e34db1a0b
+  metadata.gz: cbb53e66f5b2be03d080728ee7be1c4c9c40210796f7ab851d87e94af7519ef54c13e46c60e7a4770d5ec89a0c3c0043d60995ccc7508d43f07fd1918957161d
+  data.tar.gz: 0f3ae02d465fb1a056018bc7db0d1e47a01df19d4c6ce5ec54855fa4774edf497c1c359e5b8eb3dfc2d592e67c3b983ca06f0a21b740c8d97c6b2b38e9f8bbc2

data/README.md

@@ -1,4 +1,4 @@
-Kabutops
+Kabutops [![Code Climate](https://codeclimate.com/github/reneklacan/kabutops.png)](https://codeclimate.com/github/reneklacan/kabutops) [![Coverage](https://codeclimate.com/github/reneklacan/kabutops/coverage.png)](https://codeclimate.com/github/reneklacan/kabutops)
 ========
 
 Installation
@@ -13,7 +13,7 @@ gem install kabutops
 Or you can put it in your Gemfile
 
 ```ruby
-gem 'kabutops', '~> 0.0.5'
+gem 'kabutops', '~> 0.0.6'
 ```
 
 Basic example
@@ -41,7 +41,7 @@ class GemListCrawler < Kabutops::Crawler
     after_crawl do |resource, page|
       links = page.xpath("//a[contains(@href, '/gems?letter=#{resource[:letter]}')]")
       links.each do |link|
-        self << {
+        GemListCrawler << {
           letter: resource[:letter],
           url: "https://rubygems.org#{link['href']}",
         }

data/lib/kabutops/adapters/database_adapter.rb

@@ -7,6 +7,8 @@ module Kabutops
     class DatabaseAdapter < Base
       include Extensions::CallbackSupport
 
+      attr_reader :recipe
+
       callbacks :after_save
 
       def data &block

data/lib/kabutops/crawler.rb

@@ -20,7 +20,7 @@ module Kabutops
       end
 
       def crawl! collection=nil
-        @map ||= {}
+        @map ||= Hashie::Mash.new
 
         if storage[:status].nil?
           (collection || params[:collection] || []).each do |resource|
@@ -37,6 +37,7 @@ module Kabutops
         end
 
         key = resource[:id] || resource[:url]
+        @map ||= Hashie::Mash.new
 
         if key.nil?
           raise "url must be specified for resource"
@@ -51,26 +52,37 @@ module Kabutops
 
     def perform resource
       resource = Hashie::Mash.new(resource)
+      page = crawl(resource)
 
+      self.class.adapters.each do |adapter|
+        adapter.process(resource, page)
+      end
+    end
+
+    def << resource
+      self.class << resource
+    end
+
+    protected
+
+    def crawl resource
       content = Cachy.cache_if(self.class.params.cache, resource[:url]) do
-        agent = Mechanize.new
-        agent.set_proxy(*self.class.params[:proxy]) if self.class.params[:proxy]
+        sleep self.class.params[:wait] || 0 # wait only if value is not from cache
         agent.get(resource[:url]).body
       end
 
       page = Nokogiri::HTML(content)
-
       self.class.notify(:after_crawl, resource, page)
+      page
+    end
 
-      self.class.adapters.each do |adapter|
-        adapter.process(resource, page)
+    def agent
+      unless @agent
+        @agent = Mechanize.new
+        @agent.set_proxy(*self.class.params[:proxy]) if self.class.params[:proxy]
       end
 
-      sleep self.class.params[:wait] || 0
-    end
-
-    def << resource
-      self.class << resource
+      @agent
     end
   end
 

data/lib/kabutops/crawler_extensions/pstore_storage.rb

@@ -6,7 +6,7 @@ module Kabutops
 
     module PStoreStorage
       class Storage
-        def initialize path
+        def initialize path='.kabutopus.config.pstore'
           @storage ||= PStore.new(path)
         end
 
@@ -28,7 +28,7 @@ module Kabutops
       module ClassMethods
 
         def storage
-          @storage ||= Storage.new(".kabutopus.config.pstore")
+          @storage ||= Storage.new
         end
 
       end

data/lib/kabutops/extensions/callback_support.rb

@@ -14,18 +14,20 @@ module Kabutops
       end
 
       class Manager
+        attr_reader :map, :allowed
+
         def initialize allowed=nil
           @allowed = allowed || []
+          @map ||= Hashie::Mash.new
         end
 
         def method_missing name, *args, &block
-          return unless block_given?
+          return super unless block_given?
 
           unless @allowed.include?(name)
             raise "Invalid callback name: #{name}"
           end
 
-          @map ||= Hashie::Mash.new
           @map[name] ||= []
           @map[name] << block
         end

data/lib/kabutops/extensions/parameterable.rb

@@ -8,9 +8,6 @@ module Kabutops
 
       def self.included base
         base.extend(ClassMethods)
-        base.class_eval do
-          attr_reader :params
-        end
       end
 
       module ClassMethods
@@ -26,6 +23,10 @@ module Kabutops
               end
             end
           end
+
+          define_method :params do
+            @params ||= Hashie::Mash.new
+          end
         end
 
       end

data/lib/kabutops/version.rb

@@ -1,5 +1,5 @@
 # -*- encoding : utf-8 -*-
 
 module Kabutops
-  VERSION = '0.0.5'
+  VERSION = '0.0.6'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kabutops
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Rene Klacan