kabutops 0.0.15 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3fccc58091b11a4171dbf3ef78bf61e8a9399936
-  data.tar.gz: 72e2f5edf655110a9af6ed7d806da7f04119cf76
+  metadata.gz: 05df1a21e10c723541c9cc4514174959b794dc27
+  data.tar.gz: 6d60b8c1b4acf024f50b8c363b905c8d167f503c
 SHA512:
-  metadata.gz: 5098cd922353e30708f081e9b9f2467027cead27aba5d2045045ac413e4965703f6cb9adb454528e4c3a94de960a5542549af26f43b806773193edafa8e5dbd5
-  data.tar.gz: 7376e74da6006fd4bc4da1023e21865aa861a17306d0a99c5b47b736a2cf207a34d8f2efe7a8cd47eb3e2e05e5eea68ac010ba5d9a0bdae3b721d87fbc17fcf4
+  metadata.gz: 5cf88297902defb328c85d39233af7a8f11fc20935c193a4a9cc568d38c844f3657973ea58425db4d719b6faa943df7edc71e179e06425ef39911cadf5599d21
+  data.tar.gz: 47e08167bb5129ef665310c2d332c57ae071d9755a2c5f133b03b158d1ae1520126ef1ab8ff58c2f70564bd93ac6ccac0a613e2eae49e8ca657d63bfe6024814

data/README.md

@@ -24,7 +24,7 @@ gem install kabutops
 Or you can put it in your Gemfile
 
 ```ruby
-gem 'kabutops', '~> 0.0.15'
+gem 'kabutops', '~> 0.1.0'
 ```
 
 You will also need Redis database installed and running.

data/lib/kabutops/adapters/database_adapter.rb

@@ -12,25 +12,25 @@ module Kabutops
 
       callbacks :after_save, :save_if
 
-      def data &block
-        @recipe = Recipe.new
+      def data params={}, &block
+        @recipe = Recipe.new(params)
         @recipe.instance_eval &block
       end
 
       def process resource, page
         raise 'data block not defined' unless @recipe
 
-        result = @recipe.process(resource, page)
-        result.update(updated_at: Time.now.to_i)
-
-        save = (notify(:save_if, resource, page, result) || []).all?
-
-        if debug
-          logger.info("#{self.class.to_s} outputs:")
-          logger.info(save ? result.to_hash : 'not valid for save')
-        elsif save
-          store(result)
-          notify(:after_save, result)
+        [@recipe.process(resource, page)].flatten.each do |result|
+          result.update(updated_at: Time.now.to_i)
+          save = (notify(:save_if, resource, page, result) || []).all?
+
+          if debug
+            logger.info("#{self.class.to_s} outputs:")
+            logger.info(save ? result.to_hash : 'not valid for save')
+          elsif save
+            store(result)
+            notify(:after_save, result)
+          end
         end
       end
 

data/lib/kabutops/crawler.rb

@@ -17,7 +17,7 @@ module Kabutops
 
       params :collection, :proxy, :cache, :wait,
              :skip_existing, :agent
-      callbacks :after_crawl, :before_cache
+      callbacks :after_crawl, :before_cache, :store_if
 
       def adapters
         @adapters ||= []
@@ -77,6 +77,10 @@ module Kabutops
 
       return if page.nil?
 
+      save = (self.class.notify(:store_if, resource, page) || []).all?
+
+      return unless save
+
       adapters.each do |adapter|
         adapter.process(resource, page)
       end

data/lib/kabutops/recipe.rb

@@ -5,7 +5,8 @@ module Kabutops
   class Recipe
     attr_reader :items
 
-    def initialize
+    def initialize params={}
+      @params = Hashie::Mash.new(params)
       @items = Hashie::Mash.new
       @nested = false
     end
@@ -23,10 +24,20 @@ module Kabutops
     end
 
     def process resource, page
+      if @params[:each]
+        page.xpath(@params[:each]).map{ |n| process_one(resource, n) }
+      elsif @params[:each_css]
+        page.css(@params[:each_css]).map{ |n| process_one(resource, n) }
+      else
+        process_one(resource, page)
+      end
+    end
+
+    def process_one resource, node
       result = Hashie::Mash.new
 
       @items.each do |name, item|
-        result[name] = item.process(resource, page)
+        result[name] = item.process(resource, node)
       end
 
       result

data/lib/kabutops/spider.rb

@@ -18,6 +18,11 @@ module Kabutops
         super(collection || [{ url: params.url, }])
       end
 
+      def reset!
+        super
+        redis.keys.each{ |k| redis.del(k) }
+      end
+
       def << resource
         if resource_status(resource).nil?
           resource_status(resource, 'new')
@@ -25,6 +30,12 @@ module Kabutops
         end
       end
 
+      def follow link
+        self << {
+          url: URI.join(params.url, URI.escape(link)).to_s
+        }
+      end
+
       def resource_status resource, status=nil
         url_status(resource[:url], status)
       end
@@ -73,12 +84,10 @@ module Kabutops
 
     def after_crawl resource, page
       page.css('a').each do |a|
+        next if a['href'].nil?
+
         follow = self.class.notify(:follow_if, a['href']).any?
-        if follow
-          self << {
-            url: URI.join(params.url, URI.escape(a['href'])).to_s
-          }
-        end
+        self.class.follow(a['href']) if follow
       end
     end
   end

data/lib/kabutops/version.rb

@@ -1,5 +1,5 @@
 # -*- encoding : utf-8 -*-
 
 module Kabutops
-  VERSION = '0.0.15'
+  VERSION = '0.1.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kabutops
 version: !ruby/object:Gem::Version
-  version: 0.0.15
+  version: 0.1.0
 platform: ruby
 authors:
 - Rene Klacan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-17 00:00:00.000000000 Z
+date: 2014-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize