kabutops 0.1.7 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9f6f70b8bec88cff6caf4d04cb1e24e058e411c0
-  data.tar.gz: c7c77ccc1785e79e2b18b7c00aeae9d8d80954d3
+  metadata.gz: 6e931d9a939854fae6497f910de768971907f29a
+  data.tar.gz: 126e34c2c1eeffa8af3dca362a62381f5889cbe0
 SHA512:
-  metadata.gz: 1b85f06cbc5310797e30d7cbf4397a4ae08843e4377f478fffecb4a28db4f80884c3a9bf77542e257480fdb2c6a9ca940d35274d215b6eb0fbbf7004fad4ed39
-  data.tar.gz: f77ef30437460ca717e9e87dcdae84ff5509ff25c3983783bec803d624f7721ca796e3ec222934549b82a68c07821cbb8f6af0f74ee977fe8a91a78557bca4a4
+  metadata.gz: a869f1388c583d8e2467f096bd6e4702d95832f1f2039253d8b13ec48037e133c9fbbaf68dd44512b69edb4ca4901177757b9ceeedb52efb9aac9587e1ce0a72
+  data.tar.gz: cccb0968bcf9ce251a7ec81c8a8cbb04b5449d19c96dcf2dd58fd69c259b915f17c9e37d7ed926c229066329018b568d0221824f5ec60578eb61238a8172d4b5

data/README.md

@@ -1,19 +1,14 @@
-Kabutops [![Code Climate](https://codeclimate.com/github/reneklacan/kabutops.png)](https://codeclimate.com/github/reneklacan/kabutops) [![Coverage](https://codeclimate.com/github/reneklacan/kabutops/coverage.png)](https://codeclimate.com/github/reneklacan/kabutops)
-========
+# Kabutops [![Code Climate](https://codeclimate.com/github/reneklacan/kabutops.png)](https://codeclimate.com/github/reneklacan/kabutops) [![Coverage](https://codeclimate.com/github/reneklacan/kabutops/coverage.png)](https://codeclimate.com/github/reneklacan/kabutops)
 
 Kabutops is a ruby library whichs aims to simplify creating website crawlers.
 You can define what will be crawled and how it will be saved in the short class definition.
 
-With Kabutops you can save data easily to:
-* ElasticSearch
-* MongoDB
-* Redis
+With Kabutops you can easily save data to **ElasticSearch**.
 
 Example for every kind of database are located
 in the [examples directory](https://github.com/reneklacan/kabutops/tree/master/examples)
 
-Installation
-------------
+## Installation
 
 You can install it via gem
 
@@ -29,8 +24,7 @@ gem 'kabutops', '~> 0.1.1'
 
 You will also need Redis database installed and running.
 
-Basic example
--------------
+## Basic example
 
 Example that will crawl information about gems that start on letter Q or
 X and save them to the ElasticSearch.
@@ -129,8 +123,7 @@ Documents saved in the ElasticSearch will look like this one
 }
 ```
 
-Advanced
---------
+## Advanced
 
 ```ruby
 class SomeCrawler < Kabutops::Crawler
@@ -195,8 +188,7 @@ class SomeCrawler < Kabutops::Crawler
 end
 ```
 
-Debugging
----------
+## Debugging
 
 As we all know, crawler can't be written on the first time.
 
@@ -216,8 +208,7 @@ FruitCrawler.debug_resource { id: '123', url: '...' }
 These methods will print out what would be otherwise saved to the
 database but for this time there is no save to the database.
 
-Staying up to date
-------------------
+## Staying up to date
 
 Note: This feature is currently working only with ElasticSearch
 
@@ -248,8 +239,7 @@ GemUpdater.loop
 ruby rubygems_updater.rb
 ```
 
-Anonymity ala Tor
------------------
+## Anonymity ala Tor
 
 Anonymity can be easily achieved with [Peasant](https://github.com/reneklacan/peasant) gem.
 By following [this guide](https://github.com/reneklacan/peasant/wiki/How-to-use-Peasant-with-Tor-and-Privoxy-for-scraping)
@@ -266,8 +256,7 @@ class MyCrawler < Kabutops::Crawler
 end
 ```
 
-Javascript heavy site
----------------------
+## Javascript heavy site
 
 Crawling this kind of sites can be achieved by using non-default agent
 (default is Mechanize.new).
@@ -283,13 +272,6 @@ end
 [Bogeyman](https://github.com/reneklacan/bogeyman-ruby-client)
 is wrapper build upon Phantomjs.
 
-TODO
-----
-
-* Watchdog for Mongo
-* skip_existing feature for Mongo, Redis
-
-License
--------
+## License
 
 This library is distributed under the Beerware license.

data/lib/kabutops.rb

@@ -8,9 +8,6 @@ require 'moneta'
 require 'pstore'
 require 'mechanize'
 require 'elasticsearch'
-require 'redis'
-require 'redis-namespace'
-require 'mongo'
 require 'logger'
 
 require 'kabutops/configuration'
@@ -26,11 +23,7 @@ require 'kabutops/recipe_item'
 require 'kabutops/adapters/base'
 require 'kabutops/adapters/database_adapter'
 require 'kabutops/adapters/elastic_search'
-require 'kabutops/adapters/redis'
-require 'kabutops/adapters/mongo'
 require 'kabutops/crawler_extensions/elastic_search'
-require 'kabutops/crawler_extensions/redis'
-require 'kabutops/crawler_extensions/mongo'
 require 'kabutops/crawler_extensions/pstore_storage'
 require 'kabutops/crawler_extensions/debugging'
 require 'kabutops/crawler'

data/lib/kabutops/adapters/base.rb

@@ -5,8 +5,8 @@ module Kabutops
   module Adapters
 
     class Base
-      def initialize &block
-        instance_eval &block if block_given?
+      def initialize
+        yield if block_given?
       end
 
       def enable_debug
@@ -14,7 +14,7 @@ module Kabutops
       end
 
       def debug
-        @debug == true
+        !!@debug
       end
     end
 

data/lib/kabutops/adapters/database_adapter.rb

@@ -18,25 +18,26 @@ module Kabutops
       end
 
       def process resource, page
-        raise 'data block not defined' unless @recipe
-
-        [@recipe.process(resource, page)].flatten.each do |result|
-          result.update(updated_at: Time.now.to_i)
-          save = (notify(:save_if, resource, page, result) || []).all?
-
-          if debug
-            logger.info("#{self.class.to_s} outputs:")
-            notify(:before_save, result) if save
-            logger.info(save ? result.to_hash : 'not valid for save')
-            notify(:after_save, result) if save
-          elsif save
-            notify(:before_save, result)
-            store(result)
-            notify(:after_save, result)
-          end
+        raise 'data block not defined' unless recipe
+
+        previous = find(resource)
+
+        [recipe.process(resource, page, previous)].flatten.each do |result|
+          process_one(resource, page, result)
         end
       end
 
+      def process_one resource, page, result
+        result.update(updated_at: Time.now.to_i)
+        save = (notify(:save_if, resource, page, result) || []).all?
+
+        logger.info("#{self.class.to_s} outputs:") if debug
+        notify(:before_save, result) if save
+        logger.info(save ? result.to_hash : 'not valid for save') if debug
+        store(result) if save && !debug
+        notify(:after_save, result) if save
+      end
+
       def store result
         raise NotImplementedError
       end

data/lib/kabutops/adapters/elastic_search.rb

@@ -68,10 +68,8 @@ module Kabutops
         result['hits']['hits'].map{ |hit| hit['_source'] }
       end
 
-      protected
-
       def client
-        @@client ||= Elasticsearch::Client.new(
+        @client ||= Elasticsearch::Client.new(
           hosts: [
             {
               host: params[:host] || 'localhost',

data/lib/kabutops/crawler.rb

@@ -7,8 +7,6 @@ module Kabutops
     include CrawlerExtensions::Debugging
     include CrawlerExtensions::PStoreStorage
     include CrawlerExtensions::ElasticSearch
-    include CrawlerExtensions::Redis
-    include CrawlerExtensions::Mongo
     include Sidekiq::Worker
 
     class << self
@@ -89,8 +87,6 @@ module Kabutops
       self.class << resource
     end
 
-    protected
-
     def params
       self.class.params
     end
@@ -103,7 +99,7 @@ module Kabutops
       if e.response_code.to_i == 404
         nil
       else
-        p e.response_code
+        logger.error(e.response_code)
         raise
       end
     end

data/lib/kabutops/crawler_extensions/debugging.rb

@@ -17,7 +17,7 @@ module Kabutops
         end
 
         def debug_last count=1
-          params[:collection][(0 - count)..-1].map{ |r| debug_resource(r) }
+          params[:collection].reverse.take(count).map{ |r| debug_resource(r) }
         end
 
         def debug_all

data/lib/kabutops/crawler_extensions/pstore_storage.rb

@@ -8,6 +8,8 @@ module Kabutops
       extend Extensions::Includable
 
       class Storage
+        attr_reader :storage
+
         def initialize path='.kabutopus.config.pstore'
           @storage ||= PStore.new(path)
         end

data/lib/kabutops/extensions/callback_support.rb

@@ -10,41 +10,45 @@ module Kabutops
       class Manager
         attr_reader :map, :allowed
 
-        def initialize allowed=nil
-          @allowed = allowed || []
+        def initialize allowed=[]
+          @allowed = allowed
           @map ||= Hashie::Mash.new
         end
 
         def method_missing name, *args, &block
-          return super unless block_given? && @allowed.include?(name)
+          return super unless block_given? && allowed.include?(name)
 
-          @map[name] ||= []
-          @map[name] << block
+          map[name] ||= []
+          map[name] << block
         end
 
         def notify name, *args
-          return unless @map
+          raise "Not registered as valid callback: #{name}" unless allowed.include?(name)
+          return unless map
 
-          (@map[name] || []).map do |block|
+          (map[name] || []).map do |block|
             block.call(*args)
           end
         end
       end
 
       def callbacks &block
-        @manager ||= Manager.new(allowed_callbacks)
-        @manager.instance_eval &block
+        manager.instance_eval(&block)
       end
 
       def notify name, *args
+        manager.notify(name, *args)
+      end
+
+      def manager
+        raise 'No callbacks allowed' unless respond_to?(:allowed_callbacks)
         @manager ||= Manager.new(allowed_callbacks)
-        @manager.notify(name, *args)
       end
 
       module ClassMethods
         def callbacks *args
           define_method :allowed_callbacks do
-            args
+            args.flatten
           end
         end
       end

data/lib/kabutops/recipe.rb

@@ -3,7 +3,7 @@
 module Kabutops
 
   class Recipe
-    attr_reader :items
+    attr_reader :items, :params, :nested
 
     def initialize params={}
       @params = Hashie::Mash.new(params)
@@ -14,7 +14,7 @@ module Kabutops
     def method_missing name, *args, &block
       if block_given?
         recipe = Recipe.new
-        recipe.instance_eval &block
+        recipe.instance_eval(&block)
         @items[name] = RecipeItem.new(:recipe, recipe)
         @nested = true
       else
@@ -23,21 +23,21 @@ module Kabutops
       end
     end
 
-    def process resource, page
+    def process resource, page, previous
       if @params[:each]
-        page.xpath(@params[:each]).map{ |n| process_one(resource, n) }
+        page.xpath(@params[:each]).map{ |n| process_one(resource, n, previous) }
       elsif @params[:each_css]
-        page.css(@params[:each_css]).map{ |n| process_one(resource, n) }
+        page.css(@params[:each_css]).map{ |n| process_one(resource, n, previous) }
       else
-        process_one(resource, page)
+        process_one(resource, page, previous)
       end
     end
 
-    def process_one resource, node
+    def process_one resource, node, previous
       result = Hashie::Mash.new
 
-      @items.each do |name, item|
-        result[name] = item.process(resource, node)
+      items.each do |name, item|
+        result[name] = item.process(resource, node, previous)
       end
 
       result

data/lib/kabutops/recipe_item.rb

@@ -3,36 +3,38 @@
 module Kabutops
 
   class RecipeItem
+    TYPES = [:var, :recipe, :css, :xpath, :lambda, :proc, :const, :static]
+
     attr_reader :type, :value
 
     def initialize type, value, convert_to=nil
-      @type = type
+      raise "Unknown recipe item type '#{type}'" unless TYPES.include?(type.to_sym)
+
+      @type = type.to_sym
       @value = value
       @convert_to = convert_to
     end
 
-    def process resource, page
-      convert(get(resource, page))
+    def process resource, page, previous
+      convert(get(resource, page, previous))
     end
 
     protected
 
-    def get resource, page
-      case @type
+    def get resource, page, previous
+      case type
       when :var
-        resource[@value]
+        resource[value]
       when :recipe
-        @value.process(resource, page)
+        value.process(resource, page, previous)
       when :css
-        page.css(@value).text.gsub(/\u00a0/, ' ').strip
+        page.css(value).text.gsub(/\u00a0/, ' ').strip
       when :xpath
-        page.xpath(@value).text.gsub(/\u00a0/, ' ').strip
+        page.xpath(value).text.gsub(/\u00a0/, ' ').strip
       when :lambda, :proc
-        @value.call(resource, page)
+        value.call(resource, page, previous)
       when :const, :static
-        @value
-      else
-        raise "unknown recipe item type '#{item.type}'"
+        value
       end
     end
 
@@ -43,6 +45,7 @@ module Kabutops
       when nil then v
       when :int then v[/\d+/].to_i
       when :float then v.gsub(',', '.')[/\d+(\.\d+)?/].to_f
+      else raise "Unknown conversion type '#{@convert_to}'"
       end
     end
   end

data/lib/kabutops/version.rb

@@ -1,5 +1,5 @@
 # -*- encoding : utf-8 -*-
 
 module Kabutops
-  VERSION = '0.1.7'
+  VERSION = '0.2.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kabutops
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.2.0
 platform: ruby
 authors:
 - Rene Klacan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-19 00:00:00.000000000 Z
+date: 2015-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.6.0
+        version: '2.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.6.0
+        version: '2.7'
 - !ruby/object:Gem::Dependency
   name: cachy
   requirement: !ruby/object:Gem::Requirement
@@ -94,34 +94,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: redis
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: redis-namespace
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.4'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -137,47 +109,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.8'
 - !ruby/object:Gem::Dependency
-  name: mongo
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.10'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.10'
-- !ruby/object:Gem::Dependency
-  name: bson_ext
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
-  type: :runtime
+        version: '3.0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rspec-mocks
   requirement: !ruby/object:Gem::Requirement
@@ -205,15 +163,11 @@ files:
 - lib/kabutops/adapters/base.rb
 - lib/kabutops/adapters/database_adapter.rb
 - lib/kabutops/adapters/elastic_search.rb
-- lib/kabutops/adapters/mongo.rb
-- lib/kabutops/adapters/redis.rb
 - lib/kabutops/configuration.rb
 - lib/kabutops/crawler.rb
 - lib/kabutops/crawler_extensions/debugging.rb
 - lib/kabutops/crawler_extensions/elastic_search.rb
-- lib/kabutops/crawler_extensions/mongo.rb
 - lib/kabutops/crawler_extensions/pstore_storage.rb
-- lib/kabutops/crawler_extensions/redis.rb
 - lib/kabutops/extensions/callback_support.rb
 - lib/kabutops/extensions/includable.rb
 - lib/kabutops/extensions/logging.rb
@@ -243,10 +197,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Dead simple yet powerful Ruby crawler for easy parallel crawling with support
   for an anonymity.
 test_files: []
-has_rdoc: 

data/lib/kabutops/adapters/mongo.rb

@@ -1,49 +0,0 @@
-# -*- encoding : utf-8 -*-
-
-module Kabutops
-
-  module Adapters
-
-    class Mongo < DatabaseAdapter
-      include Extensions::Parameterable
-
-      params :host, :port, :db, :collection, :user, :password
-
-      def store result
-        existing = collection.find('id' => result[:id])
-
-        if existing.count > 0
-          existing.each do |document|
-            collection.update({'_id' => document['_id']}, result.to_hash)
-          end
-        else
-          collection.insert(result.to_hash)
-        end
-      end
-
-      protected
-
-      def client
-        @@client ||= ::Mongo::MongoClient.new(
-          params[:host] || 'localhost',
-          params[:port] || 27017,
-        )
-      end
-
-      def client_db
-        @@client_db ||= client.db(params[:db].to_s || 'kabutops')
-        if params[:user] && params[:password]
-          ok = @@client.authenticate(params[:user], params[:password])
-          raise 'mongo authentication failed' unless ok
-        end
-        @@client_db
-      end
-
-      def collection
-        @@collection ||= client_db.collection(params[:collection] || 'kabutops')
-      end
-    end
-
-  end
-
-end

data/lib/kabutops/adapters/redis.rb

@@ -1,37 +0,0 @@
-# -*- encoding : utf-8 -*-
-
-module Kabutops
-
-  module Adapters
-
-    class Redis < DatabaseAdapter
-      include Extensions::Parameterable
-
-      params :host, :port, :namespace, :db, :password
-
-      def store result
-        client[result[:id]] = JSON.dump(result.to_hash)
-      end
-
-      def find resource
-        client[resource[:id] || resource[:url]]
-      end
-
-      protected
-
-      def client
-        @@client ||= ::Redis::Namespace.new(
-          params[:namespace] || 'kabutops',
-          redis: ::Redis.new(
-            host: params[:host],
-            port: params[:port],
-            db: params[:db],
-            password: params[:password],
-          )
-        )
-      end
-    end
-
-  end
-
-end

data/lib/kabutops/crawler_extensions/mongo.rb

@@ -1,19 +0,0 @@
-# -*- encoding : utf-8 -*-
-
-module Kabutops
-
-  module CrawlerExtensions
-
-    module Mongo
-      extend Extensions::Includable
-
-      module ClassMethods
-        def mongo &block
-          adapters << Adapters::Mongo.new(&block)
-        end
-      end
-    end
-
-  end
-
-end

data/lib/kabutops/crawler_extensions/redis.rb

@@ -1,19 +0,0 @@
-# -*- encoding : utf-8 -*-
-
-module Kabutops
-
-  module CrawlerExtensions
-
-    module Redis
-      extend Extensions::Includable
-
-      module ClassMethods
-        def redis &block
-          adapters << Adapters::Redis.new(&block)
-        end
-      end
-    end
-
-  end
-
-end