kabutops 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1312dd77260f2a2f113ba6e757e9da4491caf282
-  data.tar.gz: 031d23cf3833bda39fdabe6759d22acf377ac7c3
+  metadata.gz: 23b94bb24325c0b57ed6fdbc05a94d87484e319f
+  data.tar.gz: 7175b6f9e4c0dfb1c3fad129a2574d4234e31ce2
 SHA512:
-  metadata.gz: 313a091b905a439d0b194f1b468fdccc188a99ff43b40b470bf135e58880c787bf723b70930285f05dac043d5c2a37247ec4a2f494ed098eb11b072fa24ce4c6
-  data.tar.gz: f8a3e80823d52b7e234625327d4522fd700e1296d52c699da072f31a0943c3ccbfa0f8bdaefbf40460e2bd434f7ea307ce891d4213660a5f18067a2899cf0d33
+  metadata.gz: 13e0e68d4f333526c7a9cfa23f5555df1dfa8defac859dc40317d13e5bd5bbbb981a7041ea3f0fb0845ff27ed908be23481f2e20621332c6186aaab384ef3fc1
+  data.tar.gz: b4081eaee098485fd3a92bb37e7d4b737359ec6cbdfcd3bb9131fecfee671134b3c149dee7dbf4afde34dfc3ab685636254762f33bc0b6e3d653364c7d852994

data/README.md

@@ -1,6 +1,18 @@
 Kabutops [![Code Climate](https://codeclimate.com/github/reneklacan/kabutops.png)](https://codeclimate.com/github/reneklacan/kabutops) [![Coverage](https://codeclimate.com/github/reneklacan/kabutops/coverage.png)](https://codeclimate.com/github/reneklacan/kabutops)
 ========
 
+Kabutops is a ruby library whichs aims to simplify creating website crawlers.
+You can define what will be crawled and how it will be saved in the short class definition.
+
+With Kabutops you can save data easily to:
+* ElasticSearch
+* MongoDB
+* Redis
+* SQL Databases (via Sequel)
+
+Example for every kind of database are located
+in the [examples directory](https://github.com/reneklacan/kabutops/tree/master/examples)
+
 Installation
 ------------
 
@@ -13,7 +25,7 @@ gem install kabutops
 Or you can put it in your Gemfile
 
 ```ruby
-gem 'kabutops', '~> 0.0.7'
+gem 'kabutops', '~> 0.0.8'
 ```
 
 Basic example
@@ -136,3 +148,26 @@ FruitCrawler.debug_resource { id: '123', url: '...' }
 
 These methods will print out what would be otherwise saved to the
 database but for this time there is no save to the database.
+
+Anonymity ala Tor
+-----------------
+
+Anonymity can be easily achieved with [Peasant](https://github.com/reneklacan/peasant) gem.
+By following [this guide](https://github.com/reneklacan/peasant/wiki/How-to-use-Peasant-with-Tor-and-Privoxy-for-scraping)
+you can create proxy instance that will forward requests to
+multiple tor instances.
+
+Then use Peasant proxy address in your Crawler class definition
+
+```ruby
+class MyCrawler < Kabutops::Crawler
+  ...
+  proxy 'localhost', 81818
+  ...
+end
+```
+
+License
+-------
+
+This library is distributed under the Beerware license.

data/lib/kabutops.rb

@@ -10,9 +10,14 @@ require 'mechanize'
 require 'elasticsearch'
 require 'redis'
 require 'redis-namespace'
+require 'mongo'
+require 'sequel'
+require 'mysql2'
 
 Cachy.cache_store = Moneta.new(:File, dir: 'cache') # temporary
 
+require 'kabutops/extensions/includable'
+require 'kabutops/extensions/logging'
 require 'kabutops/extensions/parameterable'
 require 'kabutops/extensions/callback_support'
 require 'kabutops/recipe'
@@ -21,8 +26,12 @@ require 'kabutops/adapters/base'
 require 'kabutops/adapters/database_adapter'
 require 'kabutops/adapters/elastic_search'
 require 'kabutops/adapters/redis'
+require 'kabutops/adapters/mongo'
+require 'kabutops/adapters/sequel'
 require 'kabutops/crawler_extensions/elastic_search'
 require 'kabutops/crawler_extensions/redis'
+require 'kabutops/crawler_extensions/mongo'
+require 'kabutops/crawler_extensions/sequel'
 require 'kabutops/crawler_extensions/pstore_storage'
 require 'kabutops/crawler_extensions/debugging'
 require 'kabutops/crawler'

data/lib/kabutops/adapters/base.rb

@@ -5,6 +5,10 @@ module Kabutops
   module Adapters
 
     class Base
+      def initialize &block
+        instance_eval &block if block_given?
+      end
+
       def enable_debug
         @debug = true
       end

data/lib/kabutops/adapters/database_adapter.rb

@@ -5,6 +5,7 @@ module Kabutops
   module Adapters
 
     class DatabaseAdapter < Base
+      include Extensions::Logging
       include Extensions::CallbackSupport
 
       attr_reader :recipe
@@ -17,10 +18,12 @@ module Kabutops
       end
 
       def process resource, page
+        raise 'data block not defined' unless @recipe
+
         result = @recipe.process(resource, page)
         if debug
-          puts "#{self.class.to_s} outputs:"
-          p result.to_hash
+          logger.info("#{self.class.to_s} outputs:")
+          logger.info(result.to_hash)
         else
           store(result)
           notify(:after_save, result)

data/lib/kabutops/adapters/mongo.rb

@@ -0,0 +1,53 @@
+# -*- encoding : utf-8 -*-
+
+module Kabutops
+
+  module Adapters
+
+    class Mongo < DatabaseAdapter
+      include Extensions::Parameterable
+
+      params :host, :port, :db, :collection, :user, :password
+
+      def store result
+        existing = collection.find('id' => result[:id])
+
+        if existing.count > 0
+          existing.each do |document|
+            collection.update({'_id' => document['_id']}, result.to_hash)
+          end
+        else
+          collection.insert(result.to_hash)
+        end
+      end
+
+      def nested?
+        true
+      end
+
+      protected
+
+      def client
+        @@client ||= ::Mongo::MongoClient.new(
+          params[:host] || 'localhost',
+          params[:port] || 27017,
+        )
+      end
+
+      def client_db
+        @@client_db ||= client.db(params[:db].to_s || 'kabutops')
+        if params[:user] && params[:password]
+          ok = @@client.authenticate(params[:user], params[:password])
+          raise 'mongo authentication failed' unless ok
+        end
+        @@client_db
+      end
+
+      def collection
+        @@collection ||= client_db.collection(params[:collection] || 'kabutops')
+      end
+    end
+
+  end
+
+end

data/lib/kabutops/adapters/sequel.rb

@@ -0,0 +1,47 @@
+# -*- encoding : utf-8 -*-
+
+module Kabutops
+
+  module Adapters
+
+    class Sequel < DatabaseAdapter
+      include Extensions::Parameterable
+
+      params :connect, :type, :host, :port,
+             :db, :user, :password, :table
+
+      def store result
+        client_table.insert(result)
+      end
+
+      def nested?
+        false
+      end
+
+      protected
+
+      def client
+        return @@client if defined?(@@client)
+
+        if params[:connect]
+          @@client ||= ::Sequel.connect(params[:connect])
+        else
+          @@client ||= ::Sequel.connect(
+            adapter: params[:type] || 'mysql2',
+            user: params[:user] || 'root',
+            password: params[:password] || 'root',
+            host: params[:host] || 'localhost',
+            port: params[:port] || 3306,
+            database: params[:db] || params[:database] || 'kabutops',
+          )
+        end
+      end
+
+      def client_table
+        @@client_table ||= client[params[:table]]
+      end
+    end
+
+  end
+
+end

data/lib/kabutops/crawler.rb

@@ -3,10 +3,13 @@
 module Kabutops
 
   class Crawler
+    include Extensions::Logging
     include CrawlerExtensions::Debugging
     include CrawlerExtensions::PStoreStorage
     include CrawlerExtensions::ElasticSearch
     include CrawlerExtensions::Redis
+    include CrawlerExtensions::Mongo
+    include CrawlerExtensions::Sequel
     include Sidekiq::Worker
 
     class << self
@@ -17,7 +20,7 @@ module Kabutops
       callbacks :after_crawl
 
       def adapters
-        @adapters || []
+        @adapters ||= []
       end
 
       def crawl! collection=nil
@@ -58,6 +61,11 @@ module Kabutops
       self.class.adapters.each do |adapter|
         adapter.process(resource, page)
       end
+    rescue Exception => e
+      logger.error(e.message)
+      logger.error(e.backtrace.join("\n"))
+      sleep self.params[:wait] || 0
+      raise e
     end
 
     def << resource

data/lib/kabutops/crawler_extensions/debugging.rb

@@ -5,13 +5,9 @@ module Kabutops
   module CrawlerExtensions
 
     module Debugging
-
-      def self.included base
-        base.extend(ClassMethods)
-      end
+      extend Extensions::Includable
 
       module ClassMethods
-
         def debug_first count=1
           params[:collection].take(count).map{ |r| debug_resource(r) }
         end
@@ -41,9 +37,7 @@ module Kabutops
         def debug
           @debug == true
         end
-
       end
-
     end
 
   end

data/lib/kabutops/crawler_extensions/elastic_search.rb

@@ -5,21 +5,13 @@ module Kabutops
   module CrawlerExtensions
 
     module ElasticSearch
-
-      def self.included base
-        base.extend(ClassMethods)
-      end
+      extend Extensions::Includable
 
       module ClassMethods
         def elasticsearch &block
-          adapter = Adapters::ElasticSearch.new
-          adapter.instance_eval &block
-
-          @adapters ||= []
-          @adapters << adapter
+          adapters << Adapters::ElasticSearch.new(&block)
         end
       end
-
     end
 
   end

data/lib/kabutops/crawler_extensions/mongo.rb

@@ -0,0 +1,19 @@
+# -*- encoding : utf-8 -*-
+
+module Kabutops
+
+  module CrawlerExtensions
+
+    module Mongo
+      extend Extensions::Includable
+
+      module ClassMethods
+        def mongo &block
+          adapters << Adapters::Mongo.new(&block)
+        end
+      end
+    end
+
+  end
+
+end

data/lib/kabutops/crawler_extensions/pstore_storage.rb

@@ -5,6 +5,8 @@ module Kabutops
   module CrawlerExtensions
 
     module PStoreStorage
+      extend Extensions::Includable
+
       class Storage
         def initialize path='.kabutopus.config.pstore'
           @storage ||= PStore.new(path)
@@ -21,22 +23,15 @@ module Kabutops
         end
       end
 
-      def self.included base
-        base.extend(ClassMethods)
-      end
-
       module ClassMethods
-
         def storage
           @storage ||= Storage.new
         end
-
       end
 
       def storage
         self.class.storage
       end
-
     end
 
   end

data/lib/kabutops/crawler_extensions/redis.rb

@@ -5,21 +5,13 @@ module Kabutops
   module CrawlerExtensions
 
     module Redis
-
-      def self.included base
-        base.extend(ClassMethods)
-      end
+      extend Extensions::Includable
 
       module ClassMethods
         def redis &block
-          adapter = Adapters::Redis.new
-          adapter.instance_eval &block
-
-          @adapters ||= []
-          @adapters << adapter
+          adapters << Adapters::Redis.new(&block)
         end
       end
-
     end
 
   end

data/lib/kabutops/crawler_extensions/sequel.rb

@@ -0,0 +1,19 @@
+# -*- encoding : utf-8 -*-
+
+module Kabutops
+
+  module CrawlerExtensions
+
+    module Sequel
+      extend Extensions::Includable
+
+      module ClassMethods
+        def sequel &block
+          adapters << Adapters::Sequel.new(&block)
+        end
+      end
+    end
+
+  end
+
+end

data/lib/kabutops/extensions/callback_support.rb

@@ -5,13 +5,7 @@ module Kabutops
   module Extensions
 
     module CallbackSupport
-
-      def self.included base
-        base.extend(ClassMethods)
-        base.class_eval do
-          attr_reader :allowed_callbacks
-        end
-      end
+      extend Includable
 
       class Manager
         attr_reader :map, :allowed
@@ -22,11 +16,7 @@ module Kabutops
         end
 
         def method_missing name, *args, &block
-          return super unless block_given?
-
-          unless @allowed.include?(name)
-            raise "Invalid callback name: #{name}"
-          end
+          return super unless block_given? && @allowed.include?(name)
 
           @map[name] ||= []
           @map[name] << block
@@ -52,15 +42,12 @@ module Kabutops
       end
 
       module ClassMethods
-
         def callbacks *args
           define_method :allowed_callbacks do
             args
           end
         end
-
       end
-
     end
 
   end

data/lib/kabutops/extensions/includable.rb

@@ -0,0 +1,25 @@
+module Kabutops
+
+  module Extensions
+
+    # inspired by ActiveSupport::Concern
+
+    module Includable
+      def append_features(base)
+        super
+        base.extend const_get(:ClassMethods) if const_defined?(:ClassMethods)
+        base.class_eval(&@_included) if instance_variable_defined?(:@_included)
+      end
+
+      def included(base = nil, &block)
+        if base.nil?
+          @_included = block
+        else
+          super
+        end
+      end
+    end
+
+  end
+
+end

data/lib/kabutops/extensions/logging.rb

@@ -0,0 +1,27 @@
+module Kabutops
+
+  module Extensions
+
+    module Logging
+      extend Includable
+
+      def logger
+        self.class.logger
+      end
+
+      module ClassMethods
+
+        def logger
+          return @@logger if defined?(@@logger)
+
+          @@logger ||= Logger.new(STDOUT)
+          #@@logger.level = Logger::WARN
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/kabutops/extensions/parameterable.rb

@@ -5,13 +5,9 @@ module Kabutops
   module Extensions
 
     module Parameterable
-
-      def self.included base
-        base.extend(ClassMethods)
-      end
+      extend Extensions::Includable
 
       module ClassMethods
-
         def params *list
           list.each do |name|
             define_method name do |*args|
@@ -28,9 +24,7 @@ module Kabutops
             @params ||= Hashie::Mash.new
           end
         end
-
       end
-
     end
 
   end

data/lib/kabutops/version.rb

@@ -1,5 +1,5 @@
 # -*- encoding : utf-8 -*-
 
 module Kabutops
-  VERSION = '0.0.7'
+  VERSION = '0.0.8'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kabutops
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - Rene Klacan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-20 00:00:00.000000000 Z
+date: 2014-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -136,6 +136,62 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.11'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.11'
+- !ruby/object:Gem::Dependency
+  name: bson_ext
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -177,13 +233,19 @@ files:
 - lib/kabutops/adapters/base.rb
 - lib/kabutops/adapters/database_adapter.rb
 - lib/kabutops/adapters/elastic_search.rb
+- lib/kabutops/adapters/mongo.rb
 - lib/kabutops/adapters/redis.rb
+- lib/kabutops/adapters/sequel.rb
 - lib/kabutops/crawler.rb
 - lib/kabutops/crawler_extensions/debugging.rb
 - lib/kabutops/crawler_extensions/elastic_search.rb
+- lib/kabutops/crawler_extensions/mongo.rb
 - lib/kabutops/crawler_extensions/pstore_storage.rb
 - lib/kabutops/crawler_extensions/redis.rb
+- lib/kabutops/crawler_extensions/sequel.rb
 - lib/kabutops/extensions/callback_support.rb
+- lib/kabutops/extensions/includable.rb
+- lib/kabutops/extensions/logging.rb
 - lib/kabutops/extensions/parameterable.rb
 - lib/kabutops/recipe.rb
 - lib/kabutops/recipe_item.rb