kabutops 0.0.10 → 0.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 41d09fa8638097298ed1c230a255086949651188
-  data.tar.gz: 3b9c0f19b9272332c940ac649b184c84e02740b2
+  metadata.gz: 491dade458c54b02dbcdd36da92010fcaa1a5463
+  data.tar.gz: 9d00eb9b6ad5e61ddb27dfa7137f0734e9eda03f
 SHA512:
-  metadata.gz: fbdb727ea93bf2048de9a0773a202d7dedcafab67f2f12e378bed53557d9b1b1855a0af45dbf877f68d69d618fc91fc12c9143e3ae8a339aa0a30636b64e1d73
-  data.tar.gz: 26ad39464ae24c5082d94b043ce4162e2de70aa025a315d3ba3d2d936325dfa200429dc5e4763f4f43b1659dbac9503d7e1c5a9f9b6e9e4cb1f4709aba1226d9
+  metadata.gz: 9d00a7d31dfa94e85c9f2d31c49d56b3df2da5fd1b0580c769724f64bad12b331bc8c87a3432071944bc427543458ffe0257743ba5f6dd1840e7e0a8674e7e88
+  data.tar.gz: aa6a67c2dcbaf8e14cf6d4a38e94b78979ef9f594ad492e5618bd3d1db36fbca305895e1c756cae2f355d21045fd82489ee315eb7150a2cac7f17d6fbc744fdb

data/README.md

@@ -8,7 +8,6 @@ With Kabutops you can save data easily to:
 * ElasticSearch
 * MongoDB
 * Redis
-* SQL Databases (via Sequel)
 
 Example for every kind of database are located
 in the [examples directory](https://github.com/reneklacan/kabutops/tree/master/examples)
@@ -25,9 +24,11 @@ gem install kabutops
 Or you can put it in your Gemfile
 
 ```ruby
-gem 'kabutops', '~> 0.0.10'
+gem 'kabutops', '~> 0.0.11'
 ```
 
+You will also need Redis database installed and running.
+
 Basic example
 -------------
 
@@ -199,12 +200,28 @@ class MyCrawler < Kabutops::Crawler
 end
 ```
 
+Javascript heavy site
+---------------------
+
+Crawling this kind of sites can be achieved by using non-default agent
+(default is Mechanize.new).
+
+```ruby
+class MyCrawler < Kabutops::Crawler
+  ...
+  agent Bogeyman::Client.new
+  ...
+end
+```
+
+[Bogeyman](https://github.com/reneklacan/bogeyman-ruby-client)
+is wrapper build upon Phantomjs.
+
 TODO
 ----
 
-* Watchdog for Mongo and Sequel
-* skip_existing for Mongo, Redis and Sequel
-* Spider
+* Watchdog for Mongo
+* skip_existing feature for Mongo, Redis
 
 License
 -------

data/lib/kabutops.rb

@@ -11,7 +11,6 @@ require 'elasticsearch'
 require 'redis'
 require 'redis-namespace'
 require 'mongo'
-require 'sequel'
 require 'mysql2'
 require 'logger'
 
@@ -30,12 +29,11 @@ require 'kabutops/adapters/database_adapter'
 require 'kabutops/adapters/elastic_search'
 require 'kabutops/adapters/redis'
 require 'kabutops/adapters/mongo'
-require 'kabutops/adapters/sequel'
 require 'kabutops/crawler_extensions/elastic_search'
 require 'kabutops/crawler_extensions/redis'
 require 'kabutops/crawler_extensions/mongo'
-require 'kabutops/crawler_extensions/sequel'
 require 'kabutops/crawler_extensions/pstore_storage'
 require 'kabutops/crawler_extensions/debugging'
 require 'kabutops/crawler'
 require 'kabutops/watchdog'
+require 'kabutops/spider'

data/lib/kabutops/configuration.rb

@@ -24,6 +24,11 @@ module Kabutops
             dev: STDOUT,
             level: Logger::DEBUG
           },
+          redis: {
+            host: 'localhost',
+            port: 6379,
+            db: 0
+          },
         )
       end
     end

data/lib/kabutops/crawler.rb

@@ -9,14 +9,14 @@ module Kabutops
     include CrawlerExtensions::ElasticSearch
     include CrawlerExtensions::Redis
     include CrawlerExtensions::Mongo
-    include CrawlerExtensions::Sequel
     include Sidekiq::Worker
 
     class << self
       include Extensions::Parameterable
       include Extensions::CallbackSupport
 
-      params :collection, :proxy, :cache, :wait, :skip_existing
+      params :collection, :proxy, :cache, :wait,
+             :skip_existing, :agent
       callbacks :after_crawl
 
       def adapters
@@ -62,29 +62,29 @@ module Kabutops
 
     def perform resource
       resource = Hashie::Mash.new(resource)
-      adapters = self.class.adapters
-
-      if self.class.params.skip_existing
-        adapters = self.class.adapters.select do |adapter|
-          if adapter.respond_to? :find
-            adapter.find(resource).nil?
-          else
-            true
-          end
-        end
 
-        return if adapters.nil?
+      adapters = self.class.adapters.select do |adapter|
+        if params.skip_existing && adapter.respond_to?(:find)
+          adapter.find(resource).nil?
+        else
+          true
+        end
       end
 
+      return if adapters.nil?
+
       page = crawl(resource)
 
       adapters.each do |adapter|
         adapter.process(resource, page)
       end
     rescue Exception => e
-      logger.error(e.message)
-      logger.error(e.backtrace.join("\n"))
-      sleep self.class.params[:wait] || 0
+      unless self.class.debug
+        logger.error(e.message)
+        logger.error(e.backtrace.join("\n"))
+      end
+
+      sleep params[:wait] || 0
       raise e
     end
 
@@ -94,10 +94,14 @@ module Kabutops
 
     protected
 
+    def params
+      self.class.params
+    end
+
     def crawl resource
       cache_key = (resource[:id] || resource[:url]).to_s
-      content = Cachy.cache_if(self.class.params.cache, cache_key) do
-        sleep self.class.params[:wait] || 0 # wait only if value is not from cache
+      content = Cachy.cache_if(params.cache, cache_key) do
+        sleep params[:wait] || 0 # wait only if value is not from cache
         agent.get(resource[:url]).body
       end
 
@@ -108,8 +112,8 @@ module Kabutops
 
     def agent
       unless @agent
-        @agent = Mechanize.new
-        @agent.set_proxy(*self.class.params[:proxy]) if self.class.params[:proxy]
+        @agent = params[:agent] || Mechanize.new
+        @agent.set_proxy(*params[:proxy]) if params[:proxy]
       end
 
       @agent

data/lib/kabutops/spider.rb

@@ -0,0 +1,82 @@
+# -*- encoding : utf-8 -*-
+
+module Kabutops
+
+  class Spider < Crawler
+    class << self
+      params :url
+      callbacks :after_crawl, :follow_if
+
+      def debug_spider
+        enable_debug
+        self.new.perform({
+          url: params[:url]
+        })
+      end
+
+      def << resource
+        if resource_status(resource).nil?
+          resource_status(resource, 'new')
+          super
+        end
+      end
+
+      def resource_status resource, status=nil
+        url_status(resource[:url], status)
+      end
+
+      def url_status url, status=nil
+        key = redis_key(url)
+
+        if status
+          redis.set(
+            key,
+            JSON.dump({
+              url: url,
+              status: status,
+            })
+          )
+        else
+          item = redis.get(key)
+          item ? JSON.parse(item)['status'] : nil
+        end
+      end
+
+      protected
+
+      def redis_key string
+        Digest::SHA256.hexdigest(string)
+      end
+
+      def redis
+        @redis ||= ::Redis::Namespace.new(
+          self.to_s,
+          redis: ::Redis.new(
+            host: Configuration[:redis][:host],
+            port: Configuration[:redis][:port],
+            db: Configuration[:redis][:db],
+          )
+        )
+      end
+    end
+
+    def crawl resource
+      page = super
+      after_crawl(resource, page)
+      self.class.resource_status(resource, 'done')
+      page
+    end
+
+    def after_crawl resource, page
+      page.css('a').each do |a|
+        follow = self.class.notify(:follow_if, a['href']).any?
+        if follow
+          self << {
+            url: a['href'],
+          }
+        end
+      end
+    end
+  end
+
+end

data/lib/kabutops/version.rb

@@ -1,5 +1,5 @@
 # -*- encoding : utf-8 -*-
 
 module Kabutops
-  VERSION = '0.0.10'
+  VERSION = '0.0.11'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kabutops
 version: !ruby/object:Gem::Version
-  version: 0.0.10
+  version: 0.0.11
 platform: ruby
 authors:
 - Rene Klacan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-07 00:00:00.000000000 Z
+date: 2014-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -150,20 +150,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.10'
-- !ruby/object:Gem::Dependency
-  name: sequel
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '4.11'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '4.11'
 - !ruby/object:Gem::Dependency
   name: bson_ext
   requirement: !ruby/object:Gem::Requirement
@@ -235,7 +221,6 @@ files:
 - lib/kabutops/adapters/elastic_search.rb
 - lib/kabutops/adapters/mongo.rb
 - lib/kabutops/adapters/redis.rb
-- lib/kabutops/adapters/sequel.rb
 - lib/kabutops/configuration.rb
 - lib/kabutops/crawler.rb
 - lib/kabutops/crawler_extensions/debugging.rb
@@ -243,13 +228,13 @@ files:
 - lib/kabutops/crawler_extensions/mongo.rb
 - lib/kabutops/crawler_extensions/pstore_storage.rb
 - lib/kabutops/crawler_extensions/redis.rb
-- lib/kabutops/crawler_extensions/sequel.rb
 - lib/kabutops/extensions/callback_support.rb
 - lib/kabutops/extensions/includable.rb
 - lib/kabutops/extensions/logging.rb
 - lib/kabutops/extensions/parameterable.rb
 - lib/kabutops/recipe.rb
 - lib/kabutops/recipe_item.rb
+- lib/kabutops/spider.rb
 - lib/kabutops/version.rb
 - lib/kabutops/watchdog.rb
 homepage: https://github.com/reneklacan/kabutops

data/lib/kabutops/adapters/sequel.rb

@@ -1,47 +0,0 @@
-# -*- encoding : utf-8 -*-
-
-module Kabutops
-
-  module Adapters
-
-    class Sequel < DatabaseAdapter
-      include Extensions::Parameterable
-
-      params :connect, :type, :host, :port,
-             :db, :user, :password, :table
-
-      def store result
-        client_table.insert(result)
-      end
-
-      def nested?
-        false
-      end
-
-      protected
-
-      def client
-        return @@client if defined?(@@client)
-
-        if params[:connect]
-          @@client ||= ::Sequel.connect(params[:connect])
-        else
-          @@client ||= ::Sequel.connect(
-            adapter: params[:type] || 'mysql2',
-            user: params[:user] || 'root',
-            password: params[:password] || 'root',
-            host: params[:host] || 'localhost',
-            port: params[:port] || 3306,
-            database: params[:db] || params[:database] || 'kabutops',
-          )
-        end
-      end
-
-      def client_table
-        @@client_table ||= client[params[:table]]
-      end
-    end
-
-  end
-
-end

data/lib/kabutops/crawler_extensions/sequel.rb

@@ -1,19 +0,0 @@
-# -*- encoding : utf-8 -*-
-
-module Kabutops
-
-  module CrawlerExtensions
-
-    module Sequel
-      extend Extensions::Includable
-
-      module ClassMethods
-        def sequel &block
-          adapters << Adapters::Sequel.new(&block)
-        end
-      end
-    end
-
-  end
-
-end