kabutops 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6e931d9a939854fae6497f910de768971907f29a
-  data.tar.gz: 126e34c2c1eeffa8af3dca362a62381f5889cbe0
+  metadata.gz: 87c6590089f52a068980373fdb2b060618b34940
+  data.tar.gz: 7505ade7b0104e76c62e73594de557cc75c07944
 SHA512:
-  metadata.gz: a869f1388c583d8e2467f096bd6e4702d95832f1f2039253d8b13ec48037e133c9fbbaf68dd44512b69edb4ca4901177757b9ceeedb52efb9aac9587e1ce0a72
-  data.tar.gz: cccb0968bcf9ce251a7ec81c8a8cbb04b5449d19c96dcf2dd58fd69c259b915f17c9e37d7ed926c229066329018b568d0221824f5ec60578eb61238a8172d4b5
+  metadata.gz: a48556126558daa3e7592fcd4eaa7767b8cf3157a4de28e862800783fed992e37dc43995506511063aa15eccf1d6f4fe9b7605fc546c07627f393f57c6c61284
+  data.tar.gz: ae9e2711c691fef7135d1e45ea132ff0bd075df7e02c2ee6371251f0388c9b7f73751c539e384d531e9a0fc05be4852fccfe36226477a3b5993ed2dbda9a6861

data/README.md

@@ -1,9 +1,9 @@
 # Kabutops [![Code Climate](https://codeclimate.com/github/reneklacan/kabutops.png)](https://codeclimate.com/github/reneklacan/kabutops) [![Coverage](https://codeclimate.com/github/reneklacan/kabutops/coverage.png)](https://codeclimate.com/github/reneklacan/kabutops)
 
-Kabutops is a ruby library whichs aims to simplify creating website crawlers.
+Kabutops is a ruby library which aims to simplify creating website crawlers.
 You can define what will be crawled and how it will be saved in the short class definition.
 
-With Kabutops you can easily save data to **ElasticSearch**.
+With Kabutops you can easily save data to **ElasticSearch 2.x**.
 
 Example for every kind of database are located
 in the [examples directory](https://github.com/reneklacan/kabutops/tree/master/examples)
@@ -19,7 +19,7 @@ gem install kabutops
 Or you can put it in your Gemfile
 
 ```ruby
-gem 'kabutops', '~> 0.1.1'
+gem 'kabutops'
 ```
 
 You will also need Redis database installed and running.

data/lib/kabutops.rb

@@ -28,4 +28,3 @@ require 'kabutops/crawler_extensions/pstore_storage'
 require 'kabutops/crawler_extensions/debugging'
 require 'kabutops/crawler'
 require 'kabutops/watchdog'
-require 'kabutops/spider'

data/lib/kabutops/adapters/base.rb

@@ -5,8 +5,8 @@ module Kabutops
   module Adapters
 
     class Base
-      def initialize
-        yield if block_given?
+      def initialize(&block)
+        instance_eval(&block) if block_given?
       end
 
       def enable_debug

data/lib/kabutops/adapters/database_adapter.rb

@@ -36,6 +36,7 @@ module Kabutops
         logger.info(save ? result.to_hash : 'not valid for save') if debug
         store(result) if save && !debug
         notify(:after_save, result) if save
+        result
       end
 
       def store result

data/lib/kabutops/crawler.rb

@@ -70,7 +70,7 @@ module Kabutops
       return if page.nil?
       return unless (self.class.notify(:store_if, resource, page) || []).all?
 
-      adapters.each do |adapter|
+      adapters.map do |adapter|
         adapter.process(resource, page)
       end
     rescue Exception => e

data/lib/kabutops/version.rb

@@ -1,5 +1,5 @@
 # -*- encoding : utf-8 -*-
 
 module Kabutops
-  VERSION = '0.2.0'
+  VERSION = '0.3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kabutops
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Rene Klacan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-01 00:00:00.000000000 Z
+date: 2015-11-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -58,42 +58,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: elasticsearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 1.0.14
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 1.0.14
 - !ruby/object:Gem::Dependency
   name: hashie
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.4'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -174,7 +174,6 @@ files:
 - lib/kabutops/extensions/parameterable.rb
 - lib/kabutops/recipe.rb
 - lib/kabutops/recipe_item.rb
-- lib/kabutops/spider.rb
 - lib/kabutops/version.rb
 - lib/kabutops/watchdog.rb
 homepage: https://github.com/reneklacan/kabutops
@@ -197,7 +196,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Dead simple yet powerful Ruby crawler for easy parallel crawling with support

data/lib/kabutops/spider.rb

@@ -1,95 +0,0 @@
-# -*- encoding : utf-8 -*-
-
-module Kabutops
-
-  class Spider < Crawler
-    class << self
-      params :url
-      callbacks :after_crawl, :before_cache, :follow_if
-
-      def debug_spider
-        enable_debug
-        self.new.perform({
-          url: params[:url]
-        })
-      end
-
-      def crawl collection=nil
-        super(collection || [{ url: params.url, }])
-      end
-
-      def reset!
-        super
-        redis.keys.each{ |k| redis.del(k) }
-      end
-
-      def << resource
-        if resource_status(resource).nil?
-          resource_status(resource, 'new')
-          super
-        end
-      end
-
-      def follow link
-        self << {
-          url: URI.join(params.url, URI.escape(link)).to_s
-        }
-      end
-
-      def resource_status resource, status=nil
-        url_status(resource[:url], status)
-      end
-
-      def url_status url, status=nil
-        key = redis_key(url)
-
-        if status
-          redis.set(
-            key,
-            JSON.dump({
-              url: url,
-              status: status,
-            })
-          )
-        else
-          item = redis.get(key)
-          item ? JSON.parse(item)['status'] : nil
-        end
-      end
-
-      protected
-
-      def redis_key string
-        Digest::SHA256.hexdigest(string)
-      end
-
-      def redis
-        @redis ||= ::Redis::Namespace.new(
-          self.to_s,
-          redis: ::Redis.new(
-            host: Configuration[:redis][:host],
-            port: Configuration[:redis][:port],
-            db: Configuration[:redis][:db],
-          )
-        )
-      end
-    end
-
-    def crawl resource
-      page = super
-      after_crawl(resource, page)
-      self.class.resource_status(resource, 'done')
-      page
-    end
-
-    def after_crawl resource, page
-      page.css('a').each do |a|
-        next if a['href'].nil?
-
-        follow = self.class.notify(:follow_if, a['href']).any?
-        self.class.follow(a['href']) if follow
-      end
-    end
-  end
-
-end