kabutops 0.0.8 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 23b94bb24325c0b57ed6fdbc05a94d87484e319f
-  data.tar.gz: 7175b6f9e4c0dfb1c3fad129a2574d4234e31ce2
+  metadata.gz: 5d0738998c869017902c5ae5da1be3dfdf6c834d
+  data.tar.gz: 12a286d8d3adeffd0bc698c80d5a761b41e4b0ff
 SHA512:
-  metadata.gz: 13e0e68d4f333526c7a9cfa23f5555df1dfa8defac859dc40317d13e5bd5bbbb981a7041ea3f0fb0845ff27ed908be23481f2e20621332c6186aaab384ef3fc1
-  data.tar.gz: b4081eaee098485fd3a92bb37e7d4b737359ec6cbdfcd3bb9131fecfee671134b3c149dee7dbf4afde34dfc3ab685636254762f33bc0b6e3d653364c7d852994
+  metadata.gz: 779f59ca039b19f6c17a63f53a4adc34de617c4e1143ab47f35e13c0c6cc731101961ade348e0046f907a39977367f5d124689a2d4faeedc43aea7fcc02d531a
+  data.tar.gz: dfeecde60aec5677485123704ef8c1e64e943c944ae340bbb994bdce5a8cd2d6581fd49353f2dfc5427913fec692088631c3d9e76bbbe4bf8f47c890b20223df

data/README.md

@@ -25,7 +25,7 @@ gem install kabutops
 Or you can put it in your Gemfile
 
 ```ruby
-gem 'kabutops', '~> 0.0.8'
+gem 'kabutops', '~> 0.0.9'
 ```
 
 Basic example
@@ -149,6 +149,38 @@ FruitCrawler.debug_resource { id: '123', url: '...' }
 These methods will print out what would be otherwise saved to the
 database but for this time there is no save to the database.
 
+Staying up to date
+------------------
+
+Note: This feature is currently working only with ElasticSearch
+
+For this purpore there is a Watchdog. Updater have to inherit from
+this class and this class can be run as a worker via sidekiq or as a
+plain ruby script as you can see below.
+
+```ruby
+class GemUpdater < Kabutops::Watchdog
+  crawler GemCrawler
+  freshness 1*24*60*60 # 1 day
+  wait 5
+
+  callbacks do
+    on_outdated do |resource|
+      puts "#{resource[:title]} outdated!"
+      GemCrawler << {
+        url: resource[:url],
+      }
+    end
+  end
+end
+
+GemUpdater.loop
+```
+
+```bash
+ruby rubygems_updater.rb
+```
+
 Anonymity ala Tor
 -----------------
 
@@ -167,6 +199,13 @@ class MyCrawler < Kabutops::Crawler
 end
 ```
 
+TODO
+----
+
+* Watchdog for Mongo and Sequel
+* skip_existing for Mongo, Redis and Sequel
+* Spider
+
 License
 -------
 

data/lib/kabutops.rb

@@ -13,6 +13,9 @@ require 'redis-namespace'
 require 'mongo'
 require 'sequel'
 require 'mysql2'
+require 'logger'
+
+require 'kabutops/configuration'
 
 Cachy.cache_store = Moneta.new(:File, dir: 'cache') # temporary
 
@@ -35,3 +38,4 @@ require 'kabutops/crawler_extensions/sequel'
 require 'kabutops/crawler_extensions/pstore_storage'
 require 'kabutops/crawler_extensions/debugging'
 require 'kabutops/crawler'
+require 'kabutops/watchdog'

data/lib/kabutops/adapters/database_adapter.rb

@@ -10,7 +10,7 @@ module Kabutops
 
       attr_reader :recipe
 
-      callbacks :after_save
+      callbacks :after_save, :save_if
 
       def data &block
         @recipe = Recipe.new
@@ -21,10 +21,14 @@ module Kabutops
         raise 'data block not defined' unless @recipe
 
         result = @recipe.process(resource, page)
+        result.update(updated_at: Time.now.to_i)
+
+        save = (notify(:save_if, resource, page, result) || []).all?
+
         if debug
           logger.info("#{self.class.to_s} outputs:")
-          logger.info(result.to_hash)
-        else
+          logger.info(save ? result.to_hash : 'not valid for save')
+        elsif save
           store(result)
           notify(:after_save, result)
         end

data/lib/kabutops/adapters/elastic_search.rb

@@ -18,6 +18,56 @@ module Kabutops
         )
       end
 
+      def find resource
+        result = client.search(
+          index: params[:index] || 'default',
+          body: {
+            query: {
+              filtered: {
+                filter: {
+                  or: [
+                    { term: { id: resource[:id] || resource[:url] } },
+                    { term: { url: resource[:url] } },
+                  ]
+                },
+              },
+            },
+          },
+          size: 5,
+        )
+        result['hits']['hits'].map{ |hit| hit['_source'] }.first
+      end
+
+      def find_outdated freshness
+        result = client.search(
+          index: params[:index] || 'default',
+          body: {
+            query: {
+              filtered: {
+                filter: {
+                  and: [
+                    {
+                      or: [
+                        { range: { updated_at: { lte: Time.now.to_i - freshness } } },
+                        { missing: { field: 'updated_at' } },
+                      ]
+                    },
+                    {
+                      or: [
+                        { range: { scheduled_update_at: { lte: Time.now.to_i - 3600 } } },
+                        { missing: { field: 'scheduled_update_at' } },
+                      ]
+                    },
+                  ]
+                },
+              },
+            },
+          },
+          size: 5,
+        )
+        result['hits']['hits'].map{ |hit| hit['_source'] }
+      end
+
       def nested?
         true
       end

data/lib/kabutops/configuration.rb

@@ -0,0 +1,32 @@
+# -*- encoding : utf-8 -*-
+
+module Kabutops
+
+  class Configuration
+    class << self
+      def config *args, &block
+        configuration.instance_eval &block
+      end
+
+      def [] key
+        configuration[key]
+      end
+
+      def []= key, value
+        configuration[key] = value
+      end
+
+      protected
+
+      def configuration
+        @configuration ||= Hashie::Mash.new(
+          logger: {
+            dev: STDOUT,
+            level: Logger::DEBUG
+          },
+        )
+      end
+    end
+  end
+
+end

data/lib/kabutops/crawler.rb

@@ -16,20 +16,32 @@ module Kabutops
       include Extensions::Parameterable
       include Extensions::CallbackSupport
 
-      params :collection, :proxy, :cache, :wait
+      params :collection, :proxy, :cache, :wait, :skip_existing
       callbacks :after_crawl
 
       def adapters
         @adapters ||= []
       end
 
+      def reset!
+        storage[:status] = nil
+      end
+
       def crawl! collection=nil
+        reset!
+        crawl(collection)
+      end
+
+      def crawl collection=nil
         @map ||= Hashie::Mash.new
 
         if storage[:status].nil?
           (collection || params[:collection] || []).each do |resource|
             self << resource
           end
+          storage[:status] = :in_progress
+        elsif storage[:status] == :in_progress
+          # pass
         end
       end
 
@@ -56,15 +68,29 @@ module Kabutops
 
     def perform resource
       resource = Hashie::Mash.new(resource)
+      adapters = self.class.adapters
+
+      if self.class.params.skip_existing
+        adapters = self.class.adapters.select do |adapter|
+          if adapter.respond_to? :find
+            adapter.find(resource).nil?
+          else
+            true
+          end
+        end
+
+        return if adapters.nil?
+      end
+
       page = crawl(resource)
 
-      self.class.adapters.each do |adapter|
+      adapters.each do |adapter|
         adapter.process(resource, page)
       end
     rescue Exception => e
       logger.error(e.message)
       logger.error(e.backtrace.join("\n"))
-      sleep self.params[:wait] || 0
+      sleep self.class.params[:wait] || 0
       raise e
     end
 
@@ -75,7 +101,8 @@ module Kabutops
     protected
 
     def crawl resource
-      content = Cachy.cache_if(self.class.params.cache, resource[:url]) do
+      cache_key = (resource[:id] || resource[:url]).to_s
+      content = Cachy.cache_if(self.class.params.cache, cache_key) do
         sleep self.class.params[:wait] || 0 # wait only if value is not from cache
         agent.get(resource[:url]).body
       end

data/lib/kabutops/extensions/logging.rb

@@ -14,8 +14,9 @@ module Kabutops
         def logger
           return @@logger if defined?(@@logger)
 
-          @@logger ||= Logger.new(STDOUT)
-          #@@logger.level = Logger::WARN
+          @@logger = Logger.new(Configuration[:logger][:dev])
+          @@logger.level = Configuration[:logger][:level]
+          @@logger
         end
 
       end

data/lib/kabutops/version.rb

@@ -1,5 +1,5 @@
 # -*- encoding : utf-8 -*-
 
 module Kabutops
-  VERSION = '0.0.8'
+  VERSION = '0.0.9'
 end

data/lib/kabutops/watchdog.rb

@@ -0,0 +1,69 @@
+module Kabutops
+  class Watchdog
+    include Extensions::Logging
+    include Sidekiq::Worker
+
+    class << self
+      include Extensions::Parameterable
+      include Extensions::CallbackSupport
+
+      params :crawler, :freshness, :wait
+      callbacks :on_outdated
+
+      def check!
+        perform_async
+      end
+
+      def check
+        new.check
+      end
+
+      def loop
+        loop do
+          sleep self.params[:wait] || 5
+          check
+        end
+      end
+    end
+
+    def check
+      logger.info "#{self.class} check started"
+
+      outdated_resources.each do |resource|
+        resource.update(scheduled_update_at: Time.now.to_i)
+
+        adapters.each do |adapter|
+          adapter.store(resource)
+        end
+
+        self.class.notify(:on_outdated, resource)
+      end
+
+      logger.info "#{self.class} check finished"
+    end
+
+    def perform
+      check
+      sleep self.class.params[:wait] || 5
+      self.class.perform_async
+    end
+
+    protected
+
+    def outdated_resources
+      adapters.map{ |a| a.find_outdated(freshness) }
+        .flatten
+        .uniq
+        .reject{ |r| (r[:scheduled_update_at] || 0) > Time.now.to_i - 3600 }
+        .map{ |r| Hashie::Mash.new(r) }
+    end
+
+    def adapters
+      self.class.params.crawler.adapters
+    end
+
+    def freshness
+      self.class.params.freshness
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kabutops
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: ruby
 authors:
 - Rene Klacan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-22 00:00:00.000000000 Z
+date: 2014-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -236,6 +236,7 @@ files:
 - lib/kabutops/adapters/mongo.rb
 - lib/kabutops/adapters/redis.rb
 - lib/kabutops/adapters/sequel.rb
+- lib/kabutops/configuration.rb
 - lib/kabutops/crawler.rb
 - lib/kabutops/crawler_extensions/debugging.rb
 - lib/kabutops/crawler_extensions/elastic_search.rb
@@ -250,6 +251,7 @@ files:
 - lib/kabutops/recipe.rb
 - lib/kabutops/recipe_item.rb
 - lib/kabutops/version.rb
+- lib/kabutops/watchdog.rb
 homepage: https://github.com/reneklacan/kabutops
 licenses:
 - Beerware