kabutops 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3558c828ca14af27797b234bab95569992e00f1e
-  data.tar.gz: 317fec36ba533f78861203c4142e4a7c7d3add84
+  metadata.gz: 29f7dfcb4dbed7228907cc636a82c97005144881
+  data.tar.gz: 9d18c9e2c070289695e1bf7f0c7f5ca773263c22
 SHA512:
-  metadata.gz: 8bf4d1916494080e06c5cda2a830f78857851c1d07228aa271b2c106896b712a54a2ef5747b881cd828d506a03a1f2d8e4db672864f888585cc3780640d317a1
-  data.tar.gz: efbef6ae896963703a2c39963b55cf57b6729d58ed1b9749d2052a5821e0366420c65b16f0a1256ef267453e2bc57732530b89a777e5ecc9669885b232cb9b9e
+  metadata.gz: 9f1961c5a5b72e3f23309f995865281875a96111f05eaa57343f669e1511c89957914e22d6d58b4919610d00a987ac087b040c7e086d44f27abdb3bc8a5fe61e
+  data.tar.gz: c3626b5fbe52eac58b8b88b637bcce0181267718bc580ab17a8f7aaf1931df95fb729646f0539c0a2da044ed4a40708e87eb58307b6e34f5ba19d592f3d9faf7

data/README.md

@@ -13,76 +13,105 @@ gem install kabutops
 Or you can put it in your Gemfile
 
 ```ruby
-gem 'kabutops', '~> 0.0.2'
+gem 'kabutops', '~> 0.0.3'
 ```
 
 Basic example
 -------------
 
-Create **fruit_crawler.rb**.
+Example that will crawl information about gems that start on letter Q or
+X and save them to the ElasticSearch.
 
 ```ruby
 require 'kabutops'
 
-class FruitCrawler < Kabutops::Crawler
-  include Sidekiq::Worker
-
-  collection (1..5).map { |id|
+class GemListCrawler < Kabutops::Crawler
+  # just two letters with the smallest amount of gems
+  collection ['Q', 'X'].map{ |letter|
                {
-                 id: id,
-                 url: "https://www.example.com/fruits/#{id}",
+                 letter: letter,
+                 url: "https://rubygems.org/gems?letter=#{letter}"
                }
-             }.shuffle
+             }
 
-  proxy '127.0.0.1', 81818
   cache true
+  wait 2 # wait two seconds after each procession (we do not want to hurt rubygems)
+
+  callbacks do
+    after_crawl do |resource, page|
+      links = page.xpath("//a[contains(@href, '/gems?letter=#{resource[:letter]}')]")
+      links.each do |link|
+        self << {
+          letter: resource[:letter],
+          url: "https://rubygems.org#{link['href']}",
+        }
+      end
+
+      links = page.xpath("//a[contains(@href, '/gems/')]")
+      links.each do |link|
+        GemCrawler << {
+          letter: resource[:letter],
+          url: "https://rubygems.org#{link['href']}",
+        }
+      end
+    end
+  end
+end
+
+class GemCrawler < Kabutops::Crawler
+  cache true
+  wait 2 # wait two seconds after each procession (we do not want to hurt rubygems)
 
   elasticsearch do
-    index :books
-    type :book
+    index :gems
+    type :gem
 
     data do
-      id :var, :id
-      url :var, :url
-      some_attr :css, 'h1.bookTitle'
-      grape :lambda, ->(resource, page) {
-        page.css('h3.fruit').split(',').first 
-      }
-
-      nested_attr do
-        apple :css, 'h1.bookTitle'
-        banana :xpath, '//table/tr/td[0]'
+      id :css, '.title > h2 > a'
+      title :css, '.title > h2 > a'
+      authors :css, '.authors > p'
+      description :css, '#markup > p'
+
+      downloads do
+        total :lambda, ->(resource, page) {
+          page.css('.downloads.counter > span > strong')[0].text.gsub(',', '').to_i
+        }
+
+        current_version :lambda, ->(resource, page) {
+          page.css('.downloads.counter > span > strong')[1].text.gsub(',', '').to_i
+        }
       end
     end
-  end
 
-  callback do |resource, page|
+    callbacks do
+      after_save do |hash|
+        puts "#{hash[:title]} saved!"
+      end
+    end
   end
 end
 
-FruitCrawler.crawl!
+GemListCrawler.crawl!
+GemCrawler.crawl!
 ```
 
 Run it via sidekiq
 
 ```bash
-bundle exec sidekiq -r ./fruit_crawler.rb -c 10
+bundle exec sidekiq -r ./rubygems_crawler.rb -c 1
 ```
 
-This example will parallely crawl specified urls and result will be
-stored to the ElasticSearch index named books as a book document.
-
-One document will look something like this
+Documents saved in the ElasticSearch will look like this one
 
 ```json
 {
-  'id': '...',
-  'url': '...',
-  'some_attr': '...',
-  'grape': '...',
-  'nested_attr': {
-    'apple': '...',
-    'banana': '...'
+  "id": "qiita_mail",
+  "title": "qiita_mail",
+  "authors": "ongaeshi",
+  "description":" Write a gem description",
+  "downloads": {
+    "total": 2493,
+    "current_version": 580
   }
 }
 ```
@@ -101,6 +130,7 @@ FruitCrawler.debug_random # will take random one
 FruitCrawler.debug_random 3 # will take 3 random resources
 FruitCrawler.debug_last # will take last from collection
 FruitCrawler.debug_last 5 # will take last 5 resources
+FruitCrawler.debug_all # guess what it will do
 FruitCrawler.debug_resource { id: '123', url: '...' }
 ```
 

data/lib/kabutops.rb

@@ -3,21 +3,19 @@ require 'sidekiq'
 require 'cachy'
 require 'moneta'
 require 'pstore'
+require 'mechanize'
 require 'elasticsearch'
 
 Cachy.cache_store = Moneta.new(:File, dir: 'cache') # temporary
 
-require 'kabutops/parameterable'
+require 'kabutops/extensions/parameterable'
+require 'kabutops/extensions/callback_support'
 require 'kabutops/recipe'
 require 'kabutops/recipe_item'
 require 'kabutops/adapters/base'
-require 'kabutops/adapters/callback'
 require 'kabutops/adapters/database_adapter'
 require 'kabutops/adapters/elastic_search'
-require 'kabutops/adapters/mysql'
-require 'kabutops/crawler_extensions/callback'
 require 'kabutops/crawler_extensions/elastic_search'
-require 'kabutops/crawler_extensions/mysql'
 require 'kabutops/crawler_extensions/pstore_storage'
 require 'kabutops/crawler_extensions/debugging'
 require 'kabutops/crawler'

data/lib/kabutops/adapters/database_adapter.rb

@@ -3,6 +3,8 @@ module Kabutops
   module Adapters
 
     class DatabaseAdapter < Base
+      include Extensions::CallbackSupport
+
       def data &block
         @recipe = Recipe.new
         @recipe.instance_eval &block
@@ -15,6 +17,7 @@ module Kabutops
           p result.to_hash
         else
           store(result)
+          notify(:after_save, result)
         end
       end
 

data/lib/kabutops/adapters/elastic_search.rb

@@ -3,7 +3,7 @@ module Kabutops
   module Adapters
 
     class ElasticSearch < DatabaseAdapter
-      include Parameterable
+      include Extensions::Parameterable
 
       params :host, :port, :index, :type
 

data/lib/kabutops/crawler.rb

@@ -3,30 +3,46 @@ module Kabutops
   class Crawler
     include CrawlerExtensions::Debugging
     include CrawlerExtensions::PStoreStorage
-    include CrawlerExtensions::Callback
     include CrawlerExtensions::ElasticSearch
+    include Sidekiq::Worker
 
     class << self
-      include Parameterable
+      include Extensions::Parameterable
+      include Extensions::CallbackSupport
 
-      params :collection, :proxy, :cache
+      params :collection, :proxy, :cache, :wait
 
       def adapters
-        @adapters
+        @adapters || []
       end
 
       def crawl! collection=nil
-        if storage[:status] == :none
-          @collection = collection || params[:collection] || []
-          @collection.each do |resource|
-            raise "url must be specified" if resource[:id].nil?
-            perform_async(resource)
+        @map ||= {}
+
+        if storage[:status].nil?
+          (collection || params[:collection] || []).each do |resource|
+            self << resource
           end
         end
       end
 
       def << resource
-        perform_async(resource)
+        if debug
+          params[:collection] ||= []
+          params[:collection] << resource
+          return
+        end
+
+        key = resource[:id] || resource[:url]
+
+        if key.nil?
+          raise "url must be specified for resource"
+        elsif @map[key]
+          # resource with an id already in map
+        else
+          perform_async(resource.to_hash)
+          @map[key] = resource
+        end
       end
     end
 
@@ -35,19 +51,23 @@ module Kabutops
 
       content = Cachy.cache_if(self.class.params.cache, resource[:url]) do
         agent = Mechanize.new
-        #agent.set_proxy(*self.class.params[:proxy])
+        agent.set_proxy(*self.class.params[:proxy]) if self.class.params[:proxy]
         agent.get(resource[:url]).body
       end
 
       page = Nokogiri::HTML(content)
 
+      self.class.notify(:after_crawl, resource, page)
+
       self.class.adapters.each do |adapter|
         adapter.process(resource, page)
       end
+
+      sleep self.class.params[:wait] || 0
     end
 
     def << resource
-      self.class.perform_async(resource.to_hash)
+      self.class << resource
     end
   end
 

data/lib/kabutops/crawler_extensions/debugging.rb

@@ -22,6 +22,10 @@ module Kabutops
           params[:collection][(0 - count)..-1].map{ |r| debug_resource(r) }
         end
 
+        def debug_all
+          params[:collection].map{ |r| debug_resource(r) }
+        end
+
         def debug_resource resource
           enable_debug
           self.new.perform(resource)

data/lib/kabutops/extensions/callback_support.rb

@@ -0,0 +1,38 @@
+module Kabutops
+
+  module Extensions
+
+    module CallbackSupport
+
+      class Manager
+        def method_missing name, *args, &block
+          return unless block_given?
+
+          @map ||= Hashie::Mash.new
+          @map[name] ||= []
+          @map[name] << block
+        end
+
+        def notify name, *args
+          return unless @map
+
+          (@map[name] || []).map do |block|
+            block.call(*args)
+          end
+        end
+      end
+
+      def callbacks &block
+        @manager ||= Manager.new
+        @manager.instance_eval &block
+      end
+
+      def notify name, *args
+        @manager ||= Manager.new
+        @manager.notify(name, *args)
+      end
+    end
+
+  end
+
+end

data/lib/kabutops/extensions/parameterable.rb

@@ -0,0 +1,37 @@
+module Kabutops
+
+  module Extensions
+
+    module Parameterable
+
+      def self.included base
+        base.extend(ClassMethods)
+        base.class_eval do
+          attr_reader :params
+        end
+      end
+
+      module ClassMethods
+
+        def params *list
+          return @params if list.empty?
+
+          list.each do |name|
+            define_method name do |*args|
+              @params ||= Hashie::Mash.new
+              if args.size == 1
+                @params[name] = args[0]
+              else
+                @params[name] = args
+              end
+            end
+          end
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/kabutops/version.rb

@@ -1,3 +1,3 @@
 module Kabutops
-  VERSION = '0.0.2'
+  VERSION = '0.0.3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kabutops
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Rene Klacan
@@ -132,17 +132,14 @@ files:
 - README.md
 - lib/kabutops.rb
 - lib/kabutops/adapters/base.rb
-- lib/kabutops/adapters/callback.rb
 - lib/kabutops/adapters/database_adapter.rb
 - lib/kabutops/adapters/elastic_search.rb
-- lib/kabutops/adapters/mysql.rb
 - lib/kabutops/crawler.rb
-- lib/kabutops/crawler_extensions/callback.rb
 - lib/kabutops/crawler_extensions/debugging.rb
 - lib/kabutops/crawler_extensions/elastic_search.rb
-- lib/kabutops/crawler_extensions/mysql.rb
 - lib/kabutops/crawler_extensions/pstore_storage.rb
-- lib/kabutops/parameterable.rb
+- lib/kabutops/extensions/callback_support.rb
+- lib/kabutops/extensions/parameterable.rb
 - lib/kabutops/recipe.rb
 - lib/kabutops/recipe_item.rb
 - lib/kabutops/version.rb

data/lib/kabutops/adapters/callback.rb

@@ -1,15 +0,0 @@
-module Kabutops
-  module Adapters
-    class Callback < Base
-      attr_accessor :block
-
-      def initialize block
-        @block = block
-      end
-
-      def process resource, page
-        block.call(resource, page)
-      end
-    end
-  end
-end

data/lib/kabutops/adapters/mysql.rb

@@ -1,17 +0,0 @@
-module Kabutops
-
-  module Adapters
-
-    class MySQL < DatabaseAdapter
-      include Parameterable
-
-      params :host, :port, :database, :user, :password, :table
-
-      def nested?
-        false
-      end
-    end
-
-  end
-
-end

data/lib/kabutops/crawler_extensions/callback.rb

@@ -1,24 +0,0 @@
-module Kabutops
-
-  module CrawlerExtensions
-
-    module Callback
-
-      def self.included base
-        base.extend(ClassMethods)
-      end
-
-      module ClassMethods
-        def callback &block
-          adapter = Adapters::Callback.new(block)
-
-          @adapters ||= []
-          @adapters << adapter
-        end
-      end
-
-    end
-
-  end
-
-end

data/lib/kabutops/crawler_extensions/mysql.rb

@@ -1,21 +0,0 @@
-module Kabutops
-
-  module CrawlerExtensions
-
-    module Mysql
-
-      def self.included base
-        base.extend(ClassMethods)
-      end
-
-      module ClassMethods
-        def mysql
-          raise NotImplementedError
-        end
-      end
-
-    end
-
-  end
-
-end

data/lib/kabutops/parameterable.rb

@@ -1,33 +0,0 @@
-module Kabutops
-
-  module Parameterable
-
-    def self.included base
-      base.extend(ClassMethods)
-      base.class_eval do
-        attr_reader :params
-      end
-    end
-
-    module ClassMethods
-
-      def params *list
-        return @params if list.empty?
-
-        list.each do |name|
-          define_method name do |*args|
-            @params ||= Hashie::Mash.new
-            if args.size == 1
-              @params[name] = args[0]
-            else
-              @params[name] = args
-            end
-          end
-        end
-      end
-
-    end
-
-  end
-
-end