daimon_skycrawlers 0.1.0 → 0.2.0

data/lib/daimon_skycrawlers/generator/templates/new/README.md.erb

@@ -15,20 +15,36 @@ TODO: Write description.
 
 1. Install dependencies
 
-        $ bundle install
+```
+$ bundle install
+```
 
 2. Create database
 
-        $ bundle exec rake db:create
-        $ bundle exec rake db:migrate
+```
+$ bundle exec rake db:create
+$ bundle exec rake db:migrate
+```
 
 3. Open new terminal and run crawler/processor
 
-        $ bundle exec ruby crawler.rb # on new terminal
-        $ bundle exec ruby processor.rb # on new terminal
+```
+$ bundle exec ruby crawler.rb # on new terminal
+$ bundle exec ruby processor.rb # on new terminal
+```
 
 4. Enqueue task
 
-        $ bundle exec ruby enqueue.rb http://example.com/
+```
+$ bundle exec ruby enqueue.rb http://example.com/
+```
 
 5. You'll see `It works with 'http://example.com'` on your terminal which runs your processor!
+
+6. You can re-enqueue task for processor
+
+```
+$ bundle exec ruby enqueue.rb response http://example.com/
+```
+
+Display `It works with 'http://example.com'` again on your terminal which runs your processor.

data/lib/daimon_skycrawlers/generator/templates/new/crawler.rb

@@ -1,13 +1,13 @@
 #!/usr/bin/env ruby
 
 require "daimon_skycrawlers/crawler"
+require "daimon_skycrawlers/crawler/default"
 
-base_url = 'http://example.com'
+require_relative "./init"
 
-crawler = DaimonSkycrawlers::Crawler.new(base_url)
-crawler.parser.append_filter do |url|
-  url.start_with?(base_url)
-end
+base_url = "http://example.com"
+
+crawler = DaimonSkycrawlers::Crawler::Default.new(base_url)
 
 DaimonSkycrawlers.register_crawler(crawler)
 

data/lib/daimon_skycrawlers/generator/templates/new/db/migrate/create_pages.rb

@@ -1,13 +1,13 @@
 class CreatePages < ActiveRecord::Migration
   def change
     create_table :pages do |t|
-    t.string :url
-    t.text :headers
-    t.binary :body, limit: 10 * 1024 ** 2 # 10MiB
-    t.datetime :last_modified_at
-    t.string :etag
+      t.string :url
+      t.text :headers
+      t.binary :body, limit: 10 * 1024 ** 2 # 10MiB
+      t.datetime :last_modified_at
+      t.string :etag
 
-    t.timestamps null: false
+      t.timestamps null: false
     end
   end
 end

data/lib/daimon_skycrawlers/generator/templates/new/enqueue.rb

@@ -1,15 +1,24 @@
 #!/usr/bin/env ruby
 
+require "thor"
+
 require "daimon_skycrawlers/crawler"
+require "daimon_skycrawlers/processor"
 
-USAGE = "Usage: #{$0} [URL]"
+require_relative "./init"
 
-if ARGV.size < 1
-  $stderr.puts "#{$0}: missing URL"
-  $stderr.puts USAGE
-  exit false
-end
+class Enqueue < Thor
+  desc "url URL [key1:value1 key2:value2...]", "Enqueue URL for URL consumer"
+  def url(url, *rest)
+    message = rest.map {|arg| arg.split(":") }.to_h
+    DaimonSkycrawlers::Crawler.enqueue_url(url, message)
+  end
 
-url = ARGV[0]
+  desc "response URL [key1:value1 key2:value2...]", "Enqueue URL for HTTP response consumer"
+  def response(url, *rest)
+    message = rest.map {|arg| arg.split(":") }.to_h
+    DaimonSkycrawlers::Processor.enqueue_http_response(url, message)
+  end
+end
 
-DaimonSkycrawlers::Crawler.enqueue_url(url)
+Enqueue.start(ARGV)

data/lib/daimon_skycrawlers/generator/templates/new/init.rb

@@ -0,0 +1,20 @@
+require "daimon_skycrawlers"
+require "daimon_skycrawlers/logger"
+require "daimon_skycrawlers/queue"
+
+DaimonSkycrawlers.configure do |config|
+  config.logger = DaimonSkycrawlers::Logger.default
+  config.crawler_interval = 1
+end
+
+DaimonSkycrawlers::Queue.configure do |config|
+  #  queue configuration
+  config.logger = DaimonSkycrawlers.configuration.logger
+  config.host = "127.0.0.1"
+  config.port = 5672
+  # config.username = 'guest'
+  # config.password = 'guest'
+  config.vhost = "/"
+  config.max_reconnect_attempts = 10
+  config.network_recovery_interval = 1.0
+end

data/lib/daimon_skycrawlers/generator/templates/new/processor.rb

@@ -2,6 +2,8 @@
 
 require "daimon_skycrawlers/processor"
 
+require_relative "./init"
+
 DaimonSkycrawlers.register_processor do |data|
   p "It works with '#{data[:url]}'"
 end

data/lib/daimon_skycrawlers/logger.rb

@@ -0,0 +1,32 @@
+require "delegate"
+require "logger"
+
+module DaimonSkycrawlers
+  class Logger < SimpleDelegator
+    class << self
+      def default
+        @default ||= DaimonSkycrawlers::Logger.new(STDOUT)
+      end
+    end
+
+    def initialize(logdev, shift_age = 0, shift_size = 1048576)
+      @log = ::Logger.new(logdev, shift_age, shift_size)
+      super(@log)
+    end
+  end
+
+  module LoggerMixin
+    def initialize
+      super
+      @log = DaimonSkycrawlers::Logger.default
+    end
+
+    class << self
+      def included(base)
+        base.module_eval do
+          attr_accessor :log
+        end
+      end
+    end
+  end
+end

data/lib/daimon_skycrawlers/processor/base.rb

@@ -0,0 +1,19 @@
+require "daimon_skycrawlers/logger"
+require "daimon_skycrawlers/config"
+
+module DaimonSkycrawlers
+  module Processor
+    class Base
+      include DaimonSkycrawlers::LoggerMixin
+      include DaimonSkycrawlers::ConfigMixin
+
+      def call(message)
+        raise "Implement this method in subclass"
+      end
+
+      def storage
+        @storage ||= DaimonSkycrawlers::Storage::RDB.new
+      end
+    end
+  end
+end

data/lib/daimon_skycrawlers/processor/default.rb

@@ -1,19 +1,22 @@
 require "daimon_skycrawlers/storage/rdb"
+require "daimon_skycrawlers/processor/base"
 
 module DaimonSkycrawlers
-  class Processor
-    class Default
+  module Processor
+    class Default < Base
       def call(message)
+        return if message[:heartbeat]
         url = message[:url]
-        storage = DaimonSkycrawlers::Storage::RDB.new
         page = storage.find(url)
         headers = JSON.parse(page.headers)
-        puts "URL: #{page.url}"
-        puts "Body: #{page.body.bytesize} bytes"
-        puts "Headers:"
-        headers.each do |key, value|
-          puts "  #{key}: #{value}"
-        end
+        headers_string = headers.map {|key, value| "  #{key}: #{value}" }.join("\n")
+        dumped_message = <<LOG
+URL: #{page.url}
+Body: #{page.body.bytesize} bytes
+Headers:
+#{headers_string}
+LOG
+        log.info(dumped_message)
       end
     end
   end

data/lib/daimon_skycrawlers/processor/spider.rb

@@ -0,0 +1,77 @@
+require "nokogiri"
+require "daimon_skycrawlers/crawler"
+
+module DaimonSkycrawlers
+  module Processor
+    class Spider < Base
+      attr_accessor :enqueue
+
+      def initialize
+        super
+        @filters = []
+        @doc = nil
+        @links = nil
+        @enqueue = true
+      end
+
+      def append_filter(filter = nil, &block)
+        if block_given?
+          @filters << block
+        else
+          @filters << filter
+        end
+      end
+
+      #
+      # @param [Hash] message Must have key :url, :depth
+      #
+      def call(message)
+        key_url = message[:url]
+        depth = Integer(message[:depth] || 2)
+        return if message[:heartbeat]
+        return if depth <= 1
+        page = storage.find(key_url)
+        @doc = Nokogiri::HTML(page.body)
+        new_message = {
+          depth: depth - 1,
+        }
+        links.each do |url|
+          enqueue_url(url, new_message)
+        end
+      end
+
+      private
+
+      def links
+        return @links if @links
+        @links = retrieve_links
+        @links
+      end
+
+      def retrieve_links
+        urls = @doc.search("a").map do |element|
+          element["href"]
+        end
+        urls.uniq!
+        apply_filters(urls) || []
+      end
+
+      def apply_filters(urls)
+        return if urls.nil?
+        return if urls.empty?
+        log.debug("Candidate URLs: #{urls.size}")
+        urls = urls.select do |url|
+          @filters.inject(true) {|memo, filter| memo & filter.call(url) }
+        end
+        log.debug("Filtered URLs: #{urls.size}")
+        urls
+      end
+
+      def enqueue_url(url, new_message)
+        return unless @enqueue
+        log.debug("Enqueue: URL:#{url}, message: #{new_message}")
+        DaimonSkycrawlers::Crawler.enqueue_url(url, new_message)
+      end
+    end
+  end
+end

data/lib/daimon_skycrawlers/processor.rb

@@ -1,17 +1,31 @@
-require 'daimon_skycrawlers'
-require 'daimon_skycrawlers/configure_songkick_queue'
-require 'daimon_skycrawlers/url_consumer'
-require 'daimon_skycrawlers/http_response_consumer'
+require "daimon_skycrawlers"
+require "daimon_skycrawlers/queue"
+require "daimon_skycrawlers/timer"
+require "daimon_skycrawlers/consumer/http_response"
 
 module DaimonSkycrawlers
-  class Processor
+  module Processor
     class << self
-      def run(process_name: 'daimon-skycrawler:http-response')
-        SongkickQueue::Worker.new(process_name, [HTTPResponseConsumer]).run
+      def run(process_name: default_process_name)
+        DaimonSkycrawlers::Timer.setup_shutdown_timer(config.queue_name_prefix, interval: config.shutdown_interval)
+        SongkickQueue::Worker.new(process_name, [DaimonSkycrawlers::Consumer::HTTPResponse]).run
       end
 
-      def enqueue_http_response(url)
-        SongkickQueue.publish('daimon-skycrawler.http-response', url: url)
+      def enqueue_http_response(url, message = {})
+        message[:url] = url
+        SongkickQueue.publish(queue_name, message)
+      end
+
+      def config
+        DaimonSkycrawlers.configuration
+      end
+
+      def queue_name
+        "#{config.queue_name_prefix}.http-response"
+      end
+
+      def default_process_name
+        "#{config.queue_name_prefix}:http-response"
       end
     end
   end

data/lib/daimon_skycrawlers/queue.rb

@@ -0,0 +1,24 @@
+require "songkick_queue"
+
+module DaimonSkycrawlers
+  class Queue
+    class << self
+      def configuration
+        @configuration ||= SongkickQueue.configure do |config|
+          config.logger = Logger.new(STDOUT)
+          config.host = "127.0.0.1"
+          config.port = 5672
+          # config.username = 'guest'
+          # config.password = 'guest'
+          config.vhost = "/"
+          config.max_reconnect_attempts = 10
+          config.network_recovery_interval = 1.0
+        end
+      end
+
+      def configure
+        yield configuration
+      end
+    end
+  end
+end

data/lib/daimon_skycrawlers/storage/base.rb

@@ -1,6 +1,12 @@
+require "daimon_skycrawlers/logger"
+require "daimon_skycrawlers/config"
+
 module DaimonSkycrawlers
   module Storage
     class Base
+      include DaimonSkycrawlers::LoggerMixin
+      include DaimonSkycrawlers::ConfigMixin
+
       def save(url, headers, body)
         raise "Implement this in subclass"
       end

data/lib/daimon_skycrawlers/timer.rb

@@ -0,0 +1,24 @@
+require "timers"
+require "daimon_skycrawlers"
+
+module DaimonSkycrawlers
+  module Timer
+    module_function
+
+    def setup_shutdown_timer(queue_name_prefix, interval: 10)
+      timers = Timers::Group.new
+      timer = timers.after(interval) do
+        Process.kill(:INT, 0)
+      end
+      Thread.new(timers) do |t|
+        loop { t.wait }
+      end
+      ActiveSupport::Notifications.subscribe("consume_message.songkick_queue") do |*args|
+        event = ActiveSupport::Notifications::Event.new(*args)
+        queue_name = event.payload[:queue_name]
+        DaimonSkycrawlers.configuration.logger.debug("Reset timer: consume message #{queue_name}")
+        timer.reset
+      end
+    end
+  end
+end

data/lib/daimon_skycrawlers/version.rb

@@ -1,3 +1,3 @@
 module DaimonSkycrawlers
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/daimon_skycrawlers.rb

@@ -1,15 +1,35 @@
-require 'bundler/setup'
+require "bundler/setup"
 
-require 'daimon_skycrawlers/version'
+require "daimon_skycrawlers/version"
+require "daimon_skycrawlers/logger"
 
 module DaimonSkycrawlers
+  Configuration = Struct.new(
+    :logger,
+    :queue_name_prefix,
+    :crawler_interval,
+    :shutdown_interval
+  )
   class << self
     def register_processor(processor = nil, &block)
-      HTTPResponseConsumer.register(processor, &block)
+      DaimonSkycrawlers::Consumer::HTTPResponse.register(processor, &block)
     end
 
     def register_crawler(crawler)
-      URLConsumer.register(crawler)
+      DaimonSkycrawlers::Consumer::URL.register(crawler)
+    end
+
+    def configuration
+      @configuration ||= DaimonSkycrawlers::Configuration.new.tap do |config|
+        config.logger = DaimonSkycrawlers::Logger.default
+        config.queue_name_prefix = "daimon-skycrawlers"
+        config.crawler_interval = 1
+        config.shutdown_interval = 10
+      end
+    end
+
+    def configure
+      yield configuration
     end
   end
 end

data/sample/spider/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+gem "rake"
+gem "daimon_skycrawlers", path: "../../"

data/sample/spider/README.md

@@ -0,0 +1,50 @@
+# spider
+
+TODO: Write description.
+
+## Requirements
+
+- Ruby
+- RabbitMQ
+- RDB
+  - PostgreSQL (default)
+  - MySQL
+  - SQLite3
+
+## Usage
+
+1. Install dependencies
+
+```
+$ bundle install
+```
+
+2. Create database
+
+```
+$ bundle exec rake db:create
+$ bundle exec rake db:migrate
+```
+
+3. Open new terminal and run crawler/processor
+
+```
+$ bundle exec ruby crawler.rb # on new terminal
+$ bundle exec ruby processor.rb # on new terminal
+```
+
+4. Enqueue task
+
+```
+$ bundle exec ruby enqueue.rb http://example.com/
+```
+
+5. You'll see `It works with 'http://example.com'` on your terminal which runs your processor!
+
+6. You can re-enqueue task for processor
+
+```
+$ bundle exec ruby enqueue.rb response http://example.com/
+```
+
+Display `It works with 'http://example.com'` again on your terminal which runs your processor.

data/sample/spider/Rakefile

@@ -0,0 +1 @@
+require "daimon_skycrawlers/tasks"

data/sample/spider/config/database.yml

@@ -0,0 +1,26 @@
+# PostgreSQL. Versions 8.2 and up are supported.
+#
+default: &default
+  adapter: postgresql
+  encoding: unicode
+  pool: 5
+
+development:
+  <<: *default
+  database: spider_development
+  #username: spider
+  #password:
+  #host: localhost
+  #port: 5432
+  #schema_search_path: myapp,sharedapp,public
+  #min_messages: notice
+
+test:
+  <<: *default
+  database: spider_test
+
+production:
+  <<: *default
+  database: spider_production
+  username: spider
+  password: <%= ENV['SPIDER_PASSWORD'] %>

data/sample/spider/crawler.rb

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "daimon_skycrawlers/crawler"
+require "daimon_skycrawlers/crawler/default"
+
+require_relative "./init"
+
+base_url = "http://www.clear-code.com/blog/"
+
+crawler = DaimonSkycrawlers::Crawler::Default.new(base_url)
+
+DaimonSkycrawlers.register_crawler(crawler)
+
+DaimonSkycrawlers::Crawler.run

data/sample/spider/db/migrate/20160830155803_create_pages.rb

@@ -0,0 +1,13 @@
+class CreatePages < ActiveRecord::Migration
+  def change
+    create_table :pages do |t|
+      t.string :url
+      t.text :headers
+      t.binary :body, limit: 10 * 1024 ** 2 # 10MiB
+      t.datetime :last_modified_at
+      t.string :etag
+
+      t.timestamps null: false
+    end
+  end
+end

data/sample/spider/db/schema.rb

@@ -0,0 +1,28 @@
+# This file is auto-generated from the current state of the database. Instead
+# of editing this file, please use the migrations feature of Active Record to
+# incrementally modify your database, and then regenerate this schema definition.
+#
+# Note that this schema.rb definition is the authoritative source for your
+# database schema. If you need to create the application database on another
+# system, you should be using db:schema:load, not running all the migrations
+# from scratch. The latter is a flawed and unsustainable approach (the more migrations
+# you'll amass, the slower it'll run and the greater likelihood for issues).
+#
+# It's strongly recommended that you check this file into your version control system.
+
+ActiveRecord::Schema.define(version: 20160830155803) do
+
+  # These are extensions that must be enabled in order to support this database
+  enable_extension "plpgsql"
+
+  create_table "pages", force: :cascade do |t|
+    t.string   "url"
+    t.text     "headers"
+    t.binary   "body"
+    t.datetime "last_modified_at"
+    t.string   "etag"
+    t.datetime "created_at",       null: false
+    t.datetime "updated_at",       null: false
+  end
+
+end

data/sample/spider/enqueue.rb

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+
+require "thor"
+
+require "daimon_skycrawlers/crawler"
+require "daimon_skycrawlers/processor"
+
+require_relative "./init"
+
+class Enqueue < Thor
+  desc "url URL [key1:value1 key2:value2...]", "Enqueue URL for URL consumer"
+  def url(url, *rest)
+    message = rest.map {|arg| arg.split(":") }.to_h
+    DaimonSkycrawlers::Crawler.enqueue_url(url, message)
+  end
+
+  desc "response URL [key1:value1 key2:value2...]", "Enqueue URL for HTTP response consumer"
+  def response(url, *rest)
+    message = rest.map {|arg| arg.split(":") }.to_h
+    DaimonSkycrawlers::Processor.enqueue_http_response(url, message)
+  end
+end
+
+Enqueue.start(ARGV)

data/sample/spider/init.rb

@@ -0,0 +1,22 @@
+require "daimon_skycrawlers"
+require "daimon_skycrawlers/logger"
+require "daimon_skycrawlers/queue"
+
+DaimonSkycrawlers.configure do |config|
+  config.logger = DaimonSkycrawlers::Logger.default
+  config.logger.level = :debug
+  config.crawler_interval = 1
+  config.shutdown_interval = 30
+end
+
+DaimonSkycrawlers::Queue.configure do |config|
+  # queue configuration
+  config.logger = DaimonSkycrawlers.configuration.logger
+  config.host = "127.0.0.1"
+  config.port = 5672
+  # config.username = 'guest'
+  # config.password = 'guest'
+  config.vhost = "/"
+  config.max_reconnect_attempts = 10
+  config.network_recovery_interval = 1.0
+end