daimon_skycrawlers 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3c9d264378336af1acadb1bec4dc2dba783d9089
-  data.tar.gz: c532cd7213889299f369ba2e8e682ea69a39172a
+  metadata.gz: 2fd637d109812fe657a771536f3c3a0041e4868a
+  data.tar.gz: 031bf50b2b72e6320ee748cb9044e177f24f5d2f
 SHA512:
-  metadata.gz: 86bcf270ee971a26cd029ab9bd9372cfbe544e15640b3275e897e8b0ec2131976db379330a047ef5ea03bf9f568c111e083a135ae1fddd3d0ead38a373086e33
-  data.tar.gz: a9aef67c6bcb70488a1ce0c5e54ed565f65e7a5490b2e0b7db87feb2960801bb829467d991ae4fb3ab52ddef145d25315ba03ca1e2178256b5147b7b52a043a2
+  metadata.gz: 59f6404ea231ecc337b1658406daa8c1e3b0d557ddc3d46c37efa91b1f7d3e0f8018d524aa3c2cc227fbee57d04ef51d7561ae879fd5d0fc719f4fca1308afb3
+  data.tar.gz: eeb06f74ae722a6cdf18d00f0aeda3e06e06199e02ddd2f0e52fedb3c7a7028c4a99e46a8c19d2a4e6aa0c3979f0602757d5598a5e1de39112457d1b5dce5679

data/README.md

@@ -2,7 +2,7 @@
 
 # DaimonSkycrawlers
 
-DaimonSkyCrawlers is a crawler framework.
+DaimonSkycrawlers is a crawler framework.
 
 ## Requirements
 
@@ -90,6 +90,55 @@ Or install it yourself as:
 
 Display `It works with 'http://example.com'` again on your terminal which runs your processor.
 
+### docker-compose
+
+1. Create project
+
+    ```
+    $ bundle exec daimon_skycrawlers new mycrawlers
+    $ cd mycrawlers
+    ```
+    or
+    ```
+    $ daimon_skycrawlers new mycrawlers
+    $ cd mycrawlers
+    ```
+
+2. Build docker images
+
+    ```
+    $ docker-compose build
+    ```
+
+3. Run docker containers
+
+    ```
+    $ docker-compose up -d
+    ```
+
+4. Run a command on docker containers
+
+    ```
+    $ docker-compose exec <service name> <command>
+    ```
+
+    For example,
+
+    ```
+    $ docker-compose exec mycrawlers-db bash
+    $ docker-compose exec mycrawlers-crawler sh
+    $ docker-compose exec mycrawlers-crawler bundle exec daimon_skycrawlers enqueue url http://example.com/
+    $ docker-compose exec mycrawlers-crawler bundle exec daimon_skycrawlers enqueue response http://example.com/
+    ```
+
+5. Shutdown docker containers
+
+    ```
+    $ docker-compose down
+    $ docker-compose down --rmi all # Remove all related images
+    ```
+
+
 ## Development
 
 After checking out the repo, run `bundle install` to install dependencies. Then, run `bundle exec rake test` to run the tests. You can also run `bundle console` for an interactive prompt that will allow you to experiment.

data/lib/daimon_skycrawlers/consumer/http_response.rb

@@ -2,6 +2,7 @@ require "songkick_queue"
 require "daimon_skycrawlers"
 require "daimon_skycrawlers/consumer/base"
 require "daimon_skycrawlers/processor/default"
+require "daimon_skycrawlers/processor/proc"
 
 module DaimonSkycrawlers
   module Consumer
@@ -27,7 +28,7 @@ module DaimonSkycrawlers
         #
         def register(processor = nil, &block)
           if block_given?
-            processors << block
+            processors << DaimonSkycrawlers::Processor::Proc.new(block)
           else
             processors << processor
           end

data/lib/daimon_skycrawlers/crawler.rb

@@ -12,7 +12,9 @@ module DaimonSkycrawlers
       # @param process_name [String] Process name
       #
       def run(process_name: default_process_name)
-        DaimonSkycrawlers::Timer.setup_shutdown_timer(config.queue_name_prefix, interval: config.shutdown_interval)
+        if config.shutdown_interval > 0
+          DaimonSkycrawlers::Timer.setup_shutdown_timer(config.queue_name_prefix, interval: config.shutdown_interval)
+        end
         SongkickQueue::Worker.new(process_name, [DaimonSkycrawlers::Consumer::URL]).run
       end
 

data/lib/daimon_skycrawlers/crawler/base.rb

@@ -112,13 +112,13 @@ module DaimonSkycrawlers
       def apply_filters(url)
         if @options[:obey_robots_txt]
           robots_txt_checker = DaimonSkycrawlers::Filter::RobotsTxtChecker.new(base_url: @base_url)
-          unless robots_txt_checker.allowed?(url)
+          unless robots_txt_checker.allowed?({ url: url })
             skip(url)
             return
           end
         end
         update_checker = DaimonSkycrawlers::Filter::UpdateChecker.new(storage: storage)
-        unless update_checker.updated?(url.to_s, connection: connection)
+        unless update_checker.updated?({ url: url.to_s }, connection: connection)
           skip(url)
           return
         end

data/lib/daimon_skycrawlers/filter/base.rb

@@ -24,7 +24,7 @@ module DaimonSkycrawlers
         @storage ||= DaimonSkycrawlers::Storage::RDB.new
       end
 
-      def call(url)
+      def call(message)
         raise NotImplementedError, "Must implement this method in subclass"
       end
     end

data/lib/daimon_skycrawlers/filter/duplicate_checker.rb

@@ -16,11 +16,12 @@ module DaimonSkycrawlers
       end
 
       #
-      # @param [String] url to check duplication. If given URL is
+      # @param [Hash] message to check duplication. If given URL is
       #        relative URL, use `@base_url + url` as absolute URL.
       # @return [true|false] Return false when duplicated, otherwise return true.
       #
-      def call(url)
+      def call(message)
+        url = message[:url]
         unless URI(url).absolute?
           url = (@base_url + url).to_s
         end
@@ -30,12 +31,12 @@ module DaimonSkycrawlers
       end
 
       #
-      # @param [String] url to check duplication. If given URL is
+      # @param [Hash] message to check duplication. If given URL is
       #        relative URL, use `@base_url + url` as absolute URL.
       # @return [true|false] Return true when duplicated, otherwise return false.
       #
-      def duplicated?(url)
-        !call(url)
+      def duplicated?(message)
+        !call(message)
       end
     end
   end

data/lib/daimon_skycrawlers/filter/robots_txt_checker.rb

@@ -11,14 +11,16 @@ module DaimonSkycrawlers
     class RobotsTxtChecker < Base
       def initialize(base_url: nil, user_agent: "DaimonSkycrawlers/#{DaimonSkycrawlers::VERSION}")
         super()
+        @base_url = base_url
         @webrobots = WebRobots.new(user_agent)
       end
 
       #
-      # @param [String] url
+      # @param [Hash] message
       # @return [true|false] Return true when web site allows to fetch the URL, otherwise return false
       #
-      def call(url)
+      def call(message)
+        url = message[:url]
         unless URI(url).absolute?
           url = (@base_url + url).to_s
         end

data/lib/daimon_skycrawlers/filter/update_checker.rb

@@ -17,11 +17,12 @@ module DaimonSkycrawlers
       end
 
       #
-      # @param [String] url
+      # @param [Hash] message
       # @param connection [Faraday]
       # @return [true|false] Return true when need update, otherwise return false
       #
-      def call(url, connection: nil)
+      def call(message, connection: nil)
+        url = message[:url]
         unless URI(url).absolute?
           url = (@base_url + url).to_s
         end

data/lib/daimon_skycrawlers/generator/new.rb

@@ -1,3 +1,4 @@
+require "securerandom"
 require "thor"
 require "rails/generators"
 require "rails/generators/actions"
@@ -16,11 +17,21 @@ module DaimonSkycrawlers
       end
 
       def create_files
+        config = {
+          password: SecureRandom.urlsafe_base64
+        }
         [
           "README.md",
           "config/database.yml",
+          "docker-compose.yml",
+          "env",
+          "env.db",
         ].each do |path|
-          template("#{path}.erb", "#{name}/#{path}")
+          if path.start_with?("env")
+            template("#{path}.erb", "#{name}/.#{path}", config)
+          else
+            template("#{path}.erb", "#{name}/#{path}", config)
+          end
         end
         migration_options = {
           destination_root: File.join(destination_root, name),
@@ -55,15 +66,36 @@ module DaimonSkycrawlers
 
       def copy_files
         [
+          "Dockerfile",
+          "Dockerfile.db",
           "Gemfile",
           "Rakefile",
           "app/crawlers/sample_crawler.rb",
+          "app/filters/sample_filter.rb",
           "app/processors/sample_processor.rb",
           "config/init.rb",
+          "services/common/docker-entrypoint.sh",
+          "services/db/init-user-db.sh"
         ].each do |path|
           copy_file(path, "#{name}/#{path}", mode: :preserve)
         end
       end
+
+      def create_directories
+        [
+          "vendor/bundle",
+          "docker-cache/bundle",
+          "docker-cache/.bundle"
+        ].each do |entry|
+          empty_directory("#{name}/#{entry}")
+        end
+      end
+
+      def display_post_message
+        puts <<MESSAGE
+Check .env and .env.db before run `docker-compose build` or `docker-compose up`.
+MESSAGE
+      end
     end
 
     class MigrationGenerator < ActiveRecord::Generators::MigrationGenerator

data/lib/daimon_skycrawlers/generator/templates/new/Dockerfile

@@ -0,0 +1,25 @@
+FROM ruby:2.3.1-alpine
+
+RUN apk --no-cache --update add build-base ruby-dev libxml2-dev postgresql-dev libcurl git
+
+RUN adduser -D -h /home/crawler -g "DaimonSkycrawlers user" -s /bin/sh crawler crawler
+
+ARG SKYCRAWLERS_ENV=production
+ARG SKYCRAWLERS_MAIN=crawler
+ENV SKYCRAWLERS_ENV=$SKYCRAWLERS_ENV \
+    SKYCRAWLERS_MAIN=$SKYCRAWLERS_MAIN \
+    BUNDLE_JOBS=4
+
+USER crawler
+WORKDIR /home/crawler
+COPY ./Gemfile* ./
+
+RUN if [ "$SKYCRAWLERS_ENV" = "production" ]; then \
+      bundle install --without development:test; \
+    fi
+
+COPY . .
+
+ADD services/common/docker-entrypoint.sh /docker-entrypoint.sh
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["$SKYCRAWLERS_MAIN"]

data/lib/daimon_skycrawlers/generator/templates/new/Dockerfile.db

@@ -0,0 +1,6 @@
+FROM postgres:9.5.4
+
+RUN localedef -i ja_JP -c -f UTF-8 -A /usr/share/locale/locale.alias ja_JP.UTF-8
+ENV LANG ja_JP.UTF-8
+
+ADD services/db/init-user-db.sh /docker-entrypoint-initdb.d/init-user-db.sh

data/lib/daimon_skycrawlers/generator/templates/new/README.md.erb

@@ -48,3 +48,37 @@ $ bin/enqueue response http://example.com/
 ```
 
 Display `It works with 'http://example.com'` again on your terminal which runs your processor.
+
+## Usage with docker-compose
+
+1. Build docker images
+
+```
+$ docker-compose build
+```
+
+2. Run docker containers
+
+```
+$ docker-compose up -d
+```
+
+3. Run a command on docker containers
+
+```
+$ docker-compose exec <service name> <command>
+```
+
+For example,
+
+```
+$ docker-compose exec <%= name %>-db bash
+$ docker-compose exec <%= name %>-crawler sh
+```
+
+4. Shutdown docker containers
+
+```
+$ docker-compose down
+$ docker-compose down --rmi all # Remove all related images
+```

data/lib/daimon_skycrawlers/generator/templates/new/app/filters/sample_filter.rb

@@ -0,0 +1,9 @@
+require "daimon_skycrawlers/filter/base"
+
+class SampleFilter < DaimonSkycrawlers::Filter::Base
+  def call(message)
+    # Imprement your filter here.
+    # If you want to crawl `url`, return true otherwise false.
+    true
+  end
+end

data/lib/daimon_skycrawlers/generator/templates/new/config/database.yml.erb

@@ -4,6 +4,7 @@ default: &default
   adapter: postgresql
   encoding: unicode
   pool: 5
+  url: <%%= ENV['DATABASE_URL'] %>
 
 development:
   <<: *default

data/lib/daimon_skycrawlers/generator/templates/new/config/init.rb

@@ -6,16 +6,22 @@ require "daimon_skycrawlers/queue"
 DaimonSkycrawlers.configure do |config|
   config.logger = DaimonSkycrawlers::Logger.default
   config.crawler_interval = 1
+  config.shutdown_interval = 300
 end
 
 DaimonSkycrawlers::Queue.configure do |config|
-  #  queue configuration
+  if ENV["CLOUDAMQP_URL"]
+    amqp_uri = URI(ENV["CLOUDAMQP_URL"])
+    config.host = amqp_uri.host
+    config.username = amqp_uri.user
+    config.password = amqp_uri.password
+    config.vhost = amqp_uri.user
+  else
+    config.port = 5672
+    config.host = ENV["SKYCRAWLERS_RABBITMQ_HOST"] || "localhost"
+    config.vhost = "/"
+  end
   config.logger = DaimonSkycrawlers.configuration.logger
-  config.host = "127.0.0.1"
-  config.port = 5672
-  # config.username = 'guest'
-  # config.password = 'guest'
-  config.vhost = "/"
   config.max_reconnect_attempts = 10
   config.network_recovery_interval = 1.0
 end

data/lib/daimon_skycrawlers/generator/templates/new/docker-compose.yml.erb

@@ -0,0 +1,35 @@
+version: "2"
+services:
+  <%= name %>-rabbitmq:
+    image: rabbitmq
+
+  <%= name %>-db:
+    build:
+      context: .
+      dockerfile: Dockerfile.db
+    env_file: .env.db
+
+  <%= name %>-common: &common
+    build:
+      context: .
+      args:
+        - SKYCRAWLERS_ENV=development
+    links:
+      - <%= name %>-rabbitmq
+      - <%= name %>-db
+    volumes:
+      - ./:/home/crawler
+      - ./docker-cache/.bundle:/home/crawler/.bundle
+      - ./docker-cache/bundle:/home/crawler/vendor/bundle
+    working_dir: /home/crawler
+    env_file: .env
+
+  <%= name %>-crawler:
+    <<: *common
+    command: crawler
+
+  <%= name %>-processor:
+    <<: *common
+    depends_on:
+      - <%= name %>-crawler
+    command: processor

data/lib/daimon_skycrawlers/generator/templates/new/env.db.erb

@@ -0,0 +1,5 @@
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=<%= config[:password] %>
+DATABASE_USER=crawler
+DATABASE_PASSWORD=<%= config[:password] %>
+DATABASE_PREFIX=<%= name %>

data/lib/daimon_skycrawlers/generator/templates/new/env.erb

@@ -0,0 +1,2 @@
+SKYCRAWLERS_RABBITMQ_HOST=<%= name %>-rabbitmq
+DATABASE_URL=postgres://crawler:<%= config[:password] %>@<%= name %>-db/xxx_development

data/lib/daimon_skycrawlers/generator/templates/new/services/common/docker-entrypoint.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+set -x
+
+MAIN=$1
+case $MAIN in
+    crawler)
+        bundle check || bundle install --retry=3 --path=vendor/bundle \
+                && bundle exec rake db:schema:load || bundle exec rake db:migrate
+        bundle exec daimon_skycrawlers exec $MAIN
+        ;;
+    processor)
+        while [ ! -e Gemfile.lock ]; do
+            sleep 5
+        done
+        bundle check || bundle install --retry=3 --path=vendor/bundle \
+                && bundle exec rake db:schema:load || bundle exec rake db:migrate
+        bundle exec daimon_skycrawlers exec $MAIN
+        ;;
+    setup)
+        bundle install --path=vendor/bundle
+        bundle exec rake db:schema:load
+        ;;
+    none)
+        echo NOP
+        ;;
+    sleep)
+        sleep 1d
+        ;;
+esac

data/lib/daimon_skycrawlers/generator/templates/new/services/db/init-user-db.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+createuser -e -U $POSTGRES_USER -d $DATABASE_USER
+createdb -e -U $POSTGRES_USER -E UTF-8 -O $DATABASE_USER $DATABASE_USER
+createdb -e -U $POSTGRES_USER -E UTF-8 -O $DATABASE_USER ${DATABASE_PREFIX}_development
+createdb -e -U $POSTGRES_USER -E UTF-8 -O $DATABASE_USER ${DATABASE_PREFIX}_test
+psql -U postgres -c "ALTER ROLE $DATABASE_USER WITH PASSWORD '$DATABASE_PASSWORD';"

data/lib/daimon_skycrawlers/processor.rb

@@ -12,7 +12,9 @@ module DaimonSkycrawlers
       # @param process_name [String] Process name
       #
       def run(process_name: default_process_name)
-        DaimonSkycrawlers::Timer.setup_shutdown_timer(config.queue_name_prefix, interval: config.shutdown_interval)
+        if config.shutdown_interval > 0
+          DaimonSkycrawlers::Timer.setup_shutdown_timer(config.queue_name_prefix, interval: config.shutdown_interval)
+        end
         SongkickQueue::Worker.new(process_name, [DaimonSkycrawlers::Consumer::HTTPResponse]).run
       end
 

data/lib/daimon_skycrawlers/processor/base.rb

@@ -21,7 +21,7 @@ module DaimonSkycrawlers
       end
 
       def process(message)
-        return unless apply_before_filters(message[:url])
+        return unless apply_before_filters(message)
         call(message)
       end
 
@@ -35,9 +35,9 @@ module DaimonSkycrawlers
 
       private
 
-      def apply_before_filters(url)
+      def apply_before_filters(message)
         @before_process_filters.all? do |filter|
-          filter.call(url)
+          filter.call(message)
         end
       end
     end

data/lib/daimon_skycrawlers/processor/proc.rb

@@ -0,0 +1,16 @@
+require "daimon_skycrawlers/processor/base"
+
+module DaimonSkycrawlers
+  module Processor
+    class Proc < Base
+      def initialize(handler)
+        super()
+        @handler = handler
+      end
+
+      def call(message)
+        @handler.call(message)
+      end
+    end
+  end
+end

data/lib/daimon_skycrawlers/tasks/database_tasks.rake

@@ -24,6 +24,8 @@
 # https://github.com/janko-m/sinatra-activerecord
 #
 
+require "erb"
+
 seed_loader = Class.new do
   def load_seed
     # load "#{ActiveRecord::Tasks::DatabaseTasks.db_dir}/seeds.rb"

data/lib/daimon_skycrawlers/version.rb

@@ -1,3 +1,3 @@
 module DaimonSkycrawlers
-  VERSION = "0.8.0"
+  VERSION = "0.9.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: daimon_skycrawlers
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors:
 - daimon developers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-01 00:00:00.000000000 Z
+date: 2016-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -328,18 +328,27 @@ files:
 - lib/daimon_skycrawlers/generator/new.rb
 - lib/daimon_skycrawlers/generator/processor.rb
 - lib/daimon_skycrawlers/generator/templates/crawler.rb.erb
+- lib/daimon_skycrawlers/generator/templates/new/Dockerfile
+- lib/daimon_skycrawlers/generator/templates/new/Dockerfile.db
 - lib/daimon_skycrawlers/generator/templates/new/Gemfile
 - lib/daimon_skycrawlers/generator/templates/new/README.md.erb
 - lib/daimon_skycrawlers/generator/templates/new/Rakefile
 - lib/daimon_skycrawlers/generator/templates/new/app/crawlers/sample_crawler.rb
+- lib/daimon_skycrawlers/generator/templates/new/app/filters/sample_filter.rb
 - lib/daimon_skycrawlers/generator/templates/new/app/processors/sample_processor.rb
 - lib/daimon_skycrawlers/generator/templates/new/config/database.yml.erb
 - lib/daimon_skycrawlers/generator/templates/new/config/init.rb
+- lib/daimon_skycrawlers/generator/templates/new/docker-compose.yml.erb
+- lib/daimon_skycrawlers/generator/templates/new/env.db.erb
+- lib/daimon_skycrawlers/generator/templates/new/env.erb
+- lib/daimon_skycrawlers/generator/templates/new/services/common/docker-entrypoint.sh
+- lib/daimon_skycrawlers/generator/templates/new/services/db/init-user-db.sh
 - lib/daimon_skycrawlers/generator/templates/processor.rb.erb
 - lib/daimon_skycrawlers/logger.rb
 - lib/daimon_skycrawlers/processor.rb
 - lib/daimon_skycrawlers/processor/base.rb
 - lib/daimon_skycrawlers/processor/default.rb
+- lib/daimon_skycrawlers/processor/proc.rb
 - lib/daimon_skycrawlers/processor/spider.rb
 - lib/daimon_skycrawlers/queue.rb
 - lib/daimon_skycrawlers/sitemap_parser.rb