daimon_skycrawlers 0.11.0 → 0.11.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7c8534d5ada19111a31249998b4ff7c935dc44c4
-  data.tar.gz: 77a7c9d78db52e9d40e412c9ced08d024caa1499
+  metadata.gz: 23b99f272ecd7f75af615fbfb696bb375d333736
+  data.tar.gz: 3f3df3ae6b5812f6b9cb676d82e09e7ecd08cd10
 SHA512:
-  metadata.gz: 205234df54f6c985aa14f97fa5e5a967e18c923d7bf0fc4b29fe3d3c40c85d88667987b6bc7ca9d389dd513b9cb235b699b4741f228ab1151aa7dcb1a5802ebb
-  data.tar.gz: e2090d4983e1ffb7434eef92b76c30795e495f79c89fe9fe3567ba4e7e8f8b650b33870b0ab718d5d1afd799de45d35d7791fbc29e43f4f76bfb23ce510b32e9
+  metadata.gz: 146c4a77653a4ad277bb573b2c6e9aae53b4a743884d7f21d1ded62bf9120e917c8ce1c7e36a5f2487dd652407df209f2facd050f47fc24885e3d7681e6dfdbc
+  data.tar.gz: 4b8512401a17ad800e4067483d28f46bbb90b96686f4b175ffa485fae3c03f6e3d7a8b15e87761d50547b0896421324715b9cc7716acd06a507831c16356194c

data/.gitignore

@@ -7,3 +7,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+/sample/*/docker-cache/

data/lib/daimon_skycrawlers/version.rb

@@ -1,3 +1,3 @@
 module DaimonSkycrawlers
-  VERSION = "0.11.0"
+  VERSION = "0.11.1"
 end

data/sample/amazon-ranking/.env

@@ -0,0 +1,5 @@
+SKYCRAWLERS_RABBITMQ_HOST=amazon-ranking-rabbitmq
+SKYCRAWLERS_RABBITMQ_PORT=5672
+DATABASE_HOST=amazon-ranking-db
+DATABASE_PORT=5432
+DATABASE_URL=postgres://crawler:jSygSCX-TOoWH08AtawoxQ@amazon-ranking-db/amazon-ranking_development

data/sample/amazon-ranking/.env.db

@@ -0,0 +1,5 @@
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=jSygSCX-TOoWH08AtawoxQ
+DATABASE_USER=crawler
+DATABASE_PASSWORD=jSygSCX-TOoWH08AtawoxQ
+DATABASE_PREFIX=amazon-ranking

data/sample/amazon-ranking/Dockerfile

@@ -0,0 +1,30 @@
+FROM ruby:2.3.1-alpine
+
+RUN apk --no-cache --update add build-base ruby-dev libxml2-dev postgresql-dev libcurl openssl git
+
+RUN adduser -D -h /home/crawler -g "DaimonSkycrawlers user" -s /bin/sh crawler crawler
+
+ARG SKYCRAWLERS_ENV=production
+ARG SKYCRAWLERS_MAIN=crawler
+ENV SKYCRAWLERS_ENV=$SKYCRAWLERS_ENV \
+    SKYCRAWLERS_MAIN=$SKYCRAWLERS_MAIN \
+    BUNDLE_JOBS=4 \
+    DOCKERIZE_VERSION=v0.3.0
+
+RUN wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
+    && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
+    && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
+
+USER crawler
+WORKDIR /home/crawler
+COPY ./Gemfile* ./
+
+RUN if [ "$SKYCRAWLERS_ENV" = "production" ]; then \
+      bundle install --without development:test; \
+    fi
+
+COPY . .
+
+ADD services/common/docker-entrypoint.sh /docker-entrypoint.sh
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["$SKYCRAWLERS_MAIN"]

data/sample/amazon-ranking/Dockerfile.db

@@ -0,0 +1,6 @@
+FROM postgres:9.5.4
+
+RUN localedef -i ja_JP -c -f UTF-8 -A /usr/share/locale/locale.alias ja_JP.UTF-8
+ENV LANG ja_JP.UTF-8
+
+ADD services/db/init-user-db.sh /docker-entrypoint-initdb.d/init-user-db.sh

data/sample/amazon-ranking/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+gem "rake"
+gem "daimon_skycrawlers"

data/sample/amazon-ranking/Gemfile.lock

@@ -0,0 +1,101 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actionpack (5.0.0.1)
+      actionview (= 5.0.0.1)
+      activesupport (= 5.0.0.1)
+      rack (~> 2.0)
+      rack-test (~> 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    actionview (5.0.0.1)
+      activesupport (= 5.0.0.1)
+      builder (~> 3.1)
+      erubis (~> 2.7.0)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    activemodel (5.0.0.1)
+      activesupport (= 5.0.0.1)
+    activerecord (5.0.0.1)
+      activemodel (= 5.0.0.1)
+      activesupport (= 5.0.0.1)
+      arel (~> 7.0)
+    activesupport (5.0.0.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (~> 0.7)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    amq-protocol (2.0.1)
+    arel (7.1.4)
+    builder (3.2.2)
+    bunny (2.6.1)
+      amq-protocol (>= 2.0.1)
+    concurrent-ruby (1.0.2)
+    daimon_skycrawlers (0.10.0)
+      activerecord
+      faraday
+      faraday_middleware
+      nokogiri
+      pg
+      railties
+      songkick_queue
+      thor
+      timers
+      typhoeus
+      webrobots
+    erubis (2.7.0)
+    ethon (0.10.1)
+      ffi (>= 1.3.0)
+    faraday (0.10.0)
+      multipart-post (>= 1.2, < 3)
+    faraday_middleware (0.10.1)
+      faraday (>= 0.7.4, < 1.0)
+    ffi (1.9.14)
+    hitimes (1.2.4)
+    i18n (0.7.0)
+    loofah (2.0.3)
+      nokogiri (>= 1.5.9)
+    method_source (0.8.2)
+    mini_portile2 (2.1.0)
+    minitest (5.10.1)
+    multipart-post (2.0.0)
+    nokogiri (1.6.8.1)
+      mini_portile2 (~> 2.1.0)
+    pg (0.19.0)
+    rack (2.0.1)
+    rack-test (0.6.3)
+      rack (>= 1.0)
+    rails-dom-testing (2.0.1)
+      activesupport (>= 4.2.0, < 6.0)
+      nokogiri (~> 1.6.0)
+    rails-html-sanitizer (1.0.3)
+      loofah (~> 2.0)
+    railties (5.0.0.1)
+      actionpack (= 5.0.0.1)
+      activesupport (= 5.0.0.1)
+      method_source
+      rake (>= 0.8.7)
+      thor (>= 0.18.1, < 2.0)
+    rake (11.3.0)
+    songkick_queue (1.0.0)
+      activesupport (>= 3.0.0)
+      bunny (~> 2.2)
+    thor (0.19.4)
+    thread_safe (0.3.5)
+    timers (4.1.2)
+      hitimes
+    typhoeus (1.1.2)
+      ethon (>= 0.9.0)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    webrobots (0.1.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  daimon_skycrawlers
+  rake
+
+BUNDLED WITH
+   1.13.5

data/sample/amazon-ranking/README.md

@@ -0,0 +1,86 @@
+# amazon-ranking
+
+https://www.amazon.co.jp/gp/bestsellers/
+
+amazonの全てのカテゴリの売れ筋ランキングを1位から20位まで抽出する。
+
+## Requirements
+
+- Ruby
+- RabbitMQ
+- RDB
+  - PostgreSQL (default)
+  - MySQL
+  - SQLite3
+
+## Usage
+
+1. Install dependencies
+
+```
+$ bundle install
+```
+
+2. Create database
+
+```
+$ bundle exec rake db:create
+$ bundle exec rake db:migrate
+```
+
+3. Open new terminal and run crawler/processor
+
+```
+$ bin/crawler   # on new terminal
+$ bin/processor # on new terminal
+```
+
+4. Enqueue task
+
+```
+$ bin/enqueue url http://example.com/
+```
+
+5. You'll see `It works with 'http://example.com'` on your terminal which runs your processor!
+
+6. You can re-enqueue task for processor
+
+```
+$ bin/enqueue response http://example.com/
+```
+
+Display `It works with 'http://example.com'` again on your terminal which runs your processor.
+
+## Usage with docker-compose
+
+1. Build docker images
+
+```
+$ docker-compose build
+```
+
+2. Run docker containers
+
+```
+$ docker-compose up -d
+```
+
+3. Run a command on docker containers
+
+```
+$ docker-compose exec <service name> <command>
+```
+
+For example,
+
+```
+$ docker-compose exec amazon-ranking-db bash
+$ docker-compose exec amazon-ranking-crawler sh
+```
+
+4. Shutdown docker containers
+
+```
+$ docker-compose down
+$ docker-compose down --rmi all # Remove all related images
+```

data/sample/amazon-ranking/Rakefile

@@ -0,0 +1 @@
+require "daimon_skycrawlers/tasks"

data/sample/amazon-ranking/app/crawlers/sample_crawler.rb

@@ -0,0 +1,8 @@
+require "daimon_skycrawlers/crawler"
+require "daimon_skycrawlers/crawler/default"
+
+base_url = "http://example.com"
+
+crawler = DaimonSkycrawlers::Crawler::Default.new(base_url)
+
+DaimonSkycrawlers.register_crawler(crawler)

data/sample/amazon-ranking/app/filters/sample_filter.rb

@@ -0,0 +1,9 @@
+require "daimon_skycrawlers/filter/base"
+
+class SampleFilter < DaimonSkycrawlers::Filter::Base
+  def call(message)
+    # Imprement your filter here.
+    # If you want to crawl `url`, return true otherwise false.
+    true
+  end
+end

data/sample/amazon-ranking/app/processors/amazon_ranking.rb

@@ -0,0 +1,37 @@
+require "daimon_skycrawlers"
+require "daimon_skycrawlers/processor"
+require "daimon_skycrawlers/processor/base"
+require "daimon_skycrawlers/processor/spider"
+
+class AmazonRanking < DaimonSkycrawlers::Processor::Base
+  Item = Struct.new(:rank, :name, :url, :star, :review)
+  def call(message)
+    url = message[:url]
+    page = storage.find(url)
+    doc = Nokogiri::HTML(page.body)
+    ranking = []
+    doc.search(".zg_itemRow").each do |item|
+      rank = item.at(".zg_rankNumber").inner_text
+      link = item.at(".zg_rankLine+a")
+      star, review = item.search(".zg_rankLine+a+div a")
+      ranking << Item.new(rank, link.inner_text, link[:href], star[:title], review.inner_text)
+    end
+    p ranking
+  end
+end
+
+spider = DaimonSkycrawlers::Processor::Spider.new
+spider.configure do |s|
+  s.link_rules = "ul#zg_browseRoot li a"
+  s.link_message = { next_processor: "AmazonRanking" }
+  s.before_process do |message|
+    message[:next_processor] != "AmazonRanking"
+  end
+end
+DaimonSkycrawlers.register_processor(spider)
+
+processor = AmazonRanking.new
+processor.before_process do |message|
+  message[:next_processor] == "AmazonRanking"
+end
+DaimonSkycrawlers.register_processor(processor)

data/sample/amazon-ranking/app/processors/sample_processor.rb

@@ -0,0 +1,5 @@
+require "daimon_skycrawlers/processor"
+
+DaimonSkycrawlers.register_processor do |data|
+  p "It works with '#{data[:url]}'"
+end

data/sample/amazon-ranking/config/database.yml

@@ -0,0 +1,27 @@
+# PostgreSQL. Versions 8.2 and up are supported.
+#
+default: &default
+  adapter: postgresql
+  encoding: unicode
+  pool: 5
+  url: <%= ENV['DATABASE_URL'] %>
+
+development:
+  <<: *default
+  database: amazon-ranking_development
+  #username: amazon-ranking
+  #password:
+  #host: localhost
+  #port: 5432
+  #schema_search_path: myapp,sharedapp,public
+  #min_messages: notice
+
+test:
+  <<: *default
+  database: amazon-ranking_test
+
+production:
+  <<: *default
+  database: amazon-ranking_production
+  username: amazon-ranking
+  password: <%= ENV['AMAZON-RANKING_PASSWORD'] %>

data/sample/amazon-ranking/config/init.rb

@@ -0,0 +1,27 @@
+require "bundler/setup"
+require "daimon_skycrawlers"
+require "daimon_skycrawlers/logger"
+require "daimon_skycrawlers/queue"
+
+DaimonSkycrawlers.configure do |config|
+  config.logger = DaimonSkycrawlers::Logger.default
+  config.crawler_interval = 1
+  config.shutdown_interval = 300
+end
+
+DaimonSkycrawlers::Queue.configure do |config|
+  if ENV["CLOUDAMQP_URL"]
+    amqp_uri = URI(ENV["CLOUDAMQP_URL"])
+    config.host = amqp_uri.host
+    config.username = amqp_uri.user
+    config.password = amqp_uri.password
+    config.vhost = amqp_uri.user
+  else
+    config.port = 5672
+    config.host = ENV["SKYCRAWLERS_RABBITMQ_HOST"] || "localhost"
+    config.vhost = "/"
+  end
+  config.logger = DaimonSkycrawlers.configuration.logger
+  config.max_reconnect_attempts = 10
+  config.network_recovery_interval = 1.0
+end

data/sample/amazon-ranking/db/migrate/20161206061241_create_pages.rb

@@ -0,0 +1,16 @@
+class CreatePages < ActiveRecord::Migration[5.0]
+  def change
+    create_table :pages do |t|
+      t.string :url
+      t.text :headers
+      t.binary :body
+      t.datetime :last_modified_at
+      t.string :etag
+
+      t.timestamps
+
+      t.index [:url]
+      t.index [:url, :updated_at]
+    end
+  end
+end

data/sample/amazon-ranking/db/schema.rb

@@ -0,0 +1,30 @@
+# This file is auto-generated from the current state of the database. Instead
+# of editing this file, please use the migrations feature of Active Record to
+# incrementally modify your database, and then regenerate this schema definition.
+#
+# Note that this schema.rb definition is the authoritative source for your
+# database schema. If you need to create the application database on another
+# system, you should be using db:schema:load, not running all the migrations
+# from scratch. The latter is a flawed and unsustainable approach (the more migrations
+# you'll amass, the slower it'll run and the greater likelihood for issues).
+#
+# It's strongly recommended that you check this file into your version control system.
+
+ActiveRecord::Schema.define(version: 20161206061241) do
+
+  # These are extensions that must be enabled in order to support this database
+  enable_extension "plpgsql"
+
+  create_table "pages", force: :cascade do |t|
+    t.string   "url"
+    t.text     "headers"
+    t.binary   "body"
+    t.datetime "last_modified_at"
+    t.string   "etag"
+    t.datetime "created_at",       null: false
+    t.datetime "updated_at",       null: false
+    t.index ["url", "updated_at"], name: "index_pages_on_url_and_updated_at", using: :btree
+    t.index ["url"], name: "index_pages_on_url", using: :btree
+  end
+
+end

data/sample/amazon-ranking/docker-compose.yml

@@ -0,0 +1,43 @@
+version: "2"
+services:
+  amazon-ranking-rabbitmq:
+    image: rabbitmq
+    volumes:
+      - amazon-ranking-rabbitmq-storage:/var/lib/rabbitmq
+
+  amazon-ranking-db:
+    build:
+      context: .
+      dockerfile: Dockerfile.db
+    volumes:
+      - amazon-ranking-db-storage:/var/lib/postgresql/data
+    env_file: .env.db
+
+  amazon-ranking-common: &common
+    build:
+      context: .
+      args:
+        - SKYCRAWLERS_ENV=development
+    links:
+      - amazon-ranking-rabbitmq
+      - amazon-ranking-db
+    volumes:
+      - ./:/home/crawler
+      - ./docker-cache/.bundle:/home/crawler/.bundle
+      - ./docker-cache/bundle:/home/crawler/vendor/bundle
+    working_dir: /home/crawler
+    env_file: .env
+
+  amazon-ranking-crawler:
+    <<: *common
+    command: crawler
+
+  amazon-ranking-processor:
+    <<: *common
+    depends_on:
+      - amazon-ranking-crawler
+    command: processor
+
+volumes:
+  amazon-ranking-db-storage:
+  amazon-ranking-rabbitmq-storage:

data/sample/amazon-ranking/services/common/docker-entrypoint.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+set -x
+
+MAIN=$1
+dockerize -timeout 60s \
+          -wait tcp://${DATABASE_HOST}:${DATABASE_PORT} \
+          -wait tcp://${SKYCRAWLERS_RABBITMQ_HOST}:${SKYCRAWLERS_RABBITMQ_PORT}
+case $MAIN in
+    crawler)
+        bundle check || bundle install --retry=3 --path=vendor/bundle \
+                && bundle exec rake db:migrate
+        bundle exec daimon_skycrawlers exec $MAIN
+        ;;
+    processor)
+        while [ ! -e Gemfile.lock ]; do
+            sleep 5
+        done
+        bundle check || bundle install --retry=3 --path=vendor/bundle \
+                && bundle exec rake db:migrate
+        bundle exec daimon_skycrawlers exec $MAIN
+        ;;
+    setup)
+        bundle install --retry=3 --path=vendor/bundle
+        bundle exec rake db:schema:load
+        ;;
+    migrate)
+        bundle exec rake db:migrate
+        ;;
+    none)
+        echo NOP
+        ;;
+    sleep)
+        sleep 1d
+        ;;
+esac

data/sample/amazon-ranking/services/db/init-user-db.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+createuser -e -U $POSTGRES_USER -d $DATABASE_USER
+createdb -e -U $POSTGRES_USER -E UTF-8 -O $DATABASE_USER $DATABASE_USER
+createdb -e -U $POSTGRES_USER -E UTF-8 -O $DATABASE_USER ${DATABASE_PREFIX}_development
+createdb -e -U $POSTGRES_USER -E UTF-8 -O $DATABASE_USER ${DATABASE_PREFIX}_test
+psql -U postgres -c "ALTER ROLE $DATABASE_USER WITH PASSWORD '$DATABASE_PASSWORD';"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: daimon_skycrawlers
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.11.1
 platform: ruby
 authors:
 - daimon developers
@@ -376,6 +376,25 @@ files:
 - lib/daimon_skycrawlers/tasks/database_tasks.rake
 - lib/daimon_skycrawlers/timer.rb
 - lib/daimon_skycrawlers/version.rb
+- sample/amazon-ranking/.env
+- sample/amazon-ranking/.env.db
+- sample/amazon-ranking/Dockerfile
+- sample/amazon-ranking/Dockerfile.db
+- sample/amazon-ranking/Gemfile
+- sample/amazon-ranking/Gemfile.lock
+- sample/amazon-ranking/README.md
+- sample/amazon-ranking/Rakefile
+- sample/amazon-ranking/app/crawlers/sample_crawler.rb
+- sample/amazon-ranking/app/filters/sample_filter.rb
+- sample/amazon-ranking/app/processors/amazon_ranking.rb
+- sample/amazon-ranking/app/processors/sample_processor.rb
+- sample/amazon-ranking/config/database.yml
+- sample/amazon-ranking/config/init.rb
+- sample/amazon-ranking/db/migrate/20161206061241_create_pages.rb
+- sample/amazon-ranking/db/schema.rb
+- sample/amazon-ranking/docker-compose.yml
+- sample/amazon-ranking/services/common/docker-entrypoint.sh
+- sample/amazon-ranking/services/db/init-user-db.sh
 - sample/itp-crawler/Gemfile
 - sample/itp-crawler/Gemfile.lock
 - sample/itp-crawler/README.md