daimon_skycrawlers 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1658e1827b07bfebb54e4d84b3a9dd21f11edd66
-  data.tar.gz: 5abf3dc3cbb8d8423b8f110e2c805ca1d919f59c
+  metadata.gz: e81cffca8abac23dba0da166eb2a1f6cef80b2c1
+  data.tar.gz: d4948c33c37ee8d9508d93c037919a784a2d21cc
 SHA512:
-  metadata.gz: 16ff307268fa11cc34454808f8bdbefded16a325e043c349ae07e48758fa8dcaae69457bbd7435e2685fa7026d8848677513780f9e1d7b42352acd54f5f33cdb
-  data.tar.gz: 841d61776c8094e6d1eebb149e72dc42a1c9a71fdc453c34f06dea465a5838c37db633bed7f79a93c3beb8827e05ca549fc3e8cd0cd8e3f9e17b52e349f5c787
+  metadata.gz: 4840d12acec8c75a13330810029a32c62827e35a5ca871b06dec0b1f80e7be9b8dd9fa08208a4b0daac4cd527a32ba1bdc1c893c815834ebbfe22750fd440997
+  data.tar.gz: 072f215aa3f245445c9ca335651fe875c238039978f3cf0eac6bc873b0d34b1c1dd4626cfa2320ce4ac5e9b1defe05ffba8ec63dcf9e5c8197472c2fa5cd9cea

data/daimon_skycrawlers.gemspec

@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "railties"
   spec.add_dependency "pg"
   spec.add_dependency "timers"
-  spec.add_dependency "sitemap-parser"
+  spec.add_dependency "typhoeus"
   spec.add_dependency "webrobots"
 
   spec.add_development_dependency "rake", "~> 10.0"

data/lib/daimon_skycrawlers/commands/enqueue.rb

@@ -1,8 +1,8 @@
 require "daimon_skycrawlers"
 require "daimon_skycrawlers/crawler"
 require "daimon_skycrawlers/processor"
+require "daimon_skycrawlers/sitemap_parser"
 require "daimon_skycrawlers/version"
-require "sitemap-parser"
 require "webrobots"
 
 module DaimonSkycrawlers
@@ -27,6 +27,7 @@ module DaimonSkycrawlers
       desc "sitemap [OPTIONS] URL", "Enqueue URLs from simtemap.xml"
       method_option("robots-txt", aliases: ["-r"], type: :boolean,
                     desc: "URL for robots.txt. Detect robots.txt automatically if URL is not robots.txt")
+      method_option("dump", type: :boolean, desc: "Dump URLs without enqueue")
       def sitemap(url)
         load_init
         if options["robots-txt"]
@@ -35,9 +36,11 @@ module DaimonSkycrawlers
         else
           sitemaps = [url]
         end
-        urls = sitemaps.flat_map do |sitemap|
-          sitemap_parser = SitemapParser.new(sitemap)
-          sitemap_parser.to_a
+        sitemap_parser = DaimonSkycrawlers::SitemapParser.new(sitemaps)
+        urls = sitemap_parser.parse
+        if options["dump"]
+          puts urls.join("\n")
+          return
         end
         urls.each do |_url|
           DaimonSkycrawlers::Crawler.enqueue_url(_url)

data/lib/daimon_skycrawlers/crawler/base.rb

@@ -5,6 +5,8 @@ require "daimon_skycrawlers/logger"
 require "daimon_skycrawlers/config"
 require "daimon_skycrawlers/storage"
 require "daimon_skycrawlers/processor"
+require "daimon_skycrawlers/filter/update_checker"
+require "daimon_skycrawlers/filter/robots_txt_checker"
 
 module DaimonSkycrawlers
   module Crawler

data/lib/daimon_skycrawlers/crawler/default.rb

@@ -1,6 +1,4 @@
 require "daimon_skycrawlers/crawler/base"
-require "daimon_skycrawlers/filter/update_checker"
-require "daimon_skycrawlers/filter/robots_txt_checker"
 
 module DaimonSkycrawlers
   module Crawler

data/lib/daimon_skycrawlers/crawler.rb

@@ -24,6 +24,7 @@ module DaimonSkycrawlers
       # @return [void]
       def enqueue_url(url, message = {})
         message[:url] = url
+        config.logger.debug("#{queue_name}: #{url}")
         SongkickQueue.publish(queue_name, message)
       end
 

data/lib/daimon_skycrawlers/processor.rb

@@ -6,24 +6,51 @@ require "daimon_skycrawlers/consumer/http_response"
 module DaimonSkycrawlers
   module Processor
     class << self
+      #
+      # Run registered processors
+      #
+      # @param process_name [String] Process name
+      #
       def run(process_name: default_process_name)
         DaimonSkycrawlers::Timer.setup_shutdown_timer(config.queue_name_prefix, interval: config.shutdown_interval)
         SongkickQueue::Worker.new(process_name, [DaimonSkycrawlers::Consumer::HTTPResponse]).run
       end
 
+      #
+      # Enqueue a URL to processor queue
+      #
+      # @param [String] Specify absolute URL
+      # @param [Hash] Extra parameters for crawler
+      # @return [void]
       def enqueue_http_response(url, message = {})
         message[:url] = url
+        config.logger.debug("#{queue_name}: #{url}")
         SongkickQueue.publish(queue_name, message)
       end
 
+      #
+      # Shortcut of DaimonSkycrawlers.configuration
+      #
+      # @return [DaimonSkycrawlers::Configuration]
+      #
       def config
         DaimonSkycrawlers.configuration
       end
 
+      #
+      # Queue name for processor
+      #
+      # @return [String] Queue name
+      #
       def queue_name
         "#{config.queue_name_prefix}.http-response"
       end
 
+      #
+      # Default process name
+      #
+      # @return [String] Default process name
+      #
       def default_process_name
         "#{config.queue_name_prefix}:http-response"
       end

data/lib/daimon_skycrawlers/sitemap_parser.rb

@@ -0,0 +1,79 @@
+require "nokogiri"
+require "typhoeus"
+require "zlib"
+require "uri"
+
+module DaimonSkycrawlers
+  # Based on https://github.com/benbalter/sitemap-parser
+  class SitemapParser
+    def initialize(urls, options = {})
+      @urls = urls
+    end
+
+    def parse
+      hydra = Typhoeus::Hydra.new(max_concurrency: 1)
+      sitemap_urls = []
+      @urls.each do |url|
+        if URI(url).scheme.start_with?("http")
+          request = Typhoeus::Request.new(url, followlocation: true)
+          request.on_complete do |response|
+            sitemap_urls.concat(on_complete(response))
+          end
+          hydra.queue(request)
+        else
+          if File.exist?(url)
+            extract_urls(File.read(url))
+          end
+        end
+      end
+      hydra.run
+      sitemap_urls
+    end
+
+    private
+
+    def on_complete(response)
+      raise "HTTP requset to #{response.effective_url} failed" unless response.success?
+      raw_sitemap = inflate_response(response)
+      extract_urls(raw_sitemap)
+    end
+
+    def extract_urls(body)
+      sitemap = Nokogiri::XML(body)
+      case
+      when sitemap.at("sitemapindex")
+        urls = sitemap.search("sitemap").flat_map do |s|
+          s.at("loc").content
+        end
+        SitemapParser.new(urls).parse
+      when sitemap.at("urlset")
+        sitemap.search("url").flat_map do |url|
+          url.at("loc").content
+        end
+      else
+        raise "Malformed sitemap.xml no <sitemapindex> or <urlset>"
+      end
+    end
+
+    def inflate_response(response)
+      if compressed?(response)
+        # We cannot inflate compressed data from NTFS filesystem (NT).
+        # This can avoid errors
+        stream = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
+        stream.inflate(response.body)
+      else
+        response.body
+      end
+    end
+
+    def compressed?(response)
+      case response.headers["Content-Encoding"]&.downcase
+      when "deflate", "gzip", "x-gzip"
+        true
+      else
+        signature = response.body[0, 2]
+        signature == "\x1F\x8B".b
+      end
+    end
+  end
+end

data/lib/daimon_skycrawlers/storage/rdb.rb

@@ -9,9 +9,8 @@ module DaimonSkycrawlers
     class RDB < Base
       def initialize(config_path = "config/database.yml")
         super()
-        config = YAML.load_file(config_path)
-        environment = ENV["SKYCRAWLERS_ENV"] || "development"
-        ActiveRecord::Base.establish_connection(config[environment])
+        Base.configurations = YAML.load_file(config_path)
+        Base.establish_connection(DaimonSkycrawlers.env.to_sym)
       end
 
       #
@@ -38,7 +37,11 @@ module DaimonSkycrawlers
         Page.where(url: url).order(last_modified_at: :desc).limit(1).first
       end
 
-      class Page < ActiveRecord::Base
+      class Base < ActiveRecord::Base
+        self.abstract_class = true
+      end
+
+      class Page < Base
         self.table_name = "pages"
       end
     end

data/lib/daimon_skycrawlers/tasks/database_tasks.rake

@@ -30,24 +30,23 @@ seed_loader = Class.new do
   end
 end
 
-ActiveRecord::Tasks::DatabaseTasks.tap do |config|
-  config.root                   = Rake.application.original_dir
-  config.env                    = ENV["SKYCRAWLERS_ENV"] || "development"
-  config.db_dir                 = "db"
-  config.migrations_paths       = ["db/migrate"]
-  config.fixtures_path          = "test/fixtures"
-  config.seed_loader            = seed_loader.new
-  config.database_configuration = ActiveRecord::Base.configurations
-end
-
 # db:load_config can be overriden manually
 Rake::Task["db:seed"].enhance(["db:load_config"])
 Rake::Task["db:load_config"].clear
 
 Rake::Task.define_task("db:environment")
 Rake::Task.define_task("db:load_config") do
-  ActiveRecord::Base.configurations = YAML.load_file("config/database.yml")
+  ActiveRecord::Tasks::DatabaseTasks.tap do |config|
+    config.root                   = Rake.application.original_dir
+    config.env                    = ENV["SKYCRAWLERS_ENV"] || "development"
+    config.db_dir                 = "db"
+    config.migrations_paths       = ["db/migrate"]
+    config.fixtures_path          = "test/fixtures"
+    config.seed_loader            = seed_loader.new
+    config.database_configuration = YAML.load_file("config/database.yml")
+  end
   environment = ENV["SKYCRAWLERS_ENV"] || "development"
+  ActiveRecord::Base.configurations = ActiveRecord::Tasks::DatabaseTasks.database_configuration
   ActiveRecord::Base.establish_connection(environment.to_sym)
 end
 Rake::Task["db:test:deprecated"].clear if Rake::Task.task_defined?("db:test:deprecated")

data/lib/daimon_skycrawlers/version.rb

@@ -1,3 +1,3 @@
 module DaimonSkycrawlers
-  VERSION = "0.6.0"
+  VERSION = "0.7.0"
 end

data/lib/daimon_skycrawlers.rb

@@ -74,5 +74,12 @@ module DaimonSkycrawlers
       puts ex.message
       exit(false)
     end
+
+    #
+    # Return current environment
+    #
+    def env
+      ENV["SKYCRAWLERS_ENV"] || "development"
+    end
   end
 end

data/sample/itp-crawler/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+gem "rake"
+gem "daimon_skycrawlers", path: "../../"

data/sample/itp-crawler/Gemfile.lock

@@ -0,0 +1,106 @@
+PATH
+  remote: ../../
+  specs:
+    daimon_skycrawlers (0.6.0)
+      activerecord
+      bundler (~> 1.11)
+      faraday
+      faraday_middleware
+      nokogiri
+      pg
+      railties
+      songkick_queue
+      thor
+      timers
+      typhoeus
+      webrobots
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actionpack (5.0.0.1)
+      actionview (= 5.0.0.1)
+      activesupport (= 5.0.0.1)
+      rack (~> 2.0)
+      rack-test (~> 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    actionview (5.0.0.1)
+      activesupport (= 5.0.0.1)
+      builder (~> 3.1)
+      erubis (~> 2.7.0)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    activemodel (5.0.0.1)
+      activesupport (= 5.0.0.1)
+    activerecord (5.0.0.1)
+      activemodel (= 5.0.0.1)
+      activesupport (= 5.0.0.1)
+      arel (~> 7.0)
+    activesupport (5.0.0.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (~> 0.7)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    amq-protocol (2.0.1)
+    arel (7.1.4)
+    builder (3.2.2)
+    bunny (2.6.0)
+      amq-protocol (>= 2.0.1)
+    concurrent-ruby (1.0.2)
+    erubis (2.7.0)
+    ethon (0.9.1)
+      ffi (>= 1.3.0)
+    faraday (0.9.2)
+      multipart-post (>= 1.2, < 3)
+    faraday_middleware (0.10.0)
+      faraday (>= 0.7.4, < 0.10)
+    ffi (1.9.14)
+    hitimes (1.2.4)
+    i18n (0.7.0)
+    loofah (2.0.3)
+      nokogiri (>= 1.5.9)
+    method_source (0.8.2)
+    mini_portile2 (2.1.0)
+    minitest (5.9.1)
+    multipart-post (2.0.0)
+    nokogiri (1.6.8.1)
+      mini_portile2 (~> 2.1.0)
+    pg (0.19.0)
+    rack (2.0.1)
+    rack-test (0.6.3)
+      rack (>= 1.0)
+    rails-dom-testing (2.0.1)
+      activesupport (>= 4.2.0, < 6.0)
+      nokogiri (~> 1.6.0)
+    rails-html-sanitizer (1.0.3)
+      loofah (~> 2.0)
+    railties (5.0.0.1)
+      actionpack (= 5.0.0.1)
+      activesupport (= 5.0.0.1)
+      method_source
+      rake (>= 0.8.7)
+      thor (>= 0.18.1, < 2.0)
+    rake (11.3.0)
+    songkick_queue (1.0.0)
+      activesupport (>= 3.0.0)
+      bunny (~> 2.2)
+    thor (0.19.1)
+    thread_safe (0.3.5)
+    timers (4.1.1)
+      hitimes
+    typhoeus (0.8.0)
+      ethon (>= 0.8.0)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    webrobots (0.1.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  daimon_skycrawlers!
+  rake
+
+BUNDLED WITH
+   1.12.5

data/sample/itp-crawler/README.md

@@ -0,0 +1,50 @@
+# itp-crawler
+
+Simple crawler for [iタウンページ](http://itp.ne.jp)
+
+## Requirements
+
+- Ruby
+- RabbitMQ
+- RDB
+  - PostgreSQL (default)
+  - MySQL
+  - SQLite3
+
+## Usage
+
+1. Install dependencies
+
+    ```
+    $ bundle install
+    ```
+
+2. Create database
+
+    ```
+    $ bundle exec rake db:create
+    $ bundle exec rake db:migrate
+    $ bundle exec rake itp:db:create
+    $ bundle exec rake itp:db:migrate
+    ```
+
+3. Open new terminal and run crawler/processor
+
+    ```
+    $ bundle exec daimon_skycrawlers exec crawler   # on new terminal
+    $ bundle exec daimon_skycrawlers exec processor # on new terminal
+    ```
+
+4. Enqueue task
+
+    ```
+    $ bundle exec daimon_skycrawlers enqueue url "http://itp.ne.jp/osaka/genre_dir/niku/?num=50"
+    ```
+
+5. You'll see `It works with 'http://example.com'` on your terminal which runs your processor!
+
+6. You can re-enqueue task for processor
+
+    ```
+    $ bundle exec daimon_skycrawlers enqueue response "http://itp.ne.jp/osaka/genre_dir/niku/?num=50"
+    ```

data/sample/itp-crawler/Rakefile

@@ -0,0 +1,2 @@
+require "daimon_skycrawlers/tasks"
+require_relative "./lib/tasks"

data/sample/itp-crawler/app/crawlers/itp_crawler.rb

@@ -0,0 +1,8 @@
+require "daimon_skycrawlers"
+require "daimon_skycrawlers/crawler"
+require "daimon_skycrawlers/crawler/default"
+
+
+base_url = "http://itp.ne.jp/"
+crawler = DaimonSkycrawlers::Crawler::Default.new(base_url)
+DaimonSkycrawlers.register_crawler(crawler)

data/sample/itp-crawler/app/models/itp_base.rb

@@ -0,0 +1,5 @@
+class ItpBase < ActiveRecord::Base
+  self.abstract_class = true
+  self.configurations = YAML.load_file("config/database_itp.yml")
+  self.establish_connection(DaimonSkycrawlers.env.to_sym)
+end

data/sample/itp-crawler/app/models/itp_shop.rb

@@ -0,0 +1,5 @@
+require_relative "./itp_base"
+
+class ItpShop < ItpBase
+  self.table_name = "shops"
+end

data/sample/itp-crawler/app/processors/itp_processor.rb

@@ -0,0 +1,95 @@
+require "daimon_skycrawlers"
+require "daimon_skycrawlers/processor"
+require "daimon_skycrawlers/processor/base"
+
+require_relative "../models/itp_shop"
+
+class ItpProcessor < DaimonSkycrawlers::Processor::Base
+  def call(message)
+    key_url = message[:url]
+    page = storage.find(key_url)
+    @doc = Nokogiri::HTML(page.body.encode("UTF-8", "CP932"))
+    ItpShop.transaction do
+      prepare_shops do |shop|
+        itp_shop = ItpShop.find_or_initialize_by(itp_url: shop.itp_url)
+        itp_shop.assign_attributes(shop.to_h)
+        itp_shop.save!
+      end
+    end
+    unless %r(/pg/) =~ key_url
+      enqueue_pages(key_url)
+    end
+  end
+
+  Shop = Struct.new(:name, :description, :itp_url, :zip_code, :address, :phone)
+
+  private
+
+  def prepare_shops
+    @doc.search(".normalResultsBox").each do |shop|
+      begin
+        name_element = shop.at("section h4 .blueText") || shop.at("section h4 .brackText")
+        name = name_element.content.strip
+        description = shop.at("section p").content.strip
+        # Avoid false detection for shop address
+        if description.start_with?("住所")
+          description = nil
+        end
+        itp_path = shop.at("section h4 a").attr("href")
+        phone = shop.at("section p b").content.strip
+        address_element = shop.search("section p").detect do |element|
+          /住所/ =~ element.content
+        end
+        address_element.search("span").unlink
+        address_element.search("a").unlink
+        address_text = address_element.content.strip
+        zip_code = address_text.slice(/〒(\d{3}-\d{4})(.+)/, 1)
+        address = address_text.slice(/〒(\d{3}-\d{4})(.+)/, 2).sub(/\A[[:space:]]+/, "")
+        s = Shop.new(name,
+                     description,
+                     retrieve_individual_page_url(itp_path),
+                     zip_code,
+                     address,
+                     phone)
+        yield s
+      rescue => e
+        log.warn("#{e.class}: #{e.message}")
+        log.debug(e.backtrace.join("\n"))
+        break
+      end
+    end
+  end
+
+  # NOTE: HEAD request to itp.ne.jp is so slow
+  #
+  # /shop/KN2700060500184274/?url=%2F0663026300%2F&s_bid=KN2700060500184274&s_sid=FSP-LSR-002&s_fr=V01&s_ck=C01
+  # http://nttbj.itp.ne.jp/0663026300/index.html
+  #
+  # or
+  #
+  # /shop/KN2700060500039708/
+  # http://itp.ne.jp/shop/KN2700060500039708/
+  def retrieve_individual_page_url(path)
+    shop_id = path.slice(/\/\?url=(.+)&/, 1)
+    uri = if shop_id
+            URI("http://nttbj.itp.ne.jp/") + URI.unescape(shop_id) + "index.html"
+          else
+            URI("http://itp.ne.jp/") + path
+          end
+    uri.to_s
+  end
+
+  MAX_PAGE_NUM = 100
+
+  def enqueue_pages(base_url)
+    search_result = @doc.at("h1.searchResultHeader").content.strip.slice(/(\d+)件/, 1).to_i
+    # itp.ne.jp can displays 5000 search results.
+    2.upto([(search_result / 50), MAX_PAGE_NUM].min) do |n|
+      url = URI.join(base_url, "pg/#{n}/?num=50")
+      DaimonSkycrawlers::Crawler.enqueue_url(url.to_s)
+    end
+  end
+end
+
+processor = ItpProcessor.new
+DaimonSkycrawlers.register_processor(processor)

data/sample/itp-crawler/config/database.yml

@@ -0,0 +1,26 @@
+# PostgreSQL. Versions 8.2 and up are supported.
+#
+default: &default
+  adapter: postgresql
+  encoding: unicode
+  pool: 5
+
+development:
+  <<: *default
+  database: itp-crawler_development
+  #username: itp-crawler
+  #password:
+  #host: localhost
+  #port: 5432
+  #schema_search_path: myapp,sharedapp,public
+  #min_messages: notice
+
+test:
+  <<: *default
+  database: itp-crawler_test
+
+production:
+  <<: *default
+  database: itp-crawler_production
+  username: itp-crawler
+  password: <%= ENV['ITP-CRAWLER_PASSWORD'] %>

data/sample/itp-crawler/config/database_itp.yml

@@ -0,0 +1,26 @@
+# PostgreSQL. Versions 8.2 and up are supported.
+#
+default: &default
+  adapter: postgresql
+  encoding: unicode
+  pool: 5
+
+development:
+  <<: *default
+  database: itp-processor_development
+  #username: itp-processor
+  #password:
+  #host: localhost
+  #port: 5432
+  #schema_search_path: myapp,sharedapp,public
+  #min_messages: notice
+
+test:
+  <<: *default
+  database: itp-processor_test
+
+production:
+  <<: *default
+  database: itp-processor_production
+  username: itp-processor
+  password: <%= ENV['ITP-PROCESSOR_PASSWORD'] %>

data/sample/itp-crawler/config/init.rb

@@ -0,0 +1,22 @@
+require "bundler/setup"
+require "daimon_skycrawlers"
+require "daimon_skycrawlers/logger"
+require "daimon_skycrawlers/queue"
+
+DaimonSkycrawlers.configure do |config|
+  config.logger = DaimonSkycrawlers::Logger.default
+  config.crawler_interval = 1
+  config.shutdown_interval = 30
+end
+
+DaimonSkycrawlers::Queue.configure do |config|
+  #  queue configuration
+  config.logger = DaimonSkycrawlers.configuration.logger
+  config.host = "127.0.0.1"
+  config.port = 5672
+  # config.username = 'guest'
+  # config.password = 'guest'
+  config.vhost = "/"
+  config.max_reconnect_attempts = 10
+  config.network_recovery_interval = 1.0
+end

data/sample/itp-crawler/db/migrate/20161018044144_create_page.rb

@@ -0,0 +1,13 @@
+class CreatePage < ActiveRecord::Migration[5.0]
+  def change
+    create_table :pages do |t|
+      t.string :url
+      t.text :headers
+      t.binary :body
+      t.datetime :last_modified_at
+      t.string :etag
+
+      t.timestamps
+    end
+  end
+end

data/sample/itp-crawler/db/schema.rb

@@ -0,0 +1,28 @@
+# This file is auto-generated from the current state of the database. Instead
+# of editing this file, please use the migrations feature of Active Record to
+# incrementally modify your database, and then regenerate this schema definition.
+#
+# Note that this schema.rb definition is the authoritative source for your
+# database schema. If you need to create the application database on another
+# system, you should be using db:schema:load, not running all the migrations
+# from scratch. The latter is a flawed and unsustainable approach (the more migrations
+# you'll amass, the slower it'll run and the greater likelihood for issues).
+#
+# It's strongly recommended that you check this file into your version control system.
+
+ActiveRecord::Schema.define(version: 20161018044144) do
+
+  # These are extensions that must be enabled in order to support this database
+  enable_extension "plpgsql"
+
+  create_table "pages", force: :cascade do |t|
+    t.string   "url"
+    t.text     "headers"
+    t.binary   "body"
+    t.datetime "last_modified_at"
+    t.string   "etag"
+    t.datetime "created_at",       null: false
+    t.datetime "updated_at",       null: false
+  end
+
+end

data/sample/itp-crawler/db_itp/migrate/20161020044144_create_shop.rb

@@ -0,0 +1,14 @@
+class CreateShop < ActiveRecord::Migration[5.0]
+  def change
+    create_table :shops do |t|
+      t.string :name, index: true
+      t.text :description
+      t.string :itp_url, index: true
+      t.string :zip_code
+      t.string :address
+      t.string :phone
+
+      t.timestamps
+    end
+  end
+end

data/sample/itp-crawler/db_itp/schema.rb

@@ -0,0 +1,31 @@
+# This file is auto-generated from the current state of the database. Instead
+# of editing this file, please use the migrations feature of Active Record to
+# incrementally modify your database, and then regenerate this schema definition.
+#
+# Note that this schema.rb definition is the authoritative source for your
+# database schema. If you need to create the application database on another
+# system, you should be using db:schema:load, not running all the migrations
+# from scratch. The latter is a flawed and unsustainable approach (the more migrations
+# you'll amass, the slower it'll run and the greater likelihood for issues).
+#
+# It's strongly recommended that you check this file into your version control system.
+
+ActiveRecord::Schema.define(version: 20161020044144) do
+
+  # These are extensions that must be enabled in order to support this database
+  enable_extension "plpgsql"
+
+  create_table "shops", force: :cascade do |t|
+    t.string   "name"
+    t.text     "description"
+    t.string   "itp_url"
+    t.string   "zip_code"
+    t.string   "address"
+    t.string   "phone"
+    t.datetime "created_at",  null: false
+    t.datetime "updated_at",  null: false
+    t.index ["itp_url"], name: "index_shops_on_itp_url", using: :btree
+    t.index ["name"], name: "index_shops_on_name", using: :btree
+  end
+
+end

data/sample/itp-crawler/lib/tasks/database_tasks.rb

@@ -0,0 +1,66 @@
+require "daimon_skycrawlers"
+
+seed_loader = Class.new do
+  def load_seed
+    # load "#{ActiveRecord::Tasks::DatabaseTasks.db_dir}/seeds.rb"
+  end
+end
+
+namespace :itp do
+  namespace :db do |ns|
+    task :drop => [:load_config] do
+      ActiveRecord::Tasks::DatabaseTasks.drop_current
+    end
+
+    task :create => [:load_config] do
+      ActiveRecord::Tasks::DatabaseTasks.create_current
+    end
+
+    task :migrate => [:load_config] do
+      ActiveRecord::Tasks::DatabaseTasks.migrate
+      ns["_dump"].invoke
+    end
+
+    task :_dump do
+      ns["schema:dump"].invoke
+      ns["_dump"].reenable
+    end
+
+    task :version => [:load_config] do
+      puts "Current version: #{ActiveRecord::Migrator.current_version}"
+    end
+
+    namespace :migrate do
+      task :reset => ["db:drop", "db:create", "db:migrate"]
+    end
+
+    namespace :schema do
+      task :dump => [:load_config] do
+        require "active_record/schema_dumper"
+        filename = ENV["SCHEMA"] || File.join(ActiveRecord::Tasks::DatabaseTasks.db_dir, "schema.rb")
+        File.open(filename, "w:utf-8") do |file|
+          ActiveRecord::SchemaDumper.dump(ActiveRecord::Base.connection, file)
+        end
+        ns["schema:dump"].reenable
+      end
+
+      task :load => [:load_config] do
+        ActiveRecord::Tasks::DatabaseTasks.load_schema_current(:ruby, ENV["SCHEMA"])
+      end
+    end
+
+    task :load_config do
+      ActiveRecord::Tasks::DatabaseTasks.tap do |config|
+        config.root                   = Rake.application.original_dir
+        config.env                    = ENV["SKYCRAWLERS_ENV"] || "development"
+        config.db_dir                 = "db_itp"
+        config.migrations_paths       = ["db_itp/migrate"]
+        config.fixtures_path          = "test/fixtures"
+        config.seed_loader            = seed_loader.new
+        config.database_configuration = YAML.load_file("config/database_itp.yml")
+      end
+      ActiveRecord::Base.configurations = YAML.load_file("config/database_itp.yml")
+      ActiveRecord::Base.establish_connection(DaimonSkycrawlers.env.to_sym)
+    end
+  end
+end

data/sample/itp-crawler/lib/tasks.rb

@@ -0,0 +1 @@
+require_relative "./tasks/database_tasks"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: daimon_skycrawlers
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Ryunosuke SATO
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-18 00:00:00.000000000 Z
+date: 2016-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -151,7 +151,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: sitemap-parser
+  name: typhoeus
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -343,6 +343,7 @@ files:
 - lib/daimon_skycrawlers/processor/default.rb
 - lib/daimon_skycrawlers/processor/spider.rb
 - lib/daimon_skycrawlers/queue.rb
+- lib/daimon_skycrawlers/sitemap_parser.rb
 - lib/daimon_skycrawlers/storage.rb
 - lib/daimon_skycrawlers/storage/base.rb
 - lib/daimon_skycrawlers/storage/file.rb
@@ -352,6 +353,23 @@ files:
 - lib/daimon_skycrawlers/tasks/database_tasks.rake
 - lib/daimon_skycrawlers/timer.rb
 - lib/daimon_skycrawlers/version.rb
+- sample/itp-crawler/Gemfile
+- sample/itp-crawler/Gemfile.lock
+- sample/itp-crawler/README.md
+- sample/itp-crawler/Rakefile
+- sample/itp-crawler/app/crawlers/itp_crawler.rb
+- sample/itp-crawler/app/models/itp_base.rb
+- sample/itp-crawler/app/models/itp_shop.rb
+- sample/itp-crawler/app/processors/itp_processor.rb
+- sample/itp-crawler/config/database.yml
+- sample/itp-crawler/config/database_itp.yml
+- sample/itp-crawler/config/init.rb
+- sample/itp-crawler/db/migrate/20161018044144_create_page.rb
+- sample/itp-crawler/db/schema.rb
+- sample/itp-crawler/db_itp/migrate/20161020044144_create_shop.rb
+- sample/itp-crawler/db_itp/schema.rb
+- sample/itp-crawler/lib/tasks.rb
+- sample/itp-crawler/lib/tasks/database_tasks.rb
 - sample/spider/Gemfile
 - sample/spider/README.md
 - sample/spider/Rakefile