daimon_skycrawlers 1.0.0.pre.rc1 → 1.0.0.pre.rc2

data/lib/daimon_skycrawlers/storage.rb

@@ -1,4 +1,7 @@
 module DaimonSkycrawlers
+  #
+  # Name space for storage
+  #
   module Storage
   end
 end

data/lib/daimon_skycrawlers/storage/base.rb

@@ -3,15 +3,35 @@ require "daimon_skycrawlers/config"
 
 module DaimonSkycrawlers
   module Storage
+    #
+    # Base class of storage implementation
+    #
     class Base
       include DaimonSkycrawlers::LoggerMixin
       include DaimonSkycrawlers::ConfigMixin
 
+      #
+      # Save data to storage
+      #
+      # Override this method in subclass
+      #
+      # @param data [Hash] data has following keys
+      #   * `:url`: URL
+      #   * `:message`: Given message
+      #   * `:response`: HTTP response
+      #
       def save(data)
         raise "Implement this in subclass"
       end
 
-      def read(url)
+      #
+      # Fetch page identified by url
+      #
+      # Override this method in subclass
+      #
+      # @param url [String] the key to find data in storage
+      #
+      def read(url, message = {})
         raise "Implement this in subclass"
       end
     end

data/lib/daimon_skycrawlers/storage/file.rb

@@ -11,6 +11,14 @@ module DaimonSkycrawlers
         @base_dir = Pathname(base_dir)
       end
 
+      #
+      # Save data to files under base directory
+      #
+      # @param data [Hash] data has following keys
+      #   * `:url`: URL
+      #   * `:message`: Given message
+      #   * `:response`: HTTP response
+      #
       def save(data)
         url = data[:url]
         message = data[:message]
@@ -28,6 +36,11 @@ module DaimonSkycrawlers
         end
       end
 
+      #
+      # Read data from files under base directory
+      #
+      # @return [DaimonSkycrawlers::Storage::File::Page]
+      #
       def read(url, message)
         key = message[:key]
         headers = JSON.parse(headers_path(url, key).read)
@@ -35,6 +48,9 @@ module DaimonSkycrawlers
         Page.new(url, key, headers, body, headers["last-modified"], headers["etag"])
       end
 
+      #
+      # Page for file storage
+      #
       Page = Struct.new(:url, :key, :headers, :body, :last_modified, :etag)
 
       private

data/lib/daimon_skycrawlers/storage/null.rb

@@ -14,9 +14,9 @@ module DaimonSkycrawlers
       end
 
       #
-      # Find nothing
+      # Read nothing
       #
-      def find(url)
+      def read(url, message = {})
       end
     end
   end

data/lib/daimon_skycrawlers/storage/rdb.rb

@@ -14,12 +14,12 @@ module DaimonSkycrawlers
       end
 
       #
-      # Save
+      # Save data to RDB
       #
-      # @param [Hash] data has following keys
-      #   * :url: URL
-      #   * :message: Given message
-      #   * :response: HTTP response
+      # @param data [Hash] data has following keys
+      #   * `:url`: URL
+      #   * `:message`: Given message
+      #   * `:response`: HTTP response
       #
       def save(data)
         url = data[:url]
@@ -39,9 +39,10 @@ module DaimonSkycrawlers
       #
       # Fetch page identified by url
       #
-      # @param [String] url identity of the page
+      # @param url [String] identity of the page
+      # @param message [Hash] this hash may include `:key` to find page
       #
-      def find(url, message = {})
+      def read(url, message = {})
         key = message[:key]
         if key
           Page.where(key: key).order(updated_at: :desc).limit(1).first
@@ -50,10 +51,27 @@ module DaimonSkycrawlers
         end
       end
 
+      # @private
       class Base < ActiveRecord::Base
         self.abstract_class = true
       end
 
+      #
+      # Model represents page
+      #
+      #   * key
+      #     * The key to identify page
+      #   * url
+      #     * The URL of page
+      #   * headers
+      #     * HTTP response header
+      #   * body
+      #     * HTTP response body
+      #   * last_modified_at
+      #     * Last-Modified header
+      #   * etag
+      #     * ETag header
+      #
       class Page < Base
         self.table_name = "pages"
       end

data/lib/daimon_skycrawlers/timer.rb

@@ -2,9 +2,18 @@ require "timers"
 require "daimon_skycrawlers"
 
 module DaimonSkycrawlers
+  #
+  # Name space for timer
+  #
   module Timer
     module_function
 
+    # Setup timer for shutdown
+    #
+    # @param queue_name_prefix [String] previx of queue name
+    # @param interval [String] shutdown after this interval after the queue is empty
+    # @return [Timers::Group] timers
+    #
     def setup_shutdown_timer(queue_name_prefix, interval: 10)
       timers = Timers::Group.new
       timer = timers.after(interval) do

data/lib/daimon_skycrawlers/version.rb

@@ -1,3 +1,6 @@
 module DaimonSkycrawlers
-  VERSION = "1.0.0-rc1"
+  #
+  # Version of this library
+  #
+  VERSION = "1.0.0-rc2"
 end

data/sample/amazon-ranking/app/processors/amazon_ranking.rb

@@ -7,7 +7,7 @@ class AmazonRanking < DaimonSkycrawlers::Processor::Base
   Item = Struct.new(:rank, :name, :url, :star, :review)
   def call(message)
     url = message[:url]
-    page = storage.find(url)
+    page = storage.read(url)
     doc = Nokogiri::HTML(page.body)
     ranking = []
     doc.search(".zg_itemRow").each do |item|

data/sample/itp-crawler/app/processors/itp_processor.rb

@@ -7,7 +7,7 @@ require_relative "../models/itp_shop"
 class ItpProcessor < DaimonSkycrawlers::Processor::Base
   def call(message)
     key_url = message[:url]
-    page = storage.find(key_url)
+    page = storage.read(key_url)
     @doc = Nokogiri::HTML(page.body.encode("UTF-8", "CP932"))
     ItpShop.transaction do
       prepare_shops do |shop|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: daimon_skycrawlers
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre.rc1
+  version: 1.0.0.pre.rc2
 platform: ruby
 authors:
 - daimon developers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-31 00:00:00.000000000 Z
+date: 2017-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -248,6 +248,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: redcarpet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: tapp
   requirement: !ruby/object:Gem::Requirement
@@ -313,6 +327,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".travis.yml"
+- ".yardopts"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -344,23 +359,6 @@ files:
 - lib/daimon_skycrawlers/generator/generate.rb
 - lib/daimon_skycrawlers/generator/new.rb
 - lib/daimon_skycrawlers/generator/processor.rb
-- lib/daimon_skycrawlers/generator/templates/crawler.rb.erb
-- lib/daimon_skycrawlers/generator/templates/filter.rb.erb
-- lib/daimon_skycrawlers/generator/templates/new/Dockerfile
-- lib/daimon_skycrawlers/generator/templates/new/Dockerfile.db
-- lib/daimon_skycrawlers/generator/templates/new/Gemfile
-- lib/daimon_skycrawlers/generator/templates/new/README.md.erb
-- lib/daimon_skycrawlers/generator/templates/new/Rakefile
-- lib/daimon_skycrawlers/generator/templates/new/app/crawler.rb
-- lib/daimon_skycrawlers/generator/templates/new/app/processor.rb
-- lib/daimon_skycrawlers/generator/templates/new/config/database.yml.erb
-- lib/daimon_skycrawlers/generator/templates/new/config/init.rb
-- lib/daimon_skycrawlers/generator/templates/new/docker-compose.yml.erb
-- lib/daimon_skycrawlers/generator/templates/new/env.db.erb
-- lib/daimon_skycrawlers/generator/templates/new/env.erb
-- lib/daimon_skycrawlers/generator/templates/new/services/common/docker-entrypoint.sh
-- lib/daimon_skycrawlers/generator/templates/new/services/db/init-user-db.sh
-- lib/daimon_skycrawlers/generator/templates/processor.rb.erb
 - lib/daimon_skycrawlers/logger.rb
 - lib/daimon_skycrawlers/processor.rb
 - lib/daimon_skycrawlers/processor/base.rb
@@ -427,6 +425,23 @@ files:
 - sample/spider/config/init.rb
 - sample/spider/db/migrate/20160830155803_create_pages.rb
 - sample/spider/db/schema.rb
+- templates/crawler.rb.erb
+- templates/filter.rb.erb
+- templates/new/Dockerfile
+- templates/new/Dockerfile.db
+- templates/new/Gemfile
+- templates/new/README.md.erb
+- templates/new/Rakefile
+- templates/new/app/crawler.rb
+- templates/new/app/processor.rb
+- templates/new/config/database.yml.erb
+- templates/new/config/init.rb
+- templates/new/docker-compose.yml.erb
+- templates/new/env.db.erb
+- templates/new/env.erb
+- templates/new/services/common/docker-entrypoint.sh
+- templates/new/services/db/init-user-db.sh
+- templates/processor.rb.erb
 homepage: https://github.com/bm-sms/daimon_skycrawlers
 licenses:
 - MIT