daimon_skycrawlers 1.0.0.pre.rc1 → 1.0.0.pre.rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c9a394115d873b122e18a6970e62dfbc897dd690
-  data.tar.gz: 55fc07c13fa04c82c5e1c6a1075ffd97793b01f9
+  metadata.gz: 648bff4fe2019f82836bcf21c75dc6e29d45abc8
+  data.tar.gz: f7691acca87a0686190171806a6a07aaa8f4015f
 SHA512:
-  metadata.gz: 5ae484de037763f04487ad8c1ea4047a78a726156781cffeebf0c4c14ae0dfb70029b5f834c1414af9ee43c2b67fb7b137e5d5e335c2dee01abe9aae6ced1099
-  data.tar.gz: 63c85408015248ca66448025b2eb8ff6b65080c3da375b2e5b9723e944e3836f73285cb083088a31c1c8d60fe6176f57c47bc2d894fc7618c5853b3aebc3d500
+  metadata.gz: 378fe0060bacead511a87f637702cd5d24ce79c5acf3cba0bfac123d9dfc2cfcde41261a813dce69f9e11126d988df609a92df126f82235610db7c5a7332539b
+  data.tar.gz: '0790c05562fc71542a1cf33ca0971bbf356659d20012f7968c6c4ffd9ec9df521f0e6d009aa8256ec05a7d1b3e494baaec187d1faa8bbd34102fb73875edcfd0'

data/.yardopts

@@ -0,0 +1,11 @@
+--markup markdown
+--markup-provider redcarpet
+--format html
+--exclude /test/
+--exclude /sample/
+--exclude /db/
+--exclude /bin/
+--no-private
+-
+README.md
+LICENSE.txt

data/daimon_skycrawlers.gemspec

@@ -36,6 +36,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "test-unit-rr"
   spec.add_development_dependency "test-unit-notify"
   spec.add_development_dependency "pry"
+  spec.add_development_dependency "redcarpet"
   spec.add_development_dependency "tapp"
   spec.add_development_dependency "simplecov"
   spec.add_development_dependency "sqlite3"

data/lib/daimon_skycrawlers.rb

@@ -3,7 +3,13 @@ require "bundler/setup"
 require "daimon_skycrawlers/version"
 require "daimon_skycrawlers/logger"
 
+#
+# Name space for this library
+#
 module DaimonSkycrawlers
+  #
+  # Configuration class
+  #
   Configuration = Struct.new(
     :logger,
     :queue_name_prefix,
@@ -15,7 +21,7 @@ module DaimonSkycrawlers
     # Register a processor
     #
     # @overload register_processor(processor)
-    #   @param [Processor] processor instance which implements `call` method
+    #   @param processor [Processor] instance which implements `call` method
     #   @return [void]
     #
     # @overload register_processor
@@ -31,7 +37,7 @@ module DaimonSkycrawlers
     #
     # Register a crawler
     #
-    # @param [Crawler] crawler instance which implements `fetch` method
+    # @param crawler [Crawler] instance which implements `fetch` method
     # @return [void]
     #
     def register_crawler(crawler)
@@ -55,6 +61,20 @@ module DaimonSkycrawlers
     #
     # Configure DaimonSkycrawlers
     #
+    # ```ruby
+    # DaimonSkycrawlers.configure do |config|
+    #   config.logger = DaimonSkycrawlers::Logger.default
+    #   config.queue_name_prefix = "daimon-skycrawlers"
+    #   config.crawler_interval = 1
+    #   config.shutdown_interval = 10
+    # end
+    # ```
+    #
+    # * logger: logger instance
+    # * queue_name_prefix: prefix of queue name.
+    # * crawler_interval: crawling interval
+    # * shutdown_interval: shutdown after interval after the queue is empty
+    #
     # @return [void]
     # @yield [configuration] configure DaimonSkycrawlers
     # @yieldparam configuration [DaimonSkycrawlers::Configuration] configuration object
@@ -75,6 +95,11 @@ module DaimonSkycrawlers
       exit(false)
     end
 
+    #
+    # Load "app/crawlers/**/*.rb"
+    #
+    # @return [void]
+    #
     def load_crawlers
       Dir.glob("app/crawlers/**/*.rb") do |path|
         require(File.expand_path(path, Dir.pwd)) &&
@@ -82,6 +107,11 @@ module DaimonSkycrawlers
       end
     end
 
+    #
+    # Load "app/processors/**/*.rb"
+    #
+    # @return [void]
+    #
     def load_processors
       Dir.glob("app/processors/**/*.rb") do |path|
         require(File.expand_path(path, Dir.pwd)) &&

data/lib/daimon_skycrawlers/callbacks.rb

@@ -1,11 +1,22 @@
 module DaimonSkycrawlers
+  #
+  # This module provides simple callback system
+  #
   module Callbacks
+    # @private
     def initialize
       super
       @before_process_callbacks = []
       @after_process_callbacks = []
     end
 
+    #
+    # Register before process callback
+    #
+    # @param callback [Object] This object must respond to call
+    # @yield [message]
+    # @yieldparam message [Hash]
+    #
     def before_process(callback = nil, &block)
       if block_given?
         @before_process_callbacks << block
@@ -14,16 +25,29 @@ module DaimonSkycrawlers
       end
     end
 
-    def run_before_callbacks(message)
+    #
+    # Run registered before process callbacks
+    #
+    def run_before_process_callbacks(message)
       @before_process_callbacks.all? do |callback|
         callback.call(message)
       end
     end
 
+    #
+    # Clear all before process callbacks
+    #
     def clear_before_process_callbacks
       @before_process_callbacks = []
     end
 
+    #
+    # Register after process callback
+    #
+    # @param callback [Object] This object must respond to call
+    # @yield [message]
+    # @yieldparam message [Hash]
+    #
     def after_process(callback = nil, &block)
       if block_given?
         @after_process_callbacks << block
@@ -32,12 +56,18 @@ module DaimonSkycrawlers
       end
     end
 
-    def run_after_callbacks(message)
+    #
+    # Run registered before process callbacks
+    #
+    def run_after_process_callbacks(message)
       @after_process_callbacks.each do |callback|
         callback.call(message)
       end
     end
 
+    #
+    # Clear all after process callbacks
+    #
     def clear_after_process_callbacks
       @after_process_callbacks = []
     end

data/lib/daimon_skycrawlers/cli.rb

@@ -6,6 +6,10 @@ require "daimon_skycrawlers/commands/runner"
 require "daimon_skycrawlers/version"
 
 module DaimonSkycrawlers
+  #
+  # This class provides CLI (Command Line Interface)
+  #
+  # @private
   class CLI < Thor
     register(Generator::New, "new", "new NAME", "Create new project")
     register(Generator::Generate, "generate", "generate COMMAND", "Generate new code")

data/lib/daimon_skycrawlers/commands/enqueue.rb

@@ -7,7 +7,9 @@ require "thor"
 require "webrobots"
 
 module DaimonSkycrawlers
+  # @private
   module Commands
+    # @private
     class Enqueue < Thor
       desc "url URL [key1:value1 key2:value2...]", "Enqueue URL for URL consumer"
       def url(url, *rest)
@@ -72,7 +74,8 @@ module DaimonSkycrawlers
       method_option("type", aliases: ["-t"], type: :string, default: "url", desc: "Specify type for URLs")
       def yaml(path)
         load_init
-        YAML.load_file(path).each do |hash|
+        yaml_text = ERB.new(File.read(path), nil, "-").result(binding)
+        YAML.load(yaml_text).each do |hash|
           url = hash["url"]
           message = hash["message"] || {}
           raise "Could not find URL: #{hash}" unless url

data/lib/daimon_skycrawlers/commands/runner.rb

@@ -3,7 +3,9 @@ require "daimon_skycrawlers"
 require "daimon_skycrawlers/crawler"
 
 module DaimonSkycrawlers
+  # @private
   module Commands
+    # @private
     class Runner < Thor
       namespace "exec"
 

data/lib/daimon_skycrawlers/config.rb

@@ -1,5 +1,6 @@
 module DaimonSkycrawlers
   module ConfigMixin
+    # @private
     def initialize
       super
       @log = DaimonSkycrawlers.configuration.logger

data/lib/daimon_skycrawlers/configurable.rb

@@ -1,8 +1,13 @@
 module DaimonSkycrawlers
+  #
+  # This module provides `#configure` to construct instance
+  #
   module Configurable
     #
-    # Configure spider instance
+    # Configure instance
     #
+    # @yield [instance] give instance to the block
+    # @yieldparam instance [DaimonSkycrawlers::Crawler::Base|DaimonSkycrawlers::Processor::Base] self
     # @return [DaimonSkycrawlers::Crawler::Base|DaimonSkycrawlers::Processor::Base] self
     #
     def configure

data/lib/daimon_skycrawlers/consumer.rb

@@ -1,4 +1,7 @@
 module DaimonSkycrawlers
+  #
+  # Name space for consumer related classes
+  #
   module Consumer
   end
 end

data/lib/daimon_skycrawlers/consumer/base.rb

@@ -4,10 +4,15 @@ require "daimon_skycrawlers/config"
 
 module DaimonSkycrawlers
   module Consumer
+    #
+    # Base class for consumer
+    #
+    # @private
     class Base
       include DaimonSkycrawlers::LoggerMixin
       include DaimonSkycrawlers::ConfigMixin
 
+      # @private
       def process(message)
         raise NotImplementedError, "Must implement in subclass"
       end

data/lib/daimon_skycrawlers/consumer/http_response.rb

@@ -17,7 +17,7 @@ module DaimonSkycrawlers
         # Register a processor
         #
         # @overload register(processor)
-        #   @param [Processor] processor instance which implements `call` method
+        #   @param processor [Processor] processor instance which implements `call` method
         #   @return [void]
         #
         # @overload register

data/lib/daimon_skycrawlers/consumer/url.rb

@@ -14,7 +14,7 @@ module DaimonSkycrawlers
         #
         # Register a given crawler
         #
-        # @param [Crawler] crawler instance which implements `fetch` method
+        # @param crawler [Crawler] crawler instance which implements `fetch` method
         # @return [void]
         #
         def register(crawler)

data/lib/daimon_skycrawlers/crawler.rb

@@ -4,6 +4,9 @@ require "daimon_skycrawlers/timer"
 require "daimon_skycrawlers/consumer/url"
 
 module DaimonSkycrawlers
+  #
+  # Name space for crawler related classes
+  #
   module Crawler
     class << self
       #
@@ -21,8 +24,8 @@ module DaimonSkycrawlers
       #
       # Enqueue a URL to crawler queue
       #
-      # @param [String] Specify absolute URL
-      # @param [Hash] Extra parameters for crawler
+      # @param url [String] Specify absolute URL
+      # @param message [Hash] Extra parameters for crawler
       # @return [void]
       def enqueue_url(url, message = {})
         message[:url] = url

data/lib/daimon_skycrawlers/crawler/base.rb

@@ -18,6 +18,9 @@ module DaimonSkycrawlers
     #
     # The base class of crawler
     #
+    # A crawler implementation can inherit this class and override
+    # `#fetch` in the class.
+    #
     class Base
       include DaimonSkycrawlers::LoggerMixin
       include DaimonSkycrawlers::ConfigMixin
@@ -35,8 +38,9 @@ module DaimonSkycrawlers
       attr_reader :n_processed_urls
 
       #
-      # @param [String] Base URL for crawler
-      # @param [Hash] options for Faraday
+      # @param base_url [String] Base URL for crawler
+      # @param faraday_options [Hash] options for Faraday
+      # @param options [Hash] options for crawler
       #
       def initialize(base_url = nil, faraday_options: {}, options: {})
         super()
@@ -46,12 +50,15 @@ module DaimonSkycrawlers
         @prepare = ->(connection) {}
         @skipped = false
         @n_processed_urls = 0
+
+        setup_default_filters
+        setup_default_post_processes
       end
 
       #
       # Set up connection
       #
-      # @param [Hash] options for Faraday
+      # @param options [Hash] options for Faraday
       # @yield [faraday]
       # @yieldparam faraday [Faraday]
       #
@@ -67,6 +74,8 @@ module DaimonSkycrawlers
       # Call this method before DaimonSkycrawlers.register_crawler
       # For example, you can login before fetch URL
       #
+      # @yield [connection]
+      #
       def prepare(&block)
         @prepare = block
       end
@@ -74,26 +83,39 @@ module DaimonSkycrawlers
       #
       # Retrieve storage instance
       #
+      # @return [DaimonSkycrawlers::Storage::Base]
+      #
       def storage
         @storage ||= Storage::RDB.new
       end
 
+      #
+      # @return [true|false]
+      #
       def skipped?
         @skipped
       end
 
+      #
+      # @return [Faraday]
+      #
       def connection
         @connection ||= Faraday.new(@base_url, @faraday_options)
       end
 
+      #
+      # Process crawler sequence
+      #
+      # 1. Run registered filters
+      # 1. Prepare connection
+      # 1. Download(fetch) data from given URL
+      # 1. Run post processes (store downloaded data to storage)
+      #
       def process(message, &block)
         @skipped = false
         @n_processed_urls += 1
 
-        setup_default_filters
-        setup_default_post_processes
-
-        proceeding = run_before_callbacks(message)
+        proceeding = run_before_process_callbacks(message)
         unless proceeding
           skip(message[:url])
           return
@@ -106,18 +128,44 @@ module DaimonSkycrawlers
         @prepare.call(connection)
         response = fetch(url, message, &block)
         data = { url: url, message: message, response: response }
-        run_after_callbacks(data)
+        run_after_process_callbacks(data)
         data
       end
 
+      #
+      # Fetch URL
+      #
+      # Override this method in subclass.
+      #
+      # @param path [String] URI or path
+      # @param message [Hash] message can include anything
+      #
+      # @return [Faraday::Response] HTTP response
+      #
       def fetch(path, message = {})
         raise NotImplementedError, "Must implement this method in subclass"
       end
 
+      #
+      # GET URL with params
+      #
+      # @param path [String] URI or path
+      # @param params [Hash] query parameters
+      #
+      # @return [Faraday::Response] HTTP response
+      #
       def get(path, params = {})
         @connection.get(path, params)
       end
 
+      #
+      # POST URL with params
+      #
+      # @param path [String] URI or path
+      # @param params [Hash] query parameters
+      #
+      # @return [Faraday::Response] HTTP response
+      #
       def post(path, params = {})
         @connection.post(path, params)
       end