daimon_skycrawlers 1.0.0.pre.rc1 → 1.0.0.pre.rc2

data/lib/daimon_skycrawlers/crawler/default.rb

@@ -5,9 +5,17 @@ module DaimonSkycrawlers
     #
     # The default crawler
     #
-    # This crawler can GET given URL and store response to storage
+    # This crawler can GET/POST given URL and store response to storage
     #
     class Default < Base
+      #
+      # GET/POST given url
+      #
+      # @param url [String] URI or path
+      # @param message [Hash] message can include anything
+      #
+      # @return [Faraday::Response] HTTP response
+      #
       def fetch(url, message)
         params = message[:params] || {}
         method = message[:method] || "GET"

data/lib/daimon_skycrawlers/filter.rb

@@ -1,4 +1,7 @@
 module DaimonSkycrawlers
+  #
+  # Name space for filters
+  #
   module Filter
   end
 end

data/lib/daimon_skycrawlers/filter/base.rb

@@ -20,10 +20,22 @@ module DaimonSkycrawlers
         @storage = storage
       end
 
+      #
+      # Retrieve storage instance
+      #
       def storage
         @storage ||= DaimonSkycrawlers::Storage::RDB.new
       end
 
+      #
+      # Filter message
+      #
+      # Override this method in subclass.
+      #
+      # @param message [Hash] message can include anything
+      #
+      # @return [true|false] process the message if true otherwise skip message.
+      #
       def call(message)
         raise NotImplementedError, "Must implement this method in subclass"
       end

data/lib/daimon_skycrawlers/filter/duplicate_checker.rb

@@ -16,7 +16,7 @@ module DaimonSkycrawlers
       end
 
       #
-      # @param [Hash] message to check duplication. If given URL is
+      # @param message [Hash] message to check duplication. If given URL is
       #        relative URL, use `@base_url + url` as absolute URL.
       # @return [true|false] Return false when duplicated, otherwise return true.
       #
@@ -28,7 +28,7 @@ module DaimonSkycrawlers
       end
 
       #
-      # @param [Hash] message to check duplication. If given URL is
+      # @param message [Hash] message to check duplication. If given URL is
       #        relative URL, use `@base_url + url` as absolute URL.
       # @return [true|false] Return true when duplicated, otherwise return false.
       #

data/lib/daimon_skycrawlers/filter/robots_txt_checker.rb

@@ -16,7 +16,7 @@ module DaimonSkycrawlers
       end
 
       #
-      # @param [Hash] message
+      # @param message [Hash] check given URL is allowed or not by robots.txt
       # @return [true|false] Return true when web site allows to fetch the URL, otherwise return false
       #
       def call(message)

data/lib/daimon_skycrawlers/filter/update_checker.rb

@@ -17,13 +17,13 @@ module DaimonSkycrawlers
       end
 
       #
-      # @param [Hash] message
+      # @param message [Hash] message includes `:url`
       # @param connection [Faraday]
       # @return [true|false] Return true when need update, otherwise return false
       #
       def call(message, connection: nil)
         url = normalize_url(message[:url])
-        page = storage.find(url)
+        page = storage.read(url, message)
         return true unless page
         if connection
           response = connection.head(url)

data/lib/daimon_skycrawlers/generator/crawler.rb

@@ -1,14 +1,17 @@
 require "thor"
+require "pathname"
 
 module DaimonSkycrawlers
+  # @private
   module Generator
+    # @private
     class Crawler < Thor::Group
       include Thor::Actions
 
       argument :name
 
       def self.source_root
-        File.join(__dir__, "templates")
+        (Pathname(__dir__) + "../../../templates").to_s
       end
 
       def create_files

data/lib/daimon_skycrawlers/generator/filter.rb

@@ -1,14 +1,17 @@
 require "thor"
+require "pathname"
 
 module DaimonSkycrawlers
+  # @private
   module Generator
+    # @private
     class Filter < Thor::Group
       include Thor::Actions
 
       argument :name
 
       def self.source_root
-        File.join(__dir__, "templates")
+        (Pathname(__dir__) + "../../../templates").to_s
       end
 
       def create_files

data/lib/daimon_skycrawlers/generator/generate.rb

@@ -1,10 +1,13 @@
 require "thor"
+require "pathname"
 require "daimon_skycrawlers/generator/crawler"
 require "daimon_skycrawlers/generator/processor"
 require "daimon_skycrawlers/generator/filter"
 
 module DaimonSkycrawlers
+  # @private
   module Generator
+    # @private
     class Generate < Thor
       register(Crawler, "crawler", "crawler NAME", "Generate new crawler")
       register(Processor, "processor", "processor NAME", "Generate new processor")

data/lib/daimon_skycrawlers/generator/new.rb

@@ -1,19 +1,22 @@
 require "securerandom"
 require "thor"
+require "pathname"
 require "rails/generators"
 require "rails/generators/actions"
 require "rails/generators/active_record"
 require "rails/generators/active_record/migration/migration_generator"
 
 module DaimonSkycrawlers
+  # @private
   module Generator
+    # @private
     class New < Thor::Group
       include Thor::Actions
 
       argument :name
 
       def self.source_root
-        File.join(__dir__, "templates", "new")
+        (Pathname(__dir__) + "../../../templates/new").to_s
       end
 
       def create_files
@@ -103,6 +106,7 @@ MESSAGE
       end
     end
 
+    # @private
     class MigrationGenerator < ActiveRecord::Generators::MigrationGenerator
       def self.source_root
         ActiveRecord::Generators::MigrationGenerator.source_root

data/lib/daimon_skycrawlers/generator/processor.rb

@@ -1,14 +1,17 @@
 require "thor"
+require "pathname"
 
 module DaimonSkycrawlers
+  # @private
   module Generator
+    # @private
     class Processor < Thor::Group
       include Thor::Actions
 
       argument :name
 
       def self.source_root
-        File.join(__dir__, "templates")
+        (Pathname(__dir__) + "../../../templates").to_s
       end
 
       def create_files

data/lib/daimon_skycrawlers/logger.rb

@@ -2,13 +2,20 @@ require "delegate"
 require "logger"
 
 module DaimonSkycrawlers
+  #
+  # Logger for daimon_skycrawlers
+  #
   class Logger < SimpleDelegator
     class << self
+      #
+      # Default logger
+      #
       def default
         @default ||= DaimonSkycrawlers::Logger.new(STDOUT)
       end
     end
 
+    # @private
     def initialize(logdev, shift_age = 0, shift_size = 1048576)
       @log = ::Logger.new(logdev, shift_age, shift_size)
       super(@log)
@@ -16,6 +23,7 @@ module DaimonSkycrawlers
   end
 
   module LoggerMixin
+    # @private
     def initialize
       super
       @log = DaimonSkycrawlers::Logger.default

data/lib/daimon_skycrawlers/processor.rb

@@ -4,6 +4,9 @@ require "daimon_skycrawlers/timer"
 require "daimon_skycrawlers/consumer/http_response"
 
 module DaimonSkycrawlers
+  #
+  # Name space for processors
+  #
   module Processor
     class << self
       #
@@ -21,8 +24,8 @@ module DaimonSkycrawlers
       #
       # Enqueue a URL to processor queue
       #
-      # @param [String] Specify absolute URL
-      # @param [Hash] Extra parameters for crawler
+      # @param url [String] Specify absolute URL
+      # @param message [Hash] Extra parameters for crawler
       # @return [void]
       def enqueue_http_response(url, message = {})
         message[:url] = url

data/lib/daimon_skycrawlers/processor/base.rb

@@ -5,6 +5,12 @@ require "daimon_skycrawlers/configurable"
 
 module DaimonSkycrawlers
   module Processor
+    #
+    # The base class of processor
+    #
+    # A processor implementation can inherit this class and override
+    # `#call` in the class.
+    #
     class Base
       include DaimonSkycrawlers::LoggerMixin
       include DaimonSkycrawlers::ConfigMixin
@@ -14,16 +20,26 @@ module DaimonSkycrawlers
       def initialize
         super
         @skipped = false
+
+        setup_default_filters
       end
 
+      # @private
       def skipped?
         @skipped
       end
 
+      #
+      # Process processor sequence
+      #
+      # 1. Run registered filters
+      # 1. Process HTTP response from message
+      #
+      # @param message [Hash] parameters for processor
+      #
       def process(message)
         @skipped = false
-        setup_default_filters
-        proceeding = run_before_callbacks(message)
+        proceeding = run_before_process_callbacks(message)
         unless proceeding
           skip(message[:url])
           return
@@ -31,10 +47,20 @@ module DaimonSkycrawlers
         call(message)
       end
 
+      #
+      # Process message
+      #
+      # Override this method in subclass
+      #
+      # @param message [Hash] parameters for processor
+      #
       def call(message)
         raise "Implement this method in subclass"
       end
 
+      #
+      # Retrieve storage instance
+      #
       def storage
         @storage ||= DaimonSkycrawlers::Storage::RDB.new
       end

data/lib/daimon_skycrawlers/processor/default.rb

@@ -3,10 +3,16 @@ require "daimon_skycrawlers/processor/base"
 
 module DaimonSkycrawlers
   module Processor
+    #
+    # Very simple processor
+    #
     class Default < Base
+      #
+      # Display page information
+      #
       def call(message)
         url = message[:url]
-        page = storage.find(url)
+        page = storage.read(url, message)
         headers = JSON.parse(page.headers)
         headers_string = headers.map {|key, value| "  #{key}: #{value}" }.join("\n")
         dumped_message = <<LOG

data/lib/daimon_skycrawlers/processor/proc.rb

@@ -2,12 +2,18 @@ require "daimon_skycrawlers/processor/base"
 
 module DaimonSkycrawlers
   module Processor
+    #
+    # Processor for Proc
+    #
     class Proc < Base
       def initialize(handler)
         super()
         @handler = handler
       end
 
+      #
+      # Process message
+      #
       def call(message)
         @handler.call(message)
       end

data/lib/daimon_skycrawlers/processor/spider.rb

@@ -96,13 +96,13 @@ module DaimonSkycrawlers
       end
 
       #
-      # @param [Hash] message Must have key :url, :depth
+      # @param message [Hash] Must have key :url, :depth
       #
       def call(message)
         key_url = message[:url]
         depth = Integer(message[:depth] || 2)
         return if depth <= 1
-        page = storage.find(key_url)
+        page = storage.read(key_url, message)
         @doc = Nokogiri::HTML(page.body)
         new_message = {
           depth: depth - 1,

data/lib/daimon_skycrawlers/queue.rb

@@ -1,8 +1,14 @@
 require "songkick_queue"
 
 module DaimonSkycrawlers
+  #
+  # Wrapper for queue configuration class
+  #
   class Queue
     class << self
+      #
+      # Configuration for queue
+      #
       def configuration
         @configuration ||= SongkickQueue.configure do |config|
           config.logger = Logger.new(STDOUT)
@@ -16,6 +22,31 @@ module DaimonSkycrawlers
         end
       end
 
+      #
+      # Configure queue
+      #
+      # ```ruby
+      # DaimonSkycrawlers::Queue.configure do |config|
+      #   config.logger = Logger.new(STDOUT)
+      #   config.host = "127.0.0.1"
+      #   config.port = 5672
+      #   # config.username = 'guest'
+      #   # config.password = 'guest'
+      #   config.vhost = "/"
+      #   config.max_reconnect_attempts = 10
+      #   config.network_recovery_interval = 1.0
+      # end
+      # ```
+      #
+      # * logger: logger instance for queue system
+      # * host: RabbitMQ host
+      # * port: RabbitMQ port
+      # * username: RabbitMQ username
+      # * passowrd: RabbitMQ password
+      # * vhost: virtual host used for connection
+      # * max_reconnect_attempts: The maximum number of reconnection attempts
+      # * network_recovery_interval: reconnection interval for TCP connection failures
+      #
       def configure
         yield configuration
       end

data/lib/daimon_skycrawlers/sitemap_parser.rb

@@ -4,15 +4,37 @@ require "zlib"
 require "uri"
 
 module DaimonSkycrawlers
+  #
+  # Parser for sitemap.xml
+  #
   # Based on https://github.com/benbalter/sitemap-parser
+  # See also https://www.sitemaps.org/
+  #
+  # ```ruby
+  # urls = ["https://example.com/sitemap.xml"]
+  # sitemap_parser = DaimonSkycrawlers::SitemapParser.new(urls)
+  # sitemap_urls = sitemap_parser.parse
+  # ```
+  #
   class SitemapParser
+    #
+    # Error class for SitemapParser
+    #
     class Error < StandardError
     end
 
-    def initialize(urls, options = {})
+    #
+    # @param urls [Array] List of sitemap.xml URL
+    #
+    def initialize(urls)
       @urls = urls
     end
 
+    #
+    # Fetch and parse sitemap.xml
+    #
+    # @return [Array] URLs in sitemap.xml
+    #
     def parse
       hydra = Typhoeus::Hydra.new(max_concurrency: 1)
       sitemap_urls = []