daimon_skycrawlers 0.12.0 → 1.0.0.pre.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ea06b539d343f1a13e2eb22f3cd2e9662ce3d9da
-  data.tar.gz: 39164028e5902f05a86a4c1e1f1890bbe8065dfa
+  metadata.gz: c9a394115d873b122e18a6970e62dfbc897dd690
+  data.tar.gz: 55fc07c13fa04c82c5e1c6a1075ffd97793b01f9
 SHA512:
-  metadata.gz: 22472b47e7a5e87758c2bf8712bd76d227e7c6589afd641060e93e8265fdb628abf8a236dbcf1dab0dc6390f0fd6fb680e96ff4998c4ba479f4a5ff334b15a4f
-  data.tar.gz: 8bdf2df5606221dcb984ab3850b8922f524dfbaf12d80cb38c11f65632327d0567b6c17aaff7a51da47e77e88fa27e360045dbdf38af2f3dc02cec3384f15090
+  metadata.gz: 5ae484de037763f04487ad8c1ea4047a78a726156781cffeebf0c4c14ae0dfb70029b5f834c1414af9ee43c2b67fb7b137e5d5e335c2dee01abe9aae6ced1099
+  data.tar.gz: 63c85408015248ca66448025b2eb8ff6b65080c3da375b2e5b9723e944e3836f73285cb083088a31c1c8d60fe6176f57c47bc2d894fc7618c5853b3aebc3d500

data/.travis.yml

@@ -3,7 +3,8 @@ sudo: false
 cache: bundler
 rvm:
   - 2.2.5
-  - 2.3.1
+  - 2.3.3
+  - 2.4.0
 
 before_install: gem update bundler
 

data/README.md

@@ -1,5 +1,3 @@
-## Caution!! This product is NOT production-ready.
-
 # DaimonSkycrawlers
 
 [![Gem Version](https://badge.fury.io/rb/daimon_skycrawlers.svg)](https://badge.fury.io/rb/daimon_skycrawlers)

data/Rakefile

@@ -6,7 +6,9 @@ require "daimon_skycrawlers/tasks"
 Rake::TestTask.new(:test) do |t|
   t.libs << "test"
   t.libs << "lib"
-  t.test_files = FileList["test/**/*_test.rb"]
+  t.test_files = FileList["test/**/test_*.rb"]
+  t.verbose = false
+  t.warning = false
 end
 
 task :default => [:test]

data/daimon_skycrawlers.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "webrobots"
 
   spec.add_development_dependency "bundler", "~> 1.11"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "test-unit"
   spec.add_development_dependency "test-unit-rr"
   spec.add_development_dependency "test-unit-notify"

data/lib/daimon_skycrawlers/callbacks.rb

@@ -3,6 +3,7 @@ module DaimonSkycrawlers
     def initialize
       super
       @before_process_callbacks = []
+      @after_process_callbacks = []
     end
 
     def before_process(callback = nil, &block)
@@ -22,5 +23,23 @@ module DaimonSkycrawlers
     def clear_before_process_callbacks
       @before_process_callbacks = []
     end
+
+    def after_process(callback = nil, &block)
+      if block_given?
+        @after_process_callbacks << block
+      else
+        @after_process_callbacks << callback if callback.respond_to?(:call)
+      end
+    end
+
+    def run_after_callbacks(message)
+      @after_process_callbacks.each do |callback|
+        callback.call(message)
+      end
+    end
+
+    def clear_after_process_callbacks
+      @after_process_callbacks = []
+    end
   end
 end

data/lib/daimon_skycrawlers/commands/enqueue.rb

@@ -3,6 +3,7 @@ require "daimon_skycrawlers/crawler"
 require "daimon_skycrawlers/processor"
 require "daimon_skycrawlers/sitemap_parser"
 require "daimon_skycrawlers/version"
+require "thor"
 require "webrobots"
 
 module DaimonSkycrawlers
@@ -47,6 +48,45 @@ module DaimonSkycrawlers
         end
       end
 
+      desc "list PATH", "Enqueue URLs from PATH. PATH content includes a URL per line"
+      method_option("type", aliases: ["-t"], type: :string, default: "url", desc: "Specify type for URLs")
+      def list(path)
+        load_init
+        File.open(path, "r") do |file|
+          file.each_line do |line|
+            line.chomp!
+            next if /\A#/ =~ line
+            case options["type"]
+            when "response"
+              DaimonSkycrawlers::Processor.enqueue_http_response(line)
+            when "url"
+              DaimonSkycrawlers::Crawler.enqueue_url(line)
+            else
+              raise ArgumentError, "Unknown type: #{options["type"]}"
+            end
+          end
+        end
+      end
+
+      desc "yaml PATH", "Enqueue URLs from PATH."
+      method_option("type", aliases: ["-t"], type: :string, default: "url", desc: "Specify type for URLs")
+      def yaml(path)
+        load_init
+        YAML.load_file(path).each do |hash|
+          url = hash["url"]
+          message = hash["message"] || {}
+          raise "Could not find URL: #{hash}" unless url
+          case options["type"]
+          when "response"
+            DaimonSkycrawlers::Processor.enqueue_http_response(url, message)
+          when "url"
+            DaimonSkycrawlers::Crawler.enqueue_url(url, message)
+          else
+            raise ArgumentError, "Unknown type: #{options["type"]}"
+          end
+        end
+      end
+
       private
 
       def load_init

data/lib/daimon_skycrawlers/commands/runner.rb

@@ -11,7 +11,7 @@ module DaimonSkycrawlers
       def crawler
         load_init
         load_crawlers
-        require(File.expand_path("app/crawler.rb", Dri.pwd))
+        require(File.expand_path("app/crawler.rb", Dir.pwd))
         DaimonSkycrawlers::Crawler.run
       rescue => ex
         puts ex.message

data/lib/daimon_skycrawlers/crawler/base.rb

@@ -1,5 +1,6 @@
 require "uri"
 require "faraday"
+require "typhoeus/adapters/faraday"
 
 require "daimon_skycrawlers/logger"
 require "daimon_skycrawlers/config"
@@ -10,6 +11,8 @@ require "daimon_skycrawlers/processor"
 require "daimon_skycrawlers/filter/update_checker"
 require "daimon_skycrawlers/filter/robots_txt_checker"
 
+Faraday.default_adapter = :typhoeus
+
 module DaimonSkycrawlers
   module Crawler
     #
@@ -88,10 +91,10 @@ module DaimonSkycrawlers
         @n_processed_urls += 1
 
         setup_default_filters
+        setup_default_post_processes
 
         proceeding = run_before_callbacks(message)
         unless proceeding
-          @skipped = true
           skip(message[:url])
           return
         end
@@ -101,7 +104,10 @@ module DaimonSkycrawlers
         url = (URI(connection.url_prefix) + url).to_s
 
         @prepare.call(connection)
-        fetch(url, message, &block)
+        response = fetch(url, message, &block)
+        data = { url: url, message: message, response: response }
+        run_after_callbacks(data)
+        data
       end
 
       def fetch(path, message = {})
@@ -137,8 +143,17 @@ module DaimonSkycrawlers
         end
       end
 
+      def setup_default_post_processes
+        after_process do |data|
+          storage.save(data)
+          message = data[:message]
+          url = data[:url]
+          schedule_to_process(url, message)
+        end
+      end
+
       def skip(url)
-        log.info("Skip #{url}")
+        log.info("Skipped '#{url}' by '#{self.class}'")
         @skipped = true
         schedule_to_process(url.to_s, heartbeat: true)
       end

data/lib/daimon_skycrawlers/crawler/default.rb

@@ -9,13 +9,13 @@ module DaimonSkycrawlers
     #
     class Default < Base
       def fetch(url, message)
-        response = get(url)
-        data = [url.to_s, response.headers, response.body]
-
-        yield(*data) if block_given?
-
-        storage.save(*data)
-        schedule_to_process(url.to_s, message)
+        params = message[:params] || {}
+        method = message[:method] || "GET"
+        if method == "POST"
+          post(url, params)
+        else
+          get(url, params)
+        end
       end
     end
   end

data/lib/daimon_skycrawlers/generator/new.rb

@@ -39,6 +39,7 @@ module DaimonSkycrawlers
         }
         invoke(MigrationGenerator, [
                  "CreatePages",
+                 "key:string",
                  "url:string",
                  "headers:text",
                  "body:binary",
@@ -56,6 +57,8 @@ module DaimonSkycrawlers
             <<-CODE.chomp
 #{indent}t.timestamps
 
+#{indent}t.index [:key]
+#{indent}t.index [:key, :updated_at]
 #{indent}t.index [:url]
 #{indent}t.index [:url, :updated_at]
             CODE
@@ -95,7 +98,7 @@ module DaimonSkycrawlers
 
       def display_post_message
         puts <<MESSAGE
-Check .env and .env.db before run `docker-compose build` or `docker-compose up`.
+Check .env and .env.db before running `docker-compose build` or `docker-compose up`.
 MESSAGE
       end
     end

data/lib/daimon_skycrawlers/generator/processor.rb

@@ -24,7 +24,7 @@ module DaimonSkycrawlers
 You can register your processor in `app/processor.rb` to run your processor.
 Following code snippet is useful:
 
-    processor = #{naem.classify}.new
+    processor = #{name.classify}.new
     DaimonSkycrawlers.register_processor(processor)
 
 MESSAGE

data/lib/daimon_skycrawlers/processor/base.rb

@@ -11,9 +11,23 @@ module DaimonSkycrawlers
       include DaimonSkycrawlers::Callbacks
       include DaimonSkycrawlers::Configurable
 
+      def initialize
+        super
+        @skipped = false
+      end
+
+      def skipped?
+        @skipped
+      end
+
       def process(message)
+        @skipped = false
+        setup_default_filters
         proceeding = run_before_callbacks(message)
-        return unless proceeding
+        unless proceeding
+          skip(message[:url])
+          return
+        end
         call(message)
       end
 
@@ -24,6 +38,19 @@ module DaimonSkycrawlers
       def storage
         @storage ||= DaimonSkycrawlers::Storage::RDB.new
       end
+
+      private
+
+      def setup_default_filters
+        before_process do |m|
+          !m[:heartbeat]
+        end
+      end
+
+      def skip(url)
+        log.info("Skipped '#{url}' by '#{self.class}'")
+        @skipped = true
+      end
     end
   end
 end

data/lib/daimon_skycrawlers/processor/default.rb

@@ -5,7 +5,6 @@ module DaimonSkycrawlers
   module Processor
     class Default < Base
       def call(message)
-        return if message[:heartbeat]
         url = message[:url]
         page = storage.find(url)
         headers = JSON.parse(page.headers)

data/lib/daimon_skycrawlers/processor/spider.rb

@@ -101,7 +101,6 @@ module DaimonSkycrawlers
       def call(message)
         key_url = message[:url]
         depth = Integer(message[:depth] || 2)
-        return if message[:heartbeat]
         return if depth <= 1
         page = storage.find(key_url)
         @doc = Nokogiri::HTML(page.body)

data/lib/daimon_skycrawlers/sitemap_parser.rb

@@ -6,6 +6,9 @@ require "uri"
 module DaimonSkycrawlers
   # Based on https://github.com/benbalter/sitemap-parser
   class SitemapParser
+    class Error < StandardError
+    end
+
     def initialize(urls, options = {})
       @urls = urls
     end
@@ -14,7 +17,8 @@ module DaimonSkycrawlers
       hydra = Typhoeus::Hydra.new(max_concurrency: 1)
       sitemap_urls = []
       @urls.each do |url|
-        if URI(url).scheme.start_with?("http")
+        uri = URI(url)
+        if uri.scheme && uri.scheme.start_with?("http")
           request = Typhoeus::Request.new(url, followlocation: true)
           request.on_complete do |response|
             sitemap_urls.concat(on_complete(response))
@@ -22,18 +26,21 @@ module DaimonSkycrawlers
           hydra.queue(request)
         else
           if File.exist?(url)
-            extract_urls(File.read(url))
+            sitemap_urls.concat(extract_urls(File.read(url)))
           end
         end
       end
-      hydra.run
+      loop do
+        hydra.run
+        break if hydra.queued_requests.empty?
+      end
       sitemap_urls
     end
 
     private
 
     def on_complete(response)
-      raise "HTTP requset to #{response.effective_url} failed" unless response.success?
+      raise Error, "HTTP requset to #{response.effective_url} failed. status: #{response.code}" unless response.success?
       raw_sitemap = inflate_response(response)
       extract_urls(raw_sitemap)
     end
@@ -51,7 +58,7 @@ module DaimonSkycrawlers
           url.at("loc").content
         end
       else
-        raise "Malformed sitemap.xml no <sitemapindex> or <urlset>"
+        raise Error, "Malformed sitemap.xml no <sitemapindex> or <urlset>"
       end
     end
 
@@ -72,7 +79,7 @@ module DaimonSkycrawlers
       when "deflate", "gzip", "x-gzip"
         true
       else
-        signature = response.body[0, 2]
+        signature = response.body[0, 2].b
         signature == "\x1F\x8B".b
       end
     end

data/lib/daimon_skycrawlers/storage/base.rb

@@ -7,7 +7,7 @@ module DaimonSkycrawlers
       include DaimonSkycrawlers::LoggerMixin
       include DaimonSkycrawlers::ConfigMixin
 
-      def save(url, headers, body)
+      def save(data)
         raise "Implement this in subclass"
       end
 

data/lib/daimon_skycrawlers/storage/file.rb

@@ -11,35 +11,46 @@ module DaimonSkycrawlers
         @base_dir = Pathname(base_dir)
       end
 
-      def save(url, headers, body)
+      def save(data)
+        url = data[:url]
+        message = data[:message]
+        key = message[:key]
+        response = data[:response]
+        headers = response.headers
+        body = response.body
         @base_dir.mkpath
-        body_path(url).dirname.mkpath
-        body_path(url).open("wb+") do |file|
+        body_path(url, key).dirname.mkpath
+        body_path(url, key).open("wb+") do |file|
           file.write(body)
         end
-        headers_path(url).open("wb+") do |file|
+        headers_path(url, key).open("wb+") do |file|
           file.write(JSON.generate(headers))
         end
       end
 
-      def read(url)
-        headers = JSON.parse(headers_path(url).read)
-        body = body_path(url).read
-        Page.new(url, headers, body, headers["last-modified"], headers["etag"])
+      def read(url, message)
+        key = message[:key]
+        headers = JSON.parse(headers_path(url, key).read)
+        body = body_path(url, key).read
+        Page.new(url, key, headers, body, headers["last-modified"], headers["etag"])
       end
 
-      Page = Struct.new(:url, :headers, :body, :last_modified, :etag)
+      Page = Struct.new(:url, :key, :headers, :body, :last_modified, :etag)
 
       private
 
-      def body_path(url)
+      def body_path(url, key)
         url = URI(url)
-        @base_dir + ".#{url.path}"
+        if key
+          @base_dir + key
+        else
+          @base_dir + ".#{url.path}"
+        end
       end
 
-      def headers_path(url)
+      def headers_path(url, key)
         url = URI(url)
-        Pathname("#{body_path(url)}-headers.json")
+        Pathname("#{body_path(url, key)}-headers.json")
       end
     end
   end

data/lib/daimon_skycrawlers/storage/null.rb

@@ -10,7 +10,7 @@ module DaimonSkycrawlers
       #
       # Save nothing
       #
-      def save(url, headers, body)
+      def save(data)
       end
 
       #

data/lib/daimon_skycrawlers/storage/rdb.rb

@@ -16,12 +16,20 @@ module DaimonSkycrawlers
       #
       # Save
       #
-      # @param [String] url identity of the page
-      # @param [Hash] header of URL
-      # @param [String] body
+      # @param [Hash] data has following keys
+      #   * :url: URL
+      #   * :message: Given message
+      #   * :response: HTTP response
       #
-      def save(url, headers, body)
+      def save(data)
+        url = data[:url]
+        message = data[:message]
+        key = message[:key] || url
+        response = data[:response]
+        headers = response.headers
+        body = response.body
         Page.create(url: url,
+                    key: key,
                     headers: JSON.generate(headers),
                     body: body,
                     last_modified_at: headers["last-modified"],
@@ -33,8 +41,13 @@ module DaimonSkycrawlers
       #
       # @param [String] url identity of the page
       #
-      def find(url)
-        Page.where(url: url).order(updated_at: :desc).limit(1).first
+      def find(url, message = {})
+        key = message[:key]
+        if key
+          Page.where(key: key).order(updated_at: :desc).limit(1).first
+        else
+          Page.where(url: url).order(updated_at: :desc).limit(1).first
+        end
       end
 
       class Base < ActiveRecord::Base

data/lib/daimon_skycrawlers/timer.rb

@@ -8,7 +8,11 @@ module DaimonSkycrawlers
     def setup_shutdown_timer(queue_name_prefix, interval: 10)
       timers = Timers::Group.new
       timer = timers.after(interval) do
-        Process.kill(:INT, 0)
+        if block_given?
+          yield
+        else
+          Process.kill(:INT, 0)
+        end
       end
       Thread.new(timers) do |t|
         loop { t.wait }
@@ -19,6 +23,7 @@ module DaimonSkycrawlers
         DaimonSkycrawlers.configuration.logger.debug("Reset timer: consume message #{queue_name}")
         timer.reset
       end
+      timers
     end
   end
 end

data/lib/daimon_skycrawlers/version.rb

@@ -1,3 +1,3 @@
 module DaimonSkycrawlers
-  VERSION = "0.12.0"
+  VERSION = "1.0.0-rc1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: daimon_skycrawlers
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 1.0.0.pre.rc1
 platform: ruby
 authors:
 - daimon developers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-20 00:00:00.000000000 Z
+date: 2017-01-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -184,14 +184,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -442,12 +442,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.4
+rubygems_version: 2.6.8
 signing_key: 
 specification_version: 4
 summary: This is a crawler framework.