kudzu 1.1.11 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e644c892ed66d9f00577bc01c4a47ab8c2f8650f1ad412de119efd56d3027d4a
-  data.tar.gz: afe03192b1a584ae8cf04b18404c3ff751edf5ac9a2c99289b84263891d6c187
+  metadata.gz: 232a54685462f7415227ddf7ce2efd31d0046cb5bf7756ffbffb0b473836d45d
+  data.tar.gz: e601657db196282b54e89633eebbaa4633ae1f7951fd0e7a3aa4b6ea00109d48
 SHA512:
-  metadata.gz: b57e8d06b027f03dbee49cfb93bf961a1bf7bbe62a2660da865f9aec7905c21484ffe6127840d54812e33b25f51b26229aaf01df6f0618f83d977321db7c7390
-  data.tar.gz: a39396c80a0efbfe21bc33492995c897f094d7b81c134b75bfa594a2247d980ab6b385d658a8b1a697a67cb30dbefaea92c3cfd490deb33dc00400aebadc9a9c
+  metadata.gz: 5a3dabef241ebaed7b9a80ed11abd8bbaee63e09fe674743931c4f2c36ca44e436be6e5b44ed44e68cc30899ff080faf803f5e2834074fdc3dfc9abba100248f
+  data.tar.gz: fa024d77336f25d03025d056a2d40f7c16f302658cbcf21afba5a637c9f14ab29f353c135b4c90217f6de46cbc9edb52a4bd0284982d264019762b4978b69c67

data/lib/kudzu/config.rb

@@ -5,7 +5,7 @@ module Kudzu
   class Config
     SIMPLE_CONFIGS   = [:config_file,
                         :user_agent, :thread_num, :open_timeout, :read_timeout, :keep_alive,
-                        :max_connection, :max_redirect, :max_depth, :default_request_header,
+                        :max_connection, :max_redirect, :max_depth, :max_retry, :default_request_header,
                         :politeness_delay, :handle_cookie,
                         :respect_robots_txt, :respect_nofollow, :respect_noindex,
                         :filters]
@@ -16,6 +16,7 @@ module Kudzu
                          thread_num: 1,
                          max_connection: 10,
                          max_redirect: 3,
+                         max_retry: 0,
                          politeness_delay: 0.5,
                          handle_cookie: true,
                          respect_robots_txt: true,

data/lib/kudzu/crawler.rb

@@ -107,13 +107,16 @@ module Kudzu
 
     def fetch(link, request_header)
       response = nil
-      @callback.around(:fetch, link, request_header, response) do
-        response = @agent.fetch(link.url, request_header)
-      end
-      if response.fetched?
-        Kudzu.log :info, "fetched page: #{response.status} #{response.url}"
-      else
-        Kudzu.log :info, "skipped page: #{response.status} #{response.url}"
+      (@config.max_retry.to_i + 1).times do
+        @callback.around(:fetch, link, request_header, response) do
+          response = @agent.fetch(link.url, request_header)
+        end
+        if response.fetched?
+          Kudzu.log :info, "fetched page: #{response.status} #{response.url}"
+        else
+          Kudzu.log :info, "skipped page: #{response.status} #{response.url}"
+        end
+        break if !response.fetched? || response.status_success? || response.status_redirection?
       end
       response
     rescue Exception => e
@@ -135,7 +138,7 @@ module Kudzu
 
       if @config.max_depth.nil? || link.depth < @config.max_depth.to_i
         refs = @agent.extract_refs(response)
-        enqueue_links(refs_to_links(refs, link.depth + 1)) unless refs.empty?
+        enqueue_links(refs_to_links(refs, link.depth + 1), response) unless refs.empty?
       end
 
       if @agent.filter_response?(response)
@@ -168,8 +171,8 @@ module Kudzu
       end
     end
 
-    def enqueue_links(links)
-      @callback.around(:enqueue, links) do
+    def enqueue_links(links, response = nil)
+      @callback.around(:enqueue, links, response) do
         @frontier.enqueue(links)
       end
     end

data/lib/kudzu/model/link.rb

@@ -4,6 +4,14 @@ module Kudzu
       def uri
         Addressable::URI.parse(url)
       end
+
+      def status_success?
+        200 <= status && status <= 299
+      end
+
+      def status_redirection?
+        300 <= status && status <= 399
+      end
     end
   end
 end

data/lib/kudzu/version.rb

@@ -1,3 +1,3 @@
 module Kudzu
-  VERSION = '1.1.11'
+  VERSION = '1.2.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kudzu
 version: !ruby/object:Gem::Version
-  version: 1.1.11
+  version: 1.2.0
 platform: ruby
 authors:
 - Yoshikazu Kaneta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-17 00:00:00.000000000 Z
+date: 2021-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -214,7 +214,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A simple web crawler for ruby