kudzu 1.1.7 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: adc0bdb2f1017f8f4e8abcc2f0e6416c88b2110ce9f345957c6233484505509c
-  data.tar.gz: 49a6c3166b4d499177a144987623d17e516c4a6d84674af81d883dddbfee3b5b
+  metadata.gz: 232a54685462f7415227ddf7ce2efd31d0046cb5bf7756ffbffb0b473836d45d
+  data.tar.gz: e601657db196282b54e89633eebbaa4633ae1f7951fd0e7a3aa4b6ea00109d48
 SHA512:
-  metadata.gz: 7eba18403646beab304d92b9826bb774f4d2b2f59b365402b2aeadb3ae5e05a50f0604a5c8ba7ccfe63176db82ea85bf7759697b1069adda860a8c2e4ef56bff
-  data.tar.gz: 456b279651849f18308de4ab93cf76e4644988fdbb5ac023fa32b6d5cc163298236e374c8323227d2540854f4579d08e137caf844bf49c41235dd5a112b9dc6f
+  metadata.gz: 5a3dabef241ebaed7b9a80ed11abd8bbaee63e09fe674743931c4f2c36ca44e436be6e5b44ed44e68cc30899ff080faf803f5e2834074fdc3dfc9abba100248f
+  data.tar.gz: fa024d77336f25d03025d056a2d40f7c16f302658cbcf21afba5a637c9f14ab29f353c135b4c90217f6de46cbc9edb52a4bd0284982d264019762b4978b69c67

data/lib/kudzu/agent/fetcher.rb

@@ -83,12 +83,18 @@ module Kudzu
         Response.new(url: url,
                      status: response.code.to_i,
                      body: fetched ? response.body.to_s : nil,
-                     response_header: Hash[response.each.to_a],
+                     response_header: force_header_encoding(Hash[response.each.to_a]),
                      response_time: response_time,
                      redirect_from: redirect_from,
                      fetched: fetched)
       end
 
+      def force_header_encoding(response_header)
+        response_header.each do |key, value|
+          response_header[key] = value.force_encoding('utf-8').encode('utf-8', invalid: :replace, undef: :replace)
+        end
+      end
+
       def redirection?(code)
         code = code.to_i
         300 <= code && code <= 399

data/lib/kudzu/agent/url_extractor.rb

@@ -35,7 +35,7 @@ module Kudzu
         uri.path = uri.path.gsub(%r|/{2,}|, '/')
         uri.fragment = nil
 
-        if uri.scheme.in?(%w(http https))
+        if uri.scheme.in?(%w(http https)) && Addressable::URI.parse(uri.to_s)
           uri.to_s
         else
           nil

data/lib/kudzu/common.rb

@@ -15,7 +15,8 @@ module Kudzu
         if path.end_with?('/')
           path
         else
-          File.dirname(path) + '/'
+          dir = File.dirname(path)
+          dir.end_with?('/') ? dir : dir + '/'
         end
       end
     end

data/lib/kudzu/config.rb

@@ -5,7 +5,7 @@ module Kudzu
   class Config
     SIMPLE_CONFIGS   = [:config_file,
                         :user_agent, :thread_num, :open_timeout, :read_timeout, :keep_alive,
-                        :max_connection, :max_redirect, :max_depth, :default_request_header,
+                        :max_connection, :max_redirect, :max_depth, :max_retry, :default_request_header,
                         :politeness_delay, :handle_cookie,
                         :respect_robots_txt, :respect_nofollow, :respect_noindex,
                         :filters]
@@ -16,6 +16,7 @@ module Kudzu
                          thread_num: 1,
                          max_connection: 10,
                          max_redirect: 3,
+                         max_retry: 0,
                          politeness_delay: 0.5,
                          handle_cookie: true,
                          respect_robots_txt: true,
@@ -27,7 +28,7 @@ module Kudzu
     def initialize(config = {}, &block)
       self.filters = {}
       DEFAULT_CONFIG.merge(config).each do |key, value|
-        send("#{key}=", value)
+        send("#{key}=", value) if respond_to?("#{key}=")
       end
       if config_file || block
         delegator = Delegator.new(self)

data/lib/kudzu/crawler.rb

@@ -107,13 +107,16 @@ module Kudzu
 
     def fetch(link, request_header)
       response = nil
-      @callback.around(:fetch, link, request_header, response) do
-        response = @agent.fetch(link.url, request_header)
-      end
-      if response.fetched?
-        Kudzu.log :info, "fetched page: #{response.status} #{response.url}"
-      else
-        Kudzu.log :info, "skipped page: #{response.status} #{response.url}"
+      (@config.max_retry.to_i + 1).times do
+        @callback.around(:fetch, link, request_header, response) do
+          response = @agent.fetch(link.url, request_header)
+        end
+        if response.fetched?
+          Kudzu.log :info, "fetched page: #{response.status} #{response.url}"
+        else
+          Kudzu.log :info, "skipped page: #{response.status} #{response.url}"
+        end
+        break if !response.fetched? || response.status_success? || response.status_redirection?
       end
       response
     rescue Exception => e
@@ -135,7 +138,7 @@ module Kudzu
 
       if @config.max_depth.nil? || link.depth < @config.max_depth.to_i
         refs = @agent.extract_refs(response)
-        enqueue_links(refs_to_links(refs, link.depth + 1)) unless refs.empty?
+        enqueue_links(refs_to_links(refs, link.depth + 1), response) unless refs.empty?
       end
 
       if @agent.filter_response?(response)
@@ -168,8 +171,8 @@ module Kudzu
       end
     end
 
-    def enqueue_links(links)
-      @callback.around(:enqueue, links) do
+    def enqueue_links(links, response = nil)
+      @callback.around(:enqueue, links, response) do
         @frontier.enqueue(links)
       end
     end

data/lib/kudzu/model/link.rb

@@ -4,6 +4,14 @@ module Kudzu
       def uri
         Addressable::URI.parse(url)
       end
+
+      def status_success?
+        200 <= status && status <= 299
+      end
+
+      def status_redirection?
+        300 <= status && status <= 399
+      end
     end
   end
 end

data/lib/kudzu/version.rb

@@ -1,3 +1,3 @@
 module Kudzu
-  VERSION = '1.1.7'
+  VERSION = '1.2.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kudzu
 version: !ruby/object:Gem::Version
-  version: 1.1.7
+  version: 1.2.0
 platform: ruby
 authors:
 - Yoshikazu Kaneta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-08 00:00:00.000000000 Z
+date: 2021-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -214,8 +214,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.4
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A simple web crawler for ruby