kudzu 1.1.9 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1235e19cd09b27f161411c47d33ef361732539bbd1ae1fa801fa764972590b0a
-  data.tar.gz: 366f376338f82363e2f53b6bc5d0d2d89e695581e6dcc245e60893613334dcc1
+  metadata.gz: 16db7469597ff96bff30c59c9b6da73e8c4d33b535228424675c7e7a539e4ec2
+  data.tar.gz: ee0eb79b8e5c3e4e3d3832ed1f8e9d0dc291c60f3d626495eb21055e5e23341e
 SHA512:
-  metadata.gz: 37d53924ac944bab707c73f5bb698ecfb267f273e33eda38b7f13ee01fe667b35fc6c4941c0d7864fdbe96fd80cdfe00b5699abe4a9749992cf5c426edafeade
-  data.tar.gz: d6cd02bbd42b4be05070f1ba5a76f5a2220a633ae5184118cd74c6c753707bda6fa7e3f70935a19462e25295edbecdd4ab9657094e208ae3f8d14c5313a1e12a
+  metadata.gz: 6d3a060a674cbd7ec9de94db1d94dcb9c0e6eac1f0b6bb20a798547b8eb3f29a69c0d404ca45dd7efc7f11d91de2618ac214f0d316cbe58fd69276878e37a9bf
+  data.tar.gz: b423624bc1e7ea849a98cbea9d34b8a1dcfd22f5fd7bee8ce0791e900ac805d2d448776d0a4b7cb1e0be9a9ef033f008700cf16aed62ffa287bccacbb1db0db4

data/README.md

@@ -12,7 +12,7 @@ A simple web crawler for ruby.
 
 ## Dependencies
 
-* ruby 2.3+
+* ruby 2.5+
 * libicu
 
 ## Installation

data/lib/kudzu/agent/url_extractor.rb

@@ -35,7 +35,7 @@ module Kudzu
         uri.path = uri.path.gsub(%r|/{2,}|, '/')
         uri.fragment = nil
 
-        if uri.scheme.in?(%w(http https))
+        if uri.scheme.in?(%w(http https)) && Addressable::URI.parse(uri.to_s)
           uri.to_s
         else
           nil

data/lib/kudzu/agent/util/mime_type_detector.rb

@@ -22,13 +22,13 @@ module Kudzu
           end
 
           def from_body(body)
-            mime = MimeMagic.by_magic(StringIO.new(body))
+            mime = Marcel::Magic.by_magic(StringIO.new(body))
             mime.to_s if mime
           end
 
           def from_url(url)
             uri = Addressable::URI.parse(url)
-            mime = MimeMagic.by_path(uri.basename)
+            mime = Marcel::Magic.by_path(uri.basename)
             mime.to_s if mime
           end
         end

data/lib/kudzu/config.rb

@@ -5,7 +5,7 @@ module Kudzu
   class Config
     SIMPLE_CONFIGS   = [:config_file,
                         :user_agent, :thread_num, :open_timeout, :read_timeout, :keep_alive,
-                        :max_connection, :max_redirect, :max_depth, :default_request_header,
+                        :max_connection, :max_redirect, :max_depth, :max_retry, :default_request_header,
                         :politeness_delay, :handle_cookie,
                         :respect_robots_txt, :respect_nofollow, :respect_noindex,
                         :filters]
@@ -16,6 +16,7 @@ module Kudzu
                          thread_num: 1,
                          max_connection: 10,
                          max_redirect: 3,
+                         max_retry: 0,
                          politeness_delay: 0.5,
                          handle_cookie: true,
                          respect_robots_txt: true,
@@ -27,7 +28,7 @@ module Kudzu
     def initialize(config = {}, &block)
       self.filters = {}
       DEFAULT_CONFIG.merge(config).each do |key, value|
-        send("#{key}=", value)
+        send("#{key}=", value) if respond_to?("#{key}=")
       end
       if config_file || block
         delegator = Delegator.new(self)

data/lib/kudzu/crawler.rb

@@ -107,13 +107,16 @@ module Kudzu
 
     def fetch(link, request_header)
       response = nil
-      @callback.around(:fetch, link, request_header, response) do
-        response = @agent.fetch(link.url, request_header)
-      end
-      if response.fetched?
-        Kudzu.log :info, "fetched page: #{response.status} #{response.url}"
-      else
-        Kudzu.log :info, "skipped page: #{response.status} #{response.url}"
+      (@config.max_retry.to_i + 1).times do
+        @callback.around(:fetch, link, request_header, response) do
+          response = @agent.fetch(link.url, request_header)
+        end
+        if response.fetched?
+          Kudzu.log :info, "fetched page: #{response.status} #{response.url}"
+        else
+          Kudzu.log :info, "skipped page: #{response.status} #{response.url}"
+        end
+        break if !response.fetched? || response.status_success? || response.status_redirection?
       end
       response
     rescue Exception => e
@@ -135,7 +138,7 @@ module Kudzu
 
       if @config.max_depth.nil? || link.depth < @config.max_depth.to_i
         refs = @agent.extract_refs(response)
-        enqueue_links(refs_to_links(refs, link.depth + 1)) unless refs.empty?
+        enqueue_links(refs_to_links(refs, link.depth + 1), response) unless refs.empty?
       end
 
       if @agent.filter_response?(response)
@@ -168,8 +171,8 @@ module Kudzu
       end
     end
 
-    def enqueue_links(links)
-      @callback.around(:enqueue, links) do
+    def enqueue_links(links, response = nil)
+      @callback.around(:enqueue, links, response) do
         @frontier.enqueue(links)
       end
     end

data/lib/kudzu/model/link.rb

@@ -4,6 +4,14 @@ module Kudzu
       def uri
         Addressable::URI.parse(url)
       end
+
+      def status_success?
+        200 <= status && status <= 299
+      end
+
+      def status_redirection?
+        300 <= status && status <= 399
+      end
     end
   end
 end

data/lib/kudzu/version.rb

@@ -1,3 +1,3 @@
 module Kudzu
-  VERSION = '1.1.9'
+  VERSION = '1.3.0'
 end

data/lib/kudzu.rb

@@ -2,8 +2,7 @@ require 'net/http'
 require 'http-cookie'
 require 'addressable'
 require 'nokogiri'
-require 'mimemagic'
-require 'mimemagic/overlay'
+require 'marcel'
 require 'charlock_holmes'
 
 require 'kudzu/version'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kudzu
 version: !ruby/object:Gem::Version
-  version: 1.1.9
+  version: 1.3.0
 platform: ruby
 authors:
 - Yoshikazu Kaneta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-27 00:00:00.000000000 Z
+date: 2022-01-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -53,7 +53,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: mimemagic
+  name: marcel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec-rails
   requirement: !ruby/object:Gem::Requirement
@@ -214,8 +228,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.3.3
 signing_key: 
 specification_version: 4
 summary: A simple web crawler for ruby