kudzu 1.3.0 → 1.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 16db7469597ff96bff30c59c9b6da73e8c4d33b535228424675c7e7a539e4ec2
-  data.tar.gz: ee0eb79b8e5c3e4e3d3832ed1f8e9d0dc291c60f3d626495eb21055e5e23341e
+  metadata.gz: a481432d23d78a3eb20587d82f440dc0e3e7375ad605ac8b7afc9f65250ba19f
+  data.tar.gz: e45cf1a70ca8f151b3e9b1298d33e07e5101a939aedea5aa959dfce3309bdcba
 SHA512:
-  metadata.gz: 6d3a060a674cbd7ec9de94db1d94dcb9c0e6eac1f0b6bb20a798547b8eb3f29a69c0d404ca45dd7efc7f11d91de2618ac214f0d316cbe58fd69276878e37a9bf
-  data.tar.gz: b423624bc1e7ea849a98cbea9d34b8a1dcfd22f5fd7bee8ce0791e900ac805d2d448776d0a4b7cb1e0be9a9ef033f008700cf16aed62ffa287bccacbb1db0db4
+  metadata.gz: 4e31a988ca6f30ba60d4832c65376b62f3e285d3204da35ded55d52a4bbf64c9cef8a89b59e2653832f9369af7b57842d30fbf03d8f0003e2efdb0be3ac46b1e
+  data.tar.gz: a950a0462839e0b8b115f3b46fd267b55d56001c151cc4893f2c00d17910ee5a5c4cc109d70023d95bbabd89df354e0cd6167010c5b60491121a9386316aa09e

data/lib/kudzu/adapter/memory/frontier.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   module Adapter
     module Memory

data/lib/kudzu/adapter/memory/model/link.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   module Adapter
     module Memory

data/lib/kudzu/adapter/memory/model/page.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   module Adapter
     module Memory

data/lib/kudzu/adapter/memory/repository.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   module Adapter
     module Memory

data/lib/kudzu/adapter/memory.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 Dir[File.join(__dir__, 'memory/**/*.rb')].each do |file|
   require_relative file
 end

data/lib/kudzu/agent/all.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 Dir[File.join(__dir__, '**/*.rb')].each do |file|
   require_relative file
 end

data/lib/kudzu/agent/fetcher.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   class Agent
     class Fetcher

data/lib/kudzu/agent/http/connection.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   class Agent
     class Http

data/lib/kudzu/agent/http/connection_pool.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   class Agent
     class Http

data/lib/kudzu/agent/page_filterer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   class Agent
     class PageFilterer

data/lib/kudzu/agent/reference.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   class Agent
     class Reference < Kudzu::Model::Base

data/lib/kudzu/agent/response.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   class Agent
     class Response < Kudzu::Model::Base

data/lib/kudzu/agent/robots/parser.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   class Agent
     class Robots

data/lib/kudzu/agent/robots/txt.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   class Agent
     class Robots

data/lib/kudzu/agent/robots.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   class Agent
     class Robots

data/lib/kudzu/agent/sleeper.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   class Agent
     class Sleeper

data/lib/kudzu/agent/url_extractor.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   class Agent
     class UrlExtractor

data/lib/kudzu/agent/url_filterer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   class Agent
     class UrlFilterer

data/lib/kudzu/agent/util/charset_detector.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   class Agent
     class Util

data/lib/kudzu/agent/util/content_type_parser.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   class Agent
     class Util

data/lib/kudzu/agent/util/matcher.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   class Agent
     class Util

data/lib/kudzu/agent/util/mime_type_detector.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   class Agent
     class Util

data/lib/kudzu/agent/util/title_parser.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   class Agent
     class Util

data/lib/kudzu/agent.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'agent/all'
 
 module Kudzu
@@ -30,13 +32,21 @@ module Kudzu
     end
 
     def extract_refs(response)
+      return [] unless redirect_url_allowed?(response)
       refs = @url_extractor.extract(response)
       @url_filterer.filter(refs, response.url)
     end
 
     def filter_response?(response)
-      return false if response.redirect_from && !@url_filterer.allowed?(response.url, response.redirect_from)
+      return true unless redirect_url_allowed?(response)
       !@page_filterer.allowed?(response)
     end
+
+    private
+
+    def redirect_url_allowed?(response)
+      return true if response.redirect_from.nil? || response.redirect_from.empty?
+      @url_filterer.allowed?(response.url, response.redirect_from)
+    end
   end
 end

data/lib/kudzu/callback.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   class Callback
     CALLBACKS = [:on_success,      # 2xx

data/lib/kudzu/common.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   class Common
     class << self

data/lib/kudzu/config/filter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   class Config
     class Filter

data/lib/kudzu/config.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'ostruct'
 require_relative 'config/filter'
 

data/lib/kudzu/crawler.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'model/all'
 require_relative 'adapter/memory'
 require_relative 'agent'

data/lib/kudzu/model/all.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 Dir[File.join(__dir__, '*.rb')].each do |file|
   require_relative file
 end

data/lib/kudzu/model/base.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   module Model
     class Base

data/lib/kudzu/model/link.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   module Model
     module Link

data/lib/kudzu/model/page.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   module Model
     module Page

data/lib/kudzu/thread_pool.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
   class ThreadPool
     def initialize(size)

data/lib/kudzu/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Kudzu
-  VERSION = '1.3.0'
+  VERSION = '1.3.2'
 end

data/lib/kudzu.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'net/http'
 require 'http-cookie'
 require 'addressable'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kudzu
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.3.2
 platform: ruby
 authors:
 - Yoshikazu Kaneta
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-01-17 00:00:00.000000000 Z
+date: 2024-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -213,7 +213,7 @@ homepage: https://github.com/kanety/kudzu
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -229,7 +229,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.3.3
-signing_key: 
+signing_key:
 specification_version: 4
 summary: A simple web crawler for ruby
 test_files: []