kudzu 1.2.0 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 232a54685462f7415227ddf7ce2efd31d0046cb5bf7756ffbffb0b473836d45d
-  data.tar.gz: e601657db196282b54e89633eebbaa4633ae1f7951fd0e7a3aa4b6ea00109d48
+  metadata.gz: 2893a15f2faff7f0697f7ce39f16a6f0e9c0972645c5e19aafb5751153432480
+  data.tar.gz: a8b9b0230be54ca43852e5b0700e1ea329258cbffcba284772f8d025ee14cbcf
 SHA512:
-  metadata.gz: 5a3dabef241ebaed7b9a80ed11abd8bbaee63e09fe674743931c4f2c36ca44e436be6e5b44ed44e68cc30899ff080faf803f5e2834074fdc3dfc9abba100248f
-  data.tar.gz: fa024d77336f25d03025d056a2d40f7c16f302658cbcf21afba5a637c9f14ab29f353c135b4c90217f6de46cbc9edb52a4bd0284982d264019762b4978b69c67
+  metadata.gz: 568b1bf560acb842ba3fa768c68078be503f271f7b1eed383ac67e8529bc568fc5ba720fc35f3c4e6973c07beede61baf39710d33ddfc13d01d75dbf14e2b090
+  data.tar.gz: 2537befd681ca91912003aad92b02ae69666afcc9fc816b330064c34547e0b94c5981d86ba369f45644b6161cb1fdbc980aea49aef7ec33e2be5a7b698597545

data/README.md

@@ -12,7 +12,7 @@ A simple web crawler for ruby.
 
 ## Dependencies
 
-* ruby 2.3+
+* ruby 2.5+
 * libicu
 
 ## Installation

data/lib/kudzu/agent/util/mime_type_detector.rb

@@ -22,13 +22,13 @@ module Kudzu
           end
 
           def from_body(body)
-            mime = MimeMagic.by_magic(StringIO.new(body))
+            mime = Marcel::Magic.by_magic(StringIO.new(body))
             mime.to_s if mime
           end
 
           def from_url(url)
             uri = Addressable::URI.parse(url)
-            mime = MimeMagic.by_path(uri.basename)
+            mime = Marcel::Magic.by_path(uri.basename)
             mime.to_s if mime
           end
         end

data/lib/kudzu/agent.rb

@@ -30,13 +30,21 @@ module Kudzu
     end
 
     def extract_refs(response)
+      return [] unless redirect_url_allowed?(response)
       refs = @url_extractor.extract(response)
       @url_filterer.filter(refs, response.url)
     end
 
     def filter_response?(response)
-      return false if response.redirect_from && !@url_filterer.allowed?(response.url, response.redirect_from)
+      return true unless redirect_url_allowed?(response)
       !@page_filterer.allowed?(response)
     end
+
+    private
+
+    def redirect_url_allowed?(response)
+      return true if response.redirect_from.nil? || response.redirect_from.empty?
+      @url_filterer.allowed?(response.url, response.redirect_from)
+    end
   end
 end

data/lib/kudzu/version.rb

@@ -1,3 +1,3 @@
 module Kudzu
-  VERSION = '1.2.0'
+  VERSION = '1.3.1'
 end

data/lib/kudzu.rb

@@ -2,8 +2,7 @@ require 'net/http'
 require 'http-cookie'
 require 'addressable'
 require 'nokogiri'
-require 'mimemagic'
-require 'mimemagic/overlay'
+require 'marcel'
 require 'charlock_holmes'
 
 require 'kudzu/version'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kudzu
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.1
 platform: ruby
 authors:
 - Yoshikazu Kaneta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-24 00:00:00.000000000 Z
+date: 2023-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -53,7 +53,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: mimemagic
+  name: marcel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec-rails
   requirement: !ruby/object:Gem::Requirement
@@ -214,7 +228,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.3.3
 signing_key: 
 specification_version: 4
 summary: A simple web crawler for ruby