kudzu 1.1.6 → 1.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8aa1feb6080a9006dc33afd5af9bd4428c22a80d465c9b80410e1734719ba1d5
-  data.tar.gz: 9f293ed4f43141e6250600da8f90dd900d73ebed7c53dbb225ffea5e6fb5dd5d
+  metadata.gz: adc0bdb2f1017f8f4e8abcc2f0e6416c88b2110ce9f345957c6233484505509c
+  data.tar.gz: 49a6c3166b4d499177a144987623d17e516c4a6d84674af81d883dddbfee3b5b
 SHA512:
-  metadata.gz: 7d3012bd8053cc69337a3e1ac5c68740f8406ae8fb38239ac5e7749b017b1a10bdf3b3e2271cd9e873073be021a514b97fd245ea636d67e27c9051580a7a4ffe
-  data.tar.gz: 3abe96edc58f1c94a8dfeba9959f86fa7b602d4f7fe5b9c55acf2cb10dadd36e5997fa2a7075ac8bce0673460f9c67cc0c0bb94585c70070937ed0ca428e942b
+  metadata.gz: 7eba18403646beab304d92b9826bb774f4d2b2f59b365402b2aeadb3ae5e05a50f0604a5c8ba7ccfe63176db82ea85bf7759697b1069adda860a8c2e4ef56bff
+  data.tar.gz: 456b279651849f18308de4ab93cf76e4644988fdbb5ac023fa32b6d5cc163298236e374c8323227d2540854f4579d08e137caf844bf49c41235dd5a112b9dc6f

data/lib/kudzu/agent/util/mime_type_detector.rb

@@ -22,13 +22,13 @@ module Kudzu
           end
 
           def from_body(body)
-            mime = MIME.check_magics(StringIO.new(body))
+            mime = MimeMagic.by_magic(StringIO.new(body))
             mime.to_s if mime
           end
 
           def from_url(url)
             uri = Addressable::URI.parse(url)
-            mime = MIME.check_globs(uri.basename)
+            mime = MimeMagic.by_path(uri.basename)
             mime.to_s if mime
           end
         end

data/lib/kudzu/model/page.rb

@@ -91,9 +91,9 @@ module Kudzu
       private
 
       def decode_body(body)
-        if text?
-          if find_encoding
-            body.dup.force_encoding(charset).encode('utf-8', invalid: :replace, undef: :replace)
+        if body && text?
+          if enc = find_encoding(body)
+            body.dup.force_encoding(enc).encode('utf-8', invalid: :replace, undef: :replace)
           else
             body.dup.encode('utf-8', invalid: :replace, undef: :replace)
           end
@@ -102,10 +102,22 @@ module Kudzu
         end
       end
 
-      def find_encoding
-        Encoding.find(charset)
-      rescue
-        nil
+      def find_encoding(body)
+        begin
+          enc = Encoding.find(charset)
+        rescue ArgumentError
+          return nil
+        end
+
+        if enc == Encoding::Shift_JIS
+          Encoding::CP932
+        elsif enc == Encoding::EUC_JP
+          require 'nkf'
+          guessed = NKF.guess(body)
+          [Encoding::EUCJP_MS, Encoding::CP51932].include?(guessed) ? guessed : enc
+        else
+          enc
+        end
       end
     end
   end

data/lib/kudzu/version.rb

@@ -1,3 +1,3 @@
 module Kudzu
-  VERSION = '1.1.6'
+  VERSION = '1.1.7'
 end

data/lib/kudzu.rb

@@ -2,7 +2,8 @@ require 'net/http'
 require 'http-cookie'
 require 'addressable'
 require 'nokogiri'
-require 'shared-mime-info'
+require 'mimemagic'
+require 'mimemagic/overlay'
 require 'charlock_holmes'
 
 require 'kudzu/version'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kudzu
 version: !ruby/object:Gem::Version
-  version: 1.1.6
+  version: 1.1.7
 platform: ruby
 authors:
 - Yoshikazu Kaneta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-20 00:00:00.000000000 Z
+date: 2018-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -53,21 +53,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: shared-mime-info
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: mime-types
+  name: mimemagic
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="