web_loader 1.3.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 20b0243bdaa4253679016dcb4d6d9635e1369a5839e35acc68029b565da1658f
-  data.tar.gz: 5493a45ead30c5a64a0a12f19cfe8e6a93f782d3f913b9e8ef61268f4a2d3f14
+  metadata.gz: 25257053739d26c811dc6fe61337deb27582549fcade84cc94da4a2b841cc399
+  data.tar.gz: 5f1fc1172b8252bf552caa9131e495065c5420d8773c9ce944843999c914df4f
 SHA512:
-  metadata.gz: 83f3217dd671d4c61e748433f363526fd12caa3eacd653a0ff93d34f14dd9e64b16e9d81cc6ade637797ccacb21415beabd99649cb1f052d29764a4e1b9285e4
-  data.tar.gz: 061061d36f771d8d9170f73f0d029854f73363e38478ee0b9502908770ccc57864ae056cc01fa82a2b33796a602b8a5501ff051645015dcb582472dd24152be8
+  metadata.gz: 31d4b8dc8fb95aaa1585e4009183b4a2ac4612aa3d4fe3cf65f8f369340bdf375a3ee410ca4e26d9471d31272f8fc40447e2adccd633b6f8a7000be23f87111e
+  data.tar.gz: e4a6b5b6fee9a8a9f3a971c50fd1345a09674544c90e2ec063290e8cac067e38374f19f2395377cc84bbb8649f107dea64c6a9484dde9a67c5a9c3fa79aaece6

data/Gemfile.lock

@@ -1,12 +1,12 @@
 PATH
   remote: .
   specs:
-    web_loader (1.3.0)
+    web_loader (1.4.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    minitest (5.22.0)
+    minitest (5.22.2)
     rake (13.1.0)
 
 PLATFORMS

data/lib/web_loader/command.rb

@@ -87,10 +87,11 @@ module WebLoader
           encoding = response.type_params['charset']
           body = toutf8(body, encoding)
         end
-        if @use_cache
-          log("Write cache: #{url}", @verbose)
-          Cache.write(@cache_dir, url, response.code, body)
-        end
+
+#        if @use_cache
+        log("Write cache: #{url}", @verbose)
+        Cache.write(@cache_dir, url, response.code, body)
+        #        end
         result = body
       when Net::HTTPRedirection
         result = load(to_redirect_url(uri, response['location']), redirect_count - 1)

data/lib/web_loader/utils.rb

@@ -2,9 +2,30 @@ module WebLoader
   module Utils
     UTF_8 = 'UTF-8'
 
+    def detect_charset(str)
+      # charsetが指定されていない場合内容からcharsetを判定する
+      # https://learn.microsoft.com/en-us/windows/release-health/status-windows-11-22h2 の場合この処理がないと文字化け
+      # charsetがサーバーから返されず、ASCII-8BITとして判定される。それをKconv.toutf8で変換すると文字化けする
+      # metaタグのcharsetはUTF-8なのでこれを使えば正しいはず
+      charset = nil
+      # Nokogiriの場合 https://qiita.com/tetoralynx/items/273560ad6f75bb685935
+      # <meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i
+      if str =~ /<meta.*?charset=["']*([^"']+)/i
+        charset =  $1
+      end
+      charset
+    end
+    # テストのためにmodule_functionを使用
+    module_function :detect_charset
+
     def toutf8_charset(str, charset)
       # charsetが指定されていない場合はnil
-      return nil if charset.to_s.length == 0
+      if charset.to_s.length == 0
+        charset = detect_charset(str)
+      end
+      if charset.to_s.length == 0
+        return nil
+      end
 
       result = nil
       begin

data/lib/web_loader/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module WebLoader
-  VERSION = "1.3.0"
+  VERSION = "1.4.0"
 end

data/web_loader.iml

@@ -11,21 +11,21 @@
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
     <orderEntry type="module-library">
-      <library name="minitest (v5.20.0) [path][gem]" type="rubylib">
+      <library name="minitest (v5.22.0) [path][gem]" type="rubylib">
         <properties>
           <option name="version" value="4" />
         </properties>
         <CLASSES>
-          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/lib" />
-          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/test" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.0/lib" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.0/test" />
         </CLASSES>
         <JAVADOC />
         <SOURCES>
-          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/lib" />
-          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/test" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.0/lib" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.0/test" />
         </SOURCES>
         <excluded>
-          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/test" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.0/test" />
         </excluded>
       </library>
     </orderEntry>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_loader
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.0
 platform: ruby
 authors:
 - src
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-02-06 00:00:00.000000000 Z
+date: 2024-02-28 00:00:00.000000000 Z
 dependencies: []
 description: Web loader.
 email: