web_loader 1.3.0 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 20b0243bdaa4253679016dcb4d6d9635e1369a5839e35acc68029b565da1658f
-  data.tar.gz: 5493a45ead30c5a64a0a12f19cfe8e6a93f782d3f913b9e8ef61268f4a2d3f14
+  metadata.gz: 128484ee899230f9f654c1589e9132e2c8c65dc4f4d0063e963b84745cba85eb
+  data.tar.gz: fc0ba12028b10363d650758c416f6db5fa2ebc548573f71be05de87e9687f6d9
 SHA512:
-  metadata.gz: 83f3217dd671d4c61e748433f363526fd12caa3eacd653a0ff93d34f14dd9e64b16e9d81cc6ade637797ccacb21415beabd99649cb1f052d29764a4e1b9285e4
-  data.tar.gz: 061061d36f771d8d9170f73f0d029854f73363e38478ee0b9502908770ccc57864ae056cc01fa82a2b33796a602b8a5501ff051645015dcb582472dd24152be8
+  metadata.gz: e37ed2b6b418122f39b504f276c230e13c7f95eee7d072f645612c90f71c849af6a33543b7cb4f3d97685d5e96b73d85c42fac0fb578766e947ae24d1ba6a60b
+  data.tar.gz: 2a3f21c1e98c700c354a14f0481351ee7d54df33ebe24d4e3f51dad92d5bc42de6102cfb49bcc04e65fbb2cc665a2da36a1c6cf657881427fcb48d4ed42333d8

data/Gemfile.lock

@@ -1,12 +1,12 @@
 PATH
   remote: .
   specs:
-    web_loader (1.3.0)
+    web_loader (1.4.1)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    minitest (5.22.0)
+    minitest (5.22.2)
     rake (13.1.0)
 
 PLATFORMS

data/lib/web_loader/command.rb

@@ -31,11 +31,13 @@ module WebLoader
       @binary = false
       @verbose = false
       @cache_limit = CACHE_LIMIT
+      @always_write_cache = false
     end
 
     attr_reader :load_cache_page
     attr_accessor :use_cache, :cache_dir, :binary, :user_agent, :verbose
     attr_accessor :cache_limit
+    attr_accessor :always_write_cache
 
     def load_retry(url, retry_count = DEFAULT_RETRY)
       load(url, DEFAULT_REDIRECT, retry_count)
@@ -87,7 +89,8 @@ module WebLoader
           encoding = response.type_params['charset']
           body = toutf8(body, encoding)
         end
-        if @use_cache
+
+        if @use_cache || @always_write_cache
           log("Write cache: #{url}", @verbose)
           Cache.write(@cache_dir, url, response.code, body)
         end

data/lib/web_loader/utils.rb

@@ -2,9 +2,30 @@ module WebLoader
   module Utils
     UTF_8 = 'UTF-8'
 
+    def detect_charset(str)
+      # charsetが指定されていない場合内容からcharsetを判定する
+      # https://learn.microsoft.com/en-us/windows/release-health/status-windows-11-22h2 の場合この処理がないと文字化け
+      # charsetがサーバーから返されず、ASCII-8BITとして判定される。それをKconv.toutf8で変換すると文字化けする
+      # metaタグのcharsetはUTF-8なのでこれを使えば正しいはず
+      charset = nil
+      # Nokogiriの場合 https://qiita.com/tetoralynx/items/273560ad6f75bb685935
+      # <meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i
+      if str =~ /<meta.*?charset=["']*([^"']+)/i
+        charset =  $1
+      end
+      charset
+    end
+    # テストのためにmodule_functionを使用
+    module_function :detect_charset
+
     def toutf8_charset(str, charset)
       # charsetが指定されていない場合はnil
-      return nil if charset.to_s.length == 0
+      if charset.to_s.length == 0
+        charset = detect_charset(str)
+      end
+      if charset.to_s.length == 0
+        return nil
+      end
 
       result = nil
       begin

data/lib/web_loader/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module WebLoader
-  VERSION = "1.3.0"
+  VERSION = "1.4.1"
 end

data/web_loader.iml

@@ -11,21 +11,21 @@
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
     <orderEntry type="module-library">
-      <library name="minitest (v5.20.0) [path][gem]" type="rubylib">
+      <library name="minitest (v5.22.2) [path][gem]" type="rubylib">
         <properties>
           <option name="version" value="4" />
         </properties>
         <CLASSES>
-          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/lib" />
-          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/test" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/lib" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/test" />
         </CLASSES>
         <JAVADOC />
         <SOURCES>
-          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/lib" />
-          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/test" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/lib" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/test" />
         </SOURCES>
         <excluded>
-          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/test" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/test" />
         </excluded>
       </library>
     </orderEntry>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_loader
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.1
 platform: ruby
 authors:
 - src
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-02-06 00:00:00.000000000 Z
+date: 2024-02-28 00:00:00.000000000 Z
 dependencies: []
 description: Web loader.
 email: