web_loader 1.2.1 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa46cae0813efddd510f37385b7e1f3b4e78288405586af295b1ec6dec2a86af
-  data.tar.gz: f1f9caa1a7b767394d201abbc32ba05d7663a6e1c3af803112db252dc5b163c8
+  metadata.gz: 25257053739d26c811dc6fe61337deb27582549fcade84cc94da4a2b841cc399
+  data.tar.gz: 5f1fc1172b8252bf552caa9131e495065c5420d8773c9ce944843999c914df4f
 SHA512:
-  metadata.gz: 7cbf32092d6ad8d26954b3c3dc97a4f32ec930b9598b2cc326d5dffa39d3367fde8575757a2e878ebc6eac9c13155b082f121f730fb8884506beab2b118e383a
-  data.tar.gz: 3a38e4744ace8a488fcd4c2af85c2e19ec453de8e6d85573ddafc3bebff5a4da66ea50c711ae7a5a24349b3d5f90811d831e247c02b8c7f64b73e7a5b8679a1e
+  metadata.gz: 31d4b8dc8fb95aaa1585e4009183b4a2ac4612aa3d4fe3cf65f8f369340bdf375a3ee410ca4e26d9471d31272f8fc40447e2adccd633b6f8a7000be23f87111e
+  data.tar.gz: e4a6b5b6fee9a8a9f3a971c50fd1345a09674544c90e2ec063290e8cac067e38374f19f2395377cc84bbb8649f107dea64c6a9484dde9a67c5a9c3fa79aaece6

data/Gemfile.lock

@@ -1,12 +1,12 @@
 PATH
   remote: .
   specs:
-    web_loader (1.2.1)
+    web_loader (1.4.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    minitest (5.20.0)
+    minitest (5.22.2)
     rake (13.1.0)
 
 PLATFORMS

data/lib/web_loader/cache.rb

@@ -12,7 +12,6 @@ module WebLoader
   class Cache
     PREFIX = "__cache__"
     #    CACHE_LIMIT = 3600
-    CACHE_LIMIT = 60
 
     def self.basename(url)
       Digest::MD5.hexdigest(url)
@@ -43,11 +42,11 @@ module WebLoader
       File.write(content_path, content)
     end
 
-    def self.clear(dir)
+    def self.clear(dir, cache_limit)
       Dir.glob("#{dir}/#{PREFIX}*.{yml,html}").each do |path|
         diff = Time.now - File.mtime(path)
         # 1時間以上昔のキャッシュは使用しない
-        too_old_cache = diff > CACHE_LIMIT
+        too_old_cache = diff > cache_limit
         FileUtils.rm(path) if too_old_cache
       end
     end

data/lib/web_loader/command.rb

@@ -12,6 +12,7 @@ module WebLoader
     DEFAULT_RETRY = 3
     DEFAULT_REDIRECT = 10
     DEFAULT_SLEEP = 10
+    CACHE_LIMIT = 3600 # キャッシュが有効な秒数。デフォルトは1時間とする
 
     def self.save_image(url, file)
       # キャッシュせず単に保存する
@@ -29,10 +30,12 @@ module WebLoader
       @user_agent = "#{USER_AGENT}/#{VERSION}"
       @binary = false
       @verbose = false
+      @cache_limit = CACHE_LIMIT
     end
 
     attr_reader :load_cache_page
     attr_accessor :use_cache, :cache_dir, :binary, :user_agent, :verbose
+    attr_accessor :cache_limit
 
     def load_retry(url, retry_count = DEFAULT_RETRY)
       load(url, DEFAULT_REDIRECT, retry_count)
@@ -84,10 +87,11 @@ module WebLoader
           encoding = response.type_params['charset']
           body = toutf8(body, encoding)
         end
-        if @use_cache
-          log("Write cache: #{url}", @verbose)
-          Cache.write(@cache_dir, url, response.code, body)
-        end
+
+#        if @use_cache
+        log("Write cache: #{url}", @verbose)
+        Cache.write(@cache_dir, url, response.code, body)
+        #        end
         result = body
       when Net::HTTPRedirection
         result = load(to_redirect_url(uri, response['location']), redirect_count - 1)
@@ -116,7 +120,7 @@ module WebLoader
     private
     def try_load_cache(url)
       return nil unless @use_cache
-      Cache.clear(@cache_dir)
+      Cache.clear(@cache_dir, @cache_limit)
       Cache.load_content(@cache_dir, url)
     end
 

data/lib/web_loader/utils.rb

@@ -2,9 +2,30 @@ module WebLoader
   module Utils
     UTF_8 = 'UTF-8'
 
+    def detect_charset(str)
+      # charsetが指定されていない場合内容からcharsetを判定する
+      # https://learn.microsoft.com/en-us/windows/release-health/status-windows-11-22h2 の場合この処理がないと文字化け
+      # charsetがサーバーから返されず、ASCII-8BITとして判定される。それをKconv.toutf8で変換すると文字化けする
+      # metaタグのcharsetはUTF-8なのでこれを使えば正しいはず
+      charset = nil
+      # Nokogiriの場合 https://qiita.com/tetoralynx/items/273560ad6f75bb685935
+      # <meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i
+      if str =~ /<meta.*?charset=["']*([^"']+)/i
+        charset =  $1
+      end
+      charset
+    end
+    # テストのためにmodule_functionを使用
+    module_function :detect_charset
+
     def toutf8_charset(str, charset)
       # charsetが指定されていない場合はnil
-      return nil if charset.to_s.length == 0
+      if charset.to_s.length == 0
+        charset = detect_charset(str)
+      end
+      if charset.to_s.length == 0
+        return nil
+      end
 
       result = nil
       begin

data/lib/web_loader/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module WebLoader
-  VERSION = "1.2.1"
+  VERSION = "1.4.0"
 end

data/web_loader.iml

@@ -1,5 +1,8 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <module type="RUBY_MODULE" version="4">
+  <component name="ModuleRunConfigurationManager">
+    <shared />
+  </component>
   <component name="NewModuleRootManager" inherit-compiler-output="true">
     <exclude-output />
     <content url="file://$MODULE_DIR$">
@@ -8,21 +11,21 @@
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
     <orderEntry type="module-library">
-      <library name="minitest (v5.20.0) [path][gem]" type="rubylib">
+      <library name="minitest (v5.22.0) [path][gem]" type="rubylib">
         <properties>
           <option name="version" value="4" />
         </properties>
         <CLASSES>
-          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/lib" />
-          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/test" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.0/lib" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.0/test" />
         </CLASSES>
         <JAVADOC />
         <SOURCES>
-          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/lib" />
-          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/test" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.0/lib" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.0/test" />
         </SOURCES>
         <excluded>
-          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/test" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.0/test" />
         </excluded>
       </library>
     </orderEntry>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_loader
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.4.0
 platform: ruby
 authors:
 - src
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-11-23 00:00:00.000000000 Z
+date: 2024-02-28 00:00:00.000000000 Z
 dependencies: []
 description: Web loader.
 email: