web_loader 1.5.0 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 972dc054f042a2f2da8e3e84f8dc0963343765bd316c11ed285302551c071de0
-  data.tar.gz: 8711c2f6d20926d94960e0ad924aad1c6931551a2f1e30c9fdbc6c89d285bf15
+  metadata.gz: af7dbb07ea7ffe94d18c04d91c1f53564123b6ed794945d04ed2dd441483f2ad
+  data.tar.gz: 3e1be0e448488fb0fc8cdf0aa53e5751c0f1fc33261315b3a53882d356e42492
 SHA512:
-  metadata.gz: e7e52952ef16318d2e792521044dcd614c8828311bc49557b67948f843ce4934b7c5c79ccdbc53357ce63a3deeb6f7ad32ec6201bb274386b7bf481f5b31956e
-  data.tar.gz: b32129726e055b55b26fda3f3f9ebac71f7b2b3f3ca916a2c612dc1439195ac40052d0d7015b31663a4184fe7a279934b6fbb372ddae9cf0193cdb93a3ab7545
+  metadata.gz: 6c1cc94232fdca9458d416c90ce6832fdb08be2a6a90f15705645711566de931ca76a47650ce241589d76620ccfe9c0af02c069e7abd990ad2f6a2643220bb14
+  data.tar.gz: 227a97a262168cdc5a1f473901d274b52ef60f388ff89ca3fda07c24f725f5c977ff343a9f24d8bdc70634ae0f916ff84a98b1215f1129f5e6264999b05df007

data/.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="RbsMissingTypeSignature" enabled="false" level="WEAK WARNING" enabled_by_default="false" />
+  </profile>
+</component>

data/Gemfile.lock

@@ -1,12 +1,12 @@
 PATH
   remote: .
   specs:
-    web_loader (1.5.0)
+    web_loader (1.7.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    minitest (5.22.2)
+    minitest (5.22.3)
     rake (13.1.0)
 
 PLATFORMS

data/lib/web_loader/command.rb

@@ -33,6 +33,7 @@ module WebLoader
       @cache_limit = CACHE_LIMIT
       @always_write_cache = false
       @response = nil
+      @logger = nil
     end
 
     attr_reader :load_cache_page
@@ -40,6 +41,7 @@ module WebLoader
     attr_accessor :cache_limit
     attr_accessor :always_write_cache
     attr_reader :response
+    attr_accessor :logger
 
     def load_retry(url, retry_count = DEFAULT_RETRY)
       load(url, DEFAULT_REDIRECT, retry_count)
@@ -47,19 +49,19 @@ module WebLoader
 
     def load(url, redirect_count = DEFAULT_REDIRECT, retry_count = 0)
       raise ArgumentError, 'HTTP redirect too deep' if redirect_count == 0
-      log("Load: #{url}", @verbose)
+      log("Load: #{url}")
 
       ##### キャッシュの読み込み
       @load_cache_page = false
       content = try_load_cache(url)
       if content
-        log("Load cache: #{url}", @verbose)
+        log("Load cache: #{url}")
         @load_cache_page = true
         return content
       end
 
       ##### サーバーからロード
-      log("Load server: #{url}", @verbose)
+      log("Load server: #{url}")
       uri = URI.parse(url)
       http = Net::HTTP.new(uri.host, uri.port)
       if uri.scheme == 'https'
@@ -71,7 +73,7 @@ module WebLoader
         @response = http.get(uri.request_uri, 'User-Agent' => @user_agent) # request_uri=path + '?' + query
       rescue Net::ReadTimeout
         # タイムアウトした場合リトライ可能ならばsleepした後に再度ロード実行
-        log("Read timeout: #{url}", @verbose)
+        log("Read timeout: #{url}")
         if retry_count > 0
           sleep DEFAULT_SLEEP
           return load(url, redirect_count , retry_count - 1)
@@ -93,14 +95,14 @@ module WebLoader
         end
 
         if @use_cache || @always_write_cache
-          log("Write cache: #{url}", @verbose)
+          log("Write cache: #{url}")
           Cache.write(@cache_dir, url, @response.code, body)
         end
         result = body
       when Net::HTTPRedirection
         result = load(to_redirect_url(uri, @response['location']), redirect_count - 1)
-      # when Net::HTTPNotFound
-      #   result = nil
+        # when Net::HTTPNotFound
+        #   result = nil
       when Net::HTTPTooManyRequests, Net::ReadTimeout
         # 上記以外のレスポンスの場合、リトライ可能ならばsleepした後に再度ロード実行
         if retry_count > 0
@@ -108,17 +110,16 @@ module WebLoader
           if @response.is_a?(Net::HTTPTooManyRequests)
             # HTTPTooManyRequestsならばretry-afterで指定された値を取得。
             sleep_for = @response.header['retry-after'].to_i + 10
-            log("Rate limit: #{uri} #{@response.header.to_hash} (429 Too Many Requests). Sleeping #{sleep_for} seconds and retry (##{retry_count}).", @verbose)
+            log("Rate limit: #{uri} #{@response.header.to_hash} (429 Too Many Requests). Sleeping #{sleep_for} seconds and retry (##{retry_count}).")
           else
-            log("Unknown response: #{uri} #{@response.inspect}. Sleeping #{sleep_for} seconds and retry (##{retry_count}).", @verbose)
+            log("Unknown response: #{uri} #{@response.inspect}. Sleeping #{sleep_for} seconds and retry (##{retry_count}).")
           end
           sleep sleep_for
           result = load(url, redirect_count , retry_count - 1)
         end
-
+      else
         # それ以外は対応した例外を発生
         log("error #{url}", true)
-        @response.value
       end
       result
     end
@@ -130,8 +131,13 @@ module WebLoader
       Cache.load_content(@cache_dir, url)
     end
 
-    def log(msg, put_log)
-      puts msg if put_log
+    def log(msg, put_log = @verbose)
+      return unless put_log
+      if @logger
+        @logger.info(msg)
+      else
+        puts msg
+      end
     end
   end
 end

data/lib/web_loader/utils.rb

@@ -13,6 +13,10 @@ module WebLoader
       if str =~ /<meta.*?charset=["']*([^"']+)/i
         charset =  $1
       end
+      if charset =~ /Shift_JIS/i
+        # Shift_JISの場合、実際はWindows-31J(Windowsの標準コードの場合が多いはず)
+        charset = "Windows-31J"
+      end
       charset
     end
     # テストのためにmodule_functionを使用
@@ -45,6 +49,7 @@ module WebLoader
       end
       result
     end
+    module_function :toutf8_charset
 
     def toutf8(str, charset)
       # 2022/04/04(月)

data/lib/web_loader/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module WebLoader
-  VERSION = "1.5.0"
+  VERSION = "1.7.0"
 end

data/web_loader.iml

@@ -11,21 +11,21 @@
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
     <orderEntry type="module-library">
-      <library name="minitest (v5.22.2) [path][gem]" type="rubylib">
+      <library name="minitest (v5.22.3) [path][gem]" type="rubylib">
         <properties>
           <option name="version" value="4" />
         </properties>
         <CLASSES>
-          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/lib" />
-          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/test" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.3/lib" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.3/test" />
         </CLASSES>
         <JAVADOC />
         <SOURCES>
-          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/lib" />
-          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/test" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.3/lib" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.3/test" />
         </SOURCES>
         <excluded>
-          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/test" />
+          <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.3/test" />
         </excluded>
       </library>
     </orderEntry>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: web_loader
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.7.0
 platform: ruby
 authors:
 - src
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-03-26 00:00:00.000000000 Z
+date: 2024-03-29 00:00:00.000000000 Z
 dependencies: []
 description: Web loader.
 email:
@@ -19,6 +19,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".idea/.gitignore"
+- ".idea/inspectionProfiles/Project_Default.xml"
 - ".idea/misc.xml"
 - ".idea/modules.xml"
 - ".idea/vcs.xml"