web_loader 1.3.0 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +2 -2
  3. data/lib/web_loader/command.rb +4 -1
  4. data/lib/web_loader/utils.rb +22 -1
  5. data/lib/web_loader/version.rb +1 -1
  6. data/web_loader.iml +6 -6
  7. metadata +2 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 20b0243bdaa4253679016dcb4d6d9635e1369a5839e35acc68029b565da1658f
4
- data.tar.gz: 5493a45ead30c5a64a0a12f19cfe8e6a93f782d3f913b9e8ef61268f4a2d3f14
3
+ metadata.gz: 128484ee899230f9f654c1589e9132e2c8c65dc4f4d0063e963b84745cba85eb
4
+ data.tar.gz: fc0ba12028b10363d650758c416f6db5fa2ebc548573f71be05de87e9687f6d9
5
5
  SHA512:
6
- metadata.gz: 83f3217dd671d4c61e748433f363526fd12caa3eacd653a0ff93d34f14dd9e64b16e9d81cc6ade637797ccacb21415beabd99649cb1f052d29764a4e1b9285e4
7
- data.tar.gz: 061061d36f771d8d9170f73f0d029854f73363e38478ee0b9502908770ccc57864ae056cc01fa82a2b33796a602b8a5501ff051645015dcb582472dd24152be8
6
+ metadata.gz: e37ed2b6b418122f39b504f276c230e13c7f95eee7d072f645612c90f71c849af6a33543b7cb4f3d97685d5e96b73d85c42fac0fb578766e947ae24d1ba6a60b
7
+ data.tar.gz: 2a3f21c1e98c700c354a14f0481351ee7d54df33ebe24d4e3f51dad92d5bc42de6102cfb49bcc04e65fbb2cc665a2da36a1c6cf657881427fcb48d4ed42333d8
data/Gemfile.lock CHANGED
@@ -1,12 +1,12 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- web_loader (1.3.0)
4
+ web_loader (1.4.1)
5
5
 
6
6
  GEM
7
7
  remote: https://rubygems.org/
8
8
  specs:
9
- minitest (5.22.0)
9
+ minitest (5.22.2)
10
10
  rake (13.1.0)
11
11
 
12
12
  PLATFORMS
data/lib/web_loader/command.rb CHANGED
@@ -31,11 +31,13 @@ module WebLoader
31
31
  @binary = false
32
32
  @verbose = false
33
33
  @cache_limit = CACHE_LIMIT
34
+ @always_write_cache = false
34
35
  end
35
36
 
36
37
  attr_reader :load_cache_page
37
38
  attr_accessor :use_cache, :cache_dir, :binary, :user_agent, :verbose
38
39
  attr_accessor :cache_limit
40
+ attr_accessor :always_write_cache
39
41
 
40
42
  def load_retry(url, retry_count = DEFAULT_RETRY)
41
43
  load(url, DEFAULT_REDIRECT, retry_count)
@@ -87,7 +89,8 @@ module WebLoader
87
89
  encoding = response.type_params['charset']
88
90
  body = toutf8(body, encoding)
89
91
  end
90
- if @use_cache
92
+
93
+ if @use_cache || @always_write_cache
91
94
  log("Write cache: #{url}", @verbose)
92
95
  Cache.write(@cache_dir, url, response.code, body)
93
96
  end
data/lib/web_loader/utils.rb CHANGED
@@ -2,9 +2,30 @@ module WebLoader
2
2
  module Utils
3
3
  UTF_8 = 'UTF-8'
4
4
 
5
+ def detect_charset(str)
6
+ # charsetが指定されていない場合内容からcharsetを判定する
7
+ # https://learn.microsoft.com/en-us/windows/release-health/status-windows-11-22h2 の場合この処理がないと文字化け
8
+ # charsetがサーバーから返されず、ASCII-8BITとして判定される。それをKconv.toutf8で変換すると文字化けする
9
+ # metaタグのcharsetはUTF-8なのでこれを使えば正しいはず
10
+ charset = nil
11
+ # Nokogiriの場合 https://qiita.com/tetoralynx/items/273560ad6f75bb685935
12
+ # <meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i
13
+ if str =~ /<meta.*?charset=["']*([^"']+)/i
14
+ charset = $1
15
+ end
16
+ charset
17
+ end
18
+ # テストのためにmodule_functionを使用
19
+ module_function :detect_charset
20
+
5
21
  def toutf8_charset(str, charset)
6
22
  # charsetが指定されていない場合はnil
7
- return nil if charset.to_s.length == 0
23
+ if charset.to_s.length == 0
24
+ charset = detect_charset(str)
25
+ end
26
+ if charset.to_s.length == 0
27
+ return nil
28
+ end
8
29
 
9
30
  result = nil
10
31
  begin
data/lib/web_loader/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module WebLoader
4
- VERSION = "1.3.0"
4
+ VERSION = "1.4.1"
5
5
  end
data/web_loader.iml CHANGED
@@ -11,21 +11,21 @@
11
11
  <orderEntry type="inheritedJdk" />
12
12
  <orderEntry type="sourceFolder" forTests="false" />
13
13
  <orderEntry type="module-library">
14
- <library name="minitest (v5.20.0) [path][gem]" type="rubylib">
14
+ <library name="minitest (v5.22.2) [path][gem]" type="rubylib">
15
15
  <properties>
16
16
  <option name="version" value="4" />
17
17
  </properties>
18
18
  <CLASSES>
19
- <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/lib" />
20
- <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/test" />
19
+ <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/lib" />
20
+ <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/test" />
21
21
  </CLASSES>
22
22
  <JAVADOC />
23
23
  <SOURCES>
24
- <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/lib" />
25
- <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/test" />
24
+ <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/lib" />
25
+ <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/test" />
26
26
  </SOURCES>
27
27
  <excluded>
28
- <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/test" />
28
+ <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/test" />
29
29
  </excluded>
30
30
  </library>
31
31
  </orderEntry>
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: web_loader
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.3.0
4
+ version: 1.4.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - src
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-02-06 00:00:00.000000000 Z
11
+ date: 2024-02-28 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: Web loader.
14
14
  email: