web_loader 1.3.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 20b0243bdaa4253679016dcb4d6d9635e1369a5839e35acc68029b565da1658f
4
- data.tar.gz: 5493a45ead30c5a64a0a12f19cfe8e6a93f782d3f913b9e8ef61268f4a2d3f14
3
+ metadata.gz: 25257053739d26c811dc6fe61337deb27582549fcade84cc94da4a2b841cc399
4
+ data.tar.gz: 5f1fc1172b8252bf552caa9131e495065c5420d8773c9ce944843999c914df4f
5
5
  SHA512:
6
- metadata.gz: 83f3217dd671d4c61e748433f363526fd12caa3eacd653a0ff93d34f14dd9e64b16e9d81cc6ade637797ccacb21415beabd99649cb1f052d29764a4e1b9285e4
7
- data.tar.gz: 061061d36f771d8d9170f73f0d029854f73363e38478ee0b9502908770ccc57864ae056cc01fa82a2b33796a602b8a5501ff051645015dcb582472dd24152be8
6
+ metadata.gz: 31d4b8dc8fb95aaa1585e4009183b4a2ac4612aa3d4fe3cf65f8f369340bdf375a3ee410ca4e26d9471d31272f8fc40447e2adccd633b6f8a7000be23f87111e
7
+ data.tar.gz: e4a6b5b6fee9a8a9f3a971c50fd1345a09674544c90e2ec063290e8cac067e38374f19f2395377cc84bbb8649f107dea64c6a9484dde9a67c5a9c3fa79aaece6
data/Gemfile.lock CHANGED
@@ -1,12 +1,12 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- web_loader (1.3.0)
4
+ web_loader (1.4.0)
5
5
 
6
6
  GEM
7
7
  remote: https://rubygems.org/
8
8
  specs:
9
- minitest (5.22.0)
9
+ minitest (5.22.2)
10
10
  rake (13.1.0)
11
11
 
12
12
  PLATFORMS
@@ -87,10 +87,11 @@ module WebLoader
87
87
  encoding = response.type_params['charset']
88
88
  body = toutf8(body, encoding)
89
89
  end
90
- if @use_cache
91
- log("Write cache: #{url}", @verbose)
92
- Cache.write(@cache_dir, url, response.code, body)
93
- end
90
+
91
+ # if @use_cache
92
+ log("Write cache: #{url}", @verbose)
93
+ Cache.write(@cache_dir, url, response.code, body)
94
+ # end
94
95
  result = body
95
96
  when Net::HTTPRedirection
96
97
  result = load(to_redirect_url(uri, response['location']), redirect_count - 1)
@@ -2,9 +2,30 @@ module WebLoader
2
2
  module Utils
3
3
  UTF_8 = 'UTF-8'
4
4
 
5
+ def detect_charset(str)
6
+ # charsetが指定されていない場合内容からcharsetを判定する
7
+ # https://learn.microsoft.com/en-us/windows/release-health/status-windows-11-22h2 の場合この処理がないと文字化け
8
+ # charsetがサーバーから返されず、ASCII-8BITとして判定される。それをKconv.toutf8で変換すると文字化けする
9
+ # metaタグのcharsetはUTF-8なのでこれを使えば正しいはず
10
+ charset = nil
11
+ # Nokogiriの場合 https://qiita.com/tetoralynx/items/273560ad6f75bb685935
12
+ # <meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i
13
+ if str =~ /<meta.*?charset=["']*([^"']+)/i
14
+ charset = $1
15
+ end
16
+ charset
17
+ end
18
+ # テストのためにmodule_functionを使用
19
+ module_function :detect_charset
20
+
5
21
  def toutf8_charset(str, charset)
6
22
  # charsetが指定されていない場合はnil
7
- return nil if charset.to_s.length == 0
23
+ if charset.to_s.length == 0
24
+ charset = detect_charset(str)
25
+ end
26
+ if charset.to_s.length == 0
27
+ return nil
28
+ end
8
29
 
9
30
  result = nil
10
31
  begin
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module WebLoader
4
- VERSION = "1.3.0"
4
+ VERSION = "1.4.0"
5
5
  end
data/web_loader.iml CHANGED
@@ -11,21 +11,21 @@
11
11
  <orderEntry type="inheritedJdk" />
12
12
  <orderEntry type="sourceFolder" forTests="false" />
13
13
  <orderEntry type="module-library">
14
- <library name="minitest (v5.20.0) [path][gem]" type="rubylib">
14
+ <library name="minitest (v5.22.0) [path][gem]" type="rubylib">
15
15
  <properties>
16
16
  <option name="version" value="4" />
17
17
  </properties>
18
18
  <CLASSES>
19
- <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/lib" />
20
- <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/test" />
19
+ <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.0/lib" />
20
+ <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.0/test" />
21
21
  </CLASSES>
22
22
  <JAVADOC />
23
23
  <SOURCES>
24
- <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/lib" />
25
- <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/test" />
24
+ <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.0/lib" />
25
+ <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.0/test" />
26
26
  </SOURCES>
27
27
  <excluded>
28
- <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.20.0/test" />
28
+ <root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.0/test" />
29
29
  </excluded>
30
30
  </library>
31
31
  </orderEntry>
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: web_loader
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.3.0
4
+ version: 1.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - src
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-02-06 00:00:00.000000000 Z
11
+ date: 2024-02-28 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: Web loader.
14
14
  email: