web_loader 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +2 -2
- data/lib/web_loader/command.rb +5 -4
- data/lib/web_loader/utils.rb +22 -1
- data/lib/web_loader/version.rb +1 -1
- data/web_loader.iml +6 -6
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 25257053739d26c811dc6fe61337deb27582549fcade84cc94da4a2b841cc399
|
4
|
+
data.tar.gz: 5f1fc1172b8252bf552caa9131e495065c5420d8773c9ce944843999c914df4f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 31d4b8dc8fb95aaa1585e4009183b4a2ac4612aa3d4fe3cf65f8f369340bdf375a3ee410ca4e26d9471d31272f8fc40447e2adccd633b6f8a7000be23f87111e
|
7
|
+
data.tar.gz: e4a6b5b6fee9a8a9f3a971c50fd1345a09674544c90e2ec063290e8cac067e38374f19f2395377cc84bbb8649f107dea64c6a9484dde9a67c5a9c3fa79aaece6
|
data/Gemfile.lock
CHANGED
data/lib/web_loader/command.rb
CHANGED
@@ -87,10 +87,11 @@ module WebLoader
|
|
87
87
|
encoding = response.type_params['charset']
|
88
88
|
body = toutf8(body, encoding)
|
89
89
|
end
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
90
|
+
|
91
|
+
# if @use_cache
|
92
|
+
log("Write cache: #{url}", @verbose)
|
93
|
+
Cache.write(@cache_dir, url, response.code, body)
|
94
|
+
# end
|
94
95
|
result = body
|
95
96
|
when Net::HTTPRedirection
|
96
97
|
result = load(to_redirect_url(uri, response['location']), redirect_count - 1)
|
data/lib/web_loader/utils.rb
CHANGED
@@ -2,9 +2,30 @@ module WebLoader
|
|
2
2
|
module Utils
|
3
3
|
UTF_8 = 'UTF-8'
|
4
4
|
|
5
|
+
def detect_charset(str)
|
6
|
+
# charsetが指定されていない場合内容からcharsetを判定する
|
7
|
+
# https://learn.microsoft.com/en-us/windows/release-health/status-windows-11-22h2 の場合この処理がないと文字化け
|
8
|
+
# charsetがサーバーから返されず、ASCII-8BITとして判定される。それをKconv.toutf8で変換すると文字化けする
|
9
|
+
# metaタグのcharsetはUTF-8なのでこれを使えば正しいはず
|
10
|
+
charset = nil
|
11
|
+
# Nokogiriの場合 https://qiita.com/tetoralynx/items/273560ad6f75bb685935
|
12
|
+
# <meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i
|
13
|
+
if str =~ /<meta.*?charset=["']*([^"']+)/i
|
14
|
+
charset = $1
|
15
|
+
end
|
16
|
+
charset
|
17
|
+
end
|
18
|
+
# テストのためにmodule_functionを使用
|
19
|
+
module_function :detect_charset
|
20
|
+
|
5
21
|
def toutf8_charset(str, charset)
|
6
22
|
# charsetが指定されていない場合はnil
|
7
|
-
|
23
|
+
if charset.to_s.length == 0
|
24
|
+
charset = detect_charset(str)
|
25
|
+
end
|
26
|
+
if charset.to_s.length == 0
|
27
|
+
return nil
|
28
|
+
end
|
8
29
|
|
9
30
|
result = nil
|
10
31
|
begin
|
data/lib/web_loader/version.rb
CHANGED
data/web_loader.iml
CHANGED
@@ -11,21 +11,21 @@
|
|
11
11
|
<orderEntry type="inheritedJdk" />
|
12
12
|
<orderEntry type="sourceFolder" forTests="false" />
|
13
13
|
<orderEntry type="module-library">
|
14
|
-
<library name="minitest (v5.
|
14
|
+
<library name="minitest (v5.22.0) [path][gem]" type="rubylib">
|
15
15
|
<properties>
|
16
16
|
<option name="version" value="4" />
|
17
17
|
</properties>
|
18
18
|
<CLASSES>
|
19
|
-
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.
|
20
|
-
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.
|
19
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.0/lib" />
|
20
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.0/test" />
|
21
21
|
</CLASSES>
|
22
22
|
<JAVADOC />
|
23
23
|
<SOURCES>
|
24
|
-
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.
|
25
|
-
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.
|
24
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.0/lib" />
|
25
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.0/test" />
|
26
26
|
</SOURCES>
|
27
27
|
<excluded>
|
28
|
-
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.
|
28
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.0/test" />
|
29
29
|
</excluded>
|
30
30
|
</library>
|
31
31
|
</orderEntry>
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: web_loader
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- src
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-02-
|
11
|
+
date: 2024-02-28 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Web loader.
|
14
14
|
email:
|