web_loader 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.idea/.gitignore +2 -0
- data/Gemfile.lock +1 -1
- data/lib/web_loader/command.rb +23 -17
- data/lib/web_loader/version.rb +1 -1
- data/web_loader.iml +39 -6
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 972dc054f042a2f2da8e3e84f8dc0963343765bd316c11ed285302551c071de0
|
|
4
|
+
data.tar.gz: 8711c2f6d20926d94960e0ad924aad1c6931551a2f1e30c9fdbc6c89d285bf15
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e7e52952ef16318d2e792521044dcd614c8828311bc49557b67948f843ce4934b7c5c79ccdbc53357ce63a3deeb6f7ad32ec6201bb274386b7bf481f5b31956e
|
|
7
|
+
data.tar.gz: b32129726e055b55b26fda3f3f9ebac71f7b2b3f3ca916a2c612dc1439195ac40052d0d7015b31663a4184fe7a279934b6fbb372ddae9cf0193cdb93a3ab7545
|
data/.idea/.gitignore
CHANGED
data/Gemfile.lock
CHANGED
data/lib/web_loader/command.rb
CHANGED
|
@@ -31,11 +31,15 @@ module WebLoader
|
|
|
31
31
|
@binary = false
|
|
32
32
|
@verbose = false
|
|
33
33
|
@cache_limit = CACHE_LIMIT
|
|
34
|
+
@always_write_cache = false
|
|
35
|
+
@response = nil
|
|
34
36
|
end
|
|
35
37
|
|
|
36
38
|
attr_reader :load_cache_page
|
|
37
39
|
attr_accessor :use_cache, :cache_dir, :binary, :user_agent, :verbose
|
|
38
40
|
attr_accessor :cache_limit
|
|
41
|
+
attr_accessor :always_write_cache
|
|
42
|
+
attr_reader :response
|
|
39
43
|
|
|
40
44
|
def load_retry(url, retry_count = DEFAULT_RETRY)
|
|
41
45
|
load(url, DEFAULT_REDIRECT, retry_count)
|
|
@@ -62,9 +66,9 @@ module WebLoader
|
|
|
62
66
|
http.use_ssl = true
|
|
63
67
|
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
|
64
68
|
end
|
|
65
|
-
response = nil
|
|
69
|
+
@response = nil
|
|
66
70
|
begin
|
|
67
|
-
response = http.get(uri.request_uri, 'User-Agent' => @user_agent) # request_uri=path + '?' + query
|
|
71
|
+
@response = http.get(uri.request_uri, 'User-Agent' => @user_agent) # request_uri=path + '?' + query
|
|
68
72
|
rescue Net::ReadTimeout
|
|
69
73
|
# タイムアウトした場合リトライ可能ならばsleepした後に再度ロード実行
|
|
70
74
|
log("Read timeout: #{url}", @verbose)
|
|
@@ -76,35 +80,37 @@ module WebLoader
|
|
|
76
80
|
|
|
77
81
|
##### レスポンスの処理
|
|
78
82
|
result = nil
|
|
79
|
-
case response
|
|
83
|
+
case @response
|
|
80
84
|
when Net::HTTPSuccess
|
|
81
|
-
# responseがNet::HTTPSuccessのサブクラスの場合成功とみなし読み込んだ内容を返す
|
|
82
|
-
body = response.body
|
|
85
|
+
# @responseがNet::HTTPSuccessのサブクラスの場合成功とみなし読み込んだ内容を返す
|
|
86
|
+
body = @response.body
|
|
83
87
|
unless @binary
|
|
84
88
|
# デフォルトでは ASCII-8BITが帰ってくる。
|
|
85
89
|
# Content-Typeのcharsetとみなす。
|
|
86
90
|
# https://bugs.ruby-lang.org/issues/2567
|
|
87
|
-
encoding = response.type_params['charset']
|
|
91
|
+
encoding = @response.type_params['charset']
|
|
88
92
|
body = toutf8(body, encoding)
|
|
89
93
|
end
|
|
90
94
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
+
if @use_cache || @always_write_cache
|
|
96
|
+
log("Write cache: #{url}", @verbose)
|
|
97
|
+
Cache.write(@cache_dir, url, @response.code, body)
|
|
98
|
+
end
|
|
95
99
|
result = body
|
|
96
100
|
when Net::HTTPRedirection
|
|
97
|
-
result = load(to_redirect_url(uri, response['location']), redirect_count - 1)
|
|
98
|
-
|
|
101
|
+
result = load(to_redirect_url(uri, @response['location']), redirect_count - 1)
|
|
102
|
+
# when Net::HTTPNotFound
|
|
103
|
+
# result = nil
|
|
104
|
+
when Net::HTTPTooManyRequests, Net::ReadTimeout
|
|
99
105
|
# 上記以外のレスポンスの場合、リトライ可能ならばsleepした後に再度ロード実行
|
|
100
106
|
if retry_count > 0
|
|
101
107
|
sleep_for = 10
|
|
102
|
-
if response.is_a?(Net::HTTPTooManyRequests)
|
|
108
|
+
if @response.is_a?(Net::HTTPTooManyRequests)
|
|
103
109
|
# HTTPTooManyRequestsならばretry-afterで指定された値を取得。
|
|
104
|
-
sleep_for = response.header['retry-after'].to_i + 10
|
|
105
|
-
log("Rate limit: #{uri} #{response.header.to_hash} (429 Too Many Requests). Sleeping #{sleep_for} seconds and retry (##{retry_count}).", @verbose)
|
|
110
|
+
sleep_for = @response.header['retry-after'].to_i + 10
|
|
111
|
+
log("Rate limit: #{uri} #{@response.header.to_hash} (429 Too Many Requests). Sleeping #{sleep_for} seconds and retry (##{retry_count}).", @verbose)
|
|
106
112
|
else
|
|
107
|
-
log("Unknown response: #{uri} #{response.inspect}. Sleeping #{sleep_for} seconds and retry (##{retry_count}).", @verbose)
|
|
113
|
+
log("Unknown response: #{uri} #{@response.inspect}. Sleeping #{sleep_for} seconds and retry (##{retry_count}).", @verbose)
|
|
108
114
|
end
|
|
109
115
|
sleep sleep_for
|
|
110
116
|
result = load(url, redirect_count , retry_count - 1)
|
|
@@ -112,7 +118,7 @@ module WebLoader
|
|
|
112
118
|
|
|
113
119
|
# それ以外は対応した例外を発生
|
|
114
120
|
log("error #{url}", true)
|
|
115
|
-
response.value
|
|
121
|
+
@response.value
|
|
116
122
|
end
|
|
117
123
|
result
|
|
118
124
|
end
|
data/lib/web_loader/version.rb
CHANGED
data/web_loader.iml
CHANGED
|
@@ -11,21 +11,21 @@
|
|
|
11
11
|
<orderEntry type="inheritedJdk" />
|
|
12
12
|
<orderEntry type="sourceFolder" forTests="false" />
|
|
13
13
|
<orderEntry type="module-library">
|
|
14
|
-
<library name="minitest (v5.22.
|
|
14
|
+
<library name="minitest (v5.22.2) [path][gem]" type="rubylib">
|
|
15
15
|
<properties>
|
|
16
16
|
<option name="version" value="4" />
|
|
17
17
|
</properties>
|
|
18
18
|
<CLASSES>
|
|
19
|
-
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.
|
|
20
|
-
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.
|
|
19
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/lib" />
|
|
20
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/test" />
|
|
21
21
|
</CLASSES>
|
|
22
22
|
<JAVADOC />
|
|
23
23
|
<SOURCES>
|
|
24
|
-
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.
|
|
25
|
-
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.
|
|
24
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/lib" />
|
|
25
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/test" />
|
|
26
26
|
</SOURCES>
|
|
27
27
|
<excluded>
|
|
28
|
-
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.
|
|
28
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/test" />
|
|
29
29
|
</excluded>
|
|
30
30
|
</library>
|
|
31
31
|
</orderEntry>
|
|
@@ -57,4 +57,37 @@
|
|
|
57
57
|
<RakeTaskImpl id="rake" />
|
|
58
58
|
</option>
|
|
59
59
|
</component>
|
|
60
|
+
<component name="RakeTasksCache-v2">
|
|
61
|
+
<option name="myRootTask">
|
|
62
|
+
<RakeTaskImpl id="rake">
|
|
63
|
+
<subtasks>
|
|
64
|
+
<RakeTaskImpl description="Build web_loader-1.4.1.gem into the pkg directory" fullCommand="build" id="build" />
|
|
65
|
+
<RakeTaskImpl id="build">
|
|
66
|
+
<subtasks>
|
|
67
|
+
<RakeTaskImpl description="Generate SHA512 checksum if web_loader-1.4.1.gem into the checksums directory" fullCommand="build:checksum" id="checksum" />
|
|
68
|
+
</subtasks>
|
|
69
|
+
</RakeTaskImpl>
|
|
70
|
+
<RakeTaskImpl description="Remove any temporary products" fullCommand="clean" id="clean" />
|
|
71
|
+
<RakeTaskImpl description="Remove any generated files" fullCommand="clobber" id="clobber" />
|
|
72
|
+
<RakeTaskImpl description="Build and install web_loader-1.4.1.gem into system gems" fullCommand="install" id="install" />
|
|
73
|
+
<RakeTaskImpl id="install">
|
|
74
|
+
<subtasks>
|
|
75
|
+
<RakeTaskImpl description="Build and install web_loader-1.4.1.gem into system gems without network access" fullCommand="install:local" id="local" />
|
|
76
|
+
</subtasks>
|
|
77
|
+
</RakeTaskImpl>
|
|
78
|
+
<RakeTaskImpl description="Create tag v1.4.1 and build and push web_loader-1.4.1.gem to rubygems.org" fullCommand="release[remote]" id="release[remote]" />
|
|
79
|
+
<RakeTaskImpl description="Run tests" fullCommand="test" id="test" />
|
|
80
|
+
<RakeTaskImpl description="" fullCommand="default" id="default" />
|
|
81
|
+
<RakeTaskImpl description="" fullCommand="release" id="release" />
|
|
82
|
+
<RakeTaskImpl id="release">
|
|
83
|
+
<subtasks>
|
|
84
|
+
<RakeTaskImpl description="" fullCommand="release:guard_clean" id="guard_clean" />
|
|
85
|
+
<RakeTaskImpl description="" fullCommand="release:rubygem_push" id="rubygem_push" />
|
|
86
|
+
<RakeTaskImpl description="" fullCommand="release:source_control_push" id="source_control_push" />
|
|
87
|
+
</subtasks>
|
|
88
|
+
</RakeTaskImpl>
|
|
89
|
+
</subtasks>
|
|
90
|
+
</RakeTaskImpl>
|
|
91
|
+
</option>
|
|
92
|
+
</component>
|
|
60
93
|
</module>
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: web_loader
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.5.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- src
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-
|
|
11
|
+
date: 2024-03-26 00:00:00.000000000 Z
|
|
12
12
|
dependencies: []
|
|
13
13
|
description: Web loader.
|
|
14
14
|
email:
|