web_loader 1.4.0 → 1.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.idea/.gitignore +2 -0
- data/Gemfile.lock +1 -1
- data/lib/web_loader/command.rb +23 -17
- data/lib/web_loader/version.rb +1 -1
- data/web_loader.iml +39 -6
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 972dc054f042a2f2da8e3e84f8dc0963343765bd316c11ed285302551c071de0
|
|
4
|
+
data.tar.gz: 8711c2f6d20926d94960e0ad924aad1c6931551a2f1e30c9fdbc6c89d285bf15
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e7e52952ef16318d2e792521044dcd614c8828311bc49557b67948f843ce4934b7c5c79ccdbc53357ce63a3deeb6f7ad32ec6201bb274386b7bf481f5b31956e
|
|
7
|
+
data.tar.gz: b32129726e055b55b26fda3f3f9ebac71f7b2b3f3ca916a2c612dc1439195ac40052d0d7015b31663a4184fe7a279934b6fbb372ddae9cf0193cdb93a3ab7545
|
data/.idea/.gitignore
CHANGED
data/Gemfile.lock
CHANGED
data/lib/web_loader/command.rb
CHANGED
|
@@ -31,11 +31,15 @@ module WebLoader
|
|
|
31
31
|
@binary = false
|
|
32
32
|
@verbose = false
|
|
33
33
|
@cache_limit = CACHE_LIMIT
|
|
34
|
+
@always_write_cache = false
|
|
35
|
+
@response = nil
|
|
34
36
|
end
|
|
35
37
|
|
|
36
38
|
attr_reader :load_cache_page
|
|
37
39
|
attr_accessor :use_cache, :cache_dir, :binary, :user_agent, :verbose
|
|
38
40
|
attr_accessor :cache_limit
|
|
41
|
+
attr_accessor :always_write_cache
|
|
42
|
+
attr_reader :response
|
|
39
43
|
|
|
40
44
|
def load_retry(url, retry_count = DEFAULT_RETRY)
|
|
41
45
|
load(url, DEFAULT_REDIRECT, retry_count)
|
|
@@ -62,9 +66,9 @@ module WebLoader
|
|
|
62
66
|
http.use_ssl = true
|
|
63
67
|
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
|
64
68
|
end
|
|
65
|
-
response = nil
|
|
69
|
+
@response = nil
|
|
66
70
|
begin
|
|
67
|
-
response = http.get(uri.request_uri, 'User-Agent' => @user_agent) # request_uri=path + '?' + query
|
|
71
|
+
@response = http.get(uri.request_uri, 'User-Agent' => @user_agent) # request_uri=path + '?' + query
|
|
68
72
|
rescue Net::ReadTimeout
|
|
69
73
|
# タイムアウトした場合リトライ可能ならばsleepした後に再度ロード実行
|
|
70
74
|
log("Read timeout: #{url}", @verbose)
|
|
@@ -76,35 +80,37 @@ module WebLoader
|
|
|
76
80
|
|
|
77
81
|
##### レスポンスの処理
|
|
78
82
|
result = nil
|
|
79
|
-
case response
|
|
83
|
+
case @response
|
|
80
84
|
when Net::HTTPSuccess
|
|
81
|
-
# responseがNet::HTTPSuccessのサブクラスの場合成功とみなし読み込んだ内容を返す
|
|
82
|
-
body = response.body
|
|
85
|
+
# @responseがNet::HTTPSuccessのサブクラスの場合成功とみなし読み込んだ内容を返す
|
|
86
|
+
body = @response.body
|
|
83
87
|
unless @binary
|
|
84
88
|
# デフォルトでは ASCII-8BITが帰ってくる。
|
|
85
89
|
# Content-Typeのcharsetとみなす。
|
|
86
90
|
# https://bugs.ruby-lang.org/issues/2567
|
|
87
|
-
encoding = response.type_params['charset']
|
|
91
|
+
encoding = @response.type_params['charset']
|
|
88
92
|
body = toutf8(body, encoding)
|
|
89
93
|
end
|
|
90
94
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
+
if @use_cache || @always_write_cache
|
|
96
|
+
log("Write cache: #{url}", @verbose)
|
|
97
|
+
Cache.write(@cache_dir, url, @response.code, body)
|
|
98
|
+
end
|
|
95
99
|
result = body
|
|
96
100
|
when Net::HTTPRedirection
|
|
97
|
-
result = load(to_redirect_url(uri, response['location']), redirect_count - 1)
|
|
98
|
-
|
|
101
|
+
result = load(to_redirect_url(uri, @response['location']), redirect_count - 1)
|
|
102
|
+
# when Net::HTTPNotFound
|
|
103
|
+
# result = nil
|
|
104
|
+
when Net::HTTPTooManyRequests, Net::ReadTimeout
|
|
99
105
|
# 上記以外のレスポンスの場合、リトライ可能ならばsleepした後に再度ロード実行
|
|
100
106
|
if retry_count > 0
|
|
101
107
|
sleep_for = 10
|
|
102
|
-
if response.is_a?(Net::HTTPTooManyRequests)
|
|
108
|
+
if @response.is_a?(Net::HTTPTooManyRequests)
|
|
103
109
|
# HTTPTooManyRequestsならばretry-afterで指定された値を取得。
|
|
104
|
-
sleep_for = response.header['retry-after'].to_i + 10
|
|
105
|
-
log("Rate limit: #{uri} #{response.header.to_hash} (429 Too Many Requests). Sleeping #{sleep_for} seconds and retry (##{retry_count}).", @verbose)
|
|
110
|
+
sleep_for = @response.header['retry-after'].to_i + 10
|
|
111
|
+
log("Rate limit: #{uri} #{@response.header.to_hash} (429 Too Many Requests). Sleeping #{sleep_for} seconds and retry (##{retry_count}).", @verbose)
|
|
106
112
|
else
|
|
107
|
-
log("Unknown response: #{uri} #{response.inspect}. Sleeping #{sleep_for} seconds and retry (##{retry_count}).", @verbose)
|
|
113
|
+
log("Unknown response: #{uri} #{@response.inspect}. Sleeping #{sleep_for} seconds and retry (##{retry_count}).", @verbose)
|
|
108
114
|
end
|
|
109
115
|
sleep sleep_for
|
|
110
116
|
result = load(url, redirect_count , retry_count - 1)
|
|
@@ -112,7 +118,7 @@ module WebLoader
|
|
|
112
118
|
|
|
113
119
|
# それ以外は対応した例外を発生
|
|
114
120
|
log("error #{url}", true)
|
|
115
|
-
response.value
|
|
121
|
+
@response.value
|
|
116
122
|
end
|
|
117
123
|
result
|
|
118
124
|
end
|
data/lib/web_loader/version.rb
CHANGED
data/web_loader.iml
CHANGED
|
@@ -11,21 +11,21 @@
|
|
|
11
11
|
<orderEntry type="inheritedJdk" />
|
|
12
12
|
<orderEntry type="sourceFolder" forTests="false" />
|
|
13
13
|
<orderEntry type="module-library">
|
|
14
|
-
<library name="minitest (v5.22.
|
|
14
|
+
<library name="minitest (v5.22.2) [path][gem]" type="rubylib">
|
|
15
15
|
<properties>
|
|
16
16
|
<option name="version" value="4" />
|
|
17
17
|
</properties>
|
|
18
18
|
<CLASSES>
|
|
19
|
-
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.
|
|
20
|
-
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.
|
|
19
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/lib" />
|
|
20
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/test" />
|
|
21
21
|
</CLASSES>
|
|
22
22
|
<JAVADOC />
|
|
23
23
|
<SOURCES>
|
|
24
|
-
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.
|
|
25
|
-
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.
|
|
24
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/lib" />
|
|
25
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/test" />
|
|
26
26
|
</SOURCES>
|
|
27
27
|
<excluded>
|
|
28
|
-
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.
|
|
28
|
+
<root url="file://$MODULE_DIR$/vendor/bundle/ruby/3.1.0/gems/minitest-5.22.2/test" />
|
|
29
29
|
</excluded>
|
|
30
30
|
</library>
|
|
31
31
|
</orderEntry>
|
|
@@ -57,4 +57,37 @@
|
|
|
57
57
|
<RakeTaskImpl id="rake" />
|
|
58
58
|
</option>
|
|
59
59
|
</component>
|
|
60
|
+
<component name="RakeTasksCache-v2">
|
|
61
|
+
<option name="myRootTask">
|
|
62
|
+
<RakeTaskImpl id="rake">
|
|
63
|
+
<subtasks>
|
|
64
|
+
<RakeTaskImpl description="Build web_loader-1.4.1.gem into the pkg directory" fullCommand="build" id="build" />
|
|
65
|
+
<RakeTaskImpl id="build">
|
|
66
|
+
<subtasks>
|
|
67
|
+
<RakeTaskImpl description="Generate SHA512 checksum if web_loader-1.4.1.gem into the checksums directory" fullCommand="build:checksum" id="checksum" />
|
|
68
|
+
</subtasks>
|
|
69
|
+
</RakeTaskImpl>
|
|
70
|
+
<RakeTaskImpl description="Remove any temporary products" fullCommand="clean" id="clean" />
|
|
71
|
+
<RakeTaskImpl description="Remove any generated files" fullCommand="clobber" id="clobber" />
|
|
72
|
+
<RakeTaskImpl description="Build and install web_loader-1.4.1.gem into system gems" fullCommand="install" id="install" />
|
|
73
|
+
<RakeTaskImpl id="install">
|
|
74
|
+
<subtasks>
|
|
75
|
+
<RakeTaskImpl description="Build and install web_loader-1.4.1.gem into system gems without network access" fullCommand="install:local" id="local" />
|
|
76
|
+
</subtasks>
|
|
77
|
+
</RakeTaskImpl>
|
|
78
|
+
<RakeTaskImpl description="Create tag v1.4.1 and build and push web_loader-1.4.1.gem to rubygems.org" fullCommand="release[remote]" id="release[remote]" />
|
|
79
|
+
<RakeTaskImpl description="Run tests" fullCommand="test" id="test" />
|
|
80
|
+
<RakeTaskImpl description="" fullCommand="default" id="default" />
|
|
81
|
+
<RakeTaskImpl description="" fullCommand="release" id="release" />
|
|
82
|
+
<RakeTaskImpl id="release">
|
|
83
|
+
<subtasks>
|
|
84
|
+
<RakeTaskImpl description="" fullCommand="release:guard_clean" id="guard_clean" />
|
|
85
|
+
<RakeTaskImpl description="" fullCommand="release:rubygem_push" id="rubygem_push" />
|
|
86
|
+
<RakeTaskImpl description="" fullCommand="release:source_control_push" id="source_control_push" />
|
|
87
|
+
</subtasks>
|
|
88
|
+
</RakeTaskImpl>
|
|
89
|
+
</subtasks>
|
|
90
|
+
</RakeTaskImpl>
|
|
91
|
+
</option>
|
|
92
|
+
</component>
|
|
60
93
|
</module>
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: web_loader
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.5.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- src
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-
|
|
11
|
+
date: 2024-03-26 00:00:00.000000000 Z
|
|
12
12
|
dependencies: []
|
|
13
13
|
description: Web loader.
|
|
14
14
|
email:
|