web_loader 1.4.1 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.idea/.gitignore +2 -0
- data/Gemfile.lock +1 -1
- data/lib/web_loader/command.rb +18 -14
- data/lib/web_loader/version.rb +1 -1
- data/web_loader.iml +33 -0
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 972dc054f042a2f2da8e3e84f8dc0963343765bd316c11ed285302551c071de0
|
|
4
|
+
data.tar.gz: 8711c2f6d20926d94960e0ad924aad1c6931551a2f1e30c9fdbc6c89d285bf15
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e7e52952ef16318d2e792521044dcd614c8828311bc49557b67948f843ce4934b7c5c79ccdbc53357ce63a3deeb6f7ad32ec6201bb274386b7bf481f5b31956e
|
|
7
|
+
data.tar.gz: b32129726e055b55b26fda3f3f9ebac71f7b2b3f3ca916a2c612dc1439195ac40052d0d7015b31663a4184fe7a279934b6fbb372ddae9cf0193cdb93a3ab7545
|
data/.idea/.gitignore
CHANGED
data/Gemfile.lock
CHANGED
data/lib/web_loader/command.rb
CHANGED
|
@@ -32,12 +32,14 @@ module WebLoader
|
|
|
32
32
|
@verbose = false
|
|
33
33
|
@cache_limit = CACHE_LIMIT
|
|
34
34
|
@always_write_cache = false
|
|
35
|
+
@response = nil
|
|
35
36
|
end
|
|
36
37
|
|
|
37
38
|
attr_reader :load_cache_page
|
|
38
39
|
attr_accessor :use_cache, :cache_dir, :binary, :user_agent, :verbose
|
|
39
40
|
attr_accessor :cache_limit
|
|
40
41
|
attr_accessor :always_write_cache
|
|
42
|
+
attr_reader :response
|
|
41
43
|
|
|
42
44
|
def load_retry(url, retry_count = DEFAULT_RETRY)
|
|
43
45
|
load(url, DEFAULT_REDIRECT, retry_count)
|
|
@@ -64,9 +66,9 @@ module WebLoader
|
|
|
64
66
|
http.use_ssl = true
|
|
65
67
|
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
|
66
68
|
end
|
|
67
|
-
response = nil
|
|
69
|
+
@response = nil
|
|
68
70
|
begin
|
|
69
|
-
response = http.get(uri.request_uri, 'User-Agent' => @user_agent) # request_uri=path + '?' + query
|
|
71
|
+
@response = http.get(uri.request_uri, 'User-Agent' => @user_agent) # request_uri=path + '?' + query
|
|
70
72
|
rescue Net::ReadTimeout
|
|
71
73
|
# タイムアウトした場合リトライ可能ならばsleepした後に再度ロード実行
|
|
72
74
|
log("Read timeout: #{url}", @verbose)
|
|
@@ -78,35 +80,37 @@ module WebLoader
|
|
|
78
80
|
|
|
79
81
|
##### レスポンスの処理
|
|
80
82
|
result = nil
|
|
81
|
-
case response
|
|
83
|
+
case @response
|
|
82
84
|
when Net::HTTPSuccess
|
|
83
|
-
# responseがNet::HTTPSuccessのサブクラスの場合成功とみなし読み込んだ内容を返す
|
|
84
|
-
body = response.body
|
|
85
|
+
# @responseがNet::HTTPSuccessのサブクラスの場合成功とみなし読み込んだ内容を返す
|
|
86
|
+
body = @response.body
|
|
85
87
|
unless @binary
|
|
86
88
|
# デフォルトでは ASCII-8BITが帰ってくる。
|
|
87
89
|
# Content-Typeのcharsetとみなす。
|
|
88
90
|
# https://bugs.ruby-lang.org/issues/2567
|
|
89
|
-
encoding = response.type_params['charset']
|
|
91
|
+
encoding = @response.type_params['charset']
|
|
90
92
|
body = toutf8(body, encoding)
|
|
91
93
|
end
|
|
92
94
|
|
|
93
95
|
if @use_cache || @always_write_cache
|
|
94
96
|
log("Write cache: #{url}", @verbose)
|
|
95
|
-
Cache.write(@cache_dir, url, response.code, body)
|
|
97
|
+
Cache.write(@cache_dir, url, @response.code, body)
|
|
96
98
|
end
|
|
97
99
|
result = body
|
|
98
100
|
when Net::HTTPRedirection
|
|
99
|
-
result = load(to_redirect_url(uri, response['location']), redirect_count - 1)
|
|
100
|
-
|
|
101
|
+
result = load(to_redirect_url(uri, @response['location']), redirect_count - 1)
|
|
102
|
+
# when Net::HTTPNotFound
|
|
103
|
+
# result = nil
|
|
104
|
+
when Net::HTTPTooManyRequests, Net::ReadTimeout
|
|
101
105
|
# 上記以外のレスポンスの場合、リトライ可能ならばsleepした後に再度ロード実行
|
|
102
106
|
if retry_count > 0
|
|
103
107
|
sleep_for = 10
|
|
104
|
-
if response.is_a?(Net::HTTPTooManyRequests)
|
|
108
|
+
if @response.is_a?(Net::HTTPTooManyRequests)
|
|
105
109
|
# HTTPTooManyRequestsならばretry-afterで指定された値を取得。
|
|
106
|
-
sleep_for = response.header['retry-after'].to_i + 10
|
|
107
|
-
log("Rate limit: #{uri} #{response.header.to_hash} (429 Too Many Requests). Sleeping #{sleep_for} seconds and retry (##{retry_count}).", @verbose)
|
|
110
|
+
sleep_for = @response.header['retry-after'].to_i + 10
|
|
111
|
+
log("Rate limit: #{uri} #{@response.header.to_hash} (429 Too Many Requests). Sleeping #{sleep_for} seconds and retry (##{retry_count}).", @verbose)
|
|
108
112
|
else
|
|
109
|
-
log("Unknown response: #{uri} #{response.inspect}. Sleeping #{sleep_for} seconds and retry (##{retry_count}).", @verbose)
|
|
113
|
+
log("Unknown response: #{uri} #{@response.inspect}. Sleeping #{sleep_for} seconds and retry (##{retry_count}).", @verbose)
|
|
110
114
|
end
|
|
111
115
|
sleep sleep_for
|
|
112
116
|
result = load(url, redirect_count , retry_count - 1)
|
|
@@ -114,7 +118,7 @@ module WebLoader
|
|
|
114
118
|
|
|
115
119
|
# それ以外は対応した例外を発生
|
|
116
120
|
log("error #{url}", true)
|
|
117
|
-
response.value
|
|
121
|
+
@response.value
|
|
118
122
|
end
|
|
119
123
|
result
|
|
120
124
|
end
|
data/lib/web_loader/version.rb
CHANGED
data/web_loader.iml
CHANGED
|
@@ -57,4 +57,37 @@
|
|
|
57
57
|
<RakeTaskImpl id="rake" />
|
|
58
58
|
</option>
|
|
59
59
|
</component>
|
|
60
|
+
<component name="RakeTasksCache-v2">
|
|
61
|
+
<option name="myRootTask">
|
|
62
|
+
<RakeTaskImpl id="rake">
|
|
63
|
+
<subtasks>
|
|
64
|
+
<RakeTaskImpl description="Build web_loader-1.4.1.gem into the pkg directory" fullCommand="build" id="build" />
|
|
65
|
+
<RakeTaskImpl id="build">
|
|
66
|
+
<subtasks>
|
|
67
|
+
<RakeTaskImpl description="Generate SHA512 checksum if web_loader-1.4.1.gem into the checksums directory" fullCommand="build:checksum" id="checksum" />
|
|
68
|
+
</subtasks>
|
|
69
|
+
</RakeTaskImpl>
|
|
70
|
+
<RakeTaskImpl description="Remove any temporary products" fullCommand="clean" id="clean" />
|
|
71
|
+
<RakeTaskImpl description="Remove any generated files" fullCommand="clobber" id="clobber" />
|
|
72
|
+
<RakeTaskImpl description="Build and install web_loader-1.4.1.gem into system gems" fullCommand="install" id="install" />
|
|
73
|
+
<RakeTaskImpl id="install">
|
|
74
|
+
<subtasks>
|
|
75
|
+
<RakeTaskImpl description="Build and install web_loader-1.4.1.gem into system gems without network access" fullCommand="install:local" id="local" />
|
|
76
|
+
</subtasks>
|
|
77
|
+
</RakeTaskImpl>
|
|
78
|
+
<RakeTaskImpl description="Create tag v1.4.1 and build and push web_loader-1.4.1.gem to rubygems.org" fullCommand="release[remote]" id="release[remote]" />
|
|
79
|
+
<RakeTaskImpl description="Run tests" fullCommand="test" id="test" />
|
|
80
|
+
<RakeTaskImpl description="" fullCommand="default" id="default" />
|
|
81
|
+
<RakeTaskImpl description="" fullCommand="release" id="release" />
|
|
82
|
+
<RakeTaskImpl id="release">
|
|
83
|
+
<subtasks>
|
|
84
|
+
<RakeTaskImpl description="" fullCommand="release:guard_clean" id="guard_clean" />
|
|
85
|
+
<RakeTaskImpl description="" fullCommand="release:rubygem_push" id="rubygem_push" />
|
|
86
|
+
<RakeTaskImpl description="" fullCommand="release:source_control_push" id="source_control_push" />
|
|
87
|
+
</subtasks>
|
|
88
|
+
</RakeTaskImpl>
|
|
89
|
+
</subtasks>
|
|
90
|
+
</RakeTaskImpl>
|
|
91
|
+
</option>
|
|
92
|
+
</component>
|
|
60
93
|
</module>
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: web_loader
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.5.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- src
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-
|
|
11
|
+
date: 2024-03-26 00:00:00.000000000 Z
|
|
12
12
|
dependencies: []
|
|
13
13
|
description: Web loader.
|
|
14
14
|
email:
|