web_loader 1.4.1 → 1.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.idea/.gitignore +2 -0
- data/Gemfile.lock +1 -1
- data/lib/web_loader/command.rb +18 -14
- data/lib/web_loader/version.rb +1 -1
- data/web_loader.iml +33 -0
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 972dc054f042a2f2da8e3e84f8dc0963343765bd316c11ed285302551c071de0
|
|
4
|
+
data.tar.gz: 8711c2f6d20926d94960e0ad924aad1c6931551a2f1e30c9fdbc6c89d285bf15
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e7e52952ef16318d2e792521044dcd614c8828311bc49557b67948f843ce4934b7c5c79ccdbc53357ce63a3deeb6f7ad32ec6201bb274386b7bf481f5b31956e
|
|
7
|
+
data.tar.gz: b32129726e055b55b26fda3f3f9ebac71f7b2b3f3ca916a2c612dc1439195ac40052d0d7015b31663a4184fe7a279934b6fbb372ddae9cf0193cdb93a3ab7545
|
data/.idea/.gitignore
CHANGED
data/Gemfile.lock
CHANGED
data/lib/web_loader/command.rb
CHANGED
|
@@ -32,12 +32,14 @@ module WebLoader
|
|
|
32
32
|
@verbose = false
|
|
33
33
|
@cache_limit = CACHE_LIMIT
|
|
34
34
|
@always_write_cache = false
|
|
35
|
+
@response = nil
|
|
35
36
|
end
|
|
36
37
|
|
|
37
38
|
attr_reader :load_cache_page
|
|
38
39
|
attr_accessor :use_cache, :cache_dir, :binary, :user_agent, :verbose
|
|
39
40
|
attr_accessor :cache_limit
|
|
40
41
|
attr_accessor :always_write_cache
|
|
42
|
+
attr_reader :response
|
|
41
43
|
|
|
42
44
|
def load_retry(url, retry_count = DEFAULT_RETRY)
|
|
43
45
|
load(url, DEFAULT_REDIRECT, retry_count)
|
|
@@ -64,9 +66,9 @@ module WebLoader
|
|
|
64
66
|
http.use_ssl = true
|
|
65
67
|
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
|
66
68
|
end
|
|
67
|
-
response = nil
|
|
69
|
+
@response = nil
|
|
68
70
|
begin
|
|
69
|
-
response = http.get(uri.request_uri, 'User-Agent' => @user_agent) # request_uri=path + '?' + query
|
|
71
|
+
@response = http.get(uri.request_uri, 'User-Agent' => @user_agent) # request_uri=path + '?' + query
|
|
70
72
|
rescue Net::ReadTimeout
|
|
71
73
|
# タイムアウトした場合リトライ可能ならばsleepした後に再度ロード実行
|
|
72
74
|
log("Read timeout: #{url}", @verbose)
|
|
@@ -78,35 +80,37 @@ module WebLoader
|
|
|
78
80
|
|
|
79
81
|
##### レスポンスの処理
|
|
80
82
|
result = nil
|
|
81
|
-
case response
|
|
83
|
+
case @response
|
|
82
84
|
when Net::HTTPSuccess
|
|
83
|
-
# responseがNet::HTTPSuccessのサブクラスの場合成功とみなし読み込んだ内容を返す
|
|
84
|
-
body = response.body
|
|
85
|
+
# @responseがNet::HTTPSuccessのサブクラスの場合成功とみなし読み込んだ内容を返す
|
|
86
|
+
body = @response.body
|
|
85
87
|
unless @binary
|
|
86
88
|
# デフォルトでは ASCII-8BITが帰ってくる。
|
|
87
89
|
# Content-Typeのcharsetとみなす。
|
|
88
90
|
# https://bugs.ruby-lang.org/issues/2567
|
|
89
|
-
encoding = response.type_params['charset']
|
|
91
|
+
encoding = @response.type_params['charset']
|
|
90
92
|
body = toutf8(body, encoding)
|
|
91
93
|
end
|
|
92
94
|
|
|
93
95
|
if @use_cache || @always_write_cache
|
|
94
96
|
log("Write cache: #{url}", @verbose)
|
|
95
|
-
Cache.write(@cache_dir, url, response.code, body)
|
|
97
|
+
Cache.write(@cache_dir, url, @response.code, body)
|
|
96
98
|
end
|
|
97
99
|
result = body
|
|
98
100
|
when Net::HTTPRedirection
|
|
99
|
-
result = load(to_redirect_url(uri, response['location']), redirect_count - 1)
|
|
100
|
-
|
|
101
|
+
result = load(to_redirect_url(uri, @response['location']), redirect_count - 1)
|
|
102
|
+
# when Net::HTTPNotFound
|
|
103
|
+
# result = nil
|
|
104
|
+
when Net::HTTPTooManyRequests, Net::ReadTimeout
|
|
101
105
|
# 上記以外のレスポンスの場合、リトライ可能ならばsleepした後に再度ロード実行
|
|
102
106
|
if retry_count > 0
|
|
103
107
|
sleep_for = 10
|
|
104
|
-
if response.is_a?(Net::HTTPTooManyRequests)
|
|
108
|
+
if @response.is_a?(Net::HTTPTooManyRequests)
|
|
105
109
|
# HTTPTooManyRequestsならばretry-afterで指定された値を取得。
|
|
106
|
-
sleep_for = response.header['retry-after'].to_i + 10
|
|
107
|
-
log("Rate limit: #{uri} #{response.header.to_hash} (429 Too Many Requests). Sleeping #{sleep_for} seconds and retry (##{retry_count}).", @verbose)
|
|
110
|
+
sleep_for = @response.header['retry-after'].to_i + 10
|
|
111
|
+
log("Rate limit: #{uri} #{@response.header.to_hash} (429 Too Many Requests). Sleeping #{sleep_for} seconds and retry (##{retry_count}).", @verbose)
|
|
108
112
|
else
|
|
109
|
-
log("Unknown response: #{uri} #{response.inspect}. Sleeping #{sleep_for} seconds and retry (##{retry_count}).", @verbose)
|
|
113
|
+
log("Unknown response: #{uri} #{@response.inspect}. Sleeping #{sleep_for} seconds and retry (##{retry_count}).", @verbose)
|
|
110
114
|
end
|
|
111
115
|
sleep sleep_for
|
|
112
116
|
result = load(url, redirect_count , retry_count - 1)
|
|
@@ -114,7 +118,7 @@ module WebLoader
|
|
|
114
118
|
|
|
115
119
|
# それ以外は対応した例外を発生
|
|
116
120
|
log("error #{url}", true)
|
|
117
|
-
response.value
|
|
121
|
+
@response.value
|
|
118
122
|
end
|
|
119
123
|
result
|
|
120
124
|
end
|
data/lib/web_loader/version.rb
CHANGED
data/web_loader.iml
CHANGED
|
@@ -57,4 +57,37 @@
|
|
|
57
57
|
<RakeTaskImpl id="rake" />
|
|
58
58
|
</option>
|
|
59
59
|
</component>
|
|
60
|
+
<component name="RakeTasksCache-v2">
|
|
61
|
+
<option name="myRootTask">
|
|
62
|
+
<RakeTaskImpl id="rake">
|
|
63
|
+
<subtasks>
|
|
64
|
+
<RakeTaskImpl description="Build web_loader-1.4.1.gem into the pkg directory" fullCommand="build" id="build" />
|
|
65
|
+
<RakeTaskImpl id="build">
|
|
66
|
+
<subtasks>
|
|
67
|
+
<RakeTaskImpl description="Generate SHA512 checksum if web_loader-1.4.1.gem into the checksums directory" fullCommand="build:checksum" id="checksum" />
|
|
68
|
+
</subtasks>
|
|
69
|
+
</RakeTaskImpl>
|
|
70
|
+
<RakeTaskImpl description="Remove any temporary products" fullCommand="clean" id="clean" />
|
|
71
|
+
<RakeTaskImpl description="Remove any generated files" fullCommand="clobber" id="clobber" />
|
|
72
|
+
<RakeTaskImpl description="Build and install web_loader-1.4.1.gem into system gems" fullCommand="install" id="install" />
|
|
73
|
+
<RakeTaskImpl id="install">
|
|
74
|
+
<subtasks>
|
|
75
|
+
<RakeTaskImpl description="Build and install web_loader-1.4.1.gem into system gems without network access" fullCommand="install:local" id="local" />
|
|
76
|
+
</subtasks>
|
|
77
|
+
</RakeTaskImpl>
|
|
78
|
+
<RakeTaskImpl description="Create tag v1.4.1 and build and push web_loader-1.4.1.gem to rubygems.org" fullCommand="release[remote]" id="release[remote]" />
|
|
79
|
+
<RakeTaskImpl description="Run tests" fullCommand="test" id="test" />
|
|
80
|
+
<RakeTaskImpl description="" fullCommand="default" id="default" />
|
|
81
|
+
<RakeTaskImpl description="" fullCommand="release" id="release" />
|
|
82
|
+
<RakeTaskImpl id="release">
|
|
83
|
+
<subtasks>
|
|
84
|
+
<RakeTaskImpl description="" fullCommand="release:guard_clean" id="guard_clean" />
|
|
85
|
+
<RakeTaskImpl description="" fullCommand="release:rubygem_push" id="rubygem_push" />
|
|
86
|
+
<RakeTaskImpl description="" fullCommand="release:source_control_push" id="source_control_push" />
|
|
87
|
+
</subtasks>
|
|
88
|
+
</RakeTaskImpl>
|
|
89
|
+
</subtasks>
|
|
90
|
+
</RakeTaskImpl>
|
|
91
|
+
</option>
|
|
92
|
+
</component>
|
|
60
93
|
</module>
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: web_loader
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.5.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- src
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-
|
|
11
|
+
date: 2024-03-26 00:00:00.000000000 Z
|
|
12
12
|
dependencies: []
|
|
13
13
|
description: Web loader.
|
|
14
14
|
email:
|