webarchive 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -0
- data/bin/webarchive +6 -0
- data/lib/webarchive/version.rb +1 -1
- data/lib/webarchive.rb +9 -1
- metadata +13 -13
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 16eba058e7574f02d9e444a6d72b2aefb41b55fb0acb9adc1b6eef492cef306d
|
4
|
+
data.tar.gz: 33411ba62ea808f72b930ef52a5f112f4df80b4febb1d2bd88c7017a62785550
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8787995fe6bdc059e2275371ed33eb332a8f1c99aa099b995a20805258ed2d05983507a8f2958e104edcc31e7259bc3be68d22f838c917294687533718a799c5
|
7
|
+
data.tar.gz: f9e73ef39c3cb49b6bd25569cc2549292982aa47e71e1c31ce735eb350e8a1fe0882fe8cb0a3d9ee23d3b6075d33ec149a951f4d70f9095d4b8f03bc8c306e6c
|
data/README.md
CHANGED
@@ -39,6 +39,7 @@ It has optional command-line parameters:
|
|
39
39
|
Usage: webarchive [options]
|
40
40
|
-w, --wait=SECONDS wait for SECONDS between requests [default: 5.0]
|
41
41
|
-r, --retry=N retry for N times when failed [default: 5]
|
42
|
+
-t, --timeout=SECONDS timeout after SECONDS [default: 60.0]
|
42
43
|
--[no-]history record history [default: enabled]
|
43
44
|
-d, --debug add debug output, implies verbose
|
44
45
|
--verbose
|
data/bin/webarchive
CHANGED
@@ -7,6 +7,7 @@ require 'optparse'
|
|
7
7
|
|
8
8
|
wait = 5.0
|
9
9
|
retry_ = 5
|
10
|
+
timeout = 60.0
|
10
11
|
debug = false
|
11
12
|
verbose = false
|
12
13
|
history = true
|
@@ -21,6 +22,10 @@ OptionParser.new do |opt|
|
|
21
22
|
"retry for N times when failed [default: #{retry_}]") do |v|
|
22
23
|
retry_ = v.to_i
|
23
24
|
end
|
25
|
+
opt.on('-t', '--timeout=SECONDS',
|
26
|
+
"timeout after SECONDS [default: #{timeout}]") do |v|
|
27
|
+
timeout = v.to_i
|
28
|
+
end
|
24
29
|
opt.on('--[no-]history',
|
25
30
|
"record history [default: #{history ? 'enabled' : 'disabled'}]") do |v|
|
26
31
|
history = v
|
@@ -41,4 +46,5 @@ tagline += ' (debug)' if debug
|
|
41
46
|
warn tagline if verbose
|
42
47
|
|
43
48
|
WebArchive.launch(wait_secs: wait, max_retry: retry_, history: history,
|
49
|
+
read_timeout_secs: timeout,
|
44
50
|
debug: debug, verbose: verbose)
|
data/lib/webarchive/version.rb
CHANGED
data/lib/webarchive.rb
CHANGED
@@ -322,8 +322,13 @@ module WebArchive
|
|
322
322
|
# Launch the CLI
|
323
323
|
# @return [Concurrent::Promises::Future]
|
324
324
|
def self.launch(wait_secs: 1, max_retry: 3,
|
325
|
+
read_timeout_secs: 15,
|
325
326
|
redirect: false, canonical_uri: true,
|
326
327
|
history: true, debug: false, verbose: false)
|
328
|
+
|
329
|
+
args = method(__method__).parameters.map{ |k, v| [v, binding.local_variable_get(v)] }
|
330
|
+
puts "Arguments: #{args.inspect}" if verbose
|
331
|
+
|
327
332
|
wait_secs = 0 if wait_secs.negative?
|
328
333
|
max_retry = 0 if max_retry.negative?
|
329
334
|
|
@@ -341,7 +346,8 @@ module WebArchive
|
|
341
346
|
# prepare queues
|
342
347
|
client.add_queue(
|
343
348
|
ArchiveQueue.new('archive.org (logged out)', wait_secs) do |uri|
|
344
|
-
URI.parse('https://web.archive.org/save/' + uri)
|
349
|
+
u = URI.parse('https://web.archive.org/save/' + uri)
|
350
|
+
u.open(read_timeout: read_timeout_secs) do |f|
|
345
351
|
if f.meta['content-location'] && verbose
|
346
352
|
puts "<https://web.archive.org#{f.meta['content-location']}>"
|
347
353
|
elsif verbose
|
@@ -354,6 +360,7 @@ module WebArchive
|
|
354
360
|
client.add_queue(
|
355
361
|
ArchiveQueue.new('megalodon.jp', wait_secs) do |uri|
|
356
362
|
agent = Mechanize.new
|
363
|
+
agent.read_timeout = read_timeout_secs
|
357
364
|
page = agent.get('https://megalodon.jp/pc/?' +
|
358
365
|
Addressable::URI.form_encode(url: uri))
|
359
366
|
form = page.forms.first
|
@@ -379,6 +386,7 @@ module WebArchive
|
|
379
386
|
client.add_queue(
|
380
387
|
ArchiveQueue.new('archive.today', wait_secs) do |uri|
|
381
388
|
agent = Mechanize.new
|
389
|
+
agent.read_timeout = read_timeout_secs
|
382
390
|
agent.follow_meta_refresh = true
|
383
391
|
|
384
392
|
page = agent.get('https://archive.is/')
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: webarchive
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yusuke Matsubara
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2025-01-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: addressable
|
@@ -30,14 +30,14 @@ dependencies:
|
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: 1.
|
33
|
+
version: 1.3.0
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: 1.
|
40
|
+
version: 1.3.0
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: fast_trie
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -58,14 +58,14 @@ dependencies:
|
|
58
58
|
requirements:
|
59
59
|
- - "~>"
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: 2.
|
61
|
+
version: 2.14.0
|
62
62
|
type: :runtime
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
66
|
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: 2.
|
68
|
+
version: 2.14.0
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: net-http-persistent
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -128,42 +128,42 @@ dependencies:
|
|
128
128
|
requirements:
|
129
129
|
- - "~>"
|
130
130
|
- !ruby/object:Gem::Version
|
131
|
-
version: '3.
|
131
|
+
version: '3.13'
|
132
132
|
type: :development
|
133
133
|
prerelease: false
|
134
134
|
version_requirements: !ruby/object:Gem::Requirement
|
135
135
|
requirements:
|
136
136
|
- - "~>"
|
137
137
|
- !ruby/object:Gem::Version
|
138
|
-
version: '3.
|
138
|
+
version: '3.13'
|
139
139
|
- !ruby/object:Gem::Dependency
|
140
140
|
name: rubocop
|
141
141
|
requirement: !ruby/object:Gem::Requirement
|
142
142
|
requirements:
|
143
143
|
- - "~>"
|
144
144
|
- !ruby/object:Gem::Version
|
145
|
-
version:
|
145
|
+
version: '1.70'
|
146
146
|
type: :development
|
147
147
|
prerelease: false
|
148
148
|
version_requirements: !ruby/object:Gem::Requirement
|
149
149
|
requirements:
|
150
150
|
- - "~>"
|
151
151
|
- !ruby/object:Gem::Version
|
152
|
-
version:
|
152
|
+
version: '1.70'
|
153
153
|
- !ruby/object:Gem::Dependency
|
154
154
|
name: webmock
|
155
155
|
requirement: !ruby/object:Gem::Requirement
|
156
156
|
requirements:
|
157
157
|
- - "~>"
|
158
158
|
- !ruby/object:Gem::Version
|
159
|
-
version: '3.
|
159
|
+
version: '3.24'
|
160
160
|
type: :development
|
161
161
|
prerelease: false
|
162
162
|
version_requirements: !ruby/object:Gem::Requirement
|
163
163
|
requirements:
|
164
164
|
- - "~>"
|
165
165
|
- !ruby/object:Gem::Version
|
166
|
-
version: '3.
|
166
|
+
version: '3.24'
|
167
167
|
- !ruby/object:Gem::Dependency
|
168
168
|
name: pry
|
169
169
|
requirement: !ruby/object:Gem::Requirement
|
@@ -225,7 +225,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
225
225
|
- !ruby/object:Gem::Version
|
226
226
|
version: '0'
|
227
227
|
requirements: []
|
228
|
-
rubygems_version: 3.
|
228
|
+
rubygems_version: 3.4.19
|
229
229
|
signing_key:
|
230
230
|
specification_version: 4
|
231
231
|
summary: webarchive - CUI tool to archive URIs
|