webarchive 0.1.4 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c9fc5065342461039496f77e140585d7023878acf1053ac820bc262fdf6917db
4
- data.tar.gz: d093202d5cf905a359d7abb7af6cbf67a88898217493a7239eef5d63015ae831
3
+ metadata.gz: 16eba058e7574f02d9e444a6d72b2aefb41b55fb0acb9adc1b6eef492cef306d
4
+ data.tar.gz: 33411ba62ea808f72b930ef52a5f112f4df80b4febb1d2bd88c7017a62785550
5
5
  SHA512:
6
- metadata.gz: 3eedec1586a6a33d9a6a95b63dc98ef3b0fde2a000389743b807f2a074c74b05341b2870acc3d20ee7b994f3cfcbb931e9451f02d70a5645ec327bce6ac80679
7
- data.tar.gz: 1c8b60d67cce9e228c8785a79df5bc056bf88141c7bf7dd7c83018aa0fc28d11ca101bfd78af4ea9289761797fee214dfa2831be7e6acf45a6d887356927c0ba
6
+ metadata.gz: 8787995fe6bdc059e2275371ed33eb332a8f1c99aa099b995a20805258ed2d05983507a8f2958e104edcc31e7259bc3be68d22f838c917294687533718a799c5
7
+ data.tar.gz: f9e73ef39c3cb49b6bd25569cc2549292982aa47e71e1c31ce735eb350e8a1fe0882fe8cb0a3d9ee23d3b6075d33ec149a951f4d70f9095d4b8f03bc8c306e6c
data/README.md CHANGED
@@ -39,6 +39,7 @@ It has optional command-line parameters:
39
39
  Usage: webarchive [options]
40
40
  -w, --wait=SECONDS wait for SECONDS between requests [default: 5.0]
41
41
  -r, --retry=N retry for N times when failed [default: 5]
42
+ -t, --timeout=SECONDS timeout after SECONDS [default: 60.0]
42
43
  --[no-]history record history [default: enabled]
43
44
  -d, --debug add debug output, implies verbose
44
45
  --verbose
data/bin/webarchive CHANGED
@@ -7,6 +7,7 @@ require 'optparse'
7
7
 
8
8
  wait = 5.0
9
9
  retry_ = 5
10
+ timeout = 60.0
10
11
  debug = false
11
12
  verbose = false
12
13
  history = true
@@ -21,6 +22,10 @@ OptionParser.new do |opt|
21
22
  "retry for N times when failed [default: #{retry_}]") do |v|
22
23
  retry_ = v.to_i
23
24
  end
25
+ opt.on('-t', '--timeout=SECONDS',
26
+ "timeout after SECONDS [default: #{timeout}]") do |v|
27
+ timeout = v.to_i
28
+ end
24
29
  opt.on('--[no-]history',
25
30
  "record history [default: #{history ? 'enabled' : 'disabled'}]") do |v|
26
31
  history = v
@@ -41,4 +46,5 @@ tagline += ' (debug)' if debug
41
46
  warn tagline if verbose
42
47
 
43
48
  WebArchive.launch(wait_secs: wait, max_retry: retry_, history: history,
49
+ read_timeout_secs: timeout,
44
50
  debug: debug, verbose: verbose)
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module WebArchive
4
- VERSION = '0.1.4'
4
+ VERSION = '0.1.5'
5
5
  end
data/lib/webarchive.rb CHANGED
@@ -322,8 +322,13 @@ module WebArchive
322
322
  # Launch the CLI
323
323
  # @return [Concurrent::Promises::Future]
324
324
  def self.launch(wait_secs: 1, max_retry: 3,
325
+ read_timeout_secs: 15,
325
326
  redirect: false, canonical_uri: true,
326
327
  history: true, debug: false, verbose: false)
328
+
329
+ args = method(__method__).parameters.map{ |k, v| [v, binding.local_variable_get(v)] }
330
+ puts "Arguments: #{args.inspect}" if verbose
331
+
327
332
  wait_secs = 0 if wait_secs.negative?
328
333
  max_retry = 0 if max_retry.negative?
329
334
 
@@ -341,7 +346,8 @@ module WebArchive
341
346
  # prepare queues
342
347
  client.add_queue(
343
348
  ArchiveQueue.new('archive.org (logged out)', wait_secs) do |uri|
344
- URI.parse('https://web.archive.org/save/' + uri).open do |f|
349
+ u = URI.parse('https://web.archive.org/save/' + uri)
350
+ u.open(read_timeout: read_timeout_secs) do |f|
345
351
  if f.meta['content-location'] && verbose
346
352
  puts "<https://web.archive.org#{f.meta['content-location']}>"
347
353
  elsif verbose
@@ -354,6 +360,7 @@ module WebArchive
354
360
  client.add_queue(
355
361
  ArchiveQueue.new('megalodon.jp', wait_secs) do |uri|
356
362
  agent = Mechanize.new
363
+ agent.read_timeout = read_timeout_secs
357
364
  page = agent.get('https://megalodon.jp/pc/?' +
358
365
  Addressable::URI.form_encode(url: uri))
359
366
  form = page.forms.first
@@ -379,6 +386,7 @@ module WebArchive
379
386
  client.add_queue(
380
387
  ArchiveQueue.new('archive.today', wait_secs) do |uri|
381
388
  agent = Mechanize.new
389
+ agent.read_timeout = read_timeout_secs
382
390
  agent.follow_meta_refresh = true
383
391
 
384
392
  page = agent.get('https://archive.is/')
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: webarchive
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.4
4
+ version: 0.1.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yusuke Matsubara
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-06-30 00:00:00.000000000 Z
11
+ date: 2025-01-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: addressable
@@ -30,14 +30,14 @@ dependencies:
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: 1.1.6
33
+ version: 1.3.0
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: 1.1.6
40
+ version: 1.3.0
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: fast_trie
43
43
  requirement: !ruby/object:Gem::Requirement
@@ -58,14 +58,14 @@ dependencies:
58
58
  requirements:
59
59
  - - "~>"
60
60
  - !ruby/object:Gem::Version
61
- version: 2.8.0
61
+ version: 2.14.0
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
66
  - - "~>"
67
67
  - !ruby/object:Gem::Version
68
- version: 2.8.0
68
+ version: 2.14.0
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: net-http-persistent
71
71
  requirement: !ruby/object:Gem::Requirement
@@ -128,42 +128,42 @@ dependencies:
128
128
  requirements:
129
129
  - - "~>"
130
130
  - !ruby/object:Gem::Version
131
- version: '3.10'
131
+ version: '3.13'
132
132
  type: :development
133
133
  prerelease: false
134
134
  version_requirements: !ruby/object:Gem::Requirement
135
135
  requirements:
136
136
  - - "~>"
137
137
  - !ruby/object:Gem::Version
138
- version: '3.10'
138
+ version: '3.13'
139
139
  - !ruby/object:Gem::Dependency
140
140
  name: rubocop
141
141
  requirement: !ruby/object:Gem::Requirement
142
142
  requirements:
143
143
  - - "~>"
144
144
  - !ruby/object:Gem::Version
145
- version: 0.81.0
145
+ version: '1.70'
146
146
  type: :development
147
147
  prerelease: false
148
148
  version_requirements: !ruby/object:Gem::Requirement
149
149
  requirements:
150
150
  - - "~>"
151
151
  - !ruby/object:Gem::Version
152
- version: 0.81.0
152
+ version: '1.70'
153
153
  - !ruby/object:Gem::Dependency
154
154
  name: webmock
155
155
  requirement: !ruby/object:Gem::Requirement
156
156
  requirements:
157
157
  - - "~>"
158
158
  - !ruby/object:Gem::Version
159
- version: '3.13'
159
+ version: '3.24'
160
160
  type: :development
161
161
  prerelease: false
162
162
  version_requirements: !ruby/object:Gem::Requirement
163
163
  requirements:
164
164
  - - "~>"
165
165
  - !ruby/object:Gem::Version
166
- version: '3.13'
166
+ version: '3.24'
167
167
  - !ruby/object:Gem::Dependency
168
168
  name: pry
169
169
  requirement: !ruby/object:Gem::Requirement
@@ -225,7 +225,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
225
225
  - !ruby/object:Gem::Version
226
226
  version: '0'
227
227
  requirements: []
228
- rubygems_version: 3.3.7
228
+ rubygems_version: 3.4.19
229
229
  signing_key:
230
230
  specification_version: 4
231
231
  summary: webarchive - CUI tool to archive URIs