wayback_machine_downloader 0.4.1 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 98ea352b6eae2e49f915634245d65f109a864ed0
4
- data.tar.gz: 5eee04e5f319e23709ef94f6677c194deaa81640
3
+ metadata.gz: dfe1b1274056f709121477fb93bdaa45b1580745
4
+ data.tar.gz: 1c97898ad292a2a2a89377c19e8e96f623a4ce2b
5
5
  SHA512:
6
- metadata.gz: 6788c3f2e764e1ebe02ee967e4653814b1fe8de316ffc8eb307dbd85eabc5c6b867b4abda5b2dd36300cd5e96f353429740b206d6ad871cd1b78b760f99ca029
7
- data.tar.gz: 7795541cd32a39a7b78875a8a841ec8124df01aa38b9504e8f29109d2d431b1299d76170624739786d3ebf28fa8eeacd1081ed496498e69ef6fa40f31dfc29c0
6
+ metadata.gz: 0c196f8e23887413a6f694f88548ff960b85115d3b5b0fccdb120b5b6b7efd9a2ad23ebdf24e34c8f5ab66cfdfd7c7c4b36db034f14ae864da6d392509bb6873
7
+ data.tar.gz: 0f0a42c3d5d0679c11eabacae2420ed901119af4a18246798320dc836dae4f9c64d57500dd6edf424d6c9ae8612b93162e57551fd76d3d27ec40910617bdbcbc
@@ -30,6 +30,10 @@ option_parser = OptionParser.new do |opts|
30
30
  options[:exclude_filter] = t
31
31
  end
32
32
 
33
+ opts.on("-a", "--all", "Expand downloading to error files (40x and 50x) and redirections (30x)") do |t|
34
+ options[:all] = true
35
+ end
36
+
33
37
  opts.on("-v", "--version", "Display version") do |t|
34
38
  options[:version] = t
35
39
  end
@@ -8,9 +8,9 @@ require_relative 'wayback_machine_downloader/to_regex'
8
8
 
9
9
  class WaybackMachineDownloader
10
10
 
11
- VERSION = "0.4.1"
11
+ VERSION = "0.4.2"
12
12
 
13
- attr_accessor :base_url, :from_timestamp, :to_timestamp, :only_filter, :exclude_filter
13
+ attr_accessor :base_url, :from_timestamp, :to_timestamp, :only_filter, :exclude_filter, :all
14
14
 
15
15
  def initialize params
16
16
  @base_url = params[:base_url]
@@ -18,6 +18,7 @@ class WaybackMachineDownloader
18
18
  @to_timestamp = params[:to_timestamp].to_i
19
19
  @only_filter = params[:only_filter]
20
20
  @exclude_filter = params[:exclude_filter]
21
+ @all = params[:all]
21
22
  end
22
23
 
23
24
  def backup_name
@@ -55,7 +56,10 @@ class WaybackMachineDownloader
55
56
  end
56
57
 
57
58
  def get_file_list_curated
58
- parameters_for_wayback_machine_api = "&fl=timestamp,original&fastLatest=true&filter=statuscode:200&collapse=original"
59
+ parameters_for_wayback_machine_api = "&fl=timestamp,original&collapse=original"
60
+ unless @all
61
+ parameters_for_wayback_machine_api += "&filter=statuscode:200"
62
+ end
59
63
  if @from_timestamp and @from_timestamp != 0
60
64
  parameters_for_wayback_machine_api += "&from=" + @from_timestamp.to_s
61
65
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: wayback_machine_downloader
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.1
4
+ version: 0.4.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - hartator
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-07-30 00:00:00.000000000 Z
11
+ date: 2016-07-31 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake