wayback_machine_downloader 0.4.1 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 98ea352b6eae2e49f915634245d65f109a864ed0
-  data.tar.gz: 5eee04e5f319e23709ef94f6677c194deaa81640
+  metadata.gz: dfe1b1274056f709121477fb93bdaa45b1580745
+  data.tar.gz: 1c97898ad292a2a2a89377c19e8e96f623a4ce2b
 SHA512:
-  metadata.gz: 6788c3f2e764e1ebe02ee967e4653814b1fe8de316ffc8eb307dbd85eabc5c6b867b4abda5b2dd36300cd5e96f353429740b206d6ad871cd1b78b760f99ca029
-  data.tar.gz: 7795541cd32a39a7b78875a8a841ec8124df01aa38b9504e8f29109d2d431b1299d76170624739786d3ebf28fa8eeacd1081ed496498e69ef6fa40f31dfc29c0
+  metadata.gz: 0c196f8e23887413a6f694f88548ff960b85115d3b5b0fccdb120b5b6b7efd9a2ad23ebdf24e34c8f5ab66cfdfd7c7c4b36db034f14ae864da6d392509bb6873
+  data.tar.gz: 0f0a42c3d5d0679c11eabacae2420ed901119af4a18246798320dc836dae4f9c64d57500dd6edf424d6c9ae8612b93162e57551fd76d3d27ec40910617bdbcbc

data/bin/wayback_machine_downloader

@@ -30,6 +30,10 @@ option_parser = OptionParser.new do |opts|
     options[:exclude_filter] = t
   end
 
+  opts.on("-a", "--all", "Expand downloading to error files (40x and 50x) and redirections (30x)") do |t|
+    options[:all] = true
+  end
+
   opts.on("-v", "--version", "Display version") do |t|
     options[:version] = t
   end

data/lib/wayback_machine_downloader.rb

@@ -8,9 +8,9 @@ require_relative 'wayback_machine_downloader/to_regex'
 
 class WaybackMachineDownloader
 
-  VERSION = "0.4.1"
+  VERSION = "0.4.2"
 
-  attr_accessor :base_url, :from_timestamp, :to_timestamp, :only_filter, :exclude_filter
+  attr_accessor :base_url, :from_timestamp, :to_timestamp, :only_filter, :exclude_filter, :all
 
   def initialize params
     @base_url = params[:base_url]
@@ -18,6 +18,7 @@ class WaybackMachineDownloader
     @to_timestamp = params[:to_timestamp].to_i
     @only_filter = params[:only_filter]
     @exclude_filter = params[:exclude_filter]
+    @all = params[:all]
   end
 
   def backup_name
@@ -55,7 +56,10 @@ class WaybackMachineDownloader
   end
 
   def get_file_list_curated
-    parameters_for_wayback_machine_api = "&fl=timestamp,original&fastLatest=true&filter=statuscode:200&collapse=original"
+    parameters_for_wayback_machine_api = "&fl=timestamp,original&collapse=original"
+    unless @all
+      parameters_for_wayback_machine_api += "&filter=statuscode:200"
+    end
     if @from_timestamp and @from_timestamp != 0
       parameters_for_wayback_machine_api += "&from=" + @from_timestamp.to_s
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wayback_machine_downloader
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors:
 - hartator
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-30 00:00:00.000000000 Z
+date: 2016-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake