wayback_machine_downloader 2.1.1 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 48f524cedc0e9f66c7b0acca132a71557a327ea2
-  data.tar.gz: 1d70bb2a76cd07c82c08674fdc96b543caec48c0
+  metadata.gz: 77c3cb9b38d076dbaa7c30790c828ed63dacf2a6
+  data.tar.gz: 1e47ae74b6d2c0bbbe2ae99bc4622115c1a6496c
 SHA512:
-  metadata.gz: 26eb05cbeebd911502bd01513535c7cc2d4ad0fe3850adc0205ca4f649351e56855af66915d86c501fb8be64963fe1d409d013d8afcd24064cc15673b2cc0854
-  data.tar.gz: 0dbbd54b4b4ab231adcae908bbf6cd3865768590263e767fe5e45fb3a9d70676c337f79aba576378272ddc14647ecd06fc26820fc1dec8cb52704aa6740582b7
+  metadata.gz: 0326d70c1cf269099418b37302b01b22433931e217cf6d37b1bfd9fd6bcd1ced45e80ff2d0aad72d7c695ac4c25fa59d5e512a4b4674f182d4602647602ccb23
+  data.tar.gz: 9a0bf4eed69041ce156075a7f206237f9c1f5974b63fcb23370c19139a996d5fd143c1ff1fbe4640ce9cf5d36e36ed2e20916db504cbff9d0949f72f4086626d

data/bin/wayback_machine_downloader

@@ -18,6 +18,10 @@ option_parser = OptionParser.new do |opts|
     options[:directory] = t
   end
 
+  opts.on("-s", "--all-timestamps", "Download all snapshots/timestamps for a given website") do |t|
+    options[:all_timestamps] = true
+  end
+  
   opts.on("-f", "--from TIMESTAMP", Integer, "Only files on or after timestamp supplied (ie. 20060716231334)") do |t|
     options[:from_timestamp] = t
   end

data/lib/wayback_machine_downloader.rb

@@ -14,9 +14,9 @@ class WaybackMachineDownloader
 
   include ArchiveAPI
 
-  VERSION = "2.1.1"
+  VERSION = "2.2.0"
 
-  attr_accessor :base_url, :exact_url, :directory,
+  attr_accessor :base_url, :exact_url, :directory, :all_timestamps,
     :from_timestamp, :to_timestamp, :only_filter, :exclude_filter, 
     :all, :maximum_pages, :threads_count
 
@@ -24,6 +24,7 @@ class WaybackMachineDownloader
     @base_url = params[:base_url]
     @exact_url = params[:exact_url]
     @directory = params[:directory]
+    @all_timestamps = params[:all_timestamps]
     @from_timestamp = params[:from_timestamp].to_i
     @to_timestamp = params[:to_timestamp].to_i
     @only_filter = params[:only_filter]
@@ -127,12 +128,49 @@ class WaybackMachineDownloader
     file_list_curated
   end
 
+  def get_file_list_all_timestamps
+    file_list_curated = Hash.new
+    get_all_snapshots_to_consider.each_line do |line|
+      next unless line.include?('/')
+      file_timestamp = line[0..13].to_i
+      file_url = line[15..-2]
+      file_id = file_url.split('/')[3..-1].join('/')
+      file_id_and_timestamp = [file_timestamp, file_id].join('/')
+      file_id_and_timestamp = CGI::unescape file_id_and_timestamp 
+      file_id_and_timestamp = file_id_and_timestamp.tidy_bytes unless file_id_and_timestamp == ""
+      if file_id.nil?
+        puts "Malformed file url, ignoring: #{file_url}"
+      else
+        if match_exclude_filter(file_url)
+          puts "File url matches exclude filter, ignoring: #{file_url}"
+        elsif not match_only_filter(file_url)
+          puts "File url doesn't match only filter, ignoring: #{file_url}"
+        elsif file_list_curated[file_id_and_timestamp]
+          puts "Duplicate file and timestamp combo, ignoring: #{file_id}" if @verbose
+        else
+          file_list_curated[file_id_and_timestamp] = {file_url: file_url, timestamp: file_timestamp}
+        end
+      end
+    end
+    puts "file_list_curated: " + file_list_curated.count.to_s
+    file_list_curated
+  end
+
+
   def get_file_list_by_timestamp
-    file_list_curated = get_file_list_curated
-    file_list_curated = file_list_curated.sort_by { |k,v| v[:timestamp] }.reverse
-    file_list_curated.map do |file_remote_info|
-      file_remote_info[1][:file_id] = file_remote_info[0]
-      file_remote_info[1]
+    if @all_timestamps
+      file_list_curated = get_file_list_all_timestamps
+      file_list_curated.map do |file_remote_info|
+        file_remote_info[1][:file_id] = file_remote_info[0]
+        file_remote_info[1]
+      end
+    else
+      file_list_curated = get_file_list_curated
+      file_list_curated = file_list_curated.sort_by { |k,v| v[:timestamp] }.reverse
+      file_list_curated.map do |file_remote_info|
+        file_remote_info[1][:file_id] = file_remote_info[0]
+        file_remote_info[1]
+      end
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wayback_machine_downloader
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.2.0
 platform: ruby
 authors:
 - hartator
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-12 00:00:00.000000000 Z
+date: 2017-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake