wayback_machine_downloader 0.1.8 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 785e30db377ae0b9d7a51a0de48bc1ac6f84de2e
-  data.tar.gz: 3f88ba144987f62fed74c0271ef6e2f55985f280
+  metadata.gz: 20560ddf17786139b6ad1bd4e01d4b5e4e41dafe
+  data.tar.gz: 3d8357785e11e81b20123a97e55f084a0442f03b
 SHA512:
-  metadata.gz: 7acc4a0d9c18ff2625cbada39ded4b2f5b919afb10d4bda9f7f8dc010fd650b374f8288e8d482c06af9bceca418d65739ab00312f49cb45f72556a5faa70b3b9
-  data.tar.gz: e945379f4517828d60e3d5ccb515f5a47a54d2961b417c8adec3d3193f6c009d09f7e7d945596aab0f5a9de32c367465bc1fc40542c2a6bf87f8c10135ddc0bc
+  metadata.gz: 333c6bcfcce0ab972d4dc927af130eb9992ebff1c12a45caee68a109aa459c76a0f6a4ace5058612742559b731876cd3af64f3aa2402b790ec756e2dc54a179a
+  data.tar.gz: 3dddac44a54eaf44a396f94c5428fa68da97a2e92bd2ec1c25f55eb569a5b2f64cd3082e429393be3893be29779faa4387a74952cd99fef09161d1a8dc070148

data/bin/wayback_machine_downloader

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
-require 'wayback_machine_downloader'
+require_relative '../lib/wayback_machine_downloader'
 require 'optparse'
 
 options = {}

data/lib/wayback_machine_downloader.rb

@@ -3,7 +3,7 @@ require 'fileutils'
 
 class WaybackMachineDownloader
 
-  VERSION = "0.1.8"
+  VERSION = "0.1.9"
 
   attr_accessor :base_url, :timestamp
 
@@ -21,35 +21,48 @@ class WaybackMachineDownloader
   end
 
   def get_file_list_curated
-    file_list_raw = open "http://web.archive.org/cdx/search/xd?url=#{@base_url}/*"
+    index_file_list_raw =  open "http://web.archive.org/cdx/search/xd?url=#{@base_url}"
+    all_file_list_raw = open "http://web.archive.org/cdx/search/xd?url=#{@base_url}/*"
     file_list_curated = Hash.new
-    file_list_raw.each_line do |line|
-      line = line.split(' ')
-      file_timestamp = line[1].to_i
-      file_url = line[2]
-      file_id = file_url.split('/')[3..-1].join('/')
-      file_id = URI.unescape file_id
-      if @timestamp == 0 or file_timestamp <= @timestamp
-        if file_list_curated[file_id]
-          unless file_list_curated[file_id][:timestamp] > file_timestamp
+    [index_file_list_raw, all_file_list_raw].each do |file|
+      file.each_line do |line|
+        line = line.split(' ')
+        file_timestamp = line[1].to_i
+        file_url = line[2]
+        file_id = file_url.split('/')[3..-1].join('/')
+        file_id = URI.unescape file_id
+        if @timestamp == 0 or file_timestamp <= @timestamp
+          if file_list_curated[file_id]
+            unless file_list_curated[file_id][:timestamp] > file_timestamp
+              file_list_curated[file_id] = {file_url: file_url, timestamp: file_timestamp}
+            end
+          else
             file_list_curated[file_id] = {file_url: file_url, timestamp: file_timestamp}
           end
-        else
-          file_list_curated[file_id] = {file_url: file_url, timestamp: file_timestamp}
         end
       end
     end
     file_list_curated
   end
 
+  def file_list_by_timestamp
+    file_list_curated = get_file_list_curated
+    file_list_curated = file_list_curated.sort_by { |k,v| v[:timestamp] }.reverse
+    file_list_curated.map do |file_remote_info|
+      file_remote_info[1][:file_id] = file_remote_info[0]
+      file_remote_info[1]
+    end
+  end
+
   def download_files
     puts "Downlading #{@base_url} from Wayback Machine..."
     puts
     file_list_curated = get_file_list_curated
     count = 0
-    file_list_curated.each do |file_id, file_remote_info|
+    file_list_by_timestamp.each do |file_remote_info|
       count += 1
       file_url = file_remote_info[:file_url]
+      file_id = file_remote_info[:file_id]
       file_path_elements = file_id.split('/')
       if file_id == ""
         dir_path = backup_path

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wayback_machine_downloader
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.9
 platform: ruby
 authors:
 - hartator