wayback_machine_downloader 0.4.9 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 73c0ec7ba53bc62a4ef801970325bcc56144e04d
-  data.tar.gz: 9096da258417570d757decce4ca2a847f39febac
+  metadata.gz: 6dd8b2f20e56a00a8601a4e4f9ecd9e4305bcce7
+  data.tar.gz: 3274121f9abb86d37fc9820bb8465f608ec78366
 SHA512:
-  metadata.gz: 27407c32b02bfc2d1ffb267c6b654552a37fa17bb0605605bae4a2dd0fe89573a1e0e092ee3bc00faad97895d0e31277c7db0e675ea846c4e8be908d0d3b317f
-  data.tar.gz: 2155e4e85e4192b252c5b92b46237b824b34c6b905d1446586452163255687fc946e1e433cf03c764b6df0c71065be302f742c5a0b631ed4ef748c6ed1cc2d77
+  metadata.gz: 739d5f8a16e0a0d8f6ba81ff6eddffb9bd7d9337e02bd9215cce2e31a033e67037a0bf3678192f4872e891f52dff240565a0b8e202b92ad38c1d6d6ca3940878
+  data.tar.gz: 98f4387c00da2c5c827e7cf9b2f63d51a0c29f98f7b5bf2a2f30c36552e524294f824a74bc9124f6b1d7ed098f6a7afcc6508de68e1cc6249fa8ff737a36a441

data/bin/wayback_machine_downloader

@@ -38,6 +38,10 @@ option_parser = OptionParser.new do |opts|
     options[:list] = true
   end
 
+  opts.on("-c", "--concurrency NUMBER", Integer, "Number of multiple files to dowload at a time. Default is one file at a time. (ie. 20)") do |t|
+    options[:threads_count] = t
+  end
+
   opts.on("-v", "--version", "Display version") do |t|
     options[:version] = t
   end

data/lib/wayback_machine_downloader.rb

@@ -9,9 +9,9 @@ require_relative 'wayback_machine_downloader/to_regex'
 
 class WaybackMachineDownloader
 
-  VERSION = "0.4.9"
+  VERSION = "0.5.0"
 
-  attr_accessor :base_url, :from_timestamp, :to_timestamp, :only_filter, :exclude_filter, :all, :list
+  attr_accessor :base_url, :from_timestamp, :to_timestamp, :only_filter, :exclude_filter, :all, :list, :threads_count
 
   def initialize params
     @base_url = params[:base_url]
@@ -21,6 +21,7 @@ class WaybackMachineDownloader
     @exclude_filter = params[:exclude_filter]
     @all = params[:all]
     @list = params[:list]
+    @threads_count = params[:threads_count].to_i
   end
 
   def backup_name
@@ -121,72 +122,37 @@ class WaybackMachineDownloader
   end
 
   def download_files
+    start_time = Time.now
     puts "Downloading #{@base_url} to #{backup_path} from Wayback Machine..."
     puts
-    file_list_by_timestamp = get_file_list_by_timestamp
+
     if file_list_by_timestamp.count == 0
       puts "No files to download."
       puts "Possible reasons:"
       puts "\t* Site is not in Wayback Machine Archive."
-      puts "\t* From timestamp too much in the future." if @from_timestamp and @from_timestamp != 0 
-      puts "\t* To timestamp too much in the past." if @to_timestamp and @to_timestamp != 0 
+      puts "\t* From timestamp too much in the future." if @from_timestamp and @from_timestamp != 0
+      puts "\t* To timestamp too much in the past." if @to_timestamp and @to_timestamp != 0
       puts "\t* Only filter too restrictive (#{only_filter.to_s})" if @only_filter
       puts "\t* Exclude filter too wide (#{exclude_filter.to_s})" if @exclude_filter
       return
     end
-    count = 0
-    file_list_by_timestamp.each do |file_remote_info|
-      count += 1
-      file_url = file_remote_info[:file_url]
-      file_id = file_remote_info[:file_id]
-      file_timestamp = file_remote_info[:timestamp]
-      file_path_elements = file_id.split('/')
-      if file_id == ""
-        dir_path = backup_path
-        file_path = backup_path + 'index.html'
-      elsif file_url[-1] == '/' or not file_path_elements[-1].include? '.'
-        dir_path = backup_path + file_path_elements[0..-1].join('/')
-        file_path = backup_path + file_path_elements[0..-1].join('/') + '/index.html'
-      else
-        dir_path = backup_path + file_path_elements[0..-2].join('/')
-        file_path = backup_path + file_path_elements[0..-1].join('/')
-      end
-      if Gem.win_platform?
-        file_path = file_path.gsub(/[:*?&=<>\\|]/) {|s| '%' + s.ord.to_s(16) }
-      end
-      unless File.exists? file_path
-        begin
-          structure_dir_path dir_path
-          open(file_path, "wb") do |file|
-            begin
-              open("http://web.archive.org/web/#{file_timestamp}id_/#{file_url}", "Accept-Encoding" => "plain") do |uri|
-                file.write(uri.read)
-              end
-            rescue OpenURI::HTTPError => e
-              puts "#{file_url} # #{e}"
-              if @all
-                file.write(e.io.read)
-                puts "#{file_path} saved anyway."
-              end
-            rescue StandardError => e
-              puts "#{file_url} # #{e}"
-            end
-          end
-        rescue StandardError => e
-          puts "#{file_url} # #{e}"
-        ensure
-          if not @all and File.exists?(file_path) and File.size(file_path) == 0
-            File.delete(file_path)
-            puts "#{file_path} was empty and was removed."
-          end
+
+    threads = []
+    @processed_file_count = 0
+    @threads_count = 1 unless @threads_count != 0
+    @threads_count.times do
+      threads << Thread.new do
+        until file_queue.empty?
+          file_remote_info = file_queue.pop(true) rescue nil
+          download_file(file_remote_info) if file_remote_info
         end
-        puts "#{file_url} -> #{file_path} (#{count}/#{file_list_by_timestamp.size})"
-      else
-        puts "#{file_url} # #{file_path} already exists. (#{count}/#{file_list_by_timestamp.size})"
       end
     end
+
+    threads.each(&:join)
+    end_time = Time.now
     puts
-    puts "Download complete, saved in #{backup_path} (#{file_list_by_timestamp.size} files)"
+    puts "Download completed in #{(end_time - start_time).round(2)}s, saved in #{backup_path} (#{file_list_by_timestamp.size} files)"
   end
 
   def structure_dir_path dir_path
@@ -212,4 +178,71 @@ class WaybackMachineDownloader
     end
   end
 
+  def download_file file_remote_info
+    file_url = file_remote_info[:file_url]
+    file_id = file_remote_info[:file_id]
+    file_timestamp = file_remote_info[:timestamp]
+    file_path_elements = file_id.split('/')
+    if file_id == ""
+      dir_path = backup_path
+      file_path = backup_path + 'index.html'
+    elsif file_url[-1] == '/' or not file_path_elements[-1].include? '.'
+      dir_path = backup_path + file_path_elements[0..-1].join('/')
+      file_path = backup_path + file_path_elements[0..-1].join('/') + '/index.html'
+    else
+      dir_path = backup_path + file_path_elements[0..-2].join('/')
+      file_path = backup_path + file_path_elements[0..-1].join('/')
+    end
+    if Gem.win_platform?
+      file_path = file_path.gsub(/[:*?&=<>\\|]/) {|s| '%' + s.ord.to_s(16) }
+    end
+    unless File.exists? file_path
+      begin
+        structure_dir_path dir_path
+        open(file_path, "wb") do |file|
+          begin
+            open("http://web.archive.org/web/#{file_timestamp}id_/#{file_url}", "Accept-Encoding" => "plain") do |uri|
+              file.write(uri.read)
+            end
+          rescue OpenURI::HTTPError => e
+            puts "#{file_url} # #{e}"
+            if @all
+              file.write(e.io.read)
+              puts "#{file_path} saved anyway."
+            end
+          rescue StandardError => e
+            puts "#{file_url} # #{e}"
+          end
+        end
+      rescue StandardError => e
+        puts "#{file_url} # #{e}"
+      ensure
+        if not @all and File.exists?(file_path) and File.size(file_path) == 0
+          File.delete(file_path)
+          puts "#{file_path} was empty and was removed."
+        end
+      end
+      semaphore.synchronize do
+        @processed_file_count += 1
+        puts "#{file_url} -> #{file_path} (#{@processed_file_count}/#{file_list_by_timestamp.size})"
+      end
+    else
+      semaphore.synchronize do
+        @processed_file_count += 1
+        puts "#{file_url} # #{file_path} already exists. (#{@processed_file_count}/#{file_list_by_timestamp.size})"
+      end
+    end
+  end
+
+  def file_queue
+    @file_queue ||= file_list_by_timestamp.each_with_object(Queue.new) { |file_info, q| q << file_info }
+  end
+
+  def file_list_by_timestamp
+    @file_list_by_timestamp ||= get_file_list_by_timestamp
+  end
+
+  def semaphore
+    @semaphore ||= Mutex.new
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wayback_machine_downloader
 version: !ruby/object:Gem::Version
-  version: 0.4.9
+  version: 0.5.0
 platform: ruby
 authors:
 - hartator
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-16 00:00:00.000000000 Z
+date: 2016-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake