wayback_machine_downloader_straw 2.3.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a8d577ca08cca3858efd95bfd879b198a57aa6262fa8e0a7f83ab4f3a362f1fc
+  data.tar.gz: ef73d81d745e7b3e9226458a66b5d54c2410db646ea85cc7145813bc26789dc7
+SHA512:
+  metadata.gz: 938e8544bb16b4afc6c81d0e4da602b5d3cd3e05482b3cc945ad3405681278fc03c7ccc1b84992b1ecafe66cf202aa71f306226b92f4b93b30b1c5c7edcbc86e
+  data.tar.gz: 8c236877be6274b9bb3c474fde9ad5a72a30abd5db3eadfc11eeae997488a2ddf27a72388b6d7d47455235ed0e084b1d14a7782911d0dd69e41f2fdcada713e2

data/bin/wayback_machine_downloader

@@ -0,0 +1,83 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/wayback_machine_downloader'
+require 'optparse'
+require 'pp'
+
+options = {}
+option_parser = OptionParser.new do |opts|
+  opts.banner = "Usage: wayback_machine_downloader http://example.com"
+
+  opts.separator ""
+  opts.separator "Download an entire website from the Wayback Machine."
+
+  opts.separator ""
+  opts.separator "Optional options:"
+
+  opts.on("-d", "--directory PATH", String, "Directory to save the downloaded files into", "Default is ./websites/ plus the domain name") do |t|
+    options[:directory] = t
+  end
+
+  opts.on("-s", "--all-timestamps", "Download all snapshots/timestamps for a given website") do |t|
+    options[:all_timestamps] = true
+  end
+  
+  opts.on("-f", "--from TIMESTAMP", Integer, "Only files on or after timestamp supplied (ie. 20060716231334)") do |t|
+    options[:from_timestamp] = t
+  end
+
+  opts.on("-t", "--to TIMESTAMP", Integer, "Only files on or before timestamp supplied (ie. 20100916231334)") do |t|
+    options[:to_timestamp] = t
+  end
+
+  opts.on("-e", "--exact-url", "Download only the url provied and not the full site") do |t|
+    options[:exact_url] = t
+  end
+
+  opts.on("-o", "--only ONLY_FILTER", String, "Restrict downloading to urls that match this filter", "(use // notation for the filter to be treated as a regex)") do |t|
+    options[:only_filter] = t
+  end
+
+  opts.on("-x", "--exclude EXCLUDE_FILTER", String, "Skip downloading of urls that match this filter", "(use // notation for the filter to be treated as a regex)") do |t|
+    options[:exclude_filter] = t
+  end
+
+  opts.on("-a", "--all", "Expand downloading to error files (40x and 50x) and redirections (30x)") do |t|
+    options[:all] = true
+  end
+
+  opts.on("-c", "--concurrency NUMBER", Integer, "Number of multiple files to download at a time", "Default is one file at a time (ie. 20)") do |t|
+    options[:threads_count] = t
+  end
+
+  opts.on("-p", "--maximum-snapshot NUMBER", Integer, "Maximum snapshot pages to consider (Default is 100)", "Count an average of 150,000 snapshots per page") do |t|
+    options[:maximum_pages] = t
+  end
+
+  opts.on("-l", "--list", "Only list file urls in a JSON format with the archived timestamps, won't download anything") do |t|
+    options[:list] = true
+  end
+
+  opts.on("-r", "--rewritten", "Downloads the rewritten Wayback Machine files instead of the original files") do |t|
+    options[:rewritten] = t
+  end
+
+  opts.on("-v", "--version", "Display version") do |t|
+    options[:version] = t
+  end
+end.parse!
+
+if (base_url = ARGV[-1])
+  options[:base_url] = base_url
+  wayback_machine_downloader = WaybackMachineDownloader.new options
+  if options[:list]
+    wayback_machine_downloader.list_files
+  else
+    wayback_machine_downloader.download_files
+  end
+elsif options[:version]
+  puts WaybackMachineDownloader::VERSION
+else
+  puts "You need to specify a website to backup. (e.g., http://example.com)"
+  puts "Run `wayback_machine_downloader --help` for more help."
+end

data/lib/wayback_machine_downloader/archive_api.rb

@@ -0,0 +1,35 @@
+require 'json'
+require 'uri'
+
+module ArchiveAPI
+
+  def get_raw_list_from_api(url, page_index, http)
+    request_url = URI("https://web.archive.org/cdx/search/xd")
+    params = [["output", "json"], ["url", url]] + parameters_for_api(page_index)
+    request_url.query = URI.encode_www_form(params)
+
+    begin
+      response = http.get(request_url)
+      body = response.body.to_s.strip
+      return [] if body.empty?
+      json = JSON.parse(body)
+
+      # Check if the response contains the header ["timestamp", "original"]
+      json.shift if json.first == ["timestamp", "original"]
+      json
+    rescue JSON::ParserError, StandardError => e
+      warn "Failed to fetch data from API: #{e.message}"
+      []
+    end
+  end
+
+  def parameters_for_api(page_index)
+    parameters = [["fl", "timestamp,original"], ["collapse", "digest"], ["gzip", "false"]]
+    parameters.push(["filter", "statuscode:200"]) unless @all
+    parameters.push(["from", @from_timestamp.to_s]) if @from_timestamp && @from_timestamp != 0
+    parameters.push(["to", @to_timestamp.to_s]) if @to_timestamp && @to_timestamp != 0
+    parameters.push(["page", page_index]) if page_index
+    parameters
+  end
+
+end

data/lib/wayback_machine_downloader/tidy_bytes.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module TidyBytes
+  # precomputing CP1252 to UTF-8 mappings for bytes 128-159
+  CP1252_MAP = (128..159).map do |byte|
+    case byte
+    when 128 then [226, 130, 172]  # EURO SIGN
+    when 130 then [226, 128, 154]  # SINGLE LOW-9 QUOTATION MARK
+    when 131 then [198, 146]       # LATIN SMALL LETTER F WITH HOOK
+    when 132 then [226, 128, 158]  # DOUBLE LOW-9 QUOTATION MARK
+    when 133 then [226, 128, 166]  # HORIZONTAL ELLIPSIS
+    when 134 then [226, 128, 160]  # DAGGER
+    when 135 then [226, 128, 161]  # DOUBLE DAGGER
+    when 136 then [203, 134]       # MODIFIER LETTER CIRCUMFLEX ACCENT
+    when 137 then [226, 128, 176]  # PER MILLE SIGN
+    when 138 then [197, 160]       # LATIN CAPITAL LETTER S WITH CARON
+    when 139 then [226, 128, 185]  # SINGLE LEFT-POINTING ANGLE QUOTATION MARK
+    when 140 then [197, 146]       # LATIN CAPITAL LIGATURE OE
+    when 142 then [197, 189]       # LATIN CAPITAL LETTER Z WITH CARON
+    when 145 then [226, 128, 152]  # LEFT SINGLE QUOTATION MARK
+    when 146 then [226, 128, 153]  # RIGHT SINGLE QUOTATION MARK
+    when 147 then [226, 128, 156]  # LEFT DOUBLE QUOTATION MARK
+    when 148 then [226, 128, 157]  # RIGHT DOUBLE QUOTATION MARK
+    when 149 then [226, 128, 162]  # BULLET
+    when 150 then [226, 128, 147]  # EN DASH
+    when 151 then [226, 128, 148]  # EM DASH
+    when 152 then [203, 156]       # SMALL TILDE
+    when 153 then [226, 132, 162]  # TRADE MARK SIGN
+    when 154 then [197, 161]       # LATIN SMALL LETTER S WITH CARON
+    when 155 then [226, 128, 186]  # SINGLE RIGHT-POINTING ANGLE QUOTATION MARK
+    when 156 then [197, 147]       # LATIN SMALL LIGATURE OE
+    when 158 then [197, 190]       # LATIN SMALL LETTER Z WITH CARON
+    when 159 then [197, 184]       # LATIN SMALL LETTER Y WITH DIAERESIS
+    end
+  end.freeze
+
+  # precomputing all possible byte conversions 
+  CP1252_TO_UTF8 = Array.new(256) do |b|
+    if (128..159).cover?(b)
+      CP1252_MAP[b - 128]&.pack('C*')
+    elsif b < 128
+      b.chr
+    else
+      b < 192 ? [194, b].pack('C*') : [195, b - 64].pack('C*')
+    end
+  end.freeze
+
+  def self.included(base)
+    base.class_eval do
+      def tidy_bytes(force = false)
+        return nil if empty?
+        
+        if force
+          buffer = String.new(capacity: bytesize)
+          each_byte { |b| buffer << CP1252_TO_UTF8[b] }
+          return buffer.force_encoding(Encoding::UTF_8)
+        end
+
+        begin
+          encode('UTF-8')
+        rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError
+          buffer = String.new(capacity: bytesize)
+          scrub { |b| CP1252_TO_UTF8[b.ord] }
+        end
+      end
+
+      def tidy_bytes!(force = false)
+        result = tidy_bytes(force)
+        result ? replace(result) : self
+      end
+    end
+  end
+end
+
+class String
+  include TidyBytes
+end

data/lib/wayback_machine_downloader/to_regex.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module ToRegex
+  module StringMixin
+    INLINE_OPTIONS = /[imxnesu]*/i.freeze
+    REGEXP_DELIMITERS = {
+      '%r{' => '}'.freeze,
+      '/' => '/'.freeze
+    }.freeze
+
+    REGEX_FLAGS = {
+      ignore_case: Regexp::IGNORECASE,
+      multiline: Regexp::MULTILINE,
+      extended: Regexp::EXTENDED
+    }.freeze
+
+    class << self
+      def literal?(str)
+        REGEXP_DELIMITERS.none? { |start, ending| str.start_with?(start) && str.match?(/#{ending}#{INLINE_OPTIONS}\z/) }
+      end
+    end
+
+    # Get a regex back
+    #
+    # Without :literal or :detect, `"foo".to_regex` will return nil.
+    #
+    # @param [optional, Hash] options
+    # @option options [true,false] :literal Treat meta characters and other regexp codes as just text; always return a regexp
+    # @option options [true,false] :detect If string starts and ends with valid regexp delimiters, treat it as a regexp; otherwise, interpret it literally
+    # @option options [true,false] :ignore_case /foo/i
+    # @option options [true,false] :multiline /foo/m
+    # @option options [true,false] :extended /foo/x
+    # @option options [true,false] :lang /foo/[nesu]
+    def to_regex(options = {})
+      args = as_regexp(options)
+      args ? Regexp.new(*args) : nil
+    end
+    # Return arguments that can be passed to `Regexp.new`
+    # @see to_regexp
+    def as_regexp(options = {})
+      raise ArgumentError, '[to_regexp] Options must be a Hash' unless options.is_a?(Hash)
+      
+      str = self
+      return if options[:detect] && str.empty?
+
+      if should_treat_as_literal?(str, options)
+        content = Regexp.escape(str)
+      elsif (delim_set = extract_delimiters(str))
+        content, options = parse_regexp_string(str, delim_set, options)
+        return unless content
+      else
+        return
+      end
+
+      build_regexp_args(content, options)
+    end
+
+    private
+
+    def should_treat_as_literal?(str, options)
+      options[:literal] || (options[:detect] && ToRegex::StringMixin.literal?(str))
+    end
+
+    def extract_delimiters(str)
+      REGEXP_DELIMITERS.find { |start, _| str.start_with?(start) }
+    end
+
+    def parse_regexp_string(str, delim_set, options)
+      start_delim, end_delim = delim_set
+      match = /\A#{start_delim}(.*)#{end_delim}(#{INLINE_OPTIONS})\z/u.match(str)
+      return unless match
+
+      content = match[1].gsub('\\/', '/')
+      parse_inline_options(match[2], options)
+      [content, options]
+    end
+
+    def parse_inline_options(inline_options, options)
+      return unless inline_options
+      options[:ignore_case] = true if inline_options.include?('i')
+      options[:multiline] = true if inline_options.include?('m')
+      options[:extended] = true if inline_options.include?('x')
+      # 'n', 'N' = none, 'e', 'E' = EUC, 's', 'S' = SJIS, 'u', 'U' = UTF-8
+      options[:lang] = inline_options.scan(/[nesu]/i).join.downcase
+    end
+
+    def build_regexp_args(content, options)
+      flags = calculate_flags(options)
+      lang = normalize_lang_option(options[:lang])
+      
+      lang.empty? ? [content, flags] : [content, flags, lang]
+    end
+
+    def calculate_flags(options)
+      REGEX_FLAGS.sum { |key, value| options[key] ? value : 0 }
+    end
+
+    def normalize_lang_option(lang)
+      return '' unless lang
+      RUBY_VERSION >= '1.9' ? lang.delete('u') : lang
+    end
+  end
+end
+
+class String
+  include ToRegex::StringMixin
+end

data/lib/wayback_machine_downloader.rb

@@ -0,0 +1,491 @@
+# encoding: UTF-8
+
+require 'thread'
+require 'net/http'
+require 'open-uri'
+require 'fileutils'
+require 'cgi'
+require 'json'
+require 'time'
+require 'concurrent-ruby'
+require 'logger'
+require_relative 'wayback_machine_downloader/tidy_bytes'
+require_relative 'wayback_machine_downloader/to_regex'
+require_relative 'wayback_machine_downloader/archive_api'
+
+class ConnectionPool
+  MAX_AGE = 300
+  CLEANUP_INTERVAL = 60
+  DEFAULT_TIMEOUT = 30
+  MAX_RETRIES = 3
+
+  def initialize(size)
+    @size = size
+    @pool = Concurrent::Map.new
+    @creation_times = Concurrent::Map.new
+    @cleanup_thread = schedule_cleanup
+  end
+
+  def with_connection(&block)
+    conn = acquire_connection
+    begin
+      yield conn
+    ensure
+      release_connection(conn)
+    end
+  end
+
+  def shutdown
+    @cleanup_thread&.exit
+    @pool.each_value { |conn| conn.finish if conn&.started? }
+    @pool.clear
+    @creation_times.clear
+  end
+
+  private
+
+  def acquire_connection
+    thread_id = Thread.current.object_id
+    conn = @pool[thread_id]
+
+    if should_create_new?(conn)
+      conn&.finish if conn&.started?
+      conn = create_connection
+      @pool[thread_id] = conn
+      @creation_times[thread_id] = Time.now
+    end
+
+    conn
+  end
+
+  def release_connection(conn)
+    return unless conn
+    if conn.started? && Time.now - @creation_times[Thread.current.object_id] > MAX_AGE
+      conn.finish
+      @pool.delete(Thread.current.object_id)
+      @creation_times.delete(Thread.current.object_id)
+    end
+  end
+
+  def should_create_new?(conn)
+    return true if conn.nil?
+    return true unless conn.started?
+    return true if Time.now - @creation_times[Thread.current.object_id] > MAX_AGE
+    false
+  end
+
+  def create_connection
+    http = Net::HTTP.new("web.archive.org", 443)
+    http.use_ssl = true
+    http.read_timeout = DEFAULT_TIMEOUT
+    http.open_timeout = DEFAULT_TIMEOUT
+    http.keep_alive_timeout = 30
+    http.max_retries = MAX_RETRIES
+    http.start
+    http
+  end
+
+  def schedule_cleanup
+    Thread.new do
+      loop do
+        cleanup_old_connections
+        sleep CLEANUP_INTERVAL
+      end
+    end
+  end
+
+  def cleanup_old_connections
+    current_time = Time.now
+    @creation_times.each do |thread_id, creation_time|
+      if current_time - creation_time > MAX_AGE
+        conn = @pool[thread_id]
+        conn&.finish if conn&.started?
+        @pool.delete(thread_id)
+        @creation_times.delete(thread_id)
+      end
+    end
+  end
+end
+
+class WaybackMachineDownloader
+
+  include ArchiveAPI
+
+  VERSION = "2.3.3"
+  DEFAULT_TIMEOUT = 30
+  MAX_RETRIES = 3
+  RETRY_DELAY = 2
+  RATE_LIMIT = 0.25  # Delay between requests in seconds
+  CONNECTION_POOL_SIZE = 10
+  MEMORY_BUFFER_SIZE = 16384  # 16KB chunks
+
+  attr_accessor :base_url, :exact_url, :directory, :all_timestamps,
+    :from_timestamp, :to_timestamp, :only_filter, :exclude_filter,
+    :all, :maximum_pages, :threads_count, :logger
+
+  def initialize params
+    validate_params(params)
+    @base_url = params[:base_url]
+    @exact_url = params[:exact_url]
+    @directory = params[:directory]
+    @all_timestamps = params[:all_timestamps]
+    @from_timestamp = params[:from_timestamp].to_i
+    @to_timestamp = params[:to_timestamp].to_i
+    @only_filter = params[:only_filter]
+    @exclude_filter = params[:exclude_filter]
+    @all = params[:all]
+    @maximum_pages = params[:maximum_pages] ? params[:maximum_pages].to_i : 100
+    @threads_count = [params[:threads_count].to_i, 1].max
+    @rewritten = params[:rewritten]
+    @timeout = params[:timeout] || DEFAULT_TIMEOUT
+    @logger = setup_logger
+    @failed_downloads = Concurrent::Array.new
+    @connection_pool = ConnectionPool.new(CONNECTION_POOL_SIZE)
+  end
+
+  def backup_name
+    if @base_url.include? '//'
+      @base_url.split('/')[2]
+    else
+      @base_url
+    end
+  end
+
+  def backup_path
+    if @directory
+      if @directory[-1] == '/'
+        @directory
+      else
+        @directory + '/'
+      end
+    else
+      'websites/' + backup_name + '/'
+    end
+  end
+
+  def match_only_filter file_url
+    if @only_filter
+      only_filter_regex = @only_filter.to_regex
+      if only_filter_regex
+        only_filter_regex =~ file_url
+      else
+        file_url.downcase.include? @only_filter.downcase
+      end
+    else
+      true
+    end
+  end
+
+  def match_exclude_filter file_url
+    if @exclude_filter
+      exclude_filter_regex = @exclude_filter.to_regex
+      if exclude_filter_regex
+        exclude_filter_regex =~ file_url
+      else
+        file_url.downcase.include? @exclude_filter.downcase
+      end
+    else
+      false
+    end
+  end
+
+  def get_all_snapshots_to_consider
+    snapshot_list_to_consider = []
+
+    @connection_pool.with_connection do |connection|
+      puts "Getting snapshot pages"
+
+      # Fetch the initial set of snapshots
+      snapshot_list_to_consider += get_raw_list_from_api(@base_url, nil, connection)
+      print "."
+
+      # Fetch additional pages if the exact URL flag is not set
+      unless @exact_url
+        @maximum_pages.times do |page_index|
+          snapshot_list = get_raw_list_from_api("#{@base_url}/*", page_index, connection)
+          break if snapshot_list.empty?
+
+          snapshot_list_to_consider += snapshot_list
+          print "."
+        end
+      end
+    end
+
+    puts " found #{snapshot_list_to_consider.length} snapshots to consider."
+    puts
+
+    snapshot_list_to_consider
+  end
+
+  def get_file_list_curated
+    file_list_curated = Hash.new
+    get_all_snapshots_to_consider.each do |file_timestamp, file_url|
+      next unless file_url.include?('/')
+      file_id = file_url.split('/')[3..-1].join('/')
+      file_id = CGI::unescape file_id
+      file_id = file_id.tidy_bytes unless file_id == ""
+      if file_id.nil?
+        puts "Malformed file url, ignoring: #{file_url}"
+      else
+        if match_exclude_filter(file_url)
+          puts "File url matches exclude filter, ignoring: #{file_url}"
+        elsif not match_only_filter(file_url)
+          puts "File url doesn't match only filter, ignoring: #{file_url}"
+        elsif file_list_curated[file_id]
+          unless file_list_curated[file_id][:timestamp] > file_timestamp
+            file_list_curated[file_id] = {file_url: file_url, timestamp: file_timestamp}
+          end
+        else
+          file_list_curated[file_id] = {file_url: file_url, timestamp: file_timestamp}
+        end
+      end
+    end
+    file_list_curated
+  end
+
+  def get_file_list_all_timestamps
+    file_list_curated = Hash.new
+    get_all_snapshots_to_consider.each do |file_timestamp, file_url|
+      next unless file_url.include?('/')
+      file_id = file_url.split('/')[3..-1].join('/')
+      file_id_and_timestamp = [file_timestamp, file_id].join('/')
+      file_id_and_timestamp = CGI::unescape file_id_and_timestamp
+      file_id_and_timestamp = file_id_and_timestamp.tidy_bytes unless file_id_and_timestamp == ""
+      if file_id.nil?
+        puts "Malformed file url, ignoring: #{file_url}"
+      else
+        if match_exclude_filter(file_url)
+          puts "File url matches exclude filter, ignoring: #{file_url}"
+        elsif not match_only_filter(file_url)
+          puts "File url doesn't match only filter, ignoring: #{file_url}"
+        elsif file_list_curated[file_id_and_timestamp]
+          puts "Duplicate file and timestamp combo, ignoring: #{file_id}" if @verbose
+        else
+          file_list_curated[file_id_and_timestamp] = {file_url: file_url, timestamp: file_timestamp}
+        end
+      end
+    end
+    puts "file_list_curated: " + file_list_curated.count.to_s
+    file_list_curated
+  end
+
+
+  def get_file_list_by_timestamp
+    if @all_timestamps
+      file_list_curated = get_file_list_all_timestamps
+      file_list_curated.map do |file_remote_info|
+        file_remote_info[1][:file_id] = file_remote_info[0]
+        file_remote_info[1]
+      end
+    else
+      file_list_curated = get_file_list_curated
+      file_list_curated = file_list_curated.sort_by { |k,v| v[:timestamp] }.reverse
+      file_list_curated.map do |file_remote_info|
+        file_remote_info[1][:file_id] = file_remote_info[0]
+        file_remote_info[1]
+      end
+    end
+  end
+
+  def list_files
+    # retrieval produces its own output
+    @orig_stdout = $stdout
+    $stdout = $stderr
+    files = get_file_list_by_timestamp
+    $stdout = @orig_stdout
+    puts "["
+    files[0...-1].each do |file|
+      puts file.to_json + ","
+    end
+    puts files[-1].to_json
+    puts "]"
+  end
+
+  def download_files
+    start_time = Time.now
+    puts "Downloading #{@base_url} to #{backup_path} from Wayback Machine archives."
+    
+    if file_list_by_timestamp.empty?
+      puts "No files to download."
+      return
+    end
+
+    total_files = file_list_by_timestamp.count
+    puts "#{total_files} files to download:"
+    
+    @processed_file_count = 0
+    @download_mutex = Mutex.new
+    
+    thread_count = [@threads_count, CONNECTION_POOL_SIZE].min
+    pool = Concurrent::FixedThreadPool.new(thread_count)
+    
+    file_list_by_timestamp.each do |file_remote_info|
+      pool.post do
+        @connection_pool.with_connection do |connection|
+          result = download_file(file_remote_info, connection)
+          @download_mutex.synchronize do
+            @processed_file_count += 1
+            puts result if result
+          end
+        end
+        sleep(RATE_LIMIT)
+      end
+    end
+
+    pool.shutdown
+    pool.wait_for_termination
+
+    end_time = Time.now
+    puts "\nDownload completed in #{(end_time - start_time).round(2)}s, saved in #{backup_path}"
+    cleanup
+  end
+
+  def structure_dir_path dir_path
+    begin
+      FileUtils::mkdir_p dir_path unless File.exist? dir_path
+    rescue Errno::EEXIST => e
+      error_to_string = e.to_s
+      puts "# #{error_to_string}"
+      if error_to_string.include? "File exists @ dir_s_mkdir - "
+        file_already_existing = error_to_string.split("File exists @ dir_s_mkdir - ")[-1]
+      elsif error_to_string.include? "File exists - "
+        file_already_existing = error_to_string.split("File exists - ")[-1]
+      else
+        raise "Unhandled directory restructure error # #{error_to_string}"
+      end
+      file_already_existing_temporary = file_already_existing + '.temp'
+      file_already_existing_permanent = file_already_existing + '/index.html'
+      FileUtils::mv file_already_existing, file_already_existing_temporary
+      FileUtils::mkdir_p file_already_existing
+      FileUtils::mv file_already_existing_temporary, file_already_existing_permanent
+      puts "#{file_already_existing} -> #{file_already_existing_permanent}"
+      structure_dir_path dir_path
+    end
+  end
+
+  def download_file (file_remote_info, http)
+    current_encoding = "".encoding
+    file_url = file_remote_info[:file_url].encode(current_encoding)
+    file_id = file_remote_info[:file_id]
+    file_timestamp = file_remote_info[:timestamp]
+    file_path_elements = file_id.split('/')
+
+    if file_id == ""
+      dir_path = backup_path
+      file_path = backup_path + 'index.html'
+    elsif file_url[-1] == '/' or not file_path_elements[-1].include? '.'
+      dir_path = backup_path + file_path_elements[0..-1].join('/')
+      file_path = backup_path + file_path_elements[0..-1].join('/') + '/index.html'
+    else
+      dir_path = backup_path + file_path_elements[0..-2].join('/')
+      file_path = backup_path + file_path_elements[0..-1].join('/')
+    end
+    if Gem.win_platform?
+      dir_path = dir_path.gsub(/[:*?&=<>\\|]/) {|s| '%' + s.ord.to_s(16) }
+      file_path = file_path.gsub(/[:*?&=<>\\|]/) {|s| '%' + s.ord.to_s(16) }
+    end
+    unless File.exist? file_path
+      begin
+        structure_dir_path dir_path
+        download_with_retry(file_path, file_url, file_timestamp, http)
+        "#{file_url} -> #{file_path} (#{@processed_file_count + 1}/#{file_list_by_timestamp.size})"
+      rescue StandardError => e
+        msg = "#{file_url} # #{e}"
+        if not @all and File.exist?(file_path) and File.size(file_path) == 0
+          File.delete(file_path)
+          msg += "\n#{file_path} was empty and was removed."
+        end
+        msg
+      end
+    else
+      "#{file_url} # #{file_path} already exists. (#{@processed_file_count + 1}/#{file_list_by_timestamp.size})"
+    end
+  end
+
+  def file_queue
+    @file_queue ||= file_list_by_timestamp.each_with_object(Queue.new) { |file_info, q| q << file_info }
+  end
+
+  def file_list_by_timestamp
+    @file_list_by_timestamp ||= get_file_list_by_timestamp
+  end
+
+  private
+
+  def validate_params(params)
+    raise ArgumentError, "Base URL is required" unless params[:base_url]
+    raise ArgumentError, "Maximum pages must be positive" if params[:maximum_pages] && params[:maximum_pages].to_i <= 0
+  end
+
+  def setup_logger
+    logger = Logger.new(STDOUT)
+    logger.level = ENV['DEBUG'] ? Logger::DEBUG : Logger::INFO
+    logger.formatter = proc do |severity, datetime, progname, msg|
+      "#{datetime.strftime('%Y-%m-%d %H:%M:%S')} [#{severity}] #{msg}\n"
+    end
+    logger
+  end
+
+  def download_with_retry(file_path, file_url, file_timestamp, connection, redirect_count = 0)
+    retries = 0
+    begin
+      wayback_url = if @rewritten
+        "https://web.archive.org/web/#{file_timestamp}/#{file_url}"
+      else  
+        "https://web.archive.org/web/#{file_timestamp}id_/#{file_url}"
+      end
+      
+      request = Net::HTTP::Get.new(URI(wayback_url))
+      request["Connection"] = "keep-alive"
+      request["User-Agent"] = "WaybackMachineDownloader/#{VERSION}"
+      
+      response = connection.request(request)
+      
+      case response
+      when Net::HTTPSuccess
+        File.open(file_path, "wb") do |file|
+          if block_given?
+            yield(response, file)
+          else
+            file.write(response.body)
+          end
+        end
+      when Net::HTTPRedirection
+        raise "Too many redirects for #{file_url}" if redirect_count >= 2
+        location = response['location']
+        @logger.warn("Redirect found for #{file_url} -> #{location}")
+        return download_with_retry(file_path, location, file_timestamp, connection, redirect_count + 1)
+      when Net::HTTPTooManyRequests
+        sleep(RATE_LIMIT * 2)
+        raise "Rate limited, retrying..."
+      when Net::HTTPNotFound
+        @logger.warn("File not found, skipping: #{file_url}")
+        return
+      else
+        raise "HTTP Error: #{response.code} #{response.message}"
+      end
+      
+    rescue StandardError => e
+      if retries < MAX_RETRIES
+        retries += 1
+        @logger.warn("Retry #{retries}/#{MAX_RETRIES} for #{file_url}: #{e.message}")
+        sleep(RETRY_DELAY * retries)
+        retry
+      else
+        @failed_downloads << {url: file_url, error: e.message}
+        raise e
+      end
+    end
+  end
+
+  def cleanup
+    @connection_pool.shutdown
+    
+    if @failed_downloads.any?
+      @logger.error("Failed downloads summary:")
+      @failed_downloads.each do |failure|
+        @logger.error("  #{failure[:url]} - #{failure[:error]}")
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification
+name: wayback_machine_downloader_straw
+version: !ruby/object:Gem::Version
+  version: 2.3.3
+platform: ruby
+authors:
+- strawberrymaster
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2025-03-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.4
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.2'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+description: Download complete websites from the Internet Archive's Wayback Machine.
+  While the Wayback Machine (archive.org) excellently preserves web history, it lacks
+  a built-in export functionality; this gem does just that, allowing you to download
+  entire archived websites. (This is a significant rewrite of the original wayback_machine_downloader
+  gem by hartator, with enhanced features and performance improvements.)
+email: strawberrymaster@vivaldi.net
+executables:
+- wayback_machine_downloader
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/wayback_machine_downloader
+- lib/wayback_machine_downloader.rb
+- lib/wayback_machine_downloader/archive_api.rb
+- lib/wayback_machine_downloader/tidy_bytes.rb
+- lib/wayback_machine_downloader/to_regex.rb
+homepage: https://github.com/StrawberryMaster/wayback-machine-downloader
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.11
+signing_key: 
+specification_version: 4
+summary: Download an entire website from the Wayback Machine.
+test_files: []