wayback_machine_downloader_straw 2.4.7 → 2.4.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 558d3187ee31faeadb08cf83e32a87307ae9d55a3327206598f27a78fb715e08
-  data.tar.gz: 9845999e0e618afde419869bb01b04277aca318aa80b238feb8252540fc16315
+  metadata.gz: cdb773f6bbfb5404d827d6c484f4b8aa0ba2b1c4aed192c86ba12f137526f814
+  data.tar.gz: a8a898bdf0640c40e6946d78e3135bb04e8fa0f76ab14a267448a514a503af2f
 SHA512:
-  metadata.gz: b323c1065ea1ab1d3c5909458cae726462ba1b88fd89effe8cd1efbdd1301d2022363c56b191f8ddbe28c0143fe87dfc94ca847865bb08e3564e29ba36f231a4
-  data.tar.gz: 00c4e775ee05e176e1048d6e5ddd9ddc436edaf9bf7fea61dbab33480e4adc499ea8a3a584e591858aad3a4c2f10096a3b6eb1ff6c8146f26ba6a3d42f104e32
+  metadata.gz: d608cf38715e49fb50bcd6aaa6b692556ef9ca0fe3c36894e0738e2e8c2487dd99d04cf7f242a053b96ced3cd7e2cf92c455f83a83113102b1dfb926c914fb58
+  data.tar.gz: 96b2ca1e63d659936a25423717aad49e8f2fa342040a4564cf425c76766dc025f09ef32b9821737a0bec8127cfee6328ecba3717286ff59c1b15c84a713752a3

data/bin/wayback_machine_downloader

@@ -84,6 +84,10 @@ option_parser = OptionParser.new do |opts|
     options[:keep] = true
   end
 
+  opts.on("--delay SECONDS", Float, "Delay between downloads in seconds (default: 0)") do |t|
+    options[:delay] = t
+  end
+
   opts.on("--rt", "--retry N", Integer, "Maximum number of retries for failed downloads (default: 3)") do |t|
     options[:max_retries] = t
   end

data/lib/wayback_machine_downloader/archive_api.rb

@@ -1,49 +1,119 @@
 require 'json'
 require 'uri'
+require 'time'
+require 'thread'
 
 module ArchiveAPI
+  DEFAULT_RATE_LIMIT_COOLDOWN = 30.0
+  DEFAULT_CDX_INTERVAL = 2.5 # 1 request every 2.5 seconds
+
+  class RateLimitError < StandardError
+    attr_reader :retry_after
+
+    def initialize(message, retry_after = nil)
+      super(message)
+      @retry_after = retry_after
+    end
+  end
+
+  @cdx_mutex = Mutex.new
+  @cdx_cv = ConditionVariable.new
+  @next_allowed_cdx_at = 0.0
+  @cdx_interval = DEFAULT_CDX_INTERVAL
+
+  class << self
+    attr_accessor :cdx_interval
+
+    # pace CDX requests to avoid exceeding the rate limit
+    def pace_cdx_request
+      @cdx_mutex.synchronize do
+        loop do
+          now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          if now < @next_allowed_cdx_at
+            wait_time = @next_allowed_cdx_at - now
+            @cdx_cv.wait(@cdx_mutex, wait_time)
+          else
+            interval = @cdx_interval || DEFAULT_CDX_INTERVAL
+            @next_allowed_cdx_at = now + interval
+            return
+          end
+        end
+      end
+    end
+
+    # extend the cooldown period for CDX requests, e.g., after receiving a 429 response
+    def extend_cdx_cooldown(seconds)
+      seconds = seconds.to_f
+      return if seconds <= 0
+
+      @cdx_mutex.synchronize do
+        now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        candidate = now + seconds
+        @next_allowed_cdx_at = candidate if candidate > @next_allowed_cdx_at
+        @cdx_cv.broadcast
+      end
+    end
+
+    def reset_cdx_limiter
+      @cdx_mutex.synchronize do
+        @next_allowed_cdx_at = 0.0
+        @cdx_cv.broadcast
+      end
+    end
+  end
 
   def get_raw_list_from_api(url, page_index, http)
     # Automatically append /* for host-only URLs
     # This is a workaround for an issue with the API and *some* domains.
     # See https://github.com/StrawberryMaster/wayback-machine-downloader/issues/6
     # But don't do this when exact_url flag is set, and never append twice
-    if url && !@exact_url
-      normalized_url = url.to_s
-      has_wildcard = normalized_url.include?('*')
-      host_and_rest = normalized_url
-        .sub(/\Ahttps?:\/\//i, '')
-        .split(/[?#]/, 2)
-        .first
-      has_path = host_and_rest.include?('/')
-
-      unless has_wildcard || has_path
-        url = "#{normalized_url}/*"
+    normalized_url = url.to_s.strip
+    
+    # strip protocol for CDX query
+    clean_url = normalized_url.sub(%r{\Ahttps?://}i, '')
+    # ensure wildcard/matchType for domain-wide crawling
+    match_type = nil
+    unless @exact_url || clean_url.include?('*')
+      if clean_url.end_with?('/')
+        clean_url = "#{clean_url}*"
+      elsif !clean_url.include?('/')
+        match_type = "prefix"
+      else
+        clean_url = "#{clean_url}/*"
       end
     end
 
     request_url = URI("https://web.archive.org/cdx/search/cdx")
-    params = [["output", "json"], ["url", url]] + parameters_for_api(page_index)
+    params = [["output", "json"], ["url", clean_url]] + parameters_for_api(page_index)
+    params << ["matchType", match_type] if match_type
     request_url.query = URI.encode_www_form(params)
 
     retries = 0
     max_retries = (@max_retries || 3)
-    delay = WaybackMachineDownloader::RETRY_DELAY rescue 2
+    base_delay = 2
 
     begin
-      request = Net::HTTP::Get.new(request_url)
-      request["User-Agent"] = "wmd-straw/#{WaybackMachineDownloader::VERSION}"
-      request["Connection"] = "keep-alive"
-      request["Accept-Encoding"] = "gzip"
-      response = http.request(request)
+      # acquire slot from the process-wide proactive pacer before sending request
+      ArchiveAPI.pace_cdx_request
 
-      case response.code.to_i
+      if HTTPX_AVAILABLE && http.is_a?(HTTPX::Session)
+        response = http.get(request_url)
+        raise response.error if response.is_a?(HTTPX::ErrorResponse)
+
+        code = response.status
+        body = response.body.to_s.strip
+      else
+        request = Net::HTTP::Get.new(request_url)
+        request["User-Agent"] = "wmd-straw/#{WaybackMachineDownloader::VERSION rescue '2.4.8'}"
+        request["Connection"] = "keep-alive"
+        request["Accept-Encoding"] = "gzip, deflate"
+        response = http.request(request)
+        code = response.code.to_i
+        body = decompress_body(response)
+      end
+
+      case code
       when 200
-        body = if response['content-encoding'] == 'gzip'
-          Zlib::GzipReader.new(StringIO.new(response.body)).read
-        else
-          response.body.to_s.strip
-        end
         return [] if body.empty?
         begin
           json = JSON.parse(body)
@@ -53,21 +123,45 @@ module ArchiveAPI
         rescue JSON::ParserError => e
           raise "Malformed JSON response: #{e.message}"
         end
-      when 429, 500, 502, 503, 504
-        raise "Server error #{response.code}: #{response.message}"
+      when 400
+        # CDX API occasionally returns 400 when page index exceeds total available pages (that is, end of pagination)
+        return []
+      when 429
+        retry_after = retry_after_seconds(response)
+        raise RateLimitError.new(
+          "Server error 429: #{response.respond_to?(:message) ? response.message : 'Too Many Requests'}",
+          retry_after
+        )
+      when 500, 502, 503, 504
+        raise "Server error #{code}: #{response.respond_to?(:message) ? response.message : ''}"
       else
-        warn "Unexpected API response #{response.code} for #{url}"
-        []
+        raise "Unexpected API response #{code} for #{url}"
       end
     rescue Net::ReadTimeout, Net::OpenTimeout, StandardError => e
       if retries < max_retries
         retries += 1
-        warn "Error talking to Wayback CDX API (#{e.class}: #{e.message}) for #{url}, retry #{retries}/#{max_retries}..."
-        sleep(delay * retries)
+        jitter = rand(0.0..1.0)
+
+        if e.is_a?(RateLimitError)
+          # if the server provided a Retry-After header, use that; otherwise, use an exponential backoff with a minimum cooldown
+          fallback = [DEFAULT_RATE_LIMIT_COOLDOWN, base_delay * (2 ** (retries - 1))].max
+          cooldown = (e.retry_after || fallback) + (e.retry_after ? 0 : jitter)
+          ArchiveAPI.extend_cdx_cooldown(cooldown)
+
+          warn "Wayback CDX API rate limited (429) for #{url}. " \
+               "Pausing CDX requests for #{cooldown.round(2)}s " \
+               "(attempt #{retries}/#{max_retries})..."
+        else
+          sleep_time = (base_delay * (2 ** (retries - 1))) + jitter
+          warn "Error talking to Wayback CDX API (#{e.class}: #{e.message}) for #{url}. " \
+               "Retrying in #{sleep_time.round(2)}s (attempt #{retries}/#{max_retries})..."
+          sleep(sleep_time)
+        end
+
         retry
       else
         warn "Giving up on Wayback CDX API for #{url} after #{max_retries} attempts. (Last error: #{e.message})"
-        []
+        raise
       end
     end
   end
@@ -78,8 +172,40 @@ module ArchiveAPI
     parameters.push(["filter", "statuscode:2..|30[12378]"]) unless @all
     parameters.push(["from", @from_timestamp.to_s]) if @from_timestamp && @from_timestamp != 0
     parameters.push(["to", @to_timestamp.to_s]) if @to_timestamp && @to_timestamp != 0
-    parameters.push(["page", page_index]) if page_index
+    parameters.push(["page", page_index.to_s]) if page_index && page_index > 0
     parameters
   end
 
-end
+  private
+
+  def retry_after_seconds(response)
+    raw = if HTTPX_AVAILABLE && defined?(HTTPX::Response) && response.is_a?(HTTPX::Response)
+            response.headers['retry-after']
+          elsif response.respond_to?(:[])
+            response['Retry-After'] || response['retry-after']
+          end
+
+    value = Array(raw).first.to_s.strip
+    return nil if value.empty?
+    return value.to_f if value.match?(/\A\d+(?:\.\d+)?\z/)
+
+    delay = Time.httpdate(value) - Time.now
+    delay.positive? ? delay : 0.0
+  rescue ArgumentError
+    nil
+  end
+
+  def decompress_body(response)
+    body = response.body.to_s
+    return body if body.empty?
+
+    case response['content-encoding']
+    when 'gzip'
+      Zlib::GzipReader.new(StringIO.new(body)).read rescue body
+    when 'deflate'
+      Zlib::Inflate.inflate(body) rescue body
+    else
+      body.strip
+    end
+  end
+end

data/lib/wayback_machine_downloader/page_requisites.rb

@@ -1,18 +1,30 @@
 module PageRequisites
   # regex to find links in href, src, url(), and srcset
   # this ignores data: URIs, mailto:, and anchors
-  ASSET_REGEX = /(?:href|src|data-src|data-url)\s*=\s*["']([^"']+)["']|url\(\s*["']?([^"'\)]+)["']?\s*\)|srcset\s*=\s*["']([^"']+)["']/i
+  ASSET_REGEX = /(?:(href|src|data-src|data-url)\s*=\s*["']([^"']+)["'])|url\(\s*["']?([^"'\)]+)["']?\s*\)|srcset\s*=\s*["']([^"']+)["']/i
+ PAGE_EXTENSIONS = %w[
+    .html .htm .shtml .shtm .xhtml
+    .asp .aspx .asa .ashx .asmx
+    .php .php3 .php4 .php5 .phtml
+    .jsp .jspx .do .action
+    .cgi .pl .cfm .dll
+  ].freeze
 
   def self.extract(html_content)
     assets = []
-    
+
     html_content.scan(ASSET_REGEX) do |match|
-      # match is an array of capture groups; find the one that matched
-      url = match.compact.first
+      attribute, attr_url, css_url, srcset_url = match
+      url = attr_url || css_url || srcset_url
       next unless url
-      
-      # handle srcset (e.g. comma separated values like "image.jpg 1x, image2.jpg 2x")
-      if url.include?(',') && (url.include?(' 1x') || url.include?(' 2w'))
+
+      # href is also used for navigation. Do not turn links to server-rendered
+      # pages into prerequisite CDX lookups; doing so can multiply API traffic
+      # dramatically on old sites that use extensions such as .php3.
+      next if attribute&.downcase == 'href' && page_url?(url)
+
+      # handle srcset (e.g. comma separated values like "image.jpg 1x, image2.jpg 2w")
+      if srcset_url
         url.split(',').each do |src_def|
           src_url = src_def.strip.split(' ').first
           assets << src_url if valid_asset?(src_url)
@@ -25,9 +37,20 @@ module PageRequisites
     assets.uniq
   end
 
+  def self.page_url?(url)
+    path = begin
+      URI.parse(url).path
+    rescue URI::InvalidURIError
+      url.to_s.split(/[?#]/, 2).first
+    end
+
+    ext = File.extname(path.to_s).downcase
+    ext.empty? || PAGE_EXTENSIONS.include?(ext)
+  end
+
   def self.valid_asset?(url)
     return false if url.strip.empty?
     return false if url.start_with?('data:', 'mailto:', '#', 'javascript:')
     true
   end
-end
+end

data/lib/wayback_machine_downloader/subdom_processor.rb

@@ -12,7 +12,7 @@ module SubdomainProcessor
     @subdomain_queue = Queue.new
     
     # scan downloaded files for subdomain links
-    initial_files = Dir.glob(File.join(backup_path, "**/*.{html,htm,css,js}"))
+    initial_files = Dir.glob(File.join(backup_path, "**/*.{html,htm,shtml,css,js,asp,aspx,ashx,php,jsp,cgi,pl}"))
     puts "Scanning #{initial_files.size} downloaded files for subdomain links..."
     
     subdomains_found = scan_files_for_subdomains(initial_files, base_domain)
@@ -40,11 +40,17 @@ module SubdomainProcessor
   private
   
   def extract_base_domain(url)
-    uri = URI.parse(url.gsub(/^https?:\/\//, '').split('/').first) rescue nil
+    # ensure the URL has a scheme for URI parsing
+    normalized_url = url.match?(/^https?:\/\//i) ? url : "http://#{url}"
+    uri = URI.parse(normalized_url) rescue nil
     return nil unless uri
     
-    host = uri.host || uri.path.split('/').first
-    host = host.downcase
+    # extract the host (and default to parsing path if host is missing)
+    host = uri.host || (uri.path || '').split('/').first
+    return nil unless host
+    
+    # strip port numbers if present
+    host = host.split(':').first.downcase
     
     # extract the base domain (e.g., "example.com" from "sub.example.com")
     parts = host.split('.')
@@ -120,7 +126,7 @@ module SubdomainProcessor
       # if we need to go deeper, scan the newly downloaded files
       if depth + 1 < max_depth
         # get all files in the subdomains directory
-        new_files = Dir.glob(File.join(backup_path, "subdomains", "**/*.{html,htm,css,js}"))
+        new_files = Dir.glob(File.join(backup_path, "subdomains", "**/*.{html,htm,shtml,css,js,asp,aspx,ashx,php,jsp,cgi,pl}"))
         new_subdomains = scan_files_for_subdomains(new_files, base_domain)
         
         # filter out already processed subdomains
@@ -235,4 +241,4 @@ module SubdomainProcessor
     
     puts "Rewrote links in #{rewritten_count} files"
   end
-end
+end

data/lib/wayback_machine_downloader/url_rewrite.rb

@@ -2,84 +2,116 @@
 
 module URLRewrite
   # server-side extensions that should work locally
-  SERVER_SIDE_EXTS = %w[.php .asp .aspx .jsp .cgi .pl .py].freeze
+  SERVER_SIDE_EXTS = %w[
+    .php .php3 .phtml 
+    .asp .aspx .ashx .asmx .asa 
+    .jsp .jspx .do .action 
+    .cgi .pl .py .cfm .shtml
+  ].freeze
+
+  def rewrite_html_attr_urls(content, root_prefix = './')
+    target_host = current_host
 
-  def rewrite_html_attr_urls(content)
     # rewrite URLs to relative paths
-    content.gsub!(/(\s(?:href|src|action|data-src|data-url)=["'])https?:\/\/web\.archive\.org\/web\/\d+(?:id_)?\/https?:\/\/[^\/]+([^"']*)(["'])/i) do
-      prefix, path, suffix = $1, $2, $3
-      path = normalize_path_for_local(path)
-      "#{prefix}#{path}#{suffix}"
+    content = content.gsub(/(\s(?:href|src|action|data-src|data-url)=["'])https?:\/\/web\.archive\.org\/web\/\d+[a-z_]*\/https?:\/\/[^\/"']+(\/[^"']*)?(["'])/i) do
+      prefix, path, suffix = $1, ($2 || "/index.html"), $3
+      local = normalize_path_for_local(path, root_prefix)
+      "#{prefix}#{local}#{suffix}"
     end
 
     # rewrite absolute URLs to same domain as relative
-    content.gsub!(/(\s(?:href|src|action|data-src|data-url)=["'])https?:\/\/[^\/]+([^"']*)(["'])/i) do
+    if target_host && !target_host.empty?
+      content = content.gsub(/(\s(?:href|src|action|data-src|data-url)=["'])https?:\/\/#{Regexp.escape(target_host)}(?::\d+)?(\/[^"']*)?(["'])/i) do
+        prefix, path, suffix = $1, ($2 || "/index.html"), $3
+        local = normalize_path_for_local(path, root_prefix)
+        "#{prefix}#{local}#{suffix}"
+      end
+    end
+
+    # rewrite root-relative URLs
+    content = content.gsub(/(\s(?:href|src|action|data-src|data-url)=["'])\/([^"'\/][^"']*)(["'])/i) do
       prefix, path, suffix = $1, $2, $3
-      path = normalize_path_for_local(path)
-      "#{prefix}#{path}#{suffix}"
+      local = normalize_path_for_local("/#{path}", root_prefix)
+      "#{prefix}#{local}#{suffix}"
     end
 
     content
   end
 
-  def rewrite_css_urls(content)
+  def rewrite_css_urls(content, root_prefix = './')
+    target_host = current_host
+
     # rewrite URLs in CSS
-    content.gsub!(/url\(\s*["']?https?:\/\/web\.archive\.org\/web\/\d+(?:id_)?\/https?:\/\/[^\/]+([^"'\)]*?)["']?\s*\)/i) do
-      path = normalize_path_for_local($1)
-      "url(\"#{path}\")"
+    content = content.gsub(/url\(\s*["']?https?:\/\/web\.archive\.org\/web\/\d+[a-z_]*\/https?:\/\/[^\/"'\)]+(\/[^"'\)]*)?["']?\s*\)/i) do
+      path = $1 || "/index.html"
+      local = normalize_path_for_local(path, root_prefix)
+      "url(\"#{local}\")"
     end
 
     # rewrite absolute URLs in CSS
-    content.gsub!(/url\(\s*["']?https?:\/\/[^\/]+([^"'\)]*?)["']?\s*\)/i) do
-      path = normalize_path_for_local($1)
-      "url(\"#{path}\")"
+    if target_host && !target_host.empty?
+      content = content.gsub(/url\(\s*["']?https?:\/\/#{Regexp.escape(target_host)}(?::\d+)?(\/[^"'\)]*)?["']?\s*\)/i) do
+        path = $1 || "/index.html"
+        local = normalize_path_for_local(path, root_prefix)
+        "url(\"#{local}\")"
+      end
+    end
+
+    # rewrite root-relative in CSS
+    content = content.gsub(/url\(\s*["']?\/([^"'\)\/][^"'\)]*?)["']?\s*\)/i) do
+      path = $1
+      local = normalize_path_for_local("/#{path}", root_prefix)
+      "url(\"#{local}\")"
     end
 
     content
   end
 
-  def rewrite_js_urls(content)
+  def rewrite_js_urls(content, root_prefix = './')
     # rewrite archive.org URLs in JavaScript strings
-    content.gsub!(/(["'])https?:\/\/web\.archive\.org\/web\/\d+(?:id_)?\/https?:\/\/[^\/]+([^"']*)(["'])/i) do
-      quote_start, path, quote_end = $1, $2, $3
-      path = normalize_path_for_local(path)
-      "#{quote_start}#{path}#{quote_end}"
+    content.gsub(/(["'])https?:\/\/web\.archive\.org\/web\/\d+[a-z_]*\/https?:\/\/[^\/"']+(\/[^"']*)?(["'])/i) do
+      quote_start, path, quote_end = $1, ($2 || "/index.html"), $3
+      local = normalize_path_for_local(path, root_prefix)
+      "#{quote_start}#{local}#{quote_end}"
     end
+  end
 
-    # rewrite absolute URLs in JavaScript
-    content.gsub!(/(["'])https?:\/\/[^\/]+([^"']*)(["'])/i) do
-      quote_start, path, quote_end = $1, $2, $3
-      next "#{quote_start}http#{$2}#{quote_end}" if $2.start_with?('s://', '://')
-      path = normalize_path_for_local(path)
-      "#{quote_start}#{path}#{quote_end}"
-    end
+  def normalize_path_for_local(path, root_prefix = './')
+    path = path.to_s.strip
+    return "#{root_prefix}index.html" if path.empty? || path == "/"
 
-    content
-  end
+    path_part, query_part = path.split('?', 2)
+    path_part = "/index.html" if path_part.nil? || path_part.empty? || path_part == "/"
 
-  private
+    # hash query parameters to match sanitize_and_prepare_id
+    if query_part && !query_part.empty?
+      q_digest = Digest::SHA256.hexdigest(query_part)[0, 12]
+      if path_part.include?('.')
+        pre, _sep, post = path_part.rpartition('.')
+        path_part = "#{pre}__q#{q_digest}.#{post}"
+      else
+        path_part = "#{path_part}__q#{q_digest}"
+      end
+    end
 
-  def normalize_path_for_local(path)
-    return "./index.html" if path.empty? || path == "/"
-    
-    # handle query strings - they're already part of the filename
-    path = path.split('?').first if path.include?('?')
-    
     # check if this is a server-side script
-    ext = File.extname(path).downcase
-    if SERVER_SIDE_EXTS.include?(ext)
-      # keep the path as-is but ensure it starts with ./
-      path = "./#{path}" unless path.start_with?('./', '/')
-    else
-      # regular file handling
-      path = "./#{path}" unless path.start_with?('./', '/')
-      
-      # if it looks like a directory, add index.html
-      if path.end_with?('/') || !path.include?('.')
-        path = "#{path.chomp('/')}/index.html"
+    ext = File.extname(path_part).downcase
+    unless SERVER_SIDE_EXTS.include?(ext)
+      if path_part.end_with?('/') || !path_part.include?('.')
+        path_part = "#{path_part.chomp('/')}/index.html"
       end
     end
-    
-    path
+
+    clean_path = path_part.sub(%r{\A/+}, '')
+    "#{root_prefix}#{clean_path}"
+  end
+
+  private
+
+  def current_host
+    return nil unless @base_url
+    clean = @base_url.to_s.sub(%r{\A\*\.}, '')
+    clean = clean.match?(%r{\Ahttps?://}i) ? clean : "http://#{clean}"
+    URI.parse(clean).host rescue nil
   end
 end

data/lib/wayback_machine_downloader.rb

@@ -4,11 +4,13 @@ require 'thread'
 require 'net/http'
 require 'fileutils'
 require 'json'
+require 'pathname'
 require 'concurrent-ruby'
 require 'logger'
 require 'zlib'
 require 'stringio'
 require 'digest'
+require 'etc'
 require_relative 'wayback_machine_downloader/tidy_bytes'
 require_relative 'wayback_machine_downloader/to_regex'
 require_relative 'wayback_machine_downloader/archive_api'
@@ -16,6 +18,13 @@ require_relative 'wayback_machine_downloader/page_requisites'
 require_relative 'wayback_machine_downloader/subdom_processor'
 require_relative 'wayback_machine_downloader/url_rewrite'
 
+begin
+  require 'httpx'
+  HTTPX_AVAILABLE = true
+rescue LoadError
+  HTTPX_AVAILABLE = false
+end
+
 class ConnectionPool
   MAX_AGE = 300
   CLEANUP_INTERVAL = 60
@@ -66,7 +75,12 @@ class ConnectionPool
   def stale?(entry)
     return true if entry.nil? || entry[:http].nil?
     http = entry[:http]
-    !http.started? || (Time.now - entry[:created_at] > MAX_AGE)
+
+    if HTTPX_AVAILABLE && http.is_a?(HTTPX::Session)
+      Time.now - entry[:created_at] > MAX_AGE
+    else
+      !http.started? || (Time.now - entry[:created_at] > MAX_AGE)
+    end
   end
 
   def build_connection_entry
@@ -74,7 +88,12 @@ class ConnectionPool
   end
 
   def safe_finish(http)
-    http.finish if http&.started?
+    return if http.nil?
+    if HTTPX_AVAILABLE && http.is_a?(HTTPX::Session)
+      http.close
+    else
+      http.finish if http&.started?
+    end
   rescue StandardError
     nil
   end
@@ -113,14 +132,37 @@ class ConnectionPool
   end
 
   def create_connection
-    http = Net::HTTP.new("web.archive.org", 443)
-    http.use_ssl = true
-    http.read_timeout = DEFAULT_TIMEOUT
-    http.open_timeout = DEFAULT_TIMEOUT
-    http.keep_alive_timeout = 30
-    http.max_retries = MAX_RETRIES
-    http.start
-    http
+    if HTTPX_AVAILABLE
+      HTTPX.with(
+        timeout: {
+          connect_timeout: DEFAULT_TIMEOUT,
+          read_timeout: DEFAULT_TIMEOUT,
+          write_timeout: DEFAULT_TIMEOUT
+        }
+      )
+    else
+      http = Net::HTTP.new("web.archive.org", 443)
+      http.use_ssl = true
+      http.read_timeout = DEFAULT_TIMEOUT
+      http.open_timeout = DEFAULT_TIMEOUT
+      http.keep_alive_timeout = 30
+      http.max_retries = MAX_RETRIES
+
+      retries = 0
+      begin
+        http.start
+      rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNRESET, Errno::ETIMEDOUT, SocketError => e
+        if retries < MAX_RETRIES
+          retries += 1
+          sleep(2 * retries)
+          retry
+        else
+          raise e
+        end
+      end
+
+      http
+    end
   end
 end
 
@@ -130,7 +172,7 @@ class WaybackMachineDownloader
   include SubdomainProcessor
   include URLRewrite
 
-  VERSION = "2.4.7"
+  VERSION = "2.4.9"
   DEFAULT_TIMEOUT = 30
   MAX_RETRIES = 3
   RETRY_DELAY = 2
@@ -164,7 +206,14 @@ class WaybackMachineDownloader
     @all = params[:all]
     @keep_duplicates = params[:keep_duplicates] || false
     @maximum_pages = params[:maximum_pages] ? params[:maximum_pages].to_i : 100
-    @threads_count = [params[:threads_count].to_i, 1].max
+    default_threads = begin
+      [Etc.nprocessors, 4].max
+    rescue StandardError
+      4
+    end
+    threads_param = params[:threads_count] ? params[:threads_count].to_i : 0
+    threads_param = default_threads if threads_param <= 0
+    @threads_count = [threads_param, 1].max
     @rewritten = params[:rewritten]
     @reset = params[:reset]
     @keep = params[:keep]
@@ -174,6 +223,7 @@ class WaybackMachineDownloader
     @connection_pool = ConnectionPool.new(CONNECTION_POOL_SIZE)
     @db_mutex = Mutex.new
     @rewrite = params[:rewrite] || false
+    @delay = params[:delay] ? params[:delay].to_f : 0.0
     @recursive_subdomains = params[:recursive_subdomains] || false
     @subdomain_depth = params[:subdomain_depth] || 1
     @snapshot_at = params[:snapshot_at] ? params[:snapshot_at].to_i : nil
@@ -181,6 +231,12 @@ class WaybackMachineDownloader
     @page_requisites = params[:page_requisites] || false
     @pending_jobs = Concurrent::AtomicFixnum.new(0)
 
+    @db_buffer = []
+    @db_buffer_mutex = Mutex.new
+    @db_last_flush = Time.now
+    @db_flush_threshold = 50  # flush to disk every 50 completed files
+    @db_flush_interval = 5.0  # or flush every 5 seconds even if count isn't met
+
     # URL for rejecting invalid/unencoded wayback urls
     @url_regexp = /^(([A-Za-z][A-Za-z0-9+.-]*):((\/\/(((([A-Za-z0-9._~-])|(%[ABCDEFabcdef0-9][ABCDEFabcdef0-9])|([!$&'('')'*+,;=]))+)(:([0-9]*))?)(((\/((([A-Za-z0-9._~-])|(%[ABCDEFabcdef0-9][ABCDEFabcdef0-9])|([!$&'('')'*+,;=])|:|@)*))*)))|((\/(((([A-Za-z0-9._~-])|(%[ABCDEFabcdef0-9][ABCDEFabcdef0-9])|([!$&'('')'*+,;=])|:|@)+)(\/((([A-Za-z0-9._~-])|(%[ABCDEFabcdef0-9][ABCDEFabcdef0-9])|([!$&'('')'*+,;=])|:|@)*))*)?))|((((([A-Za-z0-9._~-])|(%[ABCDEFabcdef0-9][ABCDEFabcdef0-9])|([!$&'('')'*+,;=])|:|@)+)(\/((([A-Za-z0-9._~-])|(%[ABCDEFabcdef0-9][ABCDEFabcdef0-9])|([!$&'('')'*+,;=])|:|@)*))*)))(\?((([A-Za-z0-9._~-])|(%[ABCDEFabcdef0-9][ABCDEFabcdef0-9])|([!$&'('')'*+,;=])|:|@)|\/|\?)*)?(\#((([A-Za-z0-9._~-])|(%[ABCDEFabcdef0-9][ABCDEFabcdef0-9])|([!$&'('')'*+,;=])|:|@)|\/|\?)*)?)$/
 
@@ -274,12 +330,8 @@ class WaybackMachineDownloader
       puts "Loading snapshot list from #{cdx_path}"
       begin
         snapshot_list_to_consider = JSON.parse(File.read(cdx_path))
-        puts "Loaded #{snapshot_list_to_consider.length} snapshots from cache."
-        puts
+        puts "Loaded #{snapshot_list_to_consider.length} snapshots from cache.\n\n"
         return Concurrent::Array.new(snapshot_list_to_consider)
-      rescue JSON::ParserError => e
-        puts "Error reading snapshot cache file #{cdx_path}: #{e.message}. Refetching..."
-        FileUtils.rm_f(cdx_path)
       rescue => e
         puts "Error loading snapshot cache #{cdx_path}: #{e.message}. Refetching..."
         FileUtils.rm_f(cdx_path)
@@ -287,89 +339,38 @@ class WaybackMachineDownloader
     end
 
     snapshot_list_to_consider = Concurrent::Array.new
-    mutex = Mutex.new
-
-    # if snapshot_at is set, limit CDX queries to snapshots at or before that timestamp
     original_to = @to_timestamp
-    if @snapshot_at
-      @to_timestamp = @snapshot_at
-    end
+    @to_timestamp = @snapshot_at if @snapshot_at
 
     puts "Getting snapshot pages from Wayback Machine API..."
 
-    # Fetch the initial set of snapshots, sequentially
+    # fetch initial page (page 0)
     @connection_pool.with_connection do |connection|
       initial_list = get_raw_list_from_api(@base_url, 0, connection)
-      initial_list ||= []
-      mutex.synchronize do
+      if initial_list && !initial_list.empty?
         snapshot_list_to_consider.concat(initial_list)
         print "."
         $stdout.flush
       end
     end
 
-    # Fetch additional pages if the exact URL flag is not set and the first page wasn't empty
+    # sequentially fetch subsequent pages
     unless @exact_url || snapshot_list_to_consider.empty?
       page_index = 1
-      batch_size = [@threads_count, 5].min
-      continue_fetching = true
-      fetch_pool = Concurrent::FixedThreadPool.new([@threads_count, 1].max)
-      begin
-        while continue_fetching && page_index < @maximum_pages
-          # Determine the range of pages to fetch in this batch
-          end_index = [page_index + batch_size, @maximum_pages].min
-          current_batch = (page_index...end_index).to_a
-
-          # Create futures for concurrent API calls
-          futures = current_batch.map do |page|
-            Concurrent::Future.execute(executor: fetch_pool) do
-              result = nil
-              @connection_pool.with_connection do |connection|
-                result = get_raw_list_from_api(@base_url, page, connection)
-              end
-              result ||= []
-              [page, result]
-            end
-          end
-
-          results = []
-
-          futures.each do |future|
-            begin
-              val = future.value
-              # only append if valid
-              if val && val.is_a?(Array) && val.first.is_a?(Integer)
-                results << val
-              end
-            rescue => e
-              puts "\nError fetching page #{future}: #{e.message}"
-            end
-          end
-
-          # Sort results by page number to maintain order
-          results.sort_by! { |page, _| page }
-
-          # Process results and check for empty pages
-          results.each do |page, result|
-            if result.nil? || result.empty?
-              continue_fetching = false
-              break
-            else
-              mutex.synchronize do
-                snapshot_list_to_consider.concat(result)
-                print "."
-                $stdout.flush
-              end
-            end
-          end
-
-          page_index = end_index
+      while page_index < @maximum_pages
+        result = nil
+        @connection_pool.with_connection do |connection|
+          result = get_raw_list_from_api(@base_url, page_index, connection)
+        end
 
-          sleep(RATE_LIMIT) if continue_fetching
+        if result.nil? || result.empty?
+          break
+        else
+          snapshot_list_to_consider.concat(result)
+          print "."
+          $stdout.flush
+          page_index += 1
         end
-      ensure
-        fetch_pool.shutdown
-        fetch_pool.wait_for_termination
       end
     end
 
@@ -378,7 +379,7 @@ class WaybackMachineDownloader
     # save the fetched list to the cache file
     begin
       FileUtils.mkdir_p(File.dirname(cdx_path))
-      File.write(cdx_path, JSON.pretty_generate(snapshot_list_to_consider.to_a)) # Convert Concurrent::Array back to Array for JSON
+      File.write(cdx_path, JSON.pretty_generate(snapshot_list_to_consider.to_a))
       puts "Saved snapshot list to #{cdx_path}"
     rescue => e
       puts "Error saving snapshot cache to #{cdx_path}: #{e.message}"
@@ -391,6 +392,19 @@ class WaybackMachineDownloader
     snapshot_list_to_consider
   end
 
+  def extract_path_and_query(file_url)
+    return "" if file_url.nil? || file_url.empty?
+    normalized = file_url.match?(%r{\Ahttps?://}i) ? file_url : "http://#{file_url}"
+    uri = URI.parse(normalized)
+    # returns path + query
+    path = uri.path.to_s
+    path += "?#{uri.query}" if uri.query && !uri.query.empty?
+    path.sub(%r{\A/}, '')
+  rescue URI::InvalidURIError
+    # for ill-formed URLs
+    file_url.sub(%r{\Ahttps?://[^/]+/?}i, '').sub(%r{\A[^/]+/?}, '')
+  end
+
   # Get a composite snapshot file list for a specific timestamp
   def get_composite_snapshot_file_list(target_timestamp)
     file_versions = {}
@@ -399,7 +413,7 @@ class WaybackMachineDownloader
       next if file_timestamp.to_i > target_timestamp
 
       # allow empty path by treating missing tail as empty string
-      raw_tail = file_url.split('/')[3..-1]&.join('/') || ''
+      raw_tail = extract_path_and_query(file_url)
       file_id = sanitize_and_prepare_id(raw_tail, file_url)
       next if file_id.nil?
       next if match_exclude_filter(file_url)
@@ -424,7 +438,7 @@ class WaybackMachineDownloader
     get_all_snapshots_to_consider.each do |file_timestamp, file_url|
       next unless file_url.include?('/')
 
-      raw_tail = file_url.split('/')[3..-1]&.join('/') || ''
+      raw_tail = extract_path_and_query(file_url)
       file_id = sanitize_and_prepare_id(raw_tail, file_url)
       if file_id.nil?
         puts "Malformed file url, ignoring: #{file_url}"
@@ -439,7 +453,7 @@ class WaybackMachineDownloader
         elsif !match_only_filter(file_url)
           puts "File url doesn't match only filter, ignoring: #{file_url}"
         elsif file_list_curated[file_id]
-          unless file_list_curated[file_id][:timestamp] > file_timestamp
+          unless file_list_curated[file_id][:timestamp].to_i > file_timestamp.to_i
             file_list_curated[file_id] = { file_url: file_url, timestamp: file_timestamp }
           end
         else
@@ -455,7 +469,7 @@ class WaybackMachineDownloader
     get_all_snapshots_to_consider.each do |file_timestamp, file_url|
       next unless file_url.include?('/')
 
-      raw_tail = file_url.split('/')[3..-1]&.join('/') || ''
+      raw_tail = extract_path_and_query(file_url)
       file_id = sanitize_and_prepare_id(raw_tail, file_url)
       if file_id.nil?
         puts "Malformed file url, ignoring: #{file_url}"
@@ -545,12 +559,40 @@ class WaybackMachineDownloader
   end
 
   def append_to_db(file_id)
-    @db_mutex.synchronize do
-      begin
-        FileUtils.mkdir_p(File.dirname(db_path))
-        File.open(db_path, 'a') { |f| f.puts(file_id) }
-      rescue => e
-        @logger.error("Failed to append downloaded file ID #{file_id} to #{db_path}: #{e.message}")
+    flush_needed = false
+
+    @db_buffer_mutex.synchronize do
+      @db_buffer << file_id
+      if @db_buffer.size >= @db_flush_threshold || (Time.now - @db_last_flush) >= @db_flush_interval
+        flush_needed = true
+      end
+    end
+
+    flush_db if flush_needed
+  end
+
+  def flush_db
+    lines_to_write = nil
+
+    @db_buffer_mutex.synchronize do
+      return if @db_buffer.empty?
+      lines_to_write = @db_buffer.dup
+      @db_buffer.clear
+      @db_last_flush = Time.now
+    end
+
+    return if lines_to_write.nil? || lines_to_write.empty?
+
+    begin
+      FileUtils.mkdir_p(File.dirname(db_path))
+      File.open(db_path, 'a') do |f|
+        lines_to_write.each { |id| f.puts(id) }
+      end
+    rescue StandardError => e
+      @logger.error("Failed to write batch to #{db_path}: #{e.message}")
+
+      @db_buffer_mutex.synchronize do
+        @db_buffer.unshift(*lines_to_write)
       end
     end
   end
@@ -587,9 +629,18 @@ class WaybackMachineDownloader
       end
     end
   end
-  
+
+  @@engine_printed = false
+
   def download_files
     start_time = Time.now
+
+    unless @@engine_printed
+      engine_name = HTTPX_AVAILABLE ? "HTTPX" : "Net::HTTP"
+      puts "Connection engine used: #{color(engine_name, :green)}"
+      @@engine_printed = true
+    end
+
     puts "Downloading #{@base_url} to #{backup_path} from Wayback Machine archives."
 
     FileUtils.mkdir_p(backup_path)
@@ -608,7 +659,7 @@ class WaybackMachineDownloader
 
     # Load IDs of already downloaded files
     downloaded_ids = load_downloaded_ids
-    
+
     # We use a thread-safe Set to track what we have queued/downloaded in this session
     # to avoid infinite loops with page requisites
     @session_downloaded_ids = Concurrent::Set.new
@@ -624,7 +675,7 @@ class WaybackMachineDownloader
     if skipped_count > 0
       puts "Found #{skipped_count} previously downloaded files, skipping them."
     end
-    
+
     if remaining_count == 0 && !@page_requisites
       puts "All matching files have already been downloaded."
       cleanup
@@ -691,6 +742,9 @@ class WaybackMachineDownloader
     download_success = false
     downloaded_path = nil
 
+    # if delay was set by the user
+    sleep(@delay) if @delay > 0
+
     # fast-path for resumed runs: if file already exists locally, avoid HTTP work entirely
     existing_path = local_path_for_file_id(file_remote_info[:file_id])
     if existing_path && File.exist?(existing_path)
@@ -702,19 +756,19 @@ class WaybackMachineDownloader
 
       append_to_db(file_remote_info[:file_id])
 
-      if @page_requisites && File.extname(existing_path) =~ /\.(html?|php|asp|aspx|jsp)$/i
+      if @page_requisites && File.extname(existing_path) =~ /\.(html?|shtml|php|asp|aspx|ashx|jsp|do|action|cgi|pl|cfm)$/i
         process_page_requisites(existing_path, file_remote_info)
       end
       return
     end
-    
+
     @connection_pool.with_connection do |connection|
       result_message, downloaded_path = download_file(file_remote_info, connection)
-      
+
       if downloaded_path && File.exist?(downloaded_path)
          download_success = true
       end
-      
+
       @download_mutex.synchronize do
         @processed_file_count += 1 if @processed_file_count < @total_to_download
         # only print if it's a "User" file or a requisite we found
@@ -724,7 +778,7 @@ class WaybackMachineDownloader
 
     if download_success
       append_to_db(file_remote_info[:file_id])
-      
+
       if @page_requisites && downloaded_path && File.extname(downloaded_path) =~ /\.(html?|php|asp|aspx|jsp)$/i
         process_page_requisites(downloaded_path, file_remote_info)
       end
@@ -732,7 +786,7 @@ class WaybackMachineDownloader
   rescue => e
     @logger.error("Error processing file #{file_remote_info[:file_url]}: #{e.message}")
   end
-  
+
   def process_page_requisites(file_path, parent_remote_info)
     return unless File.exist?(file_path)
 
@@ -744,7 +798,7 @@ class WaybackMachineDownloader
     # prepare base URI for resolving relative paths
     parent_raw = parent_remote_info[:file_url]
     parent_raw = "http://#{parent_raw}" unless parent_raw.match?(/^https?:\/\//)
-    
+
     begin
       base_uri = URI(parent_raw)
       # calculate the "root" host of the site we are downloading to compare later
@@ -778,7 +832,7 @@ class WaybackMachineDownloader
         path = resolved_uri.path
         ext = File.extname(path).downcase
         if ext.empty? || ['.html', '.htm', '.php', '.asp', '.aspx'].include?(ext)
-           next 
+           next
         end
 
         # construct the original URL to query the Wayback API
@@ -849,7 +903,7 @@ class WaybackMachineDownloader
     rescue Errno::EEXIST, Errno::ENOTDIR => e
       file_already_existing = nil
       check_path = dir_path
-      
+
       # walk up the path to find the specific file that is blocking directory creation
       while check_path != "." && check_path != "/"
         if File.exist?(check_path) && !File.directory?(check_path)
@@ -864,11 +918,11 @@ class WaybackMachineDownloader
       if file_already_existing
         file_already_existing_temporary = file_already_existing + '.temp'
         file_already_existing_permanent = file_already_existing + '/index.html'
-        
+
         FileUtils::mv file_already_existing, file_already_existing_temporary
         FileUtils::mkdir_p file_already_existing
         FileUtils::mv file_already_existing_temporary, file_already_existing_permanent
-        
+
         puts "#{file_already_existing} -> #{file_already_existing_permanent}"
         # retry the directory creation now that the path is clear
         structure_dir_path dir_path
@@ -880,13 +934,13 @@ class WaybackMachineDownloader
 
   def rewrite_local_files
     puts "Scanning #{backup_path} for files to rewrite..."
-    files = Dir.glob(File.join(backup_path, "**/*.{html,htm,css,js,php,asp,aspx,jsp}"))
-    
+    files = Dir.glob(File.join(backup_path, "**/*.{html,htm,shtml,css,js,php,asp,aspx,ashx,jsp,do,action,cgi,pl,cfm}"))
+
     puts "Found #{files.size} files. Rewriting links for local browsing..."
-    
+
     pool = Concurrent::FixedThreadPool.new(@threads_count)
     progress = Concurrent::AtomicFixnum.new(0)
-    
+
     files.each do |file_path|
       pool.post do
         rewrite_urls_to_relative(file_path)
@@ -894,7 +948,7 @@ class WaybackMachineDownloader
         print "\rProgress: #{current}/#{files.size}" if current % 100 == 0
       end
     end
-    
+
     pool.shutdown
     pool.wait_for_termination
     puts "\nFinished rewriting all files."
@@ -902,55 +956,82 @@ class WaybackMachineDownloader
 
   def rewrite_urls_to_relative(file_path)
     return unless File.exist?(file_path)
-    
+
     file_ext = File.extname(file_path).downcase
-    
+
     begin
       content = File.binread(file_path)
 
-      # detect encoding for HTML files
-      if file_ext == '.html' || file_ext == '.htm' || file_ext == '.php' || file_ext == '.asp'
-        encoding = content.match(/<meta\s+charset=["']?([^"'>]+)/i)&.captures&.first || 'UTF-8'
-        content.force_encoding(encoding) rescue content.force_encoding('UTF-8')
+      # detect encoding for HTML/server-rendered files
+      if %w[.html .htm .php .asp .jsp .shtml].include?(file_ext)
+        encoding_match = content.match(/<meta.*?charset=["'\s]?([^"'\s>;]+)/i)
+        encoding_name = encoding_match ? encoding_match.captures.first : 'UTF-8'
+
+        begin
+          encoding = Encoding.find(encoding_name)
+        rescue ArgumentError
+          encoding = Encoding::UTF_8
+        end
+
+        content.force_encoding(encoding)
       else
         content.force_encoding('UTF-8')
       end
 
+      # convert the content to valid UTF-8
+      if !content.valid_encoding? || content.encoding != Encoding::UTF_8
+        begin
+          content = content.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: '')
+        rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError, ArgumentError
+          content = content.tidy_bytes
+        end
+      end
+
+      root_prefix = site_root_relative_prefix(file_path)
+
       # URLs in HTML attributes
-      content = rewrite_html_attr_urls(content)
-      
+      content = rewrite_html_attr_urls(content, root_prefix)
+
       # URLs in CSS
-      content = rewrite_css_urls(content)
-      
+      content = rewrite_css_urls(content, root_prefix)
+
       # URLs in JavaScript
-      content = rewrite_js_urls(content)
-      
-      # for URLs that start with a single slash, make them relative
-      content.gsub!(/(\s(?:href|src|action|data-src|data-url)=["'])\/([^"'\/][^"']*)(["'])/i) do
-        prefix, path, suffix = $1, $2, $3
-        "#{prefix}./#{path}#{suffix}"
-      end
-      
-      # for URLs in CSS that start with a single slash, make them relative
-      content.gsub!(/url\(\s*["']?\/([^"'\)\/][^"'\)]*?)["']?\s*\)/i) do
-        path = $1
-        "url(\"./#{path}\")"
-      end
+      content = rewrite_js_urls(content, root_prefix)
 
       # save the modified content back to the file
       File.binwrite(file_path, content)
       puts "Rewrote URLs in #{file_path} to be relative."
     rescue Errno::ENOENT => e
       @logger.warn("Error reading file #{file_path}: #{e.message}")
+    rescue StandardError => e
+      @logger.error("Failed to rewrite URLs in #{file_path}: #{e.message}")
     end
   end
 
+  def site_root_relative_prefix(file_path)
+    file_dir = File.dirname(File.expand_path(file_path))
+    root_dir = File.expand_path(backup_path)
+
+    begin
+      relative_dir = Pathname.new(file_dir).relative_path_from(Pathname.new(root_dir)).to_s
+    rescue ArgumentError
+      return './'
+    end
+
+    return './' if relative_dir == '.' || relative_dir.empty?
+
+    depth = relative_dir.split(/[\\\/]+/).reject(&:empty?).length
+    return './' if depth <= 0
+
+    '../' * depth
+  end
+
   def download_file (file_remote_info, http)
     current_encoding = "".encoding
     file_url = file_remote_info[:file_url].encode(current_encoding)
     file_id = file_remote_info[:file_id]
     file_timestamp = file_remote_info[:timestamp]
-    
+
     # sanitize file_id to ensure it is a valid path component
     raw_path_elements = file_id.split('/')
 
@@ -964,7 +1045,7 @@ class WaybackMachineDownloader
         element
       end
     end
-    
+
     current_backup_path = backup_path
 
     if file_id == ""
@@ -993,7 +1074,7 @@ class WaybackMachineDownloader
 
       case status
       when :saved
-        if @rewrite && File.extname(file_path) =~ /\.(html?|css|js)$/i
+        if @rewrite && File.extname(file_path) =~ /\.(html?|shtml|css|js|php|asp|aspx|jsp|cgi|pl)$/i
           rewrite_urls_to_relative(file_path)
         end
         return ["#{color("[SAVED]", :green)}  #{file_url} (#{@processed_file_count + 1}/#{@total_to_download})", file_path]
@@ -1079,7 +1160,7 @@ class WaybackMachineDownloader
     end
     logger
   end
-    
+
   # safely sanitize a file id (or id+timestamp)
   def sanitize_and_prepare_id(raw, file_url)
     return nil if raw.nil?
@@ -1221,65 +1302,85 @@ class WaybackMachineDownloader
           return :skipped_not_found
       end
 
-      request = Net::HTTP::Get.new(URI(wayback_url))
-      request["Connection"] = "keep-alive"
-      request["User-Agent"] = "WaybackMachineDownloader/#{VERSION}"
-      request["Accept-Encoding"] = "gzip, deflate"
-
-      response = connection.request(request)
-
-      save_response_body = lambda do
-        File.open(file_path, "wb") do |file|
-          body = response.body
-          if response['content-encoding'] == 'gzip' && body && !body.empty?
-            begin
-              gz = Zlib::GzipReader.new(StringIO.new(body))
-              decompressed_body = gz.read
-              gz.close
-              file.write(decompressed_body)
-            rescue Zlib::GzipFile::Error => e
-              @logger.warn("Failure decompressing gzip file #{file_url}: #{e.message}. Writing raw body.")
-              file.write(body)
-            end
+      if HTTPX_AVAILABLE && connection.is_a?(HTTPX::Session)
+        response = connection.get(wayback_url)
+
+        raise response.error if response.is_a?(HTTPX::ErrorResponse)
+
+        code = response.status
+
+        save_response_body = lambda do
+          if response.respond_to?(:copy_to)
+            response.copy_to(file_path)
           else
-            file.write(body) if body
+            response.body.copy_to(file_path)
+          end
+        end
+      else
+        request = Net::HTTP::Get.new(URI(wayback_url))
+        request["Connection"] = "keep-alive"
+        request["User-Agent"] = "WaybackMachineDownloader/#{VERSION}"
+        request["Accept-Encoding"] = "gzip, deflate"
+
+        response = connection.request(request)
+        code = response.code.to_i
+
+        save_response_body = lambda do
+          File.open(file_path, "wb") do |file|
+            body = response.body
+            if response['content-encoding'] == 'gzip' && body && !body.empty?
+              begin
+                gz = Zlib::GzipReader.new(StringIO.new(body))
+                decompressed_body = gz.read
+                gz.close
+                file.write(decompressed_body)
+              rescue Zlib::GzipFile::Error => e
+                @logger.warn("Failure decompressing gzip file #{file_url}: #{e.message}. Writing raw body.")
+                file.write(body)
+              end
+            else
+              file.write(body) if body
+            end
           end
         end
       end
 
       if @all
-        case response
-        when Net::HTTPSuccess, Net::HTTPRedirection, Net::HTTPClientError, Net::HTTPServerError
+        case code
+        when 200..599
           save_response_body.call
-          if response.is_a?(Net::HTTPRedirection)
-            @logger.info("Saved redirect page for #{file_url} (status #{response.code}).")
-          elsif response.is_a?(Net::HTTPClientError) || response.is_a?(Net::HTTPServerError)
-            @logger.info("Saved error page for #{file_url} (status #{response.code}).")
+          if (300..399).cover?(code)
+            @logger.info("Saved redirect page for #{file_url} (status #{code}).")
+          elsif (400..599).cover?(code)
+            @logger.info("Saved error page for #{file_url} (status #{code}).")
           end
           return :saved
         else
           # for any other response type when --all is true, treat as an error to be retried or failed
-          raise "Unhandled HTTP response: #{response.code} #{response.message}"
+          raise "Unhandled HTTP response: #{code}"
         end
       else # not @all (our default behavior)
-        case response
-        when Net::HTTPSuccess
+        if (200..299).cover?(code)
           save_response_body.call
           return :saved
-        when Net::HTTPRedirection
+        elsif (300..399).cover?(code)
           raise "Too many redirects for #{file_url}" if redirect_count >= 5
-          location = response['location']
+          location = if HTTPX_AVAILABLE && connection.is_a?(HTTPX::Session)
+                       response.headers['location']
+                     else
+                       response['location']
+                     end
           @logger.warn("Redirect found for #{file_url} -> #{location}")
           redirected_source = resolve_redirect_source(file_url, location)
           return download_with_retry(file_path, redirected_source, file_timestamp, connection, redirect_count + 1)
-        when Net::HTTPTooManyRequests
+        elsif code == 429
           sleep(RATE_LIMIT * 2)
           raise "Rate limited, retrying..."
-        when Net::HTTPNotFound
+        elsif code == 404
           @logger.warn("File not found, skipping: #{file_url}")
           return :skipped_not_found
         else
-          raise "HTTP Error: #{response.code} #{response.message}"
+          raise "HTTP Error: #{code}"
         end
       end
 
@@ -1297,6 +1398,7 @@ class WaybackMachineDownloader
   end
 
   def cleanup
+    flush_db
     @connection_pool.shutdown
 
     if @failed_downloads.any?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wayback_machine_downloader_straw
 version: !ruby/object:Gem::Version
-  version: 2.4.7
+  version: 2.4.9
 platform: ruby
 authors:
 - strawberrymaster
@@ -87,14 +87,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.4.3
+      version: 3.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 4.0.16
 specification_version: 4
 summary: Download an entire website from the Wayback Machine.
 test_files: []