wayback_machine_downloader_straw 2.4.8 → 2.4.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 12259647dc99ef1c263b3d797098233be4f194bf1a4e9bc047e1dfb4d0fc9468
-  data.tar.gz: 5d9d6bd47efca6b43a5c1d6c2606bb2871edea0b4dc781f206bbab8c9c66aaa0
+  metadata.gz: cdb773f6bbfb5404d827d6c484f4b8aa0ba2b1c4aed192c86ba12f137526f814
+  data.tar.gz: a8a898bdf0640c40e6946d78e3135bb04e8fa0f76ab14a267448a514a503af2f
 SHA512:
-  metadata.gz: 670efe274663448a9967d2e39920ce234af1ba5c0aae82135c56f64899904f510096ac19b6367a5b90af43d3878b156a21b2fb1138a2e567041b55205fc97c67
-  data.tar.gz: 2d4d6e927664ea831c0611bbbc89d74ee6c144629f8afb2b2a58428cff1f78846832979bd994dfd97a51cff03ee52ab9c5c09beebc1b007248f1c043aa3bdd2a
+  metadata.gz: d608cf38715e49fb50bcd6aaa6b692556ef9ca0fe3c36894e0738e2e8c2487dd99d04cf7f242a053b96ced3cd7e2cf92c455f83a83113102b1dfb926c914fb58
+  data.tar.gz: 96b2ca1e63d659936a25423717aad49e8f2fa342040a4564cf425c76766dc025f09ef32b9821737a0bec8127cfee6328ecba3717286ff59c1b15c84a713752a3

data/lib/wayback_machine_downloader/archive_api.rb

@@ -1,38 +1,101 @@
 require 'json'
 require 'uri'
+require 'time'
+require 'thread'
 
 module ArchiveAPI
+  DEFAULT_RATE_LIMIT_COOLDOWN = 30.0
+  DEFAULT_CDX_INTERVAL = 2.5 # 1 request every 2.5 seconds
+
+  class RateLimitError < StandardError
+    attr_reader :retry_after
+
+    def initialize(message, retry_after = nil)
+      super(message)
+      @retry_after = retry_after
+    end
+  end
+
+  @cdx_mutex = Mutex.new
+  @cdx_cv = ConditionVariable.new
+  @next_allowed_cdx_at = 0.0
+  @cdx_interval = DEFAULT_CDX_INTERVAL
+
+  class << self
+    attr_accessor :cdx_interval
+
+    # pace CDX requests to avoid exceeding the rate limit
+    def pace_cdx_request
+      @cdx_mutex.synchronize do
+        loop do
+          now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          if now < @next_allowed_cdx_at
+            wait_time = @next_allowed_cdx_at - now
+            @cdx_cv.wait(@cdx_mutex, wait_time)
+          else
+            interval = @cdx_interval || DEFAULT_CDX_INTERVAL
+            @next_allowed_cdx_at = now + interval
+            return
+          end
+        end
+      end
+    end
+
+    # extend the cooldown period for CDX requests, e.g., after receiving a 429 response
+    def extend_cdx_cooldown(seconds)
+      seconds = seconds.to_f
+      return if seconds <= 0
+
+      @cdx_mutex.synchronize do
+        now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        candidate = now + seconds
+        @next_allowed_cdx_at = candidate if candidate > @next_allowed_cdx_at
+        @cdx_cv.broadcast
+      end
+    end
+
+    def reset_cdx_limiter
+      @cdx_mutex.synchronize do
+        @next_allowed_cdx_at = 0.0
+        @cdx_cv.broadcast
+      end
+    end
+  end
 
   def get_raw_list_from_api(url, page_index, http)
     # Automatically append /* for host-only URLs
     # This is a workaround for an issue with the API and *some* domains.
     # See https://github.com/StrawberryMaster/wayback-machine-downloader/issues/6
     # But don't do this when exact_url flag is set, and never append twice
+    normalized_url = url.to_s.strip
+    
+    # strip protocol for CDX query
+    clean_url = normalized_url.sub(%r{\Ahttps?://}i, '')
+    # ensure wildcard/matchType for domain-wide crawling
     match_type = nil
-    if url && !@exact_url
-      normalized_url = url.to_s
-      has_wildcard = normalized_url.include?('*')
-      host_and_rest = normalized_url
-        .sub(/\Ahttps?:\/\//i, '')
-        .split(/[?#]/, 2)
-        .first
-      has_path = host_and_rest.include?('/')
-
-      unless has_wildcard || has_path
+    unless @exact_url || clean_url.include?('*')
+      if clean_url.end_with?('/')
+        clean_url = "#{clean_url}*"
+      elsif !clean_url.include?('/')
         match_type = "prefix"
+      else
+        clean_url = "#{clean_url}/*"
       end
     end
 
     request_url = URI("https://web.archive.org/cdx/search/cdx")
-    params = [["output", "json"], ["url", url]] + parameters_for_api(page_index)
+    params = [["output", "json"], ["url", clean_url]] + parameters_for_api(page_index)
     params << ["matchType", match_type] if match_type
     request_url.query = URI.encode_www_form(params)
 
     retries = 0
     max_retries = (@max_retries || 3)
-    base_delay = WaybackMachineDownloader::RETRY_DELAY rescue 2
+    base_delay = 2
 
     begin
+      # acquire slot from the process-wide proactive pacer before sending request
+      ArchiveAPI.pace_cdx_request
+
       if HTTPX_AVAILABLE && http.is_a?(HTTPX::Session)
         response = http.get(request_url)
         raise response.error if response.is_a?(HTTPX::ErrorResponse)
@@ -60,27 +123,45 @@ module ArchiveAPI
         rescue JSON::ParserError => e
           raise "Malformed JSON response: #{e.message}"
         end
-      when 429, 500, 502, 503, 504
+      when 400
+        # CDX API occasionally returns 400 when page index exceeds total available pages (that is, end of pagination)
+        return []
+      when 429
+        retry_after = retry_after_seconds(response)
+        raise RateLimitError.new(
+          "Server error 429: #{response.respond_to?(:message) ? response.message : 'Too Many Requests'}",
+          retry_after
+        )
+      when 500, 502, 503, 504
         raise "Server error #{code}: #{response.respond_to?(:message) ? response.message : ''}"
       else
-        warn "Unexpected API response #{code} for #{url}"
-        []
+        raise "Unexpected API response #{code} for #{url}"
       end
     rescue Net::ReadTimeout, Net::OpenTimeout, StandardError => e
       if retries < max_retries
         retries += 1
-
         jitter = rand(0.0..1.0)
-        sleep_time = (base_delay * (2 ** (retries - 1))) + jitter
 
-        warn "Error talking to Wayback CDX API (#{e.class}: #{e.message}) for #{url}. " \
-             "Retrying in #{sleep_time.round(2)}s (attempt #{retries}/#{max_retries})..."
+        if e.is_a?(RateLimitError)
+          # if the server provided a Retry-After header, use that; otherwise, use an exponential backoff with a minimum cooldown
+          fallback = [DEFAULT_RATE_LIMIT_COOLDOWN, base_delay * (2 ** (retries - 1))].max
+          cooldown = (e.retry_after || fallback) + (e.retry_after ? 0 : jitter)
+          ArchiveAPI.extend_cdx_cooldown(cooldown)
+
+          warn "Wayback CDX API rate limited (429) for #{url}. " \
+               "Pausing CDX requests for #{cooldown.round(2)}s " \
+               "(attempt #{retries}/#{max_retries})..."
+        else
+          sleep_time = (base_delay * (2 ** (retries - 1))) + jitter
+          warn "Error talking to Wayback CDX API (#{e.class}: #{e.message}) for #{url}. " \
+               "Retrying in #{sleep_time.round(2)}s (attempt #{retries}/#{max_retries})..."
+          sleep(sleep_time)
+        end
 
-        sleep(sleep_time)
         retry
       else
         warn "Giving up on Wayback CDX API for #{url} after #{max_retries} attempts. (Last error: #{e.message})"
-        []
+        raise
       end
     end
   end
@@ -91,12 +172,29 @@ module ArchiveAPI
     parameters.push(["filter", "statuscode:2..|30[12378]"]) unless @all
     parameters.push(["from", @from_timestamp.to_s]) if @from_timestamp && @from_timestamp != 0
     parameters.push(["to", @to_timestamp.to_s]) if @to_timestamp && @to_timestamp != 0
-    parameters.push(["page", page_index]) if page_index
+    parameters.push(["page", page_index.to_s]) if page_index && page_index > 0
     parameters
   end
 
   private
 
+  def retry_after_seconds(response)
+    raw = if HTTPX_AVAILABLE && defined?(HTTPX::Response) && response.is_a?(HTTPX::Response)
+            response.headers['retry-after']
+          elsif response.respond_to?(:[])
+            response['Retry-After'] || response['retry-after']
+          end
+
+    value = Array(raw).first.to_s.strip
+    return nil if value.empty?
+    return value.to_f if value.match?(/\A\d+(?:\.\d+)?\z/)
+
+    delay = Time.httpdate(value) - Time.now
+    delay.positive? ? delay : 0.0
+  rescue ArgumentError
+    nil
+  end
+
   def decompress_body(response)
     body = response.body.to_s
     return body if body.empty?
@@ -110,4 +208,4 @@ module ArchiveAPI
       body.strip
     end
   end
-end
+end

data/lib/wayback_machine_downloader/page_requisites.rb

@@ -1,18 +1,30 @@
 module PageRequisites
   # regex to find links in href, src, url(), and srcset
   # this ignores data: URIs, mailto:, and anchors
-  ASSET_REGEX = /(?:href|src|data-src|data-url)\s*=\s*["']([^"']+)["']|url\(\s*["']?([^"'\)]+)["']?\s*\)|srcset\s*=\s*["']([^"']+)["']/i
+  ASSET_REGEX = /(?:(href|src|data-src|data-url)\s*=\s*["']([^"']+)["'])|url\(\s*["']?([^"'\)]+)["']?\s*\)|srcset\s*=\s*["']([^"']+)["']/i
+ PAGE_EXTENSIONS = %w[
+    .html .htm .shtml .shtm .xhtml
+    .asp .aspx .asa .ashx .asmx
+    .php .php3 .php4 .php5 .phtml
+    .jsp .jspx .do .action
+    .cgi .pl .cfm .dll
+  ].freeze
 
   def self.extract(html_content)
     assets = []
-    
+
     html_content.scan(ASSET_REGEX) do |match|
-      # match is an array of capture groups; find the one that matched
-      url = match.compact.first
+      attribute, attr_url, css_url, srcset_url = match
+      url = attr_url || css_url || srcset_url
       next unless url
-      
-      # handle srcset (e.g. comma separated values like "image.jpg 1x, image2.jpg 2x")
-      if url.include?(',') && (url.include?(' 1x') || url.include?(' 2w'))
+
+      # href is also used for navigation. Do not turn links to server-rendered
+      # pages into prerequisite CDX lookups; doing so can multiply API traffic
+      # dramatically on old sites that use extensions such as .php3.
+      next if attribute&.downcase == 'href' && page_url?(url)
+
+      # handle srcset (e.g. comma separated values like "image.jpg 1x, image2.jpg 2w")
+      if srcset_url
         url.split(',').each do |src_def|
           src_url = src_def.strip.split(' ').first
           assets << src_url if valid_asset?(src_url)
@@ -25,9 +37,20 @@ module PageRequisites
     assets.uniq
   end
 
+  def self.page_url?(url)
+    path = begin
+      URI.parse(url).path
+    rescue URI::InvalidURIError
+      url.to_s.split(/[?#]/, 2).first
+    end
+
+    ext = File.extname(path.to_s).downcase
+    ext.empty? || PAGE_EXTENSIONS.include?(ext)
+  end
+
   def self.valid_asset?(url)
     return false if url.strip.empty?
     return false if url.start_with?('data:', 'mailto:', '#', 'javascript:')
     true
   end
-end
+end

data/lib/wayback_machine_downloader/subdom_processor.rb

@@ -12,7 +12,7 @@ module SubdomainProcessor
     @subdomain_queue = Queue.new
     
     # scan downloaded files for subdomain links
-    initial_files = Dir.glob(File.join(backup_path, "**/*.{html,htm,css,js}"))
+    initial_files = Dir.glob(File.join(backup_path, "**/*.{html,htm,shtml,css,js,asp,aspx,ashx,php,jsp,cgi,pl}"))
     puts "Scanning #{initial_files.size} downloaded files for subdomain links..."
     
     subdomains_found = scan_files_for_subdomains(initial_files, base_domain)
@@ -126,7 +126,7 @@ module SubdomainProcessor
       # if we need to go deeper, scan the newly downloaded files
       if depth + 1 < max_depth
         # get all files in the subdomains directory
-        new_files = Dir.glob(File.join(backup_path, "subdomains", "**/*.{html,htm,css,js}"))
+        new_files = Dir.glob(File.join(backup_path, "subdomains", "**/*.{html,htm,shtml,css,js,asp,aspx,ashx,php,jsp,cgi,pl}"))
         new_subdomains = scan_files_for_subdomains(new_files, base_domain)
         
         # filter out already processed subdomains
@@ -241,4 +241,4 @@ module SubdomainProcessor
     
     puts "Rewrote links in #{rewritten_count} files"
   end
-end
+end

data/lib/wayback_machine_downloader/url_rewrite.rb

@@ -2,84 +2,116 @@
 
 module URLRewrite
   # server-side extensions that should work locally
-  SERVER_SIDE_EXTS = %w[.php .asp .aspx .jsp .cgi .pl .py].freeze
+  SERVER_SIDE_EXTS = %w[
+    .php .php3 .phtml 
+    .asp .aspx .ashx .asmx .asa 
+    .jsp .jspx .do .action 
+    .cgi .pl .py .cfm .shtml
+  ].freeze
+
+  def rewrite_html_attr_urls(content, root_prefix = './')
+    target_host = current_host
 
-  def rewrite_html_attr_urls(content)
     # rewrite URLs to relative paths
-    content.gsub!(/(\s(?:href|src|action|data-src|data-url)=["'])https?:\/\/web\.archive\.org\/web\/\d+(?:id_)?\/https?:\/\/[^\/]+([^"']*)(["'])/i) do
-      prefix, path, suffix = $1, $2, $3
-      path = normalize_path_for_local(path)
-      "#{prefix}#{path}#{suffix}"
+    content = content.gsub(/(\s(?:href|src|action|data-src|data-url)=["'])https?:\/\/web\.archive\.org\/web\/\d+[a-z_]*\/https?:\/\/[^\/"']+(\/[^"']*)?(["'])/i) do
+      prefix, path, suffix = $1, ($2 || "/index.html"), $3
+      local = normalize_path_for_local(path, root_prefix)
+      "#{prefix}#{local}#{suffix}"
     end
 
     # rewrite absolute URLs to same domain as relative
-    content.gsub!(/(\s(?:href|src|action|data-src|data-url)=["'])https?:\/\/[^\/]+([^"']*)(["'])/i) do
+    if target_host && !target_host.empty?
+      content = content.gsub(/(\s(?:href|src|action|data-src|data-url)=["'])https?:\/\/#{Regexp.escape(target_host)}(?::\d+)?(\/[^"']*)?(["'])/i) do
+        prefix, path, suffix = $1, ($2 || "/index.html"), $3
+        local = normalize_path_for_local(path, root_prefix)
+        "#{prefix}#{local}#{suffix}"
+      end
+    end
+
+    # rewrite root-relative URLs
+    content = content.gsub(/(\s(?:href|src|action|data-src|data-url)=["'])\/([^"'\/][^"']*)(["'])/i) do
       prefix, path, suffix = $1, $2, $3
-      path = normalize_path_for_local(path)
-      "#{prefix}#{path}#{suffix}"
+      local = normalize_path_for_local("/#{path}", root_prefix)
+      "#{prefix}#{local}#{suffix}"
     end
 
     content
   end
 
-  def rewrite_css_urls(content)
+  def rewrite_css_urls(content, root_prefix = './')
+    target_host = current_host
+
     # rewrite URLs in CSS
-    content.gsub!(/url\(\s*["']?https?:\/\/web\.archive\.org\/web\/\d+(?:id_)?\/https?:\/\/[^\/]+([^"'\)]*?)["']?\s*\)/i) do
-      path = normalize_path_for_local($1)
-      "url(\"#{path}\")"
+    content = content.gsub(/url\(\s*["']?https?:\/\/web\.archive\.org\/web\/\d+[a-z_]*\/https?:\/\/[^\/"'\)]+(\/[^"'\)]*)?["']?\s*\)/i) do
+      path = $1 || "/index.html"
+      local = normalize_path_for_local(path, root_prefix)
+      "url(\"#{local}\")"
     end
 
     # rewrite absolute URLs in CSS
-    content.gsub!(/url\(\s*["']?https?:\/\/[^\/]+([^"'\)]*?)["']?\s*\)/i) do
-      path = normalize_path_for_local($1)
-      "url(\"#{path}\")"
+    if target_host && !target_host.empty?
+      content = content.gsub(/url\(\s*["']?https?:\/\/#{Regexp.escape(target_host)}(?::\d+)?(\/[^"'\)]*)?["']?\s*\)/i) do
+        path = $1 || "/index.html"
+        local = normalize_path_for_local(path, root_prefix)
+        "url(\"#{local}\")"
+      end
+    end
+
+    # rewrite root-relative in CSS
+    content = content.gsub(/url\(\s*["']?\/([^"'\)\/][^"'\)]*?)["']?\s*\)/i) do
+      path = $1
+      local = normalize_path_for_local("/#{path}", root_prefix)
+      "url(\"#{local}\")"
     end
 
     content
   end
 
-  def rewrite_js_urls(content)
+  def rewrite_js_urls(content, root_prefix = './')
     # rewrite archive.org URLs in JavaScript strings
-    content.gsub!(/(["'])https?:\/\/web\.archive\.org\/web\/\d+(?:id_)?\/https?:\/\/[^\/]+([^"']*)(["'])/i) do
-      quote_start, path, quote_end = $1, $2, $3
-      path = normalize_path_for_local(path)
-      "#{quote_start}#{path}#{quote_end}"
+    content.gsub(/(["'])https?:\/\/web\.archive\.org\/web\/\d+[a-z_]*\/https?:\/\/[^\/"']+(\/[^"']*)?(["'])/i) do
+      quote_start, path, quote_end = $1, ($2 || "/index.html"), $3
+      local = normalize_path_for_local(path, root_prefix)
+      "#{quote_start}#{local}#{quote_end}"
     end
+  end
 
-    # rewrite absolute URLs in JavaScript
-    content.gsub!(/(["'])https?:\/\/[^\/]+([^"']*)(["'])/i) do
-      quote_start, path, quote_end = $1, $2, $3
-      next "#{quote_start}http#{$2}#{quote_end}" if $2.start_with?('s://', '://')
-      path = normalize_path_for_local(path)
-      "#{quote_start}#{path}#{quote_end}"
-    end
+  def normalize_path_for_local(path, root_prefix = './')
+    path = path.to_s.strip
+    return "#{root_prefix}index.html" if path.empty? || path == "/"
 
-    content
-  end
+    path_part, query_part = path.split('?', 2)
+    path_part = "/index.html" if path_part.nil? || path_part.empty? || path_part == "/"
 
-  private
+    # hash query parameters to match sanitize_and_prepare_id
+    if query_part && !query_part.empty?
+      q_digest = Digest::SHA256.hexdigest(query_part)[0, 12]
+      if path_part.include?('.')
+        pre, _sep, post = path_part.rpartition('.')
+        path_part = "#{pre}__q#{q_digest}.#{post}"
+      else
+        path_part = "#{path_part}__q#{q_digest}"
+      end
+    end
 
-  def normalize_path_for_local(path)
-    return "./index.html" if path.empty? || path == "/"
-    
-    # handle query strings - they're already part of the filename
-    path = path.split('?').first if path.include?('?')
-    
     # check if this is a server-side script
-    ext = File.extname(path).downcase
-    if SERVER_SIDE_EXTS.include?(ext)
-      # keep the path as-is but ensure it starts with ./
-      path = "./#{path}" unless path.start_with?('./', '/')
-    else
-      # regular file handling
-      path = "./#{path}" unless path.start_with?('./', '/')
-      
-      # if it looks like a directory, add index.html
-      if path.end_with?('/') || !path.include?('.')
-        path = "#{path.chomp('/')}/index.html"
+    ext = File.extname(path_part).downcase
+    unless SERVER_SIDE_EXTS.include?(ext)
+      if path_part.end_with?('/') || !path_part.include?('.')
+        path_part = "#{path_part.chomp('/')}/index.html"
       end
     end
-    
-    path
+
+    clean_path = path_part.sub(%r{\A/+}, '')
+    "#{root_prefix}#{clean_path}"
+  end
+
+  private
+
+  def current_host
+    return nil unless @base_url
+    clean = @base_url.to_s.sub(%r{\A\*\.}, '')
+    clean = clean.match?(%r{\Ahttps?://}i) ? clean : "http://#{clean}"
+    URI.parse(clean).host rescue nil
   end
 end

data/lib/wayback_machine_downloader.rb

@@ -147,7 +147,20 @@ class ConnectionPool
       http.open_timeout = DEFAULT_TIMEOUT
       http.keep_alive_timeout = 30
       http.max_retries = MAX_RETRIES
-      http.start
+
+      retries = 0
+      begin
+        http.start
+      rescue Net::OpenTimeout, Net::ReadTimeout, Errno::ECONNRESET, Errno::ETIMEDOUT, SocketError => e
+        if retries < MAX_RETRIES
+          retries += 1
+          sleep(2 * retries)
+          retry
+        else
+          raise e
+        end
+      end
+
       http
     end
   end
@@ -159,7 +172,7 @@ class WaybackMachineDownloader
   include SubdomainProcessor
   include URLRewrite
 
-  VERSION = "2.4.8"
+  VERSION = "2.4.9"
   DEFAULT_TIMEOUT = 30
   MAX_RETRIES = 3
   RETRY_DELAY = 2
@@ -317,12 +330,8 @@ class WaybackMachineDownloader
       puts "Loading snapshot list from #{cdx_path}"
       begin
         snapshot_list_to_consider = JSON.parse(File.read(cdx_path))
-        puts "Loaded #{snapshot_list_to_consider.length} snapshots from cache."
-        puts
+        puts "Loaded #{snapshot_list_to_consider.length} snapshots from cache.\n\n"
         return Concurrent::Array.new(snapshot_list_to_consider)
-      rescue JSON::ParserError => e
-        puts "Error reading snapshot cache file #{cdx_path}: #{e.message}. Refetching..."
-        FileUtils.rm_f(cdx_path)
       rescue => e
         puts "Error loading snapshot cache #{cdx_path}: #{e.message}. Refetching..."
         FileUtils.rm_f(cdx_path)
@@ -330,89 +339,38 @@ class WaybackMachineDownloader
     end
 
     snapshot_list_to_consider = Concurrent::Array.new
-    mutex = Mutex.new
-
-    # if snapshot_at is set, limit CDX queries to snapshots at or before that timestamp
     original_to = @to_timestamp
-    if @snapshot_at
-      @to_timestamp = @snapshot_at
-    end
+    @to_timestamp = @snapshot_at if @snapshot_at
 
     puts "Getting snapshot pages from Wayback Machine API..."
 
-    # Fetch the initial set of snapshots, sequentially
+    # fetch initial page (page 0)
     @connection_pool.with_connection do |connection|
       initial_list = get_raw_list_from_api(@base_url, 0, connection)
-      initial_list ||= []
-      mutex.synchronize do
+      if initial_list && !initial_list.empty?
         snapshot_list_to_consider.concat(initial_list)
         print "."
         $stdout.flush
       end
     end
 
-    # Fetch additional pages if the exact URL flag is not set and the first page wasn't empty
+    # sequentially fetch subsequent pages
     unless @exact_url || snapshot_list_to_consider.empty?
       page_index = 1
-      batch_size = [@threads_count, 5].min
-      continue_fetching = true
-      fetch_pool = Concurrent::FixedThreadPool.new([@threads_count, 1].max)
-      begin
-        while continue_fetching && page_index < @maximum_pages
-          # Determine the range of pages to fetch in this batch
-          end_index = [page_index + batch_size, @maximum_pages].min
-          current_batch = (page_index...end_index).to_a
-
-          # Create futures for concurrent API calls
-          futures = current_batch.map do |page|
-            Concurrent::Future.execute(executor: fetch_pool) do
-              result = nil
-              @connection_pool.with_connection do |connection|
-                result = get_raw_list_from_api(@base_url, page, connection)
-              end
-              result ||= []
-              [page, result]
-            end
-          end
-
-          results = []
-
-          futures.each do |future|
-            begin
-              val = future.value
-              # only append if valid
-              if val && val.is_a?(Array) && val.first.is_a?(Integer)
-                results << val
-              end
-            rescue => e
-              puts "\nError fetching page #{future}: #{e.message}"
-            end
-          end
-
-          # Sort results by page number to maintain order
-          results.sort_by! { |page, _| page }
-
-          # Process results and check for empty pages
-          results.each do |page, result|
-            if result.nil? || result.empty?
-              continue_fetching = false
-              break
-            else
-              mutex.synchronize do
-                snapshot_list_to_consider.concat(result)
-                print "."
-                $stdout.flush
-              end
-            end
-          end
-
-          page_index = end_index
+      while page_index < @maximum_pages
+        result = nil
+        @connection_pool.with_connection do |connection|
+          result = get_raw_list_from_api(@base_url, page_index, connection)
+        end
 
-          sleep(RATE_LIMIT) if continue_fetching
+        if result.nil? || result.empty?
+          break
+        else
+          snapshot_list_to_consider.concat(result)
+          print "."
+          $stdout.flush
+          page_index += 1
         end
-      ensure
-        fetch_pool.shutdown
-        fetch_pool.wait_for_termination
       end
     end
 
@@ -421,7 +379,7 @@ class WaybackMachineDownloader
     # save the fetched list to the cache file
     begin
       FileUtils.mkdir_p(File.dirname(cdx_path))
-      File.write(cdx_path, JSON.pretty_generate(snapshot_list_to_consider.to_a)) # Convert Concurrent::Array back to Array for JSON
+      File.write(cdx_path, JSON.pretty_generate(snapshot_list_to_consider.to_a))
       puts "Saved snapshot list to #{cdx_path}"
     rescue => e
       puts "Error saving snapshot cache to #{cdx_path}: #{e.message}"
@@ -434,6 +392,19 @@ class WaybackMachineDownloader
     snapshot_list_to_consider
   end
 
+  def extract_path_and_query(file_url)
+    return "" if file_url.nil? || file_url.empty?
+    normalized = file_url.match?(%r{\Ahttps?://}i) ? file_url : "http://#{file_url}"
+    uri = URI.parse(normalized)
+    # returns path + query
+    path = uri.path.to_s
+    path += "?#{uri.query}" if uri.query && !uri.query.empty?
+    path.sub(%r{\A/}, '')
+  rescue URI::InvalidURIError
+    # for ill-formed URLs
+    file_url.sub(%r{\Ahttps?://[^/]+/?}i, '').sub(%r{\A[^/]+/?}, '')
+  end
+
   # Get a composite snapshot file list for a specific timestamp
   def get_composite_snapshot_file_list(target_timestamp)
     file_versions = {}
@@ -442,7 +413,7 @@ class WaybackMachineDownloader
       next if file_timestamp.to_i > target_timestamp
 
       # allow empty path by treating missing tail as empty string
-      raw_tail = file_url.split('/')[3..-1]&.join('/') || ''
+      raw_tail = extract_path_and_query(file_url)
       file_id = sanitize_and_prepare_id(raw_tail, file_url)
       next if file_id.nil?
       next if match_exclude_filter(file_url)
@@ -467,7 +438,7 @@ class WaybackMachineDownloader
     get_all_snapshots_to_consider.each do |file_timestamp, file_url|
       next unless file_url.include?('/')
 
-      raw_tail = file_url.split('/')[3..-1]&.join('/') || ''
+      raw_tail = extract_path_and_query(file_url)
       file_id = sanitize_and_prepare_id(raw_tail, file_url)
       if file_id.nil?
         puts "Malformed file url, ignoring: #{file_url}"
@@ -482,7 +453,7 @@ class WaybackMachineDownloader
         elsif !match_only_filter(file_url)
           puts "File url doesn't match only filter, ignoring: #{file_url}"
         elsif file_list_curated[file_id]
-          unless file_list_curated[file_id][:timestamp] > file_timestamp
+          unless file_list_curated[file_id][:timestamp].to_i > file_timestamp.to_i
             file_list_curated[file_id] = { file_url: file_url, timestamp: file_timestamp }
           end
         else
@@ -498,7 +469,7 @@ class WaybackMachineDownloader
     get_all_snapshots_to_consider.each do |file_timestamp, file_url|
       next unless file_url.include?('/')
 
-      raw_tail = file_url.split('/')[3..-1]&.join('/') || ''
+      raw_tail = extract_path_and_query(file_url)
       file_id = sanitize_and_prepare_id(raw_tail, file_url)
       if file_id.nil?
         puts "Malformed file url, ignoring: #{file_url}"
@@ -785,7 +756,7 @@ class WaybackMachineDownloader
 
       append_to_db(file_remote_info[:file_id])
 
-      if @page_requisites && File.extname(existing_path) =~ /\.(html?|php|asp|aspx|jsp)$/i
+      if @page_requisites && File.extname(existing_path) =~ /\.(html?|shtml|php|asp|aspx|ashx|jsp|do|action|cgi|pl|cfm)$/i
         process_page_requisites(existing_path, file_remote_info)
       end
       return
@@ -963,7 +934,7 @@ class WaybackMachineDownloader
 
   def rewrite_local_files
     puts "Scanning #{backup_path} for files to rewrite..."
-    files = Dir.glob(File.join(backup_path, "**/*.{html,htm,css,js,php,asp,aspx,jsp}"))
+    files = Dir.glob(File.join(backup_path, "**/*.{html,htm,shtml,css,js,php,asp,aspx,ashx,jsp,do,action,cgi,pl,cfm}"))
 
     puts "Found #{files.size} files. Rewriting links for local browsing..."
 
@@ -991,8 +962,8 @@ class WaybackMachineDownloader
     begin
       content = File.binread(file_path)
 
-      # detect encoding for HTML files
-      if file_ext == '.html' || file_ext == '.htm' || file_ext == '.php' || file_ext == '.asp'
+      # detect encoding for HTML/server-rendered files
+      if %w[.html .htm .php .asp .jsp .shtml].include?(file_ext)
         encoding_match = content.match(/<meta.*?charset=["'\s]?([^"'\s>;]+)/i)
         encoding_name = encoding_match ? encoding_match.captures.first : 'UTF-8'
 
@@ -1016,28 +987,16 @@ class WaybackMachineDownloader
         end
       end
 
+      root_prefix = site_root_relative_prefix(file_path)
+
       # URLs in HTML attributes
-      content = rewrite_html_attr_urls(content)
+      content = rewrite_html_attr_urls(content, root_prefix)
 
       # URLs in CSS
-      content = rewrite_css_urls(content)
+      content = rewrite_css_urls(content, root_prefix)
 
       # URLs in JavaScript
-      content = rewrite_js_urls(content)
-
-      root_prefix = site_root_relative_prefix(file_path)
-
-      # rewrite root-absolute links to paths relative to the downloaded site root
-      content.gsub!(/(\s(?:href|src|action|data-src|data-url)=["'])\/([^"'\/][^"']*)(["'])/i) do
-        prefix, path, suffix = $1, $2, $3
-        "#{prefix}#{root_prefix}#{path}#{suffix}"
-      end
-
-      # apply the same root-relative conversion to CSS url(...) references
-      content.gsub!(/url\(\s*["']?\/([^"'\)\/][^"'\)]*?)["']?\s*\)/i) do
-        path = $1
-        "url(\"#{root_prefix}#{path}\")"
-      end
+      content = rewrite_js_urls(content, root_prefix)
 
       # save the modified content back to the file
       File.binwrite(file_path, content)
@@ -1115,7 +1074,7 @@ class WaybackMachineDownloader
 
       case status
       when :saved
-        if @rewrite && File.extname(file_path) =~ /\.(html?|css|js)$/i
+        if @rewrite && File.extname(file_path) =~ /\.(html?|shtml|css|js|php|asp|aspx|jsp|cgi|pl)$/i
           rewrite_urls_to_relative(file_path)
         end
         return ["#{color("[SAVED]", :green)}  #{file_url} (#{@processed_file_count + 1}/#{@total_to_download})", file_path]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wayback_machine_downloader_straw
 version: !ruby/object:Gem::Version
-  version: 2.4.8
+  version: 2.4.9
 platform: ruby
 authors:
 - strawberrymaster
@@ -87,14 +87,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.4.3
+      version: 3.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.10
+rubygems_version: 4.0.16
 specification_version: 4
 summary: Download an entire website from the Wayback Machine.
 test_files: []