wayback_machine_downloader_straw 2.4.6 → 2.4.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e52df092b59b0eec27b390b5b00fcfc17fab271acd6cd9df774912f57cfc4dc1
-  data.tar.gz: ce170b42caad7e8136b07c2aa5cb6e751f57dd64bd40c0addcd42a31798d0047
+  metadata.gz: 12259647dc99ef1c263b3d797098233be4f194bf1a4e9bc047e1dfb4d0fc9468
+  data.tar.gz: 5d9d6bd47efca6b43a5c1d6c2606bb2871edea0b4dc781f206bbab8c9c66aaa0
 SHA512:
-  metadata.gz: 558e9cdfc3d7d4d2081ccb49b12a96bdb64b7768697eb0a1b9a431ed1ad3017ce894975e046a6e50766928c3863797715c7b45d013b6ab7ad78bca59ea86c6d0
-  data.tar.gz: af3064f1489d32cf078fd5d87d2773700e9dfa498075f089029e0e7ec47c500c7815e84d51f426bb6fc3067bf02c9a9404da3b6f74d263c99b4ae96fc32dab35
+  metadata.gz: 670efe274663448a9967d2e39920ce234af1ba5c0aae82135c56f64899904f510096ac19b6367a5b90af43d3878b156a21b2fb1138a2e567041b55205fc97c67
+  data.tar.gz: 2d4d6e927664ea831c0611bbbc89d74ee6c144629f8afb2b2a58428cff1f78846832979bd994dfd97a51cff03ee52ab9c5c09beebc1b007248f1c043aa3bdd2a

data/bin/wayback_machine_downloader

@@ -48,6 +48,10 @@ option_parser = OptionParser.new do |opts|
     options[:all] = true
   end
 
+  opts.on("--keep-duplicates", "Do not collapse duplicate CDX captures by digest") do |t|
+    options[:keep_duplicates] = true
+  end
+
   opts.on("-c", "--concurrency NUMBER", Integer, "Number of multiple files to download at a time", "Default is one file at a time (ie. 20)") do |t|
     options[:threads_count] = t
   end
@@ -80,6 +84,10 @@ option_parser = OptionParser.new do |opts|
     options[:keep] = true
   end
 
+  opts.on("--delay SECONDS", Float, "Delay between downloads in seconds (default: 0)") do |t|
+    options[:delay] = t
+  end
+
   opts.on("--rt", "--retry N", Integer, "Maximum number of retries for failed downloads (default: 3)") do |t|
     options[:max_retries] = t
   end

data/lib/wayback_machine_downloader/archive_api.rb

@@ -4,36 +4,53 @@ require 'uri'
 module ArchiveAPI
 
   def get_raw_list_from_api(url, page_index, http)
-    # Automatically append /* if the URL doesn't contain a path after the domain
+    # Automatically append /* for host-only URLs
     # This is a workaround for an issue with the API and *some* domains.
     # See https://github.com/StrawberryMaster/wayback-machine-downloader/issues/6
-    # But don't do this when exact_url flag is set
-    if url && !url.match(/^https?:\/\/.*\//i) && !@exact_url
-      url = "#{url}/*"
+    # But don't do this when exact_url flag is set, and never append twice
+    match_type = nil
+    if url && !@exact_url
+      normalized_url = url.to_s
+      has_wildcard = normalized_url.include?('*')
+      host_and_rest = normalized_url
+        .sub(/\Ahttps?:\/\//i, '')
+        .split(/[?#]/, 2)
+        .first
+      has_path = host_and_rest.include?('/')
+
+      unless has_wildcard || has_path
+        match_type = "prefix"
+      end
     end
 
     request_url = URI("https://web.archive.org/cdx/search/cdx")
     params = [["output", "json"], ["url", url]] + parameters_for_api(page_index)
+    params << ["matchType", match_type] if match_type
     request_url.query = URI.encode_www_form(params)
 
     retries = 0
     max_retries = (@max_retries || 3)
-    delay = WaybackMachineDownloader::RETRY_DELAY rescue 2
+    base_delay = WaybackMachineDownloader::RETRY_DELAY rescue 2
 
     begin
-      request = Net::HTTP::Get.new(request_url)
-      request["User-Agent"] = "wmd-straw/#{WaybackMachineDownloader::VERSION}"
-      request["Connection"] = "keep-alive"
-      request["Accept-Encoding"] = "gzip"
-      response = http.request(request)
+      if HTTPX_AVAILABLE && http.is_a?(HTTPX::Session)
+        response = http.get(request_url)
+        raise response.error if response.is_a?(HTTPX::ErrorResponse)
 
-      case response.code.to_i
+        code = response.status
+        body = response.body.to_s.strip
+      else
+        request = Net::HTTP::Get.new(request_url)
+        request["User-Agent"] = "wmd-straw/#{WaybackMachineDownloader::VERSION rescue '2.4.8'}"
+        request["Connection"] = "keep-alive"
+        request["Accept-Encoding"] = "gzip, deflate"
+        response = http.request(request)
+        code = response.code.to_i
+        body = decompress_body(response)
+      end
+
+      case code
       when 200
-        body = if response['content-encoding'] == 'gzip'
-          Zlib::GzipReader.new(StringIO.new(response.body)).read
-        else
-          response.body.to_s.strip
-        end
         return [] if body.empty?
         begin
           json = JSON.parse(body)
@@ -44,16 +61,22 @@ module ArchiveAPI
           raise "Malformed JSON response: #{e.message}"
         end
       when 429, 500, 502, 503, 504
-        raise "Server error #{response.code}: #{response.message}"
+        raise "Server error #{code}: #{response.respond_to?(:message) ? response.message : ''}"
       else
-        warn "Unexpected API response #{response.code} for #{url}"
+        warn "Unexpected API response #{code} for #{url}"
         []
       end
     rescue Net::ReadTimeout, Net::OpenTimeout, StandardError => e
       if retries < max_retries
         retries += 1
-        warn "Error talking to Wayback CDX API (#{e.class}: #{e.message}) for #{url}, retry #{retries}/#{max_retries}..."
-        sleep(delay * retries)
+
+        jitter = rand(0.0..1.0)
+        sleep_time = (base_delay * (2 ** (retries - 1))) + jitter
+
+        warn "Error talking to Wayback CDX API (#{e.class}: #{e.message}) for #{url}. " \
+             "Retrying in #{sleep_time.round(2)}s (attempt #{retries}/#{max_retries})..."
+
+        sleep(sleep_time)
         retry
       else
         warn "Giving up on Wayback CDX API for #{url} after #{max_retries} attempts. (Last error: #{e.message})"
@@ -63,12 +86,28 @@ module ArchiveAPI
   end
 
   def parameters_for_api(page_index)
-    parameters = [["fl", "timestamp,original"], ["collapse", "digest"], ["gzip", "true"]]
-    parameters.push(["filter", "statuscode:200"]) unless @all
+    parameters = [["fl", "timestamp,original"], ["gzip", "true"]]
+    parameters.push(["collapse", "digest"]) unless @keep_duplicates || @all_timestamps
+    parameters.push(["filter", "statuscode:2..|30[12378]"]) unless @all
     parameters.push(["from", @from_timestamp.to_s]) if @from_timestamp && @from_timestamp != 0
     parameters.push(["to", @to_timestamp.to_s]) if @to_timestamp && @to_timestamp != 0
     parameters.push(["page", page_index]) if page_index
     parameters
   end
 
+  private
+
+  def decompress_body(response)
+    body = response.body.to_s
+    return body if body.empty?
+
+    case response['content-encoding']
+    when 'gzip'
+      Zlib::GzipReader.new(StringIO.new(body)).read rescue body
+    when 'deflate'
+      Zlib::Inflate.inflate(body) rescue body
+    else
+      body.strip
+    end
+  end
 end

data/lib/wayback_machine_downloader/subdom_processor.rb

@@ -40,11 +40,17 @@ module SubdomainProcessor
   private
   
   def extract_base_domain(url)
-    uri = URI.parse(url.gsub(/^https?:\/\//, '').split('/').first) rescue nil
+    # ensure the URL has a scheme for URI parsing
+    normalized_url = url.match?(/^https?:\/\//i) ? url : "http://#{url}"
+    uri = URI.parse(normalized_url) rescue nil
     return nil unless uri
     
-    host = uri.host || uri.path.split('/').first
-    host = host.downcase
+    # extract the host (and default to parsing path if host is missing)
+    host = uri.host || (uri.path || '').split('/').first
+    return nil unless host
+    
+    # strip port numbers if present
+    host = host.split(':').first.downcase
     
     # extract the base domain (e.g., "example.com" from "sub.example.com")
     parts = host.split('.')

data/lib/wayback_machine_downloader.rb

@@ -2,16 +2,15 @@
 
 require 'thread'
 require 'net/http'
-require 'open-uri'
 require 'fileutils'
-require 'cgi'
 require 'json'
-require 'time'
+require 'pathname'
 require 'concurrent-ruby'
 require 'logger'
 require 'zlib'
 require 'stringio'
 require 'digest'
+require 'etc'
 require_relative 'wayback_machine_downloader/tidy_bytes'
 require_relative 'wayback_machine_downloader/to_regex'
 require_relative 'wayback_machine_downloader/archive_api'
@@ -19,6 +18,13 @@ require_relative 'wayback_machine_downloader/page_requisites'
 require_relative 'wayback_machine_downloader/subdom_processor'
 require_relative 'wayback_machine_downloader/url_rewrite'
 
+begin
+  require 'httpx'
+  HTTPX_AVAILABLE = true
+rescue LoadError
+  HTTPX_AVAILABLE = false
+end
+
 class ConnectionPool
   MAX_AGE = 300
   CLEANUP_INTERVAL = 60
@@ -69,7 +75,12 @@ class ConnectionPool
   def stale?(entry)
     return true if entry.nil? || entry[:http].nil?
     http = entry[:http]
-    !http.started? || (Time.now - entry[:created_at] > MAX_AGE)
+
+    if HTTPX_AVAILABLE && http.is_a?(HTTPX::Session)
+      Time.now - entry[:created_at] > MAX_AGE
+    else
+      !http.started? || (Time.now - entry[:created_at] > MAX_AGE)
+    end
   end
 
   def build_connection_entry
@@ -77,7 +88,12 @@ class ConnectionPool
   end
 
   def safe_finish(http)
-    http.finish if http&.started?
+    return if http.nil?
+    if HTTPX_AVAILABLE && http.is_a?(HTTPX::Session)
+      http.close
+    else
+      http.finish if http&.started?
+    end
   rescue StandardError
     nil
   end
@@ -116,14 +132,24 @@ class ConnectionPool
   end
 
   def create_connection
-    http = Net::HTTP.new("web.archive.org", 443)
-    http.use_ssl = true
-    http.read_timeout = DEFAULT_TIMEOUT
-    http.open_timeout = DEFAULT_TIMEOUT
-    http.keep_alive_timeout = 30
-    http.max_retries = MAX_RETRIES
-    http.start
-    http
+    if HTTPX_AVAILABLE
+      HTTPX.with(
+        timeout: {
+          connect_timeout: DEFAULT_TIMEOUT,
+          read_timeout: DEFAULT_TIMEOUT,
+          write_timeout: DEFAULT_TIMEOUT
+        }
+      )
+    else
+      http = Net::HTTP.new("web.archive.org", 443)
+      http.use_ssl = true
+      http.read_timeout = DEFAULT_TIMEOUT
+      http.open_timeout = DEFAULT_TIMEOUT
+      http.keep_alive_timeout = 30
+      http.max_retries = MAX_RETRIES
+      http.start
+      http
+    end
   end
 end
 
@@ -133,7 +159,7 @@ class WaybackMachineDownloader
   include SubdomainProcessor
   include URLRewrite
 
-  VERSION = "2.4.6"
+  VERSION = "2.4.8"
   DEFAULT_TIMEOUT = 30
   MAX_RETRIES = 3
   RETRY_DELAY = 2
@@ -146,7 +172,7 @@ class WaybackMachineDownloader
 
   attr_accessor :base_url, :exact_url, :directory, :all_timestamps,
     :from_timestamp, :to_timestamp, :only_filter, :exclude_filter,
-    :all, :maximum_pages, :threads_count, :logger, :reset, :keep, :rewrite,
+    :all, :keep_duplicates, :maximum_pages, :threads_count, :logger, :reset, :keep, :rewrite,
     :snapshot_at, :page_requisites
 
   def initialize params
@@ -165,8 +191,16 @@ class WaybackMachineDownloader
     @only_filter = params[:only_filter]
     @exclude_filter = params[:exclude_filter]
     @all = params[:all]
+    @keep_duplicates = params[:keep_duplicates] || false
     @maximum_pages = params[:maximum_pages] ? params[:maximum_pages].to_i : 100
-    @threads_count = [params[:threads_count].to_i, 1].max
+    default_threads = begin
+      [Etc.nprocessors, 4].max
+    rescue StandardError
+      4
+    end
+    threads_param = params[:threads_count] ? params[:threads_count].to_i : 0
+    threads_param = default_threads if threads_param <= 0
+    @threads_count = [threads_param, 1].max
     @rewritten = params[:rewritten]
     @reset = params[:reset]
     @keep = params[:keep]
@@ -176,6 +210,7 @@ class WaybackMachineDownloader
     @connection_pool = ConnectionPool.new(CONNECTION_POOL_SIZE)
     @db_mutex = Mutex.new
     @rewrite = params[:rewrite] || false
+    @delay = params[:delay] ? params[:delay].to_f : 0.0
     @recursive_subdomains = params[:recursive_subdomains] || false
     @subdomain_depth = params[:subdomain_depth] || 1
     @snapshot_at = params[:snapshot_at] ? params[:snapshot_at].to_i : nil
@@ -183,6 +218,12 @@ class WaybackMachineDownloader
     @page_requisites = params[:page_requisites] || false
     @pending_jobs = Concurrent::AtomicFixnum.new(0)
 
+    @db_buffer = []
+    @db_buffer_mutex = Mutex.new
+    @db_last_flush = Time.now
+    @db_flush_threshold = 50  # flush to disk every 50 completed files
+    @db_flush_interval = 5.0  # or flush every 5 seconds even if count isn't met
+
     # URL for rejecting invalid/unencoded wayback urls
     @url_regexp = /^(([A-Za-z][A-Za-z0-9+.-]*):((\/\/(((([A-Za-z0-9._~-])|(%[ABCDEFabcdef0-9][ABCDEFabcdef0-9])|([!$&'('')'*+,;=]))+)(:([0-9]*))?)(((\/((([A-Za-z0-9._~-])|(%[ABCDEFabcdef0-9][ABCDEFabcdef0-9])|([!$&'('')'*+,;=])|:|@)*))*)))|((\/(((([A-Za-z0-9._~-])|(%[ABCDEFabcdef0-9][ABCDEFabcdef0-9])|([!$&'('')'*+,;=])|:|@)+)(\/((([A-Za-z0-9._~-])|(%[ABCDEFabcdef0-9][ABCDEFabcdef0-9])|([!$&'('')'*+,;=])|:|@)*))*)?))|((((([A-Za-z0-9._~-])|(%[ABCDEFabcdef0-9][ABCDEFabcdef0-9])|([!$&'('')'*+,;=])|:|@)+)(\/((([A-Za-z0-9._~-])|(%[ABCDEFabcdef0-9][ABCDEFabcdef0-9])|([!$&'('')'*+,;=])|:|@)*))*)))(\?((([A-Za-z0-9._~-])|(%[ABCDEFabcdef0-9][ABCDEFabcdef0-9])|([!$&'('')'*+,;=])|:|@)|\/|\?)*)?(\#((([A-Za-z0-9._~-])|(%[ABCDEFabcdef0-9][ABCDEFabcdef0-9])|([!$&'('')'*+,;=])|:|@)|\/|\?)*)?)$/
 
@@ -293,10 +334,8 @@ class WaybackMachineDownloader
 
     # if snapshot_at is set, limit CDX queries to snapshots at or before that timestamp
     original_to = @to_timestamp
-    skip_cache = false
     if @snapshot_at
       @to_timestamp = @snapshot_at
-      skip_cache = true
     end
 
     puts "Getting snapshot pages from Wayback Machine API..."
@@ -329,7 +368,7 @@ class WaybackMachineDownloader
             Concurrent::Future.execute(executor: fetch_pool) do
               result = nil
               @connection_pool.with_connection do |connection|
-                result = get_raw_list_from_api("#{@base_url}/*", page, connection)
+                result = get_raw_list_from_api(@base_url, page, connection)
               end
               result ||= []
               [page, result]
@@ -382,10 +421,8 @@ class WaybackMachineDownloader
     # save the fetched list to the cache file
     begin
       FileUtils.mkdir_p(File.dirname(cdx_path))
-      unless skip_cache
-        File.write(cdx_path, JSON.pretty_generate(snapshot_list_to_consider.to_a)) # Convert Concurrent::Array back to Array for JSON
-        puts "Saved snapshot list to #{cdx_path}"
-      end
+      File.write(cdx_path, JSON.pretty_generate(snapshot_list_to_consider.to_a)) # Convert Concurrent::Array back to Array for JSON
+      puts "Saved snapshot list to #{cdx_path}"
     rescue => e
       puts "Error saving snapshot cache to #{cdx_path}: #{e.message}"
     ensure
@@ -532,7 +569,16 @@ class WaybackMachineDownloader
     if File.exist?(db_path) && !@reset
       puts "Loading list of already downloaded files from #{db_path}"
       begin
-        File.foreach(db_path) { |line| downloaded_ids.add(line.strip) }
+        File.foreach(db_path) do |line|
+          id = line.strip
+          # only trust DB entries that actually exist on disk; this helps when resuming
+          path = local_path_for_file_id(id)
+          if path && File.exist?(path)
+            downloaded_ids.add(id)
+          else
+            puts "Found DB entry but file missing, will requeue: #{id}" if @logger && @logger.level == Logger::DEBUG
+          end
+        end
       rescue => e
         puts "Error reading downloaded files list #{db_path}: #{e.message}. Assuming no files downloaded."
         downloaded_ids.clear
@@ -542,12 +588,40 @@ class WaybackMachineDownloader
   end
 
   def append_to_db(file_id)
-    @db_mutex.synchronize do
-      begin
-        FileUtils.mkdir_p(File.dirname(db_path))
-        File.open(db_path, 'a') { |f| f.puts(file_id) }
-      rescue => e
-        @logger.error("Failed to append downloaded file ID #{file_id} to #{db_path}: #{e.message}")
+    flush_needed = false
+
+    @db_buffer_mutex.synchronize do
+      @db_buffer << file_id
+      if @db_buffer.size >= @db_flush_threshold || (Time.now - @db_last_flush) >= @db_flush_interval
+        flush_needed = true
+      end
+    end
+
+    flush_db if flush_needed
+  end
+
+  def flush_db
+    lines_to_write = nil
+
+    @db_buffer_mutex.synchronize do
+      return if @db_buffer.empty?
+      lines_to_write = @db_buffer.dup
+      @db_buffer.clear
+      @db_last_flush = Time.now
+    end
+
+    return if lines_to_write.nil? || lines_to_write.empty?
+
+    begin
+      FileUtils.mkdir_p(File.dirname(db_path))
+      File.open(db_path, 'a') do |f|
+        lines_to_write.each { |id| f.puts(id) }
+      end
+    rescue StandardError => e
+      @logger.error("Failed to write batch to #{db_path}: #{e.message}")
+
+      @db_buffer_mutex.synchronize do
+        @db_buffer.unshift(*lines_to_write)
       end
     end
   end
@@ -584,9 +658,18 @@ class WaybackMachineDownloader
       end
     end
   end
-  
+
+  @@engine_printed = false
+
   def download_files
     start_time = Time.now
+
+    unless @@engine_printed
+      engine_name = HTTPX_AVAILABLE ? "HTTPX" : "Net::HTTP"
+      puts "Connection engine used: #{color(engine_name, :green)}"
+      @@engine_printed = true
+    end
+
     puts "Downloading #{@base_url} to #{backup_path} from Wayback Machine archives."
 
     FileUtils.mkdir_p(backup_path)
@@ -605,7 +688,7 @@ class WaybackMachineDownloader
 
     # Load IDs of already downloaded files
     downloaded_ids = load_downloaded_ids
-    
+
     # We use a thread-safe Set to track what we have queued/downloaded in this session
     # to avoid infinite loops with page requisites
     @session_downloaded_ids = Concurrent::Set.new
@@ -621,7 +704,7 @@ class WaybackMachineDownloader
     if skipped_count > 0
       puts "Found #{skipped_count} previously downloaded files, skipping them."
     end
-    
+
     if remaining_count == 0 && !@page_requisites
       puts "All matching files have already been downloaded."
       cleanup
@@ -687,14 +770,34 @@ class WaybackMachineDownloader
   def process_single_file(file_remote_info)
     download_success = false
     downloaded_path = nil
-    
+
+    # if delay was set by the user
+    sleep(@delay) if @delay > 0
+
+    # fast-path for resumed runs: if file already exists locally, avoid HTTP work entirely
+    existing_path = local_path_for_file_id(file_remote_info[:file_id])
+    if existing_path && File.exist?(existing_path)
+      result_message = "#{color("[EXISTS]", :cyan)} #{file_remote_info[:file_url]} (#{@processed_file_count + 1}/#{@total_to_download})"
+      @download_mutex.synchronize do
+        @processed_file_count += 1 if @processed_file_count < @total_to_download
+        puts result_message
+      end
+
+      append_to_db(file_remote_info[:file_id])
+
+      if @page_requisites && File.extname(existing_path) =~ /\.(html?|php|asp|aspx|jsp)$/i
+        process_page_requisites(existing_path, file_remote_info)
+      end
+      return
+    end
+
     @connection_pool.with_connection do |connection|
       result_message, downloaded_path = download_file(file_remote_info, connection)
-      
+
       if downloaded_path && File.exist?(downloaded_path)
          download_success = true
       end
-      
+
       @download_mutex.synchronize do
         @processed_file_count += 1 if @processed_file_count < @total_to_download
         # only print if it's a "User" file or a requisite we found
@@ -704,7 +807,7 @@ class WaybackMachineDownloader
 
     if download_success
       append_to_db(file_remote_info[:file_id])
-      
+
       if @page_requisites && downloaded_path && File.extname(downloaded_path) =~ /\.(html?|php|asp|aspx|jsp)$/i
         process_page_requisites(downloaded_path, file_remote_info)
       end
@@ -712,7 +815,7 @@ class WaybackMachineDownloader
   rescue => e
     @logger.error("Error processing file #{file_remote_info[:file_url]}: #{e.message}")
   end
-  
+
   def process_page_requisites(file_path, parent_remote_info)
     return unless File.exist?(file_path)
 
@@ -724,7 +827,7 @@ class WaybackMachineDownloader
     # prepare base URI for resolving relative paths
     parent_raw = parent_remote_info[:file_url]
     parent_raw = "http://#{parent_raw}" unless parent_raw.match?(/^https?:\/\//)
-    
+
     begin
       base_uri = URI(parent_raw)
       # calculate the "root" host of the site we are downloading to compare later
@@ -758,7 +861,7 @@ class WaybackMachineDownloader
         path = resolved_uri.path
         ext = File.extname(path).downcase
         if ext.empty? || ['.html', '.htm', '.php', '.asp', '.aspx'].include?(ext)
-           next 
+           next
         end
 
         # construct the original URL to query the Wayback API
@@ -829,7 +932,7 @@ class WaybackMachineDownloader
     rescue Errno::EEXIST, Errno::ENOTDIR => e
       file_already_existing = nil
       check_path = dir_path
-      
+
       # walk up the path to find the specific file that is blocking directory creation
       while check_path != "." && check_path != "/"
         if File.exist?(check_path) && !File.directory?(check_path)
@@ -844,11 +947,11 @@ class WaybackMachineDownloader
       if file_already_existing
         file_already_existing_temporary = file_already_existing + '.temp'
         file_already_existing_permanent = file_already_existing + '/index.html'
-        
+
         FileUtils::mv file_already_existing, file_already_existing_temporary
         FileUtils::mkdir_p file_already_existing
         FileUtils::mv file_already_existing_temporary, file_already_existing_permanent
-        
+
         puts "#{file_already_existing} -> #{file_already_existing_permanent}"
         # retry the directory creation now that the path is clear
         structure_dir_path dir_path
@@ -861,12 +964,12 @@ class WaybackMachineDownloader
   def rewrite_local_files
     puts "Scanning #{backup_path} for files to rewrite..."
     files = Dir.glob(File.join(backup_path, "**/*.{html,htm,css,js,php,asp,aspx,jsp}"))
-    
+
     puts "Found #{files.size} files. Rewriting links for local browsing..."
-    
+
     pool = Concurrent::FixedThreadPool.new(@threads_count)
     progress = Concurrent::AtomicFixnum.new(0)
-    
+
     files.each do |file_path|
       pool.post do
         rewrite_urls_to_relative(file_path)
@@ -874,7 +977,7 @@ class WaybackMachineDownloader
         print "\rProgress: #{current}/#{files.size}" if current % 100 == 0
       end
     end
-    
+
     pool.shutdown
     pool.wait_for_termination
     puts "\nFinished rewriting all files."
@@ -882,39 +985,58 @@ class WaybackMachineDownloader
 
   def rewrite_urls_to_relative(file_path)
     return unless File.exist?(file_path)
-    
+
     file_ext = File.extname(file_path).downcase
-    
+
     begin
       content = File.binread(file_path)
 
       # detect encoding for HTML files
       if file_ext == '.html' || file_ext == '.htm' || file_ext == '.php' || file_ext == '.asp'
-        encoding = content.match(/<meta\s+charset=["']?([^"'>]+)/i)&.captures&.first || 'UTF-8'
-        content.force_encoding(encoding) rescue content.force_encoding('UTF-8')
+        encoding_match = content.match(/<meta.*?charset=["'\s]?([^"'\s>;]+)/i)
+        encoding_name = encoding_match ? encoding_match.captures.first : 'UTF-8'
+
+        begin
+          encoding = Encoding.find(encoding_name)
+        rescue ArgumentError
+          encoding = Encoding::UTF_8
+        end
+
+        content.force_encoding(encoding)
       else
         content.force_encoding('UTF-8')
       end
 
+      # convert the content to valid UTF-8
+      if !content.valid_encoding? || content.encoding != Encoding::UTF_8
+        begin
+          content = content.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: '')
+        rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError, ArgumentError
+          content = content.tidy_bytes
+        end
+      end
+
       # URLs in HTML attributes
       content = rewrite_html_attr_urls(content)
-      
+
       # URLs in CSS
       content = rewrite_css_urls(content)
-      
+
       # URLs in JavaScript
       content = rewrite_js_urls(content)
-      
-      # for URLs that start with a single slash, make them relative
+
+      root_prefix = site_root_relative_prefix(file_path)
+
+      # rewrite root-absolute links to paths relative to the downloaded site root
       content.gsub!(/(\s(?:href|src|action|data-src|data-url)=["'])\/([^"'\/][^"']*)(["'])/i) do
         prefix, path, suffix = $1, $2, $3
-        "#{prefix}./#{path}#{suffix}"
+        "#{prefix}#{root_prefix}#{path}#{suffix}"
       end
-      
-      # for URLs in CSS that start with a single slash, make them relative
+
+      # apply the same root-relative conversion to CSS url(...) references
       content.gsub!(/url\(\s*["']?\/([^"'\)\/][^"'\)]*?)["']?\s*\)/i) do
         path = $1
-        "url(\"./#{path}\")"
+        "url(\"#{root_prefix}#{path}\")"
       end
 
       # save the modified content back to the file
@@ -922,7 +1044,27 @@ class WaybackMachineDownloader
       puts "Rewrote URLs in #{file_path} to be relative."
     rescue Errno::ENOENT => e
       @logger.warn("Error reading file #{file_path}: #{e.message}")
+    rescue StandardError => e
+      @logger.error("Failed to rewrite URLs in #{file_path}: #{e.message}")
+    end
+  end
+
+  def site_root_relative_prefix(file_path)
+    file_dir = File.dirname(File.expand_path(file_path))
+    root_dir = File.expand_path(backup_path)
+
+    begin
+      relative_dir = Pathname.new(file_dir).relative_path_from(Pathname.new(root_dir)).to_s
+    rescue ArgumentError
+      return './'
     end
+
+    return './' if relative_dir == '.' || relative_dir.empty?
+
+    depth = relative_dir.split(/[\\\/]+/).reject(&:empty?).length
+    return './' if depth <= 0
+
+    '../' * depth
   end
 
   def download_file (file_remote_info, http)
@@ -930,7 +1072,7 @@ class WaybackMachineDownloader
     file_url = file_remote_info[:file_url].encode(current_encoding)
     file_id = file_remote_info[:file_id]
     file_timestamp = file_remote_info[:timestamp]
-    
+
     # sanitize file_id to ensure it is a valid path component
     raw_path_elements = file_id.split('/')
 
@@ -944,7 +1086,7 @@ class WaybackMachineDownloader
         element
       end
     end
-    
+
     current_backup_path = backup_path
 
     if file_id == ""
@@ -994,6 +1136,27 @@ class WaybackMachineDownloader
     end
   end
 
+  # derive the local filesystem path for a sanitized `file_id` stored in the DB
+  def local_path_for_file_id(file_id)
+    return nil if file_id.nil?
+    current_backup_path = backup_path
+
+    # file_id coming from DB is expected to already be sanitized
+    raw_path_elements = file_id.split('/')
+
+    if file_id == ""
+      dir_path = current_backup_path
+      return File.join(dir_path, 'index.html')
+    elsif file_id[-1] == '/' || (raw_path_elements.last && !raw_path_elements.last.include?('.'))
+      dir_path = File.join(current_backup_path, *raw_path_elements)
+      return File.join(dir_path, 'index.html')
+    else
+      filename = raw_path_elements.pop
+      dir_path = File.join(current_backup_path, *raw_path_elements)
+      return File.join(dir_path, filename)
+    end
+  end
+
   def color(text, color_code)
     return text if Gem.win_platform? && !ENV['ENABLE_ANSI']
     codes = { red: 31, green: 32, yellow: 33, blue: 34, magenta: 35, cyan: 36, white: 37 }
@@ -1038,7 +1201,7 @@ class WaybackMachineDownloader
     end
     logger
   end
-    
+
   # safely sanitize a file id (or id+timestamp)
   def sanitize_and_prepare_id(raw, file_url)
     return nil if raw.nil?
@@ -1111,6 +1274,46 @@ class WaybackMachineDownloader
     end
   end
 
+  def build_wayback_url(source_url, file_timestamp)
+    source = source_url.to_s
+    return source if wayback_archive_url?(source)
+
+    if source.start_with?('/web/')
+      return "https://web.archive.org#{source}"
+    end
+
+    if @rewritten
+      "https://web.archive.org/web/#{file_timestamp}/#{source}"
+    else
+      "https://web.archive.org/web/#{file_timestamp}id_/#{source}"
+    end
+  end
+
+  def wayback_archive_url?(url)
+    url.to_s.match?(%r{\Ahttps?://web\.archive\.org/web/})
+  end
+
+  def extract_original_url(url)
+    match = url.to_s.match(%r{\Ahttps?://web\.archive\.org/web/\d{1,14}(?:[a-z_]*)/(https?://.+)\z})
+    match && match[1]
+  end
+
+  def resolve_redirect_source(current_source_url, location)
+    return nil if location.nil? || location.empty?
+
+    location = location.to_s
+    return location if wayback_archive_url?(location)
+
+    if location.start_with?('/web/')
+      return "https://web.archive.org#{location}"
+    end
+
+    base_url = extract_original_url(current_source_url) || current_source_url.to_s
+    URI.join(base_url, location).to_s
+  rescue URI::InvalidURIError
+    location
+  end
+
   # wrap URL in parentheses if it contains characters that commonly break unquoted
   # Windows CMD usage (e.g., &). This is only for display; user still must quote
   # when invoking manually.
@@ -1122,11 +1325,7 @@ class WaybackMachineDownloader
   def download_with_retry(file_path, file_url, file_timestamp, connection, redirect_count = 0)
     retries = 0
     begin
-      wayback_url = if @rewritten
-        "https://web.archive.org/web/#{file_timestamp}/#{file_url}"
-      else
-        "https://web.archive.org/web/#{file_timestamp}id_/#{file_url}"
-      end
+      wayback_url = build_wayback_url(file_url, file_timestamp)
 
       # Escape characters that are not valid in URI()
       wayback_url = wayback_url.gsub(' ', '%20').gsub('[', '%5B').gsub(']', '%5D')
@@ -1144,64 +1343,85 @@ class WaybackMachineDownloader
           return :skipped_not_found
       end
 
-      request = Net::HTTP::Get.new(URI(wayback_url))
-      request["Connection"] = "keep-alive"
-      request["User-Agent"] = "WaybackMachineDownloader/#{VERSION}"
-      request["Accept-Encoding"] = "gzip, deflate"
+      if HTTPX_AVAILABLE && connection.is_a?(HTTPX::Session)
+        response = connection.get(wayback_url)
 
-      response = connection.request(request)
+        raise response.error if response.is_a?(HTTPX::ErrorResponse)
 
-      save_response_body = lambda do
-        File.open(file_path, "wb") do |file|
-          body = response.body
-          if response['content-encoding'] == 'gzip' && body && !body.empty?
-            begin
-              gz = Zlib::GzipReader.new(StringIO.new(body))
-              decompressed_body = gz.read
-              gz.close
-              file.write(decompressed_body)
-            rescue Zlib::GzipFile::Error => e
-              @logger.warn("Failure decompressing gzip file #{file_url}: #{e.message}. Writing raw body.")
-              file.write(body)
-            end
+        code = response.status
+
+        save_response_body = lambda do
+          if response.respond_to?(:copy_to)
+            response.copy_to(file_path)
           else
-            file.write(body) if body
+            response.body.copy_to(file_path)
+          end
+        end
+      else
+        request = Net::HTTP::Get.new(URI(wayback_url))
+        request["Connection"] = "keep-alive"
+        request["User-Agent"] = "WaybackMachineDownloader/#{VERSION}"
+        request["Accept-Encoding"] = "gzip, deflate"
+
+        response = connection.request(request)
+        code = response.code.to_i
+
+        save_response_body = lambda do
+          File.open(file_path, "wb") do |file|
+            body = response.body
+            if response['content-encoding'] == 'gzip' && body && !body.empty?
+              begin
+                gz = Zlib::GzipReader.new(StringIO.new(body))
+                decompressed_body = gz.read
+                gz.close
+                file.write(decompressed_body)
+              rescue Zlib::GzipFile::Error => e
+                @logger.warn("Failure decompressing gzip file #{file_url}: #{e.message}. Writing raw body.")
+                file.write(body)
+              end
+            else
+              file.write(body) if body
+            end
           end
         end
       end
 
       if @all
-        case response
-        when Net::HTTPSuccess, Net::HTTPRedirection, Net::HTTPClientError, Net::HTTPServerError
+        case code
+        when 200..599
           save_response_body.call
-          if response.is_a?(Net::HTTPRedirection)
-            @logger.info("Saved redirect page for #{file_url} (status #{response.code}).")
-          elsif response.is_a?(Net::HTTPClientError) || response.is_a?(Net::HTTPServerError)
-            @logger.info("Saved error page for #{file_url} (status #{response.code}).")
+          if (300..399).cover?(code)
+            @logger.info("Saved redirect page for #{file_url} (status #{code}).")
+          elsif (400..599).cover?(code)
+            @logger.info("Saved error page for #{file_url} (status #{code}).")
           end
           return :saved
         else
           # for any other response type when --all is true, treat as an error to be retried or failed
-          raise "Unhandled HTTP response: #{response.code} #{response.message}"
+          raise "Unhandled HTTP response: #{code}"
         end
       else # not @all (our default behavior)
-        case response
-        when Net::HTTPSuccess
+        if (200..299).cover?(code)
           save_response_body.call
           return :saved
-        when Net::HTTPRedirection
+        elsif (300..399).cover?(code)
           raise "Too many redirects for #{file_url}" if redirect_count >= 5
-          location = response['location']
+          location = if HTTPX_AVAILABLE && connection.is_a?(HTTPX::Session)
+                       response.headers['location']
+                     else
+                       response['location']
+                     end
           @logger.warn("Redirect found for #{file_url} -> #{location}")
-          return download_with_retry(file_path, location, file_timestamp, connection, redirect_count + 1)
-        when Net::HTTPTooManyRequests
+          redirected_source = resolve_redirect_source(file_url, location)
+          return download_with_retry(file_path, redirected_source, file_timestamp, connection, redirect_count + 1)
+        elsif code == 429
           sleep(RATE_LIMIT * 2)
           raise "Rate limited, retrying..."
-        when Net::HTTPNotFound
+        elsif code == 404
           @logger.warn("File not found, skipping: #{file_url}")
           return :skipped_not_found
         else
-          raise "HTTP Error: #{response.code} #{response.message}"
+          raise "HTTP Error: #{code}"
         end
       end
 
@@ -1219,6 +1439,7 @@ class WaybackMachineDownloader
   end
 
   def cleanup
+    flush_db
     @connection_pool.shutdown
 
     if @failed_downloads.any?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wayback_machine_downloader_straw
 version: !ruby/object:Gem::Version
-  version: 2.4.6
+  version: 2.4.8
 platform: ruby
 authors:
 - strawberrymaster
@@ -18,7 +18,7 @@ dependencies:
         version: '1.3'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.4
+        version: 1.3.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -28,7 +28,7 @@ dependencies:
         version: '1.3'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.4
+        version: 1.3.6
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -94,7 +94,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Download an entire website from the Wayback Machine.
 test_files: []