wayback_machine_downloader_straw 2.4.7 → 2.4.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 558d3187ee31faeadb08cf83e32a87307ae9d55a3327206598f27a78fb715e08
-  data.tar.gz: 9845999e0e618afde419869bb01b04277aca318aa80b238feb8252540fc16315
+  metadata.gz: 12259647dc99ef1c263b3d797098233be4f194bf1a4e9bc047e1dfb4d0fc9468
+  data.tar.gz: 5d9d6bd47efca6b43a5c1d6c2606bb2871edea0b4dc781f206bbab8c9c66aaa0
 SHA512:
-  metadata.gz: b323c1065ea1ab1d3c5909458cae726462ba1b88fd89effe8cd1efbdd1301d2022363c56b191f8ddbe28c0143fe87dfc94ca847865bb08e3564e29ba36f231a4
-  data.tar.gz: 00c4e775ee05e176e1048d6e5ddd9ddc436edaf9bf7fea61dbab33480e4adc499ea8a3a584e591858aad3a4c2f10096a3b6eb1ff6c8146f26ba6a3d42f104e32
+  metadata.gz: 670efe274663448a9967d2e39920ce234af1ba5c0aae82135c56f64899904f510096ac19b6367a5b90af43d3878b156a21b2fb1138a2e567041b55205fc97c67
+  data.tar.gz: 2d4d6e927664ea831c0611bbbc89d74ee6c144629f8afb2b2a58428cff1f78846832979bd994dfd97a51cff03ee52ab9c5c09beebc1b007248f1c043aa3bdd2a

data/bin/wayback_machine_downloader

@@ -84,6 +84,10 @@ option_parser = OptionParser.new do |opts|
     options[:keep] = true
   end
 
+  opts.on("--delay SECONDS", Float, "Delay between downloads in seconds (default: 0)") do |t|
+    options[:delay] = t
+  end
+
   opts.on("--rt", "--retry N", Integer, "Maximum number of retries for failed downloads (default: 3)") do |t|
     options[:max_retries] = t
   end

data/lib/wayback_machine_downloader/archive_api.rb

@@ -8,6 +8,7 @@ module ArchiveAPI
     # This is a workaround for an issue with the API and *some* domains.
     # See https://github.com/StrawberryMaster/wayback-machine-downloader/issues/6
     # But don't do this when exact_url flag is set, and never append twice
+    match_type = nil
     if url && !@exact_url
       normalized_url = url.to_s
       has_wildcard = normalized_url.include?('*')
@@ -18,32 +19,38 @@ module ArchiveAPI
       has_path = host_and_rest.include?('/')
 
       unless has_wildcard || has_path
-        url = "#{normalized_url}/*"
+        match_type = "prefix"
       end
     end
 
     request_url = URI("https://web.archive.org/cdx/search/cdx")
     params = [["output", "json"], ["url", url]] + parameters_for_api(page_index)
+    params << ["matchType", match_type] if match_type
     request_url.query = URI.encode_www_form(params)
 
     retries = 0
     max_retries = (@max_retries || 3)
-    delay = WaybackMachineDownloader::RETRY_DELAY rescue 2
+    base_delay = WaybackMachineDownloader::RETRY_DELAY rescue 2
 
     begin
-      request = Net::HTTP::Get.new(request_url)
-      request["User-Agent"] = "wmd-straw/#{WaybackMachineDownloader::VERSION}"
-      request["Connection"] = "keep-alive"
-      request["Accept-Encoding"] = "gzip"
-      response = http.request(request)
+      if HTTPX_AVAILABLE && http.is_a?(HTTPX::Session)
+        response = http.get(request_url)
+        raise response.error if response.is_a?(HTTPX::ErrorResponse)
 
-      case response.code.to_i
+        code = response.status
+        body = response.body.to_s.strip
+      else
+        request = Net::HTTP::Get.new(request_url)
+        request["User-Agent"] = "wmd-straw/#{WaybackMachineDownloader::VERSION rescue '2.4.8'}"
+        request["Connection"] = "keep-alive"
+        request["Accept-Encoding"] = "gzip, deflate"
+        response = http.request(request)
+        code = response.code.to_i
+        body = decompress_body(response)
+      end
+
+      case code
       when 200
-        body = if response['content-encoding'] == 'gzip'
-          Zlib::GzipReader.new(StringIO.new(response.body)).read
-        else
-          response.body.to_s.strip
-        end
         return [] if body.empty?
         begin
           json = JSON.parse(body)
@@ -54,16 +61,22 @@ module ArchiveAPI
           raise "Malformed JSON response: #{e.message}"
         end
       when 429, 500, 502, 503, 504
-        raise "Server error #{response.code}: #{response.message}"
+        raise "Server error #{code}: #{response.respond_to?(:message) ? response.message : ''}"
       else
-        warn "Unexpected API response #{response.code} for #{url}"
+        warn "Unexpected API response #{code} for #{url}"
         []
       end
     rescue Net::ReadTimeout, Net::OpenTimeout, StandardError => e
       if retries < max_retries
         retries += 1
-        warn "Error talking to Wayback CDX API (#{e.class}: #{e.message}) for #{url}, retry #{retries}/#{max_retries}..."
-        sleep(delay * retries)
+
+        jitter = rand(0.0..1.0)
+        sleep_time = (base_delay * (2 ** (retries - 1))) + jitter
+
+        warn "Error talking to Wayback CDX API (#{e.class}: #{e.message}) for #{url}. " \
+             "Retrying in #{sleep_time.round(2)}s (attempt #{retries}/#{max_retries})..."
+
+        sleep(sleep_time)
         retry
       else
         warn "Giving up on Wayback CDX API for #{url} after #{max_retries} attempts. (Last error: #{e.message})"
@@ -82,4 +95,19 @@ module ArchiveAPI
     parameters
   end
 
+  private
+
+  def decompress_body(response)
+    body = response.body.to_s
+    return body if body.empty?
+
+    case response['content-encoding']
+    when 'gzip'
+      Zlib::GzipReader.new(StringIO.new(body)).read rescue body
+    when 'deflate'
+      Zlib::Inflate.inflate(body) rescue body
+    else
+      body.strip
+    end
+  end
 end

data/lib/wayback_machine_downloader/subdom_processor.rb

@@ -40,11 +40,17 @@ module SubdomainProcessor
   private
   
   def extract_base_domain(url)
-    uri = URI.parse(url.gsub(/^https?:\/\//, '').split('/').first) rescue nil
+    # ensure the URL has a scheme for URI parsing
+    normalized_url = url.match?(/^https?:\/\//i) ? url : "http://#{url}"
+    uri = URI.parse(normalized_url) rescue nil
     return nil unless uri
     
-    host = uri.host || uri.path.split('/').first
-    host = host.downcase
+    # extract the host (and default to parsing path if host is missing)
+    host = uri.host || (uri.path || '').split('/').first
+    return nil unless host
+    
+    # strip port numbers if present
+    host = host.split(':').first.downcase
     
     # extract the base domain (e.g., "example.com" from "sub.example.com")
     parts = host.split('.')

data/lib/wayback_machine_downloader.rb

@@ -4,11 +4,13 @@ require 'thread'
 require 'net/http'
 require 'fileutils'
 require 'json'
+require 'pathname'
 require 'concurrent-ruby'
 require 'logger'
 require 'zlib'
 require 'stringio'
 require 'digest'
+require 'etc'
 require_relative 'wayback_machine_downloader/tidy_bytes'
 require_relative 'wayback_machine_downloader/to_regex'
 require_relative 'wayback_machine_downloader/archive_api'
@@ -16,6 +18,13 @@ require_relative 'wayback_machine_downloader/page_requisites'
 require_relative 'wayback_machine_downloader/subdom_processor'
 require_relative 'wayback_machine_downloader/url_rewrite'
 
+begin
+  require 'httpx'
+  HTTPX_AVAILABLE = true
+rescue LoadError
+  HTTPX_AVAILABLE = false
+end
+
 class ConnectionPool
   MAX_AGE = 300
   CLEANUP_INTERVAL = 60
@@ -66,7 +75,12 @@ class ConnectionPool
   def stale?(entry)
     return true if entry.nil? || entry[:http].nil?
     http = entry[:http]
-    !http.started? || (Time.now - entry[:created_at] > MAX_AGE)
+
+    if HTTPX_AVAILABLE && http.is_a?(HTTPX::Session)
+      Time.now - entry[:created_at] > MAX_AGE
+    else
+      !http.started? || (Time.now - entry[:created_at] > MAX_AGE)
+    end
   end
 
   def build_connection_entry
@@ -74,7 +88,12 @@ class ConnectionPool
   end
 
   def safe_finish(http)
-    http.finish if http&.started?
+    return if http.nil?
+    if HTTPX_AVAILABLE && http.is_a?(HTTPX::Session)
+      http.close
+    else
+      http.finish if http&.started?
+    end
   rescue StandardError
     nil
   end
@@ -113,14 +132,24 @@ class ConnectionPool
   end
 
   def create_connection
-    http = Net::HTTP.new("web.archive.org", 443)
-    http.use_ssl = true
-    http.read_timeout = DEFAULT_TIMEOUT
-    http.open_timeout = DEFAULT_TIMEOUT
-    http.keep_alive_timeout = 30
-    http.max_retries = MAX_RETRIES
-    http.start
-    http
+    if HTTPX_AVAILABLE
+      HTTPX.with(
+        timeout: {
+          connect_timeout: DEFAULT_TIMEOUT,
+          read_timeout: DEFAULT_TIMEOUT,
+          write_timeout: DEFAULT_TIMEOUT
+        }
+      )
+    else
+      http = Net::HTTP.new("web.archive.org", 443)
+      http.use_ssl = true
+      http.read_timeout = DEFAULT_TIMEOUT
+      http.open_timeout = DEFAULT_TIMEOUT
+      http.keep_alive_timeout = 30
+      http.max_retries = MAX_RETRIES
+      http.start
+      http
+    end
   end
 end
 
@@ -130,7 +159,7 @@ class WaybackMachineDownloader
   include SubdomainProcessor
   include URLRewrite
 
-  VERSION = "2.4.7"
+  VERSION = "2.4.8"
   DEFAULT_TIMEOUT = 30
   MAX_RETRIES = 3
   RETRY_DELAY = 2
@@ -164,7 +193,14 @@ class WaybackMachineDownloader
     @all = params[:all]
     @keep_duplicates = params[:keep_duplicates] || false
     @maximum_pages = params[:maximum_pages] ? params[:maximum_pages].to_i : 100
-    @threads_count = [params[:threads_count].to_i, 1].max
+    default_threads = begin
+      [Etc.nprocessors, 4].max
+    rescue StandardError
+      4
+    end
+    threads_param = params[:threads_count] ? params[:threads_count].to_i : 0
+    threads_param = default_threads if threads_param <= 0
+    @threads_count = [threads_param, 1].max
     @rewritten = params[:rewritten]
     @reset = params[:reset]
     @keep = params[:keep]
@@ -174,6 +210,7 @@ class WaybackMachineDownloader
     @connection_pool = ConnectionPool.new(CONNECTION_POOL_SIZE)
     @db_mutex = Mutex.new
     @rewrite = params[:rewrite] || false
+    @delay = params[:delay] ? params[:delay].to_f : 0.0
     @recursive_subdomains = params[:recursive_subdomains] || false
     @subdomain_depth = params[:subdomain_depth] || 1
     @snapshot_at = params[:snapshot_at] ? params[:snapshot_at].to_i : nil
@@ -181,6 +218,12 @@ class WaybackMachineDownloader
     @page_requisites = params[:page_requisites] || false
     @pending_jobs = Concurrent::AtomicFixnum.new(0)
 
+    @db_buffer = []
+    @db_buffer_mutex = Mutex.new
+    @db_last_flush = Time.now
+    @db_flush_threshold = 50  # flush to disk every 50 completed files
+    @db_flush_interval = 5.0  # or flush every 5 seconds even if count isn't met
+
     # URL for rejecting invalid/unencoded wayback urls
     @url_regexp = /^(([A-Za-z][A-Za-z0-9+.-]*):((\/\/(((([A-Za-z0-9._~-])|(%[ABCDEFabcdef0-9][ABCDEFabcdef0-9])|([!$&'('')'*+,;=]))+)(:([0-9]*))?)(((\/((([A-Za-z0-9._~-])|(%[ABCDEFabcdef0-9][ABCDEFabcdef0-9])|([!$&'('')'*+,;=])|:|@)*))*)))|((\/(((([A-Za-z0-9._~-])|(%[ABCDEFabcdef0-9][ABCDEFabcdef0-9])|([!$&'('')'*+,;=])|:|@)+)(\/((([A-Za-z0-9._~-])|(%[ABCDEFabcdef0-9][ABCDEFabcdef0-9])|([!$&'('')'*+,;=])|:|@)*))*)?))|((((([A-Za-z0-9._~-])|(%[ABCDEFabcdef0-9][ABCDEFabcdef0-9])|([!$&'('')'*+,;=])|:|@)+)(\/((([A-Za-z0-9._~-])|(%[ABCDEFabcdef0-9][ABCDEFabcdef0-9])|([!$&'('')'*+,;=])|:|@)*))*)))(\?((([A-Za-z0-9._~-])|(%[ABCDEFabcdef0-9][ABCDEFabcdef0-9])|([!$&'('')'*+,;=])|:|@)|\/|\?)*)?(\#((([A-Za-z0-9._~-])|(%[ABCDEFabcdef0-9][ABCDEFabcdef0-9])|([!$&'('')'*+,;=])|:|@)|\/|\?)*)?)$/
 
@@ -545,12 +588,40 @@ class WaybackMachineDownloader
   end
 
   def append_to_db(file_id)
-    @db_mutex.synchronize do
-      begin
-        FileUtils.mkdir_p(File.dirname(db_path))
-        File.open(db_path, 'a') { |f| f.puts(file_id) }
-      rescue => e
-        @logger.error("Failed to append downloaded file ID #{file_id} to #{db_path}: #{e.message}")
+    flush_needed = false
+
+    @db_buffer_mutex.synchronize do
+      @db_buffer << file_id
+      if @db_buffer.size >= @db_flush_threshold || (Time.now - @db_last_flush) >= @db_flush_interval
+        flush_needed = true
+      end
+    end
+
+    flush_db if flush_needed
+  end
+
+  def flush_db
+    lines_to_write = nil
+
+    @db_buffer_mutex.synchronize do
+      return if @db_buffer.empty?
+      lines_to_write = @db_buffer.dup
+      @db_buffer.clear
+      @db_last_flush = Time.now
+    end
+
+    return if lines_to_write.nil? || lines_to_write.empty?
+
+    begin
+      FileUtils.mkdir_p(File.dirname(db_path))
+      File.open(db_path, 'a') do |f|
+        lines_to_write.each { |id| f.puts(id) }
+      end
+    rescue StandardError => e
+      @logger.error("Failed to write batch to #{db_path}: #{e.message}")
+
+      @db_buffer_mutex.synchronize do
+        @db_buffer.unshift(*lines_to_write)
       end
     end
   end
@@ -587,9 +658,18 @@ class WaybackMachineDownloader
       end
     end
   end
-  
+
+  @@engine_printed = false
+
   def download_files
     start_time = Time.now
+
+    unless @@engine_printed
+      engine_name = HTTPX_AVAILABLE ? "HTTPX" : "Net::HTTP"
+      puts "Connection engine used: #{color(engine_name, :green)}"
+      @@engine_printed = true
+    end
+
     puts "Downloading #{@base_url} to #{backup_path} from Wayback Machine archives."
 
     FileUtils.mkdir_p(backup_path)
@@ -608,7 +688,7 @@ class WaybackMachineDownloader
 
     # Load IDs of already downloaded files
     downloaded_ids = load_downloaded_ids
-    
+
     # We use a thread-safe Set to track what we have queued/downloaded in this session
     # to avoid infinite loops with page requisites
     @session_downloaded_ids = Concurrent::Set.new
@@ -624,7 +704,7 @@ class WaybackMachineDownloader
     if skipped_count > 0
       puts "Found #{skipped_count} previously downloaded files, skipping them."
     end
-    
+
     if remaining_count == 0 && !@page_requisites
       puts "All matching files have already been downloaded."
       cleanup
@@ -691,6 +771,9 @@ class WaybackMachineDownloader
     download_success = false
     downloaded_path = nil
 
+    # if delay was set by the user
+    sleep(@delay) if @delay > 0
+
     # fast-path for resumed runs: if file already exists locally, avoid HTTP work entirely
     existing_path = local_path_for_file_id(file_remote_info[:file_id])
     if existing_path && File.exist?(existing_path)
@@ -707,14 +790,14 @@ class WaybackMachineDownloader
       end
       return
     end
-    
+
     @connection_pool.with_connection do |connection|
       result_message, downloaded_path = download_file(file_remote_info, connection)
-      
+
       if downloaded_path && File.exist?(downloaded_path)
          download_success = true
       end
-      
+
       @download_mutex.synchronize do
         @processed_file_count += 1 if @processed_file_count < @total_to_download
         # only print if it's a "User" file or a requisite we found
@@ -724,7 +807,7 @@ class WaybackMachineDownloader
 
     if download_success
       append_to_db(file_remote_info[:file_id])
-      
+
       if @page_requisites && downloaded_path && File.extname(downloaded_path) =~ /\.(html?|php|asp|aspx|jsp)$/i
         process_page_requisites(downloaded_path, file_remote_info)
       end
@@ -732,7 +815,7 @@ class WaybackMachineDownloader
   rescue => e
     @logger.error("Error processing file #{file_remote_info[:file_url]}: #{e.message}")
   end
-  
+
   def process_page_requisites(file_path, parent_remote_info)
     return unless File.exist?(file_path)
 
@@ -744,7 +827,7 @@ class WaybackMachineDownloader
     # prepare base URI for resolving relative paths
     parent_raw = parent_remote_info[:file_url]
     parent_raw = "http://#{parent_raw}" unless parent_raw.match?(/^https?:\/\//)
-    
+
     begin
       base_uri = URI(parent_raw)
       # calculate the "root" host of the site we are downloading to compare later
@@ -778,7 +861,7 @@ class WaybackMachineDownloader
         path = resolved_uri.path
         ext = File.extname(path).downcase
         if ext.empty? || ['.html', '.htm', '.php', '.asp', '.aspx'].include?(ext)
-           next 
+           next
         end
 
         # construct the original URL to query the Wayback API
@@ -849,7 +932,7 @@ class WaybackMachineDownloader
     rescue Errno::EEXIST, Errno::ENOTDIR => e
       file_already_existing = nil
       check_path = dir_path
-      
+
       # walk up the path to find the specific file that is blocking directory creation
       while check_path != "." && check_path != "/"
         if File.exist?(check_path) && !File.directory?(check_path)
@@ -864,11 +947,11 @@ class WaybackMachineDownloader
       if file_already_existing
         file_already_existing_temporary = file_already_existing + '.temp'
         file_already_existing_permanent = file_already_existing + '/index.html'
-        
+
         FileUtils::mv file_already_existing, file_already_existing_temporary
         FileUtils::mkdir_p file_already_existing
         FileUtils::mv file_already_existing_temporary, file_already_existing_permanent
-        
+
         puts "#{file_already_existing} -> #{file_already_existing_permanent}"
         # retry the directory creation now that the path is clear
         structure_dir_path dir_path
@@ -881,12 +964,12 @@ class WaybackMachineDownloader
   def rewrite_local_files
     puts "Scanning #{backup_path} for files to rewrite..."
     files = Dir.glob(File.join(backup_path, "**/*.{html,htm,css,js,php,asp,aspx,jsp}"))
-    
+
     puts "Found #{files.size} files. Rewriting links for local browsing..."
-    
+
     pool = Concurrent::FixedThreadPool.new(@threads_count)
     progress = Concurrent::AtomicFixnum.new(0)
-    
+
     files.each do |file_path|
       pool.post do
         rewrite_urls_to_relative(file_path)
@@ -894,7 +977,7 @@ class WaybackMachineDownloader
         print "\rProgress: #{current}/#{files.size}" if current % 100 == 0
       end
     end
-    
+
     pool.shutdown
     pool.wait_for_termination
     puts "\nFinished rewriting all files."
@@ -902,39 +985,58 @@ class WaybackMachineDownloader
 
   def rewrite_urls_to_relative(file_path)
     return unless File.exist?(file_path)
-    
+
     file_ext = File.extname(file_path).downcase
-    
+
     begin
       content = File.binread(file_path)
 
       # detect encoding for HTML files
       if file_ext == '.html' || file_ext == '.htm' || file_ext == '.php' || file_ext == '.asp'
-        encoding = content.match(/<meta\s+charset=["']?([^"'>]+)/i)&.captures&.first || 'UTF-8'
-        content.force_encoding(encoding) rescue content.force_encoding('UTF-8')
+        encoding_match = content.match(/<meta.*?charset=["'\s]?([^"'\s>;]+)/i)
+        encoding_name = encoding_match ? encoding_match.captures.first : 'UTF-8'
+
+        begin
+          encoding = Encoding.find(encoding_name)
+        rescue ArgumentError
+          encoding = Encoding::UTF_8
+        end
+
+        content.force_encoding(encoding)
       else
         content.force_encoding('UTF-8')
       end
 
+      # convert the content to valid UTF-8
+      if !content.valid_encoding? || content.encoding != Encoding::UTF_8
+        begin
+          content = content.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: '')
+        rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError, ArgumentError
+          content = content.tidy_bytes
+        end
+      end
+
       # URLs in HTML attributes
       content = rewrite_html_attr_urls(content)
-      
+
       # URLs in CSS
       content = rewrite_css_urls(content)
-      
+
       # URLs in JavaScript
       content = rewrite_js_urls(content)
-      
-      # for URLs that start with a single slash, make them relative
+
+      root_prefix = site_root_relative_prefix(file_path)
+
+      # rewrite root-absolute links to paths relative to the downloaded site root
       content.gsub!(/(\s(?:href|src|action|data-src|data-url)=["'])\/([^"'\/][^"']*)(["'])/i) do
         prefix, path, suffix = $1, $2, $3
-        "#{prefix}./#{path}#{suffix}"
+        "#{prefix}#{root_prefix}#{path}#{suffix}"
       end
-      
-      # for URLs in CSS that start with a single slash, make them relative
+
+      # apply the same root-relative conversion to CSS url(...) references
       content.gsub!(/url\(\s*["']?\/([^"'\)\/][^"'\)]*?)["']?\s*\)/i) do
         path = $1
-        "url(\"./#{path}\")"
+        "url(\"#{root_prefix}#{path}\")"
       end
 
       # save the modified content back to the file
@@ -942,15 +1044,35 @@ class WaybackMachineDownloader
       puts "Rewrote URLs in #{file_path} to be relative."
     rescue Errno::ENOENT => e
       @logger.warn("Error reading file #{file_path}: #{e.message}")
+    rescue StandardError => e
+      @logger.error("Failed to rewrite URLs in #{file_path}: #{e.message}")
     end
   end
 
+  def site_root_relative_prefix(file_path)
+    file_dir = File.dirname(File.expand_path(file_path))
+    root_dir = File.expand_path(backup_path)
+
+    begin
+      relative_dir = Pathname.new(file_dir).relative_path_from(Pathname.new(root_dir)).to_s
+    rescue ArgumentError
+      return './'
+    end
+
+    return './' if relative_dir == '.' || relative_dir.empty?
+
+    depth = relative_dir.split(/[\\\/]+/).reject(&:empty?).length
+    return './' if depth <= 0
+
+    '../' * depth
+  end
+
   def download_file (file_remote_info, http)
     current_encoding = "".encoding
     file_url = file_remote_info[:file_url].encode(current_encoding)
     file_id = file_remote_info[:file_id]
     file_timestamp = file_remote_info[:timestamp]
-    
+
     # sanitize file_id to ensure it is a valid path component
     raw_path_elements = file_id.split('/')
 
@@ -964,7 +1086,7 @@ class WaybackMachineDownloader
         element
       end
     end
-    
+
     current_backup_path = backup_path
 
     if file_id == ""
@@ -1079,7 +1201,7 @@ class WaybackMachineDownloader
     end
     logger
   end
-    
+
   # safely sanitize a file id (or id+timestamp)
   def sanitize_and_prepare_id(raw, file_url)
     return nil if raw.nil?
@@ -1221,65 +1343,85 @@ class WaybackMachineDownloader
           return :skipped_not_found
       end
 
-      request = Net::HTTP::Get.new(URI(wayback_url))
-      request["Connection"] = "keep-alive"
-      request["User-Agent"] = "WaybackMachineDownloader/#{VERSION}"
-      request["Accept-Encoding"] = "gzip, deflate"
+      if HTTPX_AVAILABLE && connection.is_a?(HTTPX::Session)
+        response = connection.get(wayback_url)
 
-      response = connection.request(request)
+        raise response.error if response.is_a?(HTTPX::ErrorResponse)
 
-      save_response_body = lambda do
-        File.open(file_path, "wb") do |file|
-          body = response.body
-          if response['content-encoding'] == 'gzip' && body && !body.empty?
-            begin
-              gz = Zlib::GzipReader.new(StringIO.new(body))
-              decompressed_body = gz.read
-              gz.close
-              file.write(decompressed_body)
-            rescue Zlib::GzipFile::Error => e
-              @logger.warn("Failure decompressing gzip file #{file_url}: #{e.message}. Writing raw body.")
-              file.write(body)
-            end
+        code = response.status
+
+        save_response_body = lambda do
+          if response.respond_to?(:copy_to)
+            response.copy_to(file_path)
           else
-            file.write(body) if body
+            response.body.copy_to(file_path)
+          end
+        end
+      else
+        request = Net::HTTP::Get.new(URI(wayback_url))
+        request["Connection"] = "keep-alive"
+        request["User-Agent"] = "WaybackMachineDownloader/#{VERSION}"
+        request["Accept-Encoding"] = "gzip, deflate"
+
+        response = connection.request(request)
+        code = response.code.to_i
+
+        save_response_body = lambda do
+          File.open(file_path, "wb") do |file|
+            body = response.body
+            if response['content-encoding'] == 'gzip' && body && !body.empty?
+              begin
+                gz = Zlib::GzipReader.new(StringIO.new(body))
+                decompressed_body = gz.read
+                gz.close
+                file.write(decompressed_body)
+              rescue Zlib::GzipFile::Error => e
+                @logger.warn("Failure decompressing gzip file #{file_url}: #{e.message}. Writing raw body.")
+                file.write(body)
+              end
+            else
+              file.write(body) if body
+            end
           end
         end
       end
 
       if @all
-        case response
-        when Net::HTTPSuccess, Net::HTTPRedirection, Net::HTTPClientError, Net::HTTPServerError
+        case code
+        when 200..599
           save_response_body.call
-          if response.is_a?(Net::HTTPRedirection)
-            @logger.info("Saved redirect page for #{file_url} (status #{response.code}).")
-          elsif response.is_a?(Net::HTTPClientError) || response.is_a?(Net::HTTPServerError)
-            @logger.info("Saved error page for #{file_url} (status #{response.code}).")
+          if (300..399).cover?(code)
+            @logger.info("Saved redirect page for #{file_url} (status #{code}).")
+          elsif (400..599).cover?(code)
+            @logger.info("Saved error page for #{file_url} (status #{code}).")
           end
           return :saved
         else
           # for any other response type when --all is true, treat as an error to be retried or failed
-          raise "Unhandled HTTP response: #{response.code} #{response.message}"
+          raise "Unhandled HTTP response: #{code}"
         end
       else # not @all (our default behavior)
-        case response
-        when Net::HTTPSuccess
+        if (200..299).cover?(code)
           save_response_body.call
           return :saved
-        when Net::HTTPRedirection
+        elsif (300..399).cover?(code)
           raise "Too many redirects for #{file_url}" if redirect_count >= 5
-          location = response['location']
+          location = if HTTPX_AVAILABLE && connection.is_a?(HTTPX::Session)
+                       response.headers['location']
+                     else
+                       response['location']
+                     end
           @logger.warn("Redirect found for #{file_url} -> #{location}")
           redirected_source = resolve_redirect_source(file_url, location)
           return download_with_retry(file_path, redirected_source, file_timestamp, connection, redirect_count + 1)
-        when Net::HTTPTooManyRequests
+        elsif code == 429
           sleep(RATE_LIMIT * 2)
           raise "Rate limited, retrying..."
-        when Net::HTTPNotFound
+        elsif code == 404
           @logger.warn("File not found, skipping: #{file_url}")
           return :skipped_not_found
         else
-          raise "HTTP Error: #{response.code} #{response.message}"
+          raise "HTTP Error: #{code}"
         end
       end
 
@@ -1297,6 +1439,7 @@ class WaybackMachineDownloader
   end
 
   def cleanup
+    flush_db
     @connection_pool.shutdown
 
     if @failed_downloads.any?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wayback_machine_downloader_straw
 version: !ruby/object:Gem::Version
-  version: 2.4.7
+  version: 2.4.8
 platform: ruby
 authors:
 - strawberrymaster
@@ -94,7 +94,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Download an entire website from the Wayback Machine.
 test_files: []