wayback_machine_downloader_straw 2.3.5 → 2.3.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d04d7ddf4b722425377ce84ad5c0f917e35553e38edb755c029ae7a2b8f8055d
-  data.tar.gz: 3d93f41ef2ba3b366a3adf071b947eabb66caf931d022b7ad8a521d3930dfe27
+  metadata.gz: b739c4ecda1e325f9d5a33872fa71a8a5103f1770cc18c7e1b46516c96c8fef6
+  data.tar.gz: 991cf1f67783f35a8da233e6d9e82edc4d933ef0229d5ecffbe8963c5d049c98
 SHA512:
-  metadata.gz: 312dcd879a3589aa0a75d47296dbe04920eee06ae2fb83c18274d918d12517c8a2064c1e9e7a3c774e656636187cd5fc510f249ae71ea6a319cb437ee8d0314b
-  data.tar.gz: 8cd5dcd421077405f920a8ff966387817450e6169aa9578a9b5f25284d4100f1f5bf77304cda727f975464782064643c96d3ee0985a2b57e16aec523c9e17429
+  metadata.gz: f9b71d59d4c5c5bdb82f58fceacd848242a34b12d15abf93c101e4d61ab8fcab46e60011b80f966b0851474160af153c92ab46db5ed2c2e80b0fec3afdc53f8c
+  data.tar.gz: 88f39d47bb8405f682ddca4236bd2e3ce93ffbfd426c2430532b904c98e7cb1593406271fa4453847ab95615adbffc36049072bd7c8b45b171e2cecb77bb41ab

data/bin/wayback_machine_downloader

@@ -62,6 +62,10 @@ option_parser = OptionParser.new do |opts|
     options[:rewritten] = true
   end
 
+  opts.on("--local", "Rewrite URLs to make them relative for local browsing") do |t|
+    options[:rewrite] = true
+  end
+
   opts.on("--reset", "Delete state files (.cdx.json, .downloaded.txt) and restart the download from scratch") do |t|
     options[:reset] = true
   end

data/lib/wayback_machine_downloader.rb

@@ -113,7 +113,7 @@ class WaybackMachineDownloader
 
   include ArchiveAPI
 
-  VERSION = "2.3.5"
+  VERSION = "2.3.7"
   DEFAULT_TIMEOUT = 30
   MAX_RETRIES = 3
   RETRY_DELAY = 2
@@ -125,7 +125,7 @@ class WaybackMachineDownloader
 
   attr_accessor :base_url, :exact_url, :directory, :all_timestamps,
     :from_timestamp, :to_timestamp, :only_filter, :exclude_filter,
-    :all, :maximum_pages, :threads_count, :logger, :reset, :keep
+    :all, :maximum_pages, :threads_count, :logger, :reset, :keep, :rewrite
 
   def initialize params
     validate_params(params)
@@ -148,6 +148,7 @@ class WaybackMachineDownloader
     @failed_downloads = Concurrent::Array.new
     @connection_pool = ConnectionPool.new(CONNECTION_POOL_SIZE)
     @db_mutex = Mutex.new
+    @rewrite = params[:rewrite] || false
 
     handle_reset
   end
@@ -476,8 +477,8 @@ class WaybackMachineDownloader
         begin
           @connection_pool.with_connection do |connection|
             result_message = download_file(file_remote_info, connection)
-            # for now, assume success if no exception and message doesn't indicate error/skip
-            if result_message && !result_message.downcase.include?('error') && !result_message.downcase.include?('failed') && !result_message.downcase.include?('skipping') && !result_message.downcase.include?('already exists')
+            # assume download success if the result message contains ' -> '
+            if result_message && result_message.include?(' -> ')
                download_success = true
             end
             @download_mutex.synchronize do
@@ -533,6 +534,101 @@ class WaybackMachineDownloader
     end
   end
 
+  def rewrite_urls_to_relative(file_path)
+    return unless File.exist?(file_path)
+    
+    file_ext = File.extname(file_path).downcase
+    
+    begin
+      content = File.binread(file_path)
+
+      if file_ext == '.html' || file_ext == '.htm'
+        encoding = content.match(/<meta\s+charset=["']?([^"'>]+)/i)&.captures&.first || 'UTF-8'
+        content.force_encoding(encoding) rescue content.force_encoding('UTF-8')
+      else
+        content.force_encoding('UTF-8')
+      end
+
+      # URLs in HTML attributes
+      content.gsub!(/(\s(?:href|src|action|data-src|data-url)=["'])https?:\/\/web\.archive\.org\/web\/[0-9]+(?:id_)?\/([^"']+)(["'])/i) do
+        prefix, url, suffix = $1, $2, $3
+        
+        if url.start_with?('http')
+          begin
+            uri = URI.parse(url)
+            path = uri.path
+            path = path[1..-1] if path.start_with?('/')
+            "#{prefix}#{path}#{suffix}"
+          rescue
+            "#{prefix}#{url}#{suffix}"
+          end
+        elsif url.start_with?('/')
+          "#{prefix}./#{url[1..-1]}#{suffix}"
+        else
+          "#{prefix}#{url}#{suffix}"
+        end
+      end
+      
+      # URLs in CSS
+      content.gsub!(/url\(\s*["']?https?:\/\/web\.archive\.org\/web\/[0-9]+(?:id_)?\/([^"'\)]+)["']?\s*\)/i) do
+        url = $1
+        
+        if url.start_with?('http')
+          begin
+            uri = URI.parse(url)
+            path = uri.path
+            path = path[1..-1] if path.start_with?('/')
+            "url(\"#{path}\")"
+          rescue
+            "url(\"#{url}\")"
+          end
+        elsif url.start_with?('/')
+          "url(\"./#{url[1..-1]}\")"
+        else
+          "url(\"#{url}\")"
+        end
+      end
+      
+      # URLs in JavaScript
+      content.gsub!(/(["'])https?:\/\/web\.archive\.org\/web\/[0-9]+(?:id_)?\/([^"']+)(["'])/i) do
+        quote_start, url, quote_end = $1, $2, $3
+        
+        if url.start_with?('http')
+          begin
+            uri = URI.parse(url)
+            path = uri.path
+            path = path[1..-1] if path.start_with?('/')
+            "#{quote_start}#{path}#{quote_end}"
+          rescue
+            "#{quote_start}#{url}#{quote_end}"
+          end
+        elsif url.start_with?('/')
+          "#{quote_start}./#{url[1..-1]}#{quote_end}"
+        else
+          "#{quote_start}#{url}#{quote_end}"
+        end
+      end
+      
+      # for URLs in HTML attributes that start with a single slash
+      content.gsub!(/(\s(?:href|src|action|data-src|data-url)=["'])\/([^"'\/][^"']*)(["'])/i) do
+        prefix, path, suffix = $1, $2, $3
+        "#{prefix}./#{path}#{suffix}"
+      end
+      
+      # for URLs in CSS that start with a single slash
+      content.gsub!(/url\(\s*["']?\/([^"'\)\/][^"'\)]*?)["']?\s*\)/i) do
+        path = $1
+        "url(\"./#{path}\")"
+      end
+
+      # save the modified content back to the file
+      File.binwrite(file_path, content)
+      puts "Rewrote URLs in #{file_path} to be relative."
+    rescue Errno::ENOENT => e
+      @logger.warn("Error reading file #{file_path}: #{e.message}")
+    end
+  end
+
   def download_file (file_remote_info, http)
     current_encoding = "".encoding
     file_url = file_remote_info[:file_url].encode(current_encoding)
@@ -563,11 +659,24 @@ class WaybackMachineDownloader
 
     begin
       structure_dir_path dir_path
-      download_with_retry(file_path, file_url, file_timestamp, http)
-      "#{file_url} -> #{file_path} (#{@processed_file_count + 1}/#{@total_to_download})"
+      status = download_with_retry(file_path, file_url, file_timestamp, http)
+
+      case status
+      when :saved
+        if @rewrite && File.extname(file_path) =~ /\.(html?|css|js)$/i
+          rewrite_urls_to_relative(file_path)
+        end
+        "#{file_url} -> #{file_path} (#{@processed_file_count + 1}/#{@total_to_download})"
+      when :skipped_not_found
+        "Skipped (not found): #{file_url} (#{@processed_file_count + 1}/#{@total_to_download})"
+      else
+        # ideally, this case should not be reached if download_with_retry behaves as expected.
+        @logger.warn("Unknown status from download_with_retry for #{file_url}: #{status}")
+        "Unknown status for #{file_url}: #{status} (#{@processed_file_count + 1}/#{@total_to_download})"
+      end
     rescue StandardError => e
       msg = "Failed: #{file_url} # #{e} (#{@processed_file_count + 1}/#{@total_to_download})"
-      if not @all and File.exist?(file_path) and File.size(file_path) == 0
+      if File.exist?(file_path) and File.size(file_path) == 0
         File.delete(file_path)
         msg += "\n#{file_path} was empty and was removed."
       end
@@ -615,8 +724,7 @@ class WaybackMachineDownloader
 
       response = connection.request(request)
 
-      case response
-      when Net::HTTPSuccess
+      save_response_body = lambda do
         File.open(file_path, "wb") do |file|
           body = response.body
           if response['content-encoding'] == 'gzip' && body && !body.empty?
@@ -626,26 +734,48 @@ class WaybackMachineDownloader
               gz.close
               file.write(decompressed_body)
             rescue Zlib::GzipFile::Error => e
-              @logger.warn("Failure decompressing gzip file #{file_url}: #{e.message}")
+              @logger.warn("Failure decompressing gzip file #{file_url}: #{e.message}. Writing raw body.")
               file.write(body)
             end
           else
             file.write(body) if body
           end
         end
-      when Net::HTTPRedirection
-        raise "Too many redirects for #{file_url}" if redirect_count >= 2
-        location = response['location']
-        @logger.warn("Redirect found for #{file_url} -> #{location}")
-        return download_with_retry(file_path, location, file_timestamp, connection, redirect_count + 1)
-      when Net::HTTPTooManyRequests
-        sleep(RATE_LIMIT * 2)
-        raise "Rate limited, retrying..."
-      when Net::HTTPNotFound
-        @logger.warn("File not found, skipping: #{file_url}")
-        return
-      else
-        raise "HTTP Error: #{response.code} #{response.message}"
+      end
+
+      if @all
+        case response
+        when Net::HTTPSuccess, Net::HTTPRedirection, Net::HTTPClientError, Net::HTTPServerError
+          save_response_body.call
+          if response.is_a?(Net::HTTPRedirection)
+            @logger.info("Saved redirect page for #{file_url} (status #{response.code}).")
+          elsif response.is_a?(Net::HTTPClientError) || response.is_a?(Net::HTTPServerError)
+            @logger.info("Saved error page for #{file_url} (status #{response.code}).")
+          end
+          return :saved
+        else
+          # for any other response type when --all is true, treat as an error to be retried or failed
+          raise "Unhandled HTTP response: #{response.code} #{response.message}"
+        end
+      else # not @all (our default behavior)
+        case response
+        when Net::HTTPSuccess
+          save_response_body.call
+          return :saved
+        when Net::HTTPRedirection
+          raise "Too many redirects for #{file_url}" if redirect_count >= 2
+          location = response['location']
+          @logger.warn("Redirect found for #{file_url} -> #{location}")
+          return download_with_retry(file_path, location, file_timestamp, connection, redirect_count + 1)
+        when Net::HTTPTooManyRequests
+          sleep(RATE_LIMIT * 2)
+          raise "Rate limited, retrying..."
+        when Net::HTTPNotFound
+          @logger.warn("File not found, skipping: #{file_url}")
+          return :skipped_not_found
+        else
+          raise "HTTP Error: #{response.code} #{response.message}"
+        end
       end
 
     rescue StandardError => e

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: wayback_machine_downloader_straw
 version: !ruby/object:Gem::Version
-  version: 2.3.5
+  version: 2.3.7
 platform: ruby
 authors:
 - strawberrymaster
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2025-04-30 00:00:00.000000000 Z
+date: 2025-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -86,7 +86,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 1.9.2
+      version: 3.4.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="