wayback_machine_downloader_straw 2.3.7 → 2.3.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b739c4ecda1e325f9d5a33872fa71a8a5103f1770cc18c7e1b46516c96c8fef6
-  data.tar.gz: 991cf1f67783f35a8da233e6d9e82edc4d933ef0229d5ecffbe8963c5d049c98
+  metadata.gz: 1df8b0394ed1e39ab2d1ef050c51c809afbc0005533e286b83389bab99a7507c
+  data.tar.gz: 500212ad8353f9252ade813c5e405e9bc0fa1f6ae4967dcfee36686caecff0fa
 SHA512:
-  metadata.gz: f9b71d59d4c5c5bdb82f58fceacd848242a34b12d15abf93c101e4d61ab8fcab46e60011b80f966b0851474160af153c92ab46db5ed2c2e80b0fec3afdc53f8c
-  data.tar.gz: 88f39d47bb8405f682ddca4236bd2e3ce93ffbfd426c2430532b904c98e7cb1593406271fa4453847ab95615adbffc36049072bd7c8b45b171e2cecb77bb41ab
+  metadata.gz: c5b855e47b617217591f385b6aeb5afd55631e10acaa39d1acf0929603717e68f350457fea2e184f17e338baeb4fd1cb6ba9ce6632967c56a80c8a66c2fbae87
+  data.tar.gz: b2c13318213a68827d47e73b8fba54ad2d52f49f73ea76726207f80d2db3bb9b77bb349980b823ac12a42f1ec1ba7705eb06affdf579970945eb655c07c955f2

data/lib/wayback_machine_downloader/archive_api.rb

@@ -4,6 +4,14 @@ require 'uri'
 module ArchiveAPI
 
   def get_raw_list_from_api(url, page_index, http)
+    # Automatically append /* if the URL doesn't contain a path after the domain
+    # This is a workaround for an issue with the API and *some* domains.
+    # See https://github.com/StrawberryMaster/wayback-machine-downloader/issues/6
+    # But don't do this when exact_url flag is set
+    if url && !url.match(/^https?:\/\/.*\//i) && !@exact_url
+      url = "#{url}/*"
+    end
+
     request_url = URI("https://web.archive.org/cdx/search/cdx")
     params = [["output", "json"], ["url", url]] + parameters_for_api(page_index)
     request_url.query = URI.encode_www_form(params)

data/lib/wayback_machine_downloader.rb

@@ -113,7 +113,7 @@ class WaybackMachineDownloader
 
   include ArchiveAPI
 
-  VERSION = "2.3.7"
+  VERSION = "2.3.9"
   DEFAULT_TIMEOUT = 30
   MAX_RETRIES = 3
   RETRY_DELAY = 2
@@ -131,7 +131,11 @@ class WaybackMachineDownloader
     validate_params(params)
     @base_url = params[:base_url]
     @exact_url = params[:exact_url]
-    @directory = params[:directory]
+    if params[:directory]
+      @directory = File.expand_path(params[:directory])
+    else
+      @directory = nil
+    end
     @all_timestamps = params[:all_timestamps]
     @from_timestamp = params[:from_timestamp].to_i
     @to_timestamp = params[:to_timestamp].to_i
@@ -154,22 +158,22 @@ class WaybackMachineDownloader
   end
 
   def backup_name
-    if @base_url.include? '//'
-      @base_url.split('/')[2]
+    url_to_process = @base_url.end_with?('/*') ? @base_url.chomp('/*') : @base_url
+    
+    if url_to_process.include? '//'
+      url_to_process.split('/')[2]
     else
-      @base_url
+      url_to_process
     end
   end
 
   def backup_path
     if @directory
-      if @directory[-1] == '/'
-        @directory
-      else
-        @directory + '/'
-      end
+      # because @directory is already an absolute path, we just ensure it exists
+      @directory
     else
-      'websites/' + backup_name + '/'
+      # ensure the default path is absolute and normalized
+      File.expand_path(File.join('websites', backup_name))
     end
   end
 
@@ -241,6 +245,7 @@ class WaybackMachineDownloader
     # Fetch the initial set of snapshots, sequentially
     @connection_pool.with_connection do |connection|
       initial_list = get_raw_list_from_api(@base_url, nil, connection)
+      initial_list ||= []
       mutex.synchronize do
         snapshot_list_to_consider.concat(initial_list)
         print "."
@@ -265,6 +270,7 @@ class WaybackMachineDownloader
             @connection_pool.with_connection do |connection|
               result = get_raw_list_from_api("#{@base_url}/*", page, connection)
             end
+            result ||= []
             [page, result]
           end
         end
@@ -284,7 +290,7 @@ class WaybackMachineDownloader
 
         # Process results and check for empty pages
         results.each do |page, result|
-          if result.empty?
+          if result.nil? || result.empty?
             continue_fetching = false
             break
           else
@@ -634,21 +640,35 @@ class WaybackMachineDownloader
     file_url = file_remote_info[:file_url].encode(current_encoding)
     file_id = file_remote_info[:file_id]
     file_timestamp = file_remote_info[:timestamp]
-    file_path_elements = file_id.split('/')
+    
+    # sanitize file_id to ensure it is a valid path component
+    raw_path_elements = file_id.split('/')
+
+    sanitized_path_elements = raw_path_elements.map do |element|
+      if Gem.win_platform?
+        # for Windows, we need to sanitize path components to avoid invalid characters
+        # this prevents issues with file names that contain characters not allowed in
+        # Windows file systems. See # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions
+        element.gsub(/[:\*?"<>\|\&\=\/\\]/, ->(match) { '%' + match.ord.to_s(16).upcase })
+      else
+        element
+      end
+    end
+    
+    current_backup_path = backup_path
 
     if file_id == ""
-      dir_path = backup_path
-      file_path = backup_path + 'index.html'
-    elsif file_url[-1] == '/' or not file_path_elements[-1].include? '.'
-      dir_path = backup_path + file_path_elements[0..-1].join('/')
-      file_path = backup_path + file_path_elements[0..-1].join('/') + '/index.html'
+      dir_path = current_backup_path
+      file_path = File.join(dir_path, 'index.html')
+    elsif file_url[-1] == '/' || (sanitized_path_elements.last && !sanitized_path_elements.last.include?('.'))
+      # if file_id is a directory, we treat it as such
+      dir_path = File.join(current_backup_path, *sanitized_path_elements)
+      file_path = File.join(dir_path, 'index.html')
     else
-      dir_path = backup_path + file_path_elements[0..-2].join('/')
-      file_path = backup_path + file_path_elements[0..-1].join('/')
-    end
-    if Gem.win_platform?
-      dir_path = dir_path.gsub(/[:*?&=<>\\|]/) {|s| '%' + s.ord.to_s(16) }
-      file_path = file_path.gsub(/[:*?&=<>\\|]/) {|s| '%' + s.ord.to_s(16) }
+      # if file_id is a file, we treat it as such
+      filename = sanitized_path_elements.pop
+      dir_path = File.join(current_backup_path, *sanitized_path_elements)
+      file_path = File.join(dir_path, filename)
     end
 
     # check existence *before* download attempt
@@ -717,6 +737,9 @@ class WaybackMachineDownloader
         "https://web.archive.org/web/#{file_timestamp}id_/#{file_url}"
       end
 
+      # Escape square brackets because they are not valid in URI()
+      wayback_url = wayback_url.gsub('[', '%5B').gsub(']', '%5D')
+
       request = Net::HTTP::Get.new(URI(wayback_url))
       request["Connection"] = "keep-alive"
       request["User-Agent"] = "WaybackMachineDownloader/#{VERSION}"

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: wayback_machine_downloader_straw
 version: !ruby/object:Gem::Version
-  version: 2.3.7
+  version: 2.3.9
 platform: ruby
 authors:
 - strawberrymaster
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2025-05-27 00:00:00.000000000 Z
+date: 2025-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -78,7 +77,6 @@ homepage: https://github.com/StrawberryMaster/wayback-machine-downloader
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -93,8 +91,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.11
-signing_key: 
+rubygems_version: 3.6.2
 specification_version: 4
 summary: Download an entire website from the Wayback Machine.
 test_files: []