wayback_machine_downloader_straw 2.4.9 → 2.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cdb773f6bbfb5404d827d6c484f4b8aa0ba2b1c4aed192c86ba12f137526f814
-  data.tar.gz: a8a898bdf0640c40e6946d78e3135bb04e8fa0f76ab14a267448a514a503af2f
+  metadata.gz: 9b8fee9861e93b51713311686022ec251db82148f071d8f8a9dc697dec67ccba
+  data.tar.gz: 2046e2f70156b107bb5c9f4d14eff5d1aab93b9c37be20ef8c88191ba6218447
 SHA512:
-  metadata.gz: d608cf38715e49fb50bcd6aaa6b692556ef9ca0fe3c36894e0738e2e8c2487dd99d04cf7f242a053b96ced3cd7e2cf92c455f83a83113102b1dfb926c914fb58
-  data.tar.gz: 96b2ca1e63d659936a25423717aad49e8f2fa342040a4564cf425c76766dc025f09ef32b9821737a0bec8127cfee6328ecba3717286ff59c1b15c84a713752a3
+  metadata.gz: 21a807f64af7eadc35b7c964124ff2c77455199313baa1d8379e99885f25788c370552e1b04202c743d924bb52c481d99367d8a570586fba9277088c41312720
+  data.tar.gz: 66f4dd29392ccc379f5747c2292afb3cef0836df7337910e2e060f5f06f4968de33b9975600ac51bb168739985d13066f13e66a9c949bd7a0439735f07cffb88

data/lib/wayback_machine_downloader/archive_api.rb

@@ -104,7 +104,7 @@ module ArchiveAPI
         body = response.body.to_s.strip
       else
         request = Net::HTTP::Get.new(request_url)
-        request["User-Agent"] = "wmd-straw/#{WaybackMachineDownloader::VERSION rescue '2.4.8'}"
+        request["User-Agent"] = "wmd-straw/#{WaybackMachineDownloader::VERSION rescue '2.5.0'}"
         request["Connection"] = "keep-alive"
         request["Accept-Encoding"] = "gzip, deflate"
         response = http.request(request)

data/lib/wayback_machine_downloader/page_requisites.rb

@@ -1,3 +1,5 @@
+require 'cgi'
+
 module PageRequisites
   # regex to find links in href, src, url(), and srcset
   # this ignores data: URIs, mailto:, and anchors
@@ -17,6 +19,7 @@ module PageRequisites
       attribute, attr_url, css_url, srcset_url = match
       url = attr_url || css_url || srcset_url
       next unless url
+      url = CGI.unescapeHTML(url).strip
 
       # href is also used for navigation. Do not turn links to server-rendered
       # pages into prerequisite CDX lookups; doing so can multiply API traffic
@@ -40,7 +43,7 @@ module PageRequisites
   def self.page_url?(url)
     path = begin
       URI.parse(url).path
-    rescue URI::InvalidURIError
+    rescue URI::Error
       url.to_s.split(/[?#]/, 2).first
     end
 
@@ -49,8 +52,9 @@ module PageRequisites
   end
 
   def self.valid_asset?(url)
-    return false if url.strip.empty?
-    return false if url.start_with?('data:', 'mailto:', '#', 'javascript:')
+    url = url.to_s.strip
+    return false if url.empty?
+    return false if url.match?(/\A(?:data|mailto|javascript):/i) || url.start_with?('#')
     true
   end
 end

data/lib/wayback_machine_downloader/url_rewrite.rb

@@ -13,26 +13,26 @@ module URLRewrite
     target_host = current_host
 
     # rewrite URLs to relative paths
-    content = content.gsub(/(\s(?:href|src|action|data-src|data-url)=["'])https?:\/\/web\.archive\.org\/web\/\d+[a-z_]*\/https?:\/\/[^\/"']+(\/[^"']*)?(["'])/i) do
-      prefix, path, suffix = $1, ($2 || "/index.html"), $3
+    content = content.gsub(/(\s(?:href|src|action|data-src|data-url)=["'])([ \t\r\n]*)https?:\/\/web\.archive\.org\/web\/\d+[a-z_]*\/https?:\/\/[^\/"'\s]+(\/[^"'\s]*)?([ \t\r\n]*)(["'])/i) do
+      prefix, leading_space, path, trailing_space, suffix = $1, $2, ($3 || "/index.html"), $4, $5
       local = normalize_path_for_local(path, root_prefix)
-      "#{prefix}#{local}#{suffix}"
+      "#{prefix}#{leading_space}#{local}#{trailing_space}#{suffix}"
     end
 
     # rewrite absolute URLs to same domain as relative
     if target_host && !target_host.empty?
-      content = content.gsub(/(\s(?:href|src|action|data-src|data-url)=["'])https?:\/\/#{Regexp.escape(target_host)}(?::\d+)?(\/[^"']*)?(["'])/i) do
-        prefix, path, suffix = $1, ($2 || "/index.html"), $3
+      content = content.gsub(/(\s(?:href|src|action|data-src|data-url)=["'])([ \t\r\n]*)https?:\/\/#{Regexp.escape(target_host)}(?::\d+)?(\/[^"'\s]*)?([ \t\r\n]*)(["'])/i) do
+        prefix, leading_space, path, trailing_space, suffix = $1, $2, ($3 || "/index.html"), $4, $5
         local = normalize_path_for_local(path, root_prefix)
-        "#{prefix}#{local}#{suffix}"
+        "#{prefix}#{leading_space}#{local}#{trailing_space}#{suffix}"
       end
     end
 
     # rewrite root-relative URLs
-    content = content.gsub(/(\s(?:href|src|action|data-src|data-url)=["'])\/([^"'\/][^"']*)(["'])/i) do
-      prefix, path, suffix = $1, $2, $3
+    content = content.gsub(/(\s(?:href|src|action|data-src|data-url)=["'])([ \t\r\n]*)\/([^"'\/\s][^"'\s]*?)([ \t\r\n]*)(["'])/i) do
+      prefix, leading_space, path, trailing_space, suffix = $1, $2, $3, $4, $5
       local = normalize_path_for_local("/#{path}", root_prefix)
-      "#{prefix}#{local}#{suffix}"
+      "#{prefix}#{leading_space}#{local}#{trailing_space}#{suffix}"
     end
 
     content
@@ -69,10 +69,10 @@ module URLRewrite
 
   def rewrite_js_urls(content, root_prefix = './')
     # rewrite archive.org URLs in JavaScript strings
-    content.gsub(/(["'])https?:\/\/web\.archive\.org\/web\/\d+[a-z_]*\/https?:\/\/[^\/"']+(\/[^"']*)?(["'])/i) do
-      quote_start, path, quote_end = $1, ($2 || "/index.html"), $3
+    content.gsub(/(["'])([ \t\r\n]*)https?:\/\/web\.archive\.org\/web\/\d+[a-z_]*\/https?:\/\/[^\/"'\s]+(\/[^"'\s]*)?([ \t\r\n]*)(["'])/i) do
+      quote_start, leading_space, path, trailing_space, quote_end = $1, $2, ($3 || "/index.html"), $4, $5
       local = normalize_path_for_local(path, root_prefix)
-      "#{quote_start}#{local}#{quote_end}"
+      "#{quote_start}#{leading_space}#{local}#{trailing_space}#{quote_end}"
     end
   end
 

data/lib/wayback_machine_downloader.rb

@@ -172,7 +172,7 @@ class WaybackMachineDownloader
   include SubdomainProcessor
   include URLRewrite
 
-  VERSION = "2.4.9"
+  VERSION = "2.5.0"
   DEFAULT_TIMEOUT = 30
   MAX_RETRIES = 3
   RETRY_DELAY = 2

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wayback_machine_downloader_straw
 version: !ruby/object:Gem::Version
-  version: 2.4.9
+  version: 2.5.0
 platform: ruby
 authors:
 - strawberrymaster
@@ -94,7 +94,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.16
+rubygems_version: 4.0.20
 specification_version: 4
 summary: Download an entire website from the Wayback Machine.
 test_files: []