archaeo 0.2.7 → 0.2.8

data/lib/archaeo/url_rewriter.rb

@@ -8,16 +8,32 @@ module Archaeo
   # Used for saving archived pages and their assets for offline
   # browsing. Converts absolute archive URLs into relative paths
   # rooted at a configurable local directory.
+  #
+  # Supports HTML attributes, srcset, inline styles, CSS url(),
+  # JavaScript string URLs, and server-side extension handling.
   class UrlRewriter
-    URL_ATTRS = %w[src href data-src poster].freeze
+    URL_ATTRS = %w[src href data-src data-url poster action].freeze
     CSS_URL_RE = /url\(\s*['"]?([^'")\s]+)['"]?\s*\)/
+    ARCHIVE_RE = %r{https?://web\.archive\.org/web/\d+(?:id_)?/}
+    JS_URL_RE = /['"](https?:\/\/web\.archive\.org\/web\/\d+(?:id_)?\/[^'"]+)['"]/
+
+    SERVER_EXTENSIONS = %w[.php .asp .aspx .jsp .cgi .pl .py].freeze
 
-    def initialize(archive_prefix, local_prefix)
+    def initialize(archive_prefix, local_prefix,
+                   rewrite_js: false, rewrite_absolute: false,
+                   server_extensions: false)
       @archive_prefix = archive_prefix.to_s
       @local_prefix = local_prefix.to_s
+      @rewrite_js = rewrite_js
+      @rewrite_absolute = rewrite_absolute
+      @server_extensions = server_extensions
     end
 
     def rewrite(url)
+      if @rewrite_absolute && url.match?(ARCHIVE_RE)
+        return rewrite_absolute_url(url)
+      end
+
       return url unless url.start_with?(@archive_prefix)
 
       relative = url.sub(@archive_prefix, "")
@@ -37,13 +53,39 @@ module Archaeo
       doc.to_html
     end
 
+    def rewrite_js(js_content)
+      return js_content unless @rewrite_js
+
+      js_content.gsub(JS_URL_RE) do
+        quote = Regexp.last_match[0][0]
+        url = Regexp.last_match[1]
+        rewritten = rewrite(url)
+        "#{quote}#{rewritten}#{quote}"
+      end
+    end
+
+    def rewrite_css(css_content)
+      css_content.gsub(CSS_URL_RE) do
+        url = Regexp.last_match[1]
+        if url.match?(ARCHIVE_RE) || url.start_with?(@archive_prefix)
+          "url('#{rewrite(url)}')"
+        else
+          Regexp.last_match[0]
+        end
+      end
+    end
+
     def rewrite_url_attrs(doc)
       URL_ATTRS.each do |attr|
         doc.css("[#{attr}]").each do |el|
           original = el[attr]
-          next unless original&.start_with?(@archive_prefix)
+          next unless original
 
-          el[attr] = rewrite(original)
+          if @rewrite_absolute && original.match?(ARCHIVE_RE)
+            el[attr] = rewrite_absolute_url(original)
+          elsif original.start_with?(@archive_prefix)
+            el[attr] = rewrite(original)
+          end
         end
       end
     end
@@ -73,8 +115,11 @@ module Archaeo
     def rewrite_css_urls(css_text)
       css_text.gsub(CSS_URL_RE) do
         url = Regexp.last_match[1]
-        rewritten = url.start_with?(@archive_prefix) ? rewrite(url) : url
-        "url('#{rewritten}')"
+        if url.match?(ARCHIVE_RE) || url.start_with?(@archive_prefix)
+          "url('#{rewrite(url)}')"
+        else
+          Regexp.last_match[0]
+        end
       end
     end
 
@@ -85,9 +130,21 @@ module Archaeo
         parts = entry.strip.split(/\s+/, 2)
         url = parts[0]
         descriptor = parts[1]
-        rewritten = url.start_with?(@archive_prefix) ? rewrite(url) : url
+
+        rewritten = if @rewrite_absolute && url.match?(ARCHIVE_RE)
+                      rewrite_absolute_url(url)
+                    elsif url.start_with?(@archive_prefix)
+                      rewrite(url)
+                    else
+                      url
+                    end
         descriptor ? "#{rewritten} #{descriptor}" : rewritten
       end.join(", ")
     end
+
+    def rewrite_absolute_url(url)
+      rest = url.sub(ARCHIVE_RE, "")
+      File.join(@local_prefix, rest)
+    end
   end
 end

data/lib/archaeo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Archaeo
-  VERSION = "0.2.7"
+  VERSION = "0.2.8"
 end

data/lib/archaeo.rb

@@ -48,4 +48,11 @@ module Archaeo
   autoload :Fetcher, "archaeo/fetcher"
   autoload :BulkDownloader, "archaeo/bulk_downloader"
   autoload :Cli, "archaeo/cli"
+  autoload :EncodingDetector, "archaeo/encoding_detector"
+  autoload :PathSanitizer, "archaeo/path_sanitizer"
+  autoload :RateLimiter, "archaeo/rate_limiter"
+  autoload :PatternFilter, "archaeo/pattern_filter"
+  autoload :CdxCache, "archaeo/cdx_cache"
+  autoload :SubdomainDiscovery, "archaeo/subdomain_discovery"
+  autoload :ArchiveHealthCheck, "archaeo/archive_health_check"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: archaeo
 version: !ruby/object:Gem::Version
-  version: 0.2.7
+  version: 0.2.8
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-05-11 00:00:00.000000000 Z
+date: 2026-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: csv
@@ -71,6 +71,7 @@ files:
 - bin/setup
 - exe/archaeo
 - lib/archaeo.rb
+- lib/archaeo/archive_health_check.rb
 - lib/archaeo/archive_url.rb
 - lib/archaeo/asset_extractor.rb
 - lib/archaeo/asset_list.rb
@@ -78,17 +79,23 @@ files:
 - lib/archaeo/availability_result.rb
 - lib/archaeo/bulk_downloader.rb
 - lib/archaeo/cdx_api.rb
+- lib/archaeo/cdx_cache.rb
 - lib/archaeo/cdx_filter.rb
 - lib/archaeo/cdx_timeline.rb
 - lib/archaeo/cli.rb
 - lib/archaeo/download_state.rb
+- lib/archaeo/encoding_detector.rb
 - lib/archaeo/fetcher.rb
 - lib/archaeo/http_client.rb
 - lib/archaeo/page.rb
 - lib/archaeo/page_bundle.rb
+- lib/archaeo/path_sanitizer.rb
+- lib/archaeo/pattern_filter.rb
+- lib/archaeo/rate_limiter.rb
 - lib/archaeo/save_api.rb
 - lib/archaeo/save_result.rb
 - lib/archaeo/snapshot.rb
+- lib/archaeo/subdomain_discovery.rb
 - lib/archaeo/timestamp.rb
 - lib/archaeo/url_normalizer.rb
 - lib/archaeo/url_rewriter.rb