archaeo 0.2.7 → 0.2.8

data/lib/archaeo/encoding_detector.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Archaeo
+  # Detects and transcodes content from legacy encodings to UTF-8.
+  #
+  # Tries a configurable list of encodings in priority order,
+  # returning the first that produces valid output. Used as a
+  # fallback when Content-Type charset and HTML meta charset are
+  # both absent.
+  class EncodingDetector
+    DEFAULT_ENCODINGS = [
+      Encoding::UTF_8,
+      Encoding::Windows_1251,
+      Encoding::GB18030,
+      Encoding::Shift_JIS,
+      Encoding::EUC_KR,
+      Encoding::ISO_8859_1,
+      Encoding::Windows_1252,
+    ].freeze
+
+    BINARY_THRESHOLD = 0.1
+    TEXT_CONTROL_BYTES = [0x09, 0x0A, 0x0D].freeze
+
+    def initialize(encodings: DEFAULT_ENCODINGS)
+      @encodings = encodings
+    end
+
+    def detect(bytes)
+      return Encoding::UTF_8 if bytes.nil? || bytes.empty?
+
+      string = bytes_to_string(bytes)
+
+      @encodings.each do |enc|
+        return enc if valid_in_encoding?(string, enc)
+      end
+
+      Encoding::UTF_8
+    end
+
+    def transcode(bytes, fallback: Encoding::UTF_8)
+      return "" if bytes.nil? || bytes.empty?
+
+      string = bytes.is_a?(String) ? bytes.dup : bytes.to_s
+      return string if string.encoding == Encoding::UTF_8 && string.valid_encoding?
+
+      binary = bytes_to_string(bytes)
+      detected = detect(bytes)
+      return binary.force_encoding(Encoding::UTF_8) if detected == Encoding::UTF_8
+
+      encode_to_utf8(binary, detected, fallback)
+    end
+
+    def binary?(bytes)
+      return false if bytes.nil? || bytes.empty?
+
+      sample = bytes.byteslice(0, [bytes.bytesize, 4096].min)
+      non_printable = sample.bytes.count do |b|
+        b < 0x20 && !TEXT_CONTROL_BYTES.include?(b)
+      end
+      non_printable.to_f / sample.bytesize > BINARY_THRESHOLD
+    end
+
+    private
+
+    def bytes_to_string(bytes)
+      case bytes
+      when String then bytes.dup.force_encoding(Encoding::ASCII_8BIT)
+      else bytes.to_s.force_encoding(Encoding::ASCII_8BIT)
+      end
+    end
+
+    def valid_in_encoding?(string, encoding)
+      candidate = string.dup.force_encoding(encoding)
+      candidate.valid_encoding?
+    rescue StandardError
+      false
+    end
+
+    def encode_to_utf8(string, source_encoding, fallback)
+      candidate = string.dup.force_encoding(source_encoding)
+      candidate.encode(Encoding::UTF_8,
+                       invalid: :replace, undef: :replace,
+                       replace: "?")
+    rescue StandardError
+      string.dup.force_encoding(fallback)
+        .encode(Encoding::UTF_8,
+                invalid: :replace, undef: :replace,
+                replace: "?")
+    end
+  end
+end

data/lib/archaeo/page.rb

@@ -130,7 +130,7 @@ module Archaeo
       html_charset = detect_html_charset
       return Encoding.find(html_charset) if html_charset
 
-      Encoding::UTF_8
+      EncodingDetector.new.detect(@raw_content)
     rescue ArgumentError
       Encoding::UTF_8
     end

data/lib/archaeo/path_sanitizer.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+require "digest"
+require "uri"
+
+module Archaeo
+  # Sanitizes URLs into safe filesystem paths.
+  #
+  # Handles recursive percent-decoding, query string hashing,
+  # segment truncation, and invalid character replacement.
+  class PathSanitizer
+    DEFAULT_MAX_SEGMENT = 200
+    HASH_LENGTH = 8
+    MAX_DECODE_ITERATIONS = 5
+
+    INVALID_CHARS = /[<>:"|?*#]/
+    SEPARATOR_RE = %r{[/\\]}
+
+    attr_reader :max_segment_length
+
+    def initialize(max_segment_length: DEFAULT_MAX_SEGMENT)
+      @max_segment_length = max_segment_length
+    end
+
+    def sanitize(url)
+      path = strip_scheme(url)
+      path = recursive_decode(path)
+      path = hash_query_strings(path)
+      clean_segments(path)
+    end
+
+    def file_id(archive_url)
+      stripped = strip_archive_prefix(archive_url)
+      sanitize(stripped)
+    end
+
+    def segment_for(path_segment)
+      cleaned = path_segment.gsub(INVALID_CHARS, "_")
+      truncate(cleaned)
+    end
+
+    private
+
+    def strip_scheme(url)
+      url.to_s.sub(%r{\Ahttps?://}, "")
+    end
+
+    def strip_archive_prefix(url)
+      url.to_s.sub(%r{\Ahttps?://web\.archive\.org/web/\d+(?:id_)?/}, "")
+        .sub(%r{\Ahttps?://}, "")
+    end
+
+    def recursive_decode(str)
+      MAX_DECODE_ITERATIONS.times do
+        decoded = decode(str)
+        return decoded if decoded == str
+
+        str = decoded
+      end
+      str
+    end
+
+    def decode(str)
+      URI.decode_www_form_component(str)
+    rescue StandardError
+      str
+    end
+
+    def hash_query_strings(path)
+      return path unless path.include?("?")
+
+      base, query = path.split("?", 2)
+      hash = Digest::SHA256.hexdigest(query)[0, HASH_LENGTH]
+      "#{base}_#{hash}"
+    end
+
+    def clean_segments(path)
+      segments = path.split(SEPARATOR_RE).reject(&:empty?)
+      return "" if segments.empty?
+
+      segments.map do |seg|
+        segment_for(seg)
+      end.join(File::SEPARATOR)
+    end
+
+    def truncate(segment)
+      return segment if segment.length <= @max_segment_length
+
+      segment[0, @max_segment_length]
+    end
+  end
+
+  # Resolves file/directory path conflicts in download targets.
+  #
+  # Detects when a file path would block creation of a needed directory
+  # (or vice versa) and resolves by relocating the file.
+  class PathConflictResolver
+    def initialize(base_dir)
+      @base_dir = base_dir
+    end
+
+    def resolve(paths)
+      conflicts = detect_conflicts(paths)
+      relocate_conflicts(conflicts)
+      paths
+    end
+
+    def conflict?(file_path)
+      return false if File.directory?(file_path)
+      return false unless File.file?(file_path)
+
+      File.exist?(file_path) && needs_directory_under?(file_path)
+    end
+
+    private
+
+    def detect_conflicts(paths)
+      conflicts = Set.new
+      paths.each do |path|
+        paths.each do |other|
+          next if path == other
+
+          prefix = path + File::SEPARATOR
+          if other.start_with?(prefix) && File.file?(path)
+            # `path` is a prefix of `other` — if `path` is a file, it blocks `other`
+            conflicts << path
+          end
+        end
+      end
+      conflicts.to_a
+    end
+
+    def relocate_conflicts(conflicts)
+      conflicts.each do |conflict_path|
+        next unless File.file?(conflict_path)
+
+        ext = File.extname(conflict_path)
+        tmp_file = "#{conflict_path}.archaeo_tmp"
+        FileUtils.mv(conflict_path, tmp_file)
+        FileUtils.mkdir_p(conflict_path)
+        new_file = File.join(conflict_path, "index#{ext}")
+        FileUtils.mv(tmp_file, new_file)
+      end
+    end
+
+    def needs_directory_under?(file_path)
+      parent = File.dirname(file_path)
+      children = Dir.glob("#{parent}/*")
+      children.any? { |c| File.directory?(c) }
+    end
+  end
+end

data/lib/archaeo/pattern_filter.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Archaeo
+  # URL pattern filter for include/exclude matching during downloads.
+  #
+  # Supports string substring matching, Regexp objects, and
+  # %r{...} and /.../ string-to-regexp conversion with inline flags.
+  class PatternFilter
+    def initialize(only: nil, exclude: nil)
+      @only_patterns = compile_patterns(Array(only))
+      @exclude_patterns = compile_patterns(Array(exclude))
+    end
+
+    def match?(url)
+      url = url.to_s
+      return false if excluded?(url)
+      return true if @only_patterns.empty?
+
+      included?(url)
+    end
+
+    def reject?(url)
+      !match?(url)
+    end
+
+    def self.to_regex(pattern)
+      case pattern
+      when Regexp then pattern
+      when String then parse_regex_string(pattern)
+      else
+        raise ArgumentError,
+              "Pattern must be String or Regexp, got #{pattern.class}"
+      end
+    end
+
+    private
+
+    def compile_patterns(patterns)
+      patterns.map { |p| self.class.to_regex(p) }
+    end
+
+    def included?(url)
+      @only_patterns.any? { |re| url.match?(re) }
+    end
+
+    def excluded?(url)
+      @exclude_patterns.any? { |re| url.match?(re) }
+    end
+
+    private_class_method def self.parse_regex_string(str)
+      stripped = str.strip
+      if stripped.start_with?("%r{") && stripped.end_with?("}")
+        body = stripped[3..-2]
+        build_regex(body)
+      elsif stripped.start_with?("/") && stripped.end_with?("/")
+        body = stripped[1..-2]
+        build_regex(body)
+      elsif stripped.start_with?("/") && stripped.length > 1
+        last_slash = stripped.rindex("/")
+        body = stripped[1...last_slash]
+        flags = stripped[(last_slash + 1)..]
+        build_regex(body, parse_flags(flags))
+      else
+        Regexp.new(Regexp.escape(stripped))
+      end
+    end
+
+    private_class_method def self.build_regex(body, options = 0)
+      Regexp.new(body, options)
+    end
+
+    private_class_method def self.parse_flags(flags)
+      options = 0
+      options |= Regexp::IGNORECASE if flags.include?("i")
+      options |= Regexp::MULTILINE if flags.include?("m")
+      options |= Regexp::EXTENDED if flags.include?("x")
+      options
+    end
+  end
+end

data/lib/archaeo/rate_limiter.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Archaeo
+  # Thread-safe request rate limiter.
+  #
+  # Enforces a minimum interval between requests to avoid hitting
+  # Wayback Machine rate limits. Supports per-host limiting and
+  # adaptive backoff when 429 responses are received.
+  class RateLimiter
+    DEFAULT_MIN_INTERVAL = 0
+
+    def initialize(min_interval: DEFAULT_MIN_INTERVAL)
+      @min_interval = min_interval.to_f
+      @mutex = Mutex.new
+      @last_request_at = 0.0
+      @host_last = {}
+    end
+
+    def wait(host: nil)
+      return if @min_interval <= 0
+
+      @mutex.synchronize do
+        if host
+          wait_for_host(host)
+        else
+          wait_global
+        end
+      end
+    end
+
+    def backoff(host: nil)
+      @mutex.synchronize do
+        if host
+          key = host.to_sym
+          current = @host_last[key] || @min_interval
+          @host_last[key] = [current * 2, 60].min
+        else
+          @min_interval = [(@min_interval * 2).clamp(0, 60), 60].min
+        end
+      end
+      wait(host: host)
+    end
+
+    def reset(host: nil)
+      @mutex.synchronize do
+        if host
+          @host_last.delete(host.to_sym)
+        else
+          @last_request_at = 0.0
+          @host_last.clear
+        end
+      end
+    end
+
+    def interval
+      @mutex.synchronize { @min_interval }
+    end
+
+    private
+
+    def wait_global
+      elapsed = now - @last_request_at
+      sleep_for(@min_interval - elapsed) if elapsed < @min_interval
+      @last_request_at = now
+    end
+
+    def wait_for_host(host)
+      key = host.to_sym
+      @host_last[key] ||= 0.0
+      host_interval = [@min_interval, 0].max
+      elapsed = now - @host_last[key]
+      sleep_for(host_interval - elapsed) if elapsed < host_interval
+      @host_last[key] = now
+    end
+
+    def sleep_for(seconds)
+      return unless seconds.positive?
+
+      sleep(seconds)
+    end
+
+    def now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/archaeo/save_api.rb

@@ -62,8 +62,13 @@ module Archaeo
 
       ts = Timestamp.parse(extract_timestamp(archive_url))
       cached = ts.to_time < start_time - 2700
-      SaveResult.new(url: url, archive_url: archive_url,
-                     timestamp: ts, cached: cached)
+      SaveResult.new(
+        url: url, archive_url: archive_url,
+        timestamp: ts, cached: cached,
+        status_code: response.status,
+        response_url: response.headers["location"],
+        response_headers: response.headers
+      )
     end
 
     def check_response_errors!(response, url)

data/lib/archaeo/save_result.rb

@@ -3,16 +3,22 @@
 module Archaeo
   # Model representing the outcome of a SavePageNow request.
   #
-  # Contains the resulting archive URL, timestamp, and whether
-  # the page was already cached in the archive.
+  # Contains the resulting archive URL, timestamp, whether
+  # the page was already cached, and response metadata.
   class SaveResult
-    attr_reader :url, :archive_url, :timestamp
+    attr_reader :url, :archive_url, :timestamp, :status_code,
+                :response_url, :response_headers
 
-    def initialize(url:, archive_url:, timestamp:, cached:)
+    def initialize(url:, archive_url:, timestamp:, cached:,
+                   status_code: nil, response_url: nil,
+                   response_headers: nil)
       @url = url
       @archive_url = archive_url
       @timestamp = timestamp ? Timestamp.coerce(timestamp) : nil
       @cached = cached
+      @status_code = status_code
+      @response_url = response_url
+      @response_headers = response_headers
     end
 
     def cached?
@@ -24,13 +30,25 @@ module Archaeo
     end
 
     def to_h
-      { url: @url, archive_url: @archive_url,
-        timestamp: @timestamp, cached: @cached }
+      {
+        url: @url,
+        archive_url: @archive_url,
+        timestamp: @timestamp,
+        cached: @cached,
+        status_code: @status_code,
+        response_url: @response_url,
+      }
     end
 
     def as_json(*)
-      { url: @url, archive_url: @archive_url,
-        timestamp: @timestamp.to_s, cached: @cached }
+      {
+        url: @url,
+        archive_url: @archive_url,
+        timestamp: @timestamp.to_s,
+        cached: @cached,
+        status_code: @status_code,
+        response_url: @response_url,
+      }
     end
 
     def to_s

data/lib/archaeo/subdomain_discovery.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Archaeo
+  # Discovers subdomains from downloaded content.
+  #
+  # Scans HTML, CSS, and JavaScript files for links to subdomains
+  # of a base domain, enabling recursive archival.
+  class SubdomainDiscovery
+    MULTI_PART_TLDS = %w[
+      co.uk com.au co.jp co.nz co.za com.br com.mx
+      com.sg co.in co.kr com.tw com.hk org.uk ac.uk
+      co.il com.ar co.id co.th com.my com.tr co.ke
+    ].freeze
+
+    HTML_URL_ATTRS = %w[href src action].freeze
+    HTML_URL_RE = /https?:\/\/([a-z0-9][-a-z0-9.]*[a-z0-9])/i
+    CSS_URL_RE = /url\(\s*['"]?(https?:\/\/[^'")\s]+)['"]?\s*\)/i
+    JS_STRING_RE = /['"](https?:\/\/[a-z0-9][-a-z0-9.]*[a-z0-9][^\s'"]*)['"]/i
+
+    def initialize(base_domain, max_depth: 1)
+      @base_domain = base_domain.to_s
+      @max_depth = max_depth
+      @visited = Set.new
+    end
+
+    def scan_content(content, content_type:)
+      urls = extract_urls(content, content_type)
+      filter_subdomains(urls)
+    end
+
+    def scan_files(directory)
+      found = Set.new
+      Dir.glob(File.join(directory, "**", "*")).each do |path|
+        next unless File.file?(path)
+
+        content = File.read(path, encoding: "UTF-8",
+                                  invalid: :replace, undef: :replace)
+        ext = File.extname(path).downcase
+        content_type = content_type_for_ext(ext)
+        next unless content_type
+
+        found.merge(scan_content(content, content_type: content_type))
+      end
+      found.to_a
+    end
+
+    def discover_recursive(directory, depth: 0)
+      return [] if depth >= @max_depth
+
+      subdomains = scan_files(directory)
+      new_subdomains = subdomains.reject { |s| @visited.include?(s) }
+      @visited.merge(new_subdomains)
+      new_subdomains
+    end
+
+    def base_domain(host)
+      parts = host.to_s.downcase.split(".")
+      return host.to_s if parts.length <= 2
+
+      MULTI_PART_TLDS.each do |tld|
+        tld_parts = tld.split(".")
+        if parts.last(tld_parts.length) == tld_parts
+          return parts.last(tld_parts.length + 1).join(".")
+        end
+      end
+
+      parts.last(2).join(".")
+    end
+
+    private
+
+    def extract_urls(content, content_type)
+      case content_type
+      when :html then extract_html_urls(content)
+      when :css  then extract_css_urls(content)
+      when :js   then extract_js_urls(content)
+      else []
+      end
+    end
+
+    def extract_html_urls(content)
+      content.scan(HTML_URL_RE).flatten.map { |h| "https://#{h}" }
+    end
+
+    def extract_css_urls(content)
+      content.scan(CSS_URL_RE).flatten
+    end
+
+    def extract_js_urls(content)
+      content.scan(JS_STRING_RE).flatten
+    end
+
+    def filter_subdomains(urls)
+      base = base_domain(@base_domain)
+      urls.filter_map do |url|
+        host = begin
+          URI.parse(url).host.to_s.downcase
+        rescue URI::InvalidURIError
+          next
+        end
+        next unless host.end_with?(".#{base}") && host != base
+
+        host
+      end.uniq
+    end
+
+    def content_type_for_ext(ext)
+      case ext
+      when ".html", ".htm" then :html
+      when ".css"          then :css
+      when ".js"           then :js
+      end
+    end
+  end
+end