archaeo 0.2.5 → 0.2.6

data/lib/archaeo/cli.rb

@@ -9,6 +9,9 @@ module Archaeo
   class Cli < Thor
     map %w[--version -v] => :version
 
+    class_option :quiet, type: :boolean, default: false,
+                         desc: "Suppress progress messages"
+
     def self.exit_on_failure?
       true
     end
@@ -65,6 +68,39 @@ module Archaeo
       end
     end
 
+    desc "before URL TIMESTAMP",
+         "Find the nearest snapshot before a timestamp"
+    option :format, desc: "Output format (url, json)", default: "url"
+    def before(url, timestamp)
+      handle_errors do
+        snap = CdxApi.new.before(url, timestamp: timestamp)
+        output_snapshot(snap)
+      end
+    end
+
+    desc "after URL TIMESTAMP",
+         "Find the nearest snapshot after a timestamp"
+    option :format, desc: "Output format (url, json)", default: "url"
+    def after(url, timestamp)
+      handle_errors do
+        snap = CdxApi.new.after(url, timestamp: timestamp)
+        output_snapshot(snap)
+      end
+    end
+
+    desc "between URL FROM TO",
+         "List snapshots in a date range"
+    option :format, desc: "Output format (table, json, csv)",
+                    default: "table"
+    def between(url, from, to)
+      fmt = validate_output_format
+      handle_errors do
+        cdx = CdxApi.new
+        snaps = cdx.between(url, from: from, to: to).to_a
+        output_formatted(snaps, fmt)
+      end
+    end
+
     desc "available URL", "Check if a URL is archived"
     option :timestamp, desc: "Check near this timestamp (YYYYMMDDHHmmss)"
     def available(url)
@@ -105,6 +141,18 @@ module Archaeo
       end
     end
 
+    desc "fetch-assets URL TIMESTAMP",
+         "Fetch a page and list its extracted assets"
+    option :format, desc: "Output format (json, table)", default: "table"
+    def fetch_assets(url, timestamp)
+      handle_errors do
+        bundle = Fetcher.new.fetch_page_with_assets(
+          url, timestamp: timestamp
+        )
+        output_assets(bundle)
+      end
+    end
+
     desc "download URL", "Download all archived snapshots of a URL"
     option :output, desc: "Output directory", default: "archive"
     option :from, desc: "Start timestamp (YYYYMMDDHHmmss)"
@@ -113,6 +161,8 @@ module Archaeo
                     desc: "Resume interrupted download"
     option :concurrency, type: :numeric, default: 1,
                          desc: "Number of parallel downloads"
+    option :dry_run, type: :boolean, default: false,
+                     desc: "Preview downloads without fetching"
     def download(url)
       handle_errors do
         downloader = BulkDownloader.new(
@@ -141,6 +191,14 @@ module Archaeo
       end
     end
 
+    desc "count URL",
+         "Count snapshots for a URL"
+    def count(url)
+      handle_errors do
+        puts CdxApi.new.count(url)
+      end
+    end
+
     CDX_OPTION_MAP = {
       from: :from,
       to: :to,
@@ -153,6 +211,10 @@ module Archaeo
 
     private
 
+    def quiet?
+      options[:quiet]
+    end
+
     def handle_errors
       yield
     rescue RateLimitError => e
@@ -214,18 +276,42 @@ module Archaeo
       end
     end
 
-    def download_with_progress(downloader, url)
-      downloader.download(
-        url,
-        from: options[:from],
-        to: options[:to],
-        resume: options[:resume],
-      ) do |current, total, snap|
-        warn "[#{current}/#{total}] " \
-             "#{snap.timestamp} #{snap.original_url}"
+    def output_assets(bundle)
+      case options[:format]
+      when "json"
+        puts bundle.assets.to_json
+      else
+        bundle.assets.to_h.each do |type, urls|
+          next if urls.empty?
+
+          puts "#{type}:"
+          urls.each { |url| puts "  #{url}" }
+        end
       end
     end
 
+    def download_with_progress(downloader, url)
+      summary = downloader.download(
+        url, from: options[:from], to: options[:to],
+             resume: options[:resume], dry_run: options[:dry_run]
+      ) { |c, t, s| print_progress(c, t, s) }
+      print_summary(summary)
+    end
+
+    def print_progress(current, total, snap)
+      return if quiet?
+
+      warn "[#{current}/#{total}] #{snap.timestamp} #{snap.original_url}"
+    end
+
+    def print_summary(summary)
+      return if quiet?
+
+      warn "Downloaded #{summary.downloaded}/#{summary.total} " \
+           "(#{summary.bytes_written} bytes) in " \
+           "#{summary.elapsed.round(1)}s"
+    end
+
     def build_cdx_options(opts)
       CDX_OPTION_MAP.each_with_object({}) do |(cli_key, api_key), result|
         value = opts[cli_key]
@@ -259,7 +345,7 @@ module Archaeo
     def write_output(path, content)
       FileUtils.mkdir_p(File.dirname(path))
       File.binwrite(path, content)
-      warn "Written to #{path}"
+      warn "Written to #{path}" unless quiet?
     end
   end
 end

data/lib/archaeo/download_state.rb

@@ -1,11 +1,12 @@
 # frozen_string_literal: true
 
+require "json"
 require "set"
 
 module Archaeo
   # Tracks download progress for resume support.
   #
-  # Persists completed snapshot timestamps to a state file within
+  # Persists completed snapshot metadata to a JSONL state file within
   # the output directory, allowing interrupted downloads to resume
   # without re-fetching already downloaded snapshots.
   class DownloadState
@@ -19,42 +20,72 @@ module Archaeo
     end
 
     def completed?(timestamp)
-      timestamps_set.include?(timestamp.to_s)
+      entries_key.include?(timestamp.to_s)
     end
 
-    def mark_completed(timestamp)
+    def mark_completed(timestamp, url: nil, bytes: nil)
       ts = timestamp.to_s
-      return if timestamps_set.include?(ts)
+      return if entries_key.include?(ts)
 
-      timestamps << ts
-      @timestamps_set = nil
+      entry = { "ts" => ts, "at" => Time.now.utc.iso8601 }
+      entry["url"] = url if url
+      entry["bytes"] = bytes if bytes
+      entries << entry
+      @entries_key = nil
       save
     end
 
+    def entry_for(timestamp)
+      entries.find { |e| e["ts"] == timestamp.to_s }
+    end
+
+    def total_bytes
+      entries.sum { |e| e["bytes"].to_i }
+    end
+
     def clear
-      @timestamps = []
-      @timestamps_set = nil
+      @entries = []
+      @entries_key = nil
       FileUtils.rm_f(@path)
     end
 
     private
 
-    def timestamps
-      @timestamps ||= load_timestamps
+    def entries
+      @entries ||= load_entries
     end
 
-    def timestamps_set
-      @timestamps_set ||= timestamps.to_set
+    def entries_key
+      @entries_key ||= entries.each_with_object(Set.new) { |e, s| s << e["ts"] }
     end
 
-    def load_timestamps
+    def load_entries
       return [] unless File.exist?(@path)
 
-      File.readlines(@path, chomp: true).reject(&:empty?)
+      first_line = File.open(@path, &:readline).strip
+      if first_line.start_with?("{")
+        parse_jsonl
+      else
+        migrate_legacy(first_line)
+      end
+    rescue EOFError
+      []
+    end
+
+    def parse_jsonl
+      File.readlines(@path, chomp: true).reject(&:empty?).map do |line|
+        JSON.parse(line)
+      end
+    end
+
+    def migrate_legacy(_first_line)
+      File.readlines(@path, chomp: true).reject(&:empty?).map do |ts|
+        { "ts" => ts }
+      end
     end
 
     def save
-      content = "#{timestamps.sort.join("\n")}\n"
+      content = "#{entries.map { |e| JSON.generate(e) }.join("\n")}\n"
       tmp_path = "#{@path}.tmp"
       File.write(tmp_path, content)
       File.rename(tmp_path, @path)

data/lib/archaeo/fetcher.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "digest"
+
 module Archaeo
   # Downloads archived content from the Wayback Machine.
   #
@@ -13,12 +15,13 @@ module Archaeo
       @client = client
     end
 
-    def fetch(url, timestamp:, identity: false)
+    def fetch(url, timestamp:, identity: false, snapshot: nil)
       url = UrlNormalizer.normalize(url)
       ts = Timestamp.coerce(timestamp)
       archive_url = ArchiveUrl.new(url, timestamp: ts,
                                         identity: identity)
       response = follow_redirects(archive_url.to_s)
+      verify_integrity!(response, snapshot) if snapshot
       build_page(response, archive_url.to_s, url, ts)
     end
 
@@ -31,6 +34,18 @@ module Archaeo
 
     private
 
+    def verify_integrity!(response, snapshot)
+      return unless snapshot.digest && !snapshot.digest.empty?
+
+      expected = snapshot.digest.delete_prefix("SHA1-")
+      actual = Digest::SHA1.hexdigest(response.body)
+      return if expected == actual
+
+      raise IntegrityError,
+            "Digest mismatch for #{snapshot.original_url}: " \
+            "expected #{expected}, got #{actual}"
+    end
+
     def build_page(response, archive_url, url, timestamp)
       Page.new(
         content: response.body,

data/lib/archaeo/http_client.rb

@@ -62,11 +62,13 @@ module Archaeo
     def initialize(timeout: DEFAULT_TIMEOUT,
                    max_retries: DEFAULT_MAX_RETRIES,
                    retry_delay: DEFAULT_RETRY_DELAY,
-                   user_agent: nil)
+                   user_agent: nil,
+                   on_request: nil)
       @timeout = timeout
       @max_retries = max_retries
       @retry_delay = retry_delay
       @user_agent = user_agent
+      @on_request = on_request
       @connections = {}
       @last_used = {}
       @mutex = Mutex.new
@@ -106,6 +108,18 @@ module Archaeo
       end
     end
 
+    def pool_stats
+      now = Time.now
+      @mutex.synchronize do
+        {
+          active_connections: @connections.size,
+          max_pool_size: MAX_POOL_SIZE,
+          hosts: @connections.keys,
+          idle_times: @last_used.transform_values { |t| (now - t).round },
+        }.freeze
+      end
+    end
+
     private
 
     def select_user_agent
@@ -222,11 +236,16 @@ module Archaeo
       value = response.headers["retry-after"]
       return nil unless value
 
-      begin
-        Integer(value)
-      rescue StandardError
-        nil
-      end
+      Integer(value)
+    rescue ArgumentError
+      parse_retry_after_date(value)
+    end
+
+    def parse_retry_after_date(value)
+      remaining = (Time.httpdate(value) - Time.now).to_i
+      [remaining, 0].max
+    rescue ArgumentError
+      nil
     end
 
     def raise_if_exhausted(retries, error)
@@ -237,11 +256,8 @@ module Archaeo
     end
 
     def execute_with_connection(uri, headers, request_class)
-      http = connection_for(uri)
-      request = request_class.new(uri)
-      headers.each { |k, v| request[k] = v }
-      raw = http.request(request)
-      build_response(raw)
+      request = build_request(uri, headers, request_class)
+      execute_tracked_request(uri, request)
     rescue *TRANSIENT_ERRORS
       raise
     rescue StandardError
@@ -249,6 +265,22 @@ module Archaeo
       raise
     end
 
+    def build_request(uri, headers, request_class)
+      request = request_class.new(uri)
+      headers.each { |k, v| request[k] = v }
+      request
+    end
+
+    def execute_tracked_request(uri, request)
+      http = connection_for(uri)
+      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      raw = http.request(request)
+      elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+      response = build_response(raw)
+      @on_request&.call(uri, elapsed, response.status, 0)
+      response
+    end
+
     def default_headers
       {
         "User-Agent" => select_user_agent,

data/lib/archaeo/page.rb

@@ -50,6 +50,10 @@ module Archaeo
       @content_type&.start_with?("text/")
     end
 
+    def css?
+      @content_type&.include?("text/css")
+    end
+
     def binary?
       !(text? || json? || html?)
     end
@@ -63,6 +67,34 @@ module Archaeo
       end
     end
 
+    def to_h
+      {
+        content_type: @content_type,
+        status_code: @status_code,
+        archive_url: @archive_url,
+        original_url: @original_url,
+        timestamp: @timestamp,
+        size: size,
+        encoding: encoding.to_s,
+      }
+    end
+
+    def as_json(*)
+      {
+        content_type: @content_type,
+        status_code: @status_code,
+        archive_url: @archive_url,
+        original_url: @original_url,
+        timestamp: @timestamp.to_s,
+        size: size,
+        encoding: encoding.to_s,
+      }
+    end
+
+    def inspect
+      "#<#{self.class.name} #{@content_type} #{size} bytes>"
+    end
+
     private
 
     def detect_encoding

data/lib/archaeo/page_bundle.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "json"
+
 module Archaeo
   # A fetched page together with all its extracted asset URLs.
   #
@@ -26,5 +28,31 @@ module Archaeo
     def asset_count
       assets.size
     end
+
+    def to_h
+      { page: @page.to_h, assets: @assets.to_h }
+    end
+
+    def as_json(*)
+      { page: @page.as_json, assets: @assets.to_h }
+    end
+
+    def to_json(*args)
+      JSON.generate(as_json, *args)
+    end
+
+    def download_assets(output_dir:, client: HttpClient.new)
+      FileUtils.mkdir_p(output_dir)
+      @assets.all.each do |url|
+        filename = File.join(output_dir,
+                             File.basename(URI.parse(url).path))
+        tmp_path = "#{filename}.tmp"
+        response = client.get(url)
+        File.binwrite(tmp_path, response.body)
+        File.rename(tmp_path, filename)
+      rescue StandardError
+        FileUtils.rm_f(tmp_path) if defined?(tmp_path)
+      end
+    end
   end
 end

data/lib/archaeo/save_result.rb

@@ -18,5 +18,24 @@ module Archaeo
     def cached?
       @cached
     end
+
+    def to_h
+      { url: @url, archive_url: @archive_url,
+        timestamp: @timestamp, cached: @cached }
+    end
+
+    def as_json(*)
+      { url: @url, archive_url: @archive_url,
+        timestamp: @timestamp.to_s, cached: @cached }
+    end
+
+    def to_s
+      label = @cached ? "Cached" : "Saved"
+      "#{label}: #{@archive_url}"
+    end
+
+    def inspect
+      "#<#{self.class.name} #{@url} cached=#{@cached}>"
+    end
   end
 end

data/lib/archaeo/snapshot.rb

@@ -29,6 +29,11 @@ module Archaeo
       ArchiveUrl.new(original_url, timestamp: @timestamp).to_s
     end
 
+    def identity_url
+      ArchiveUrl.new(original_url, timestamp: @timestamp,
+                                   identity: true).to_s
+    end
+
     def blocked?
       @status_code == BLOCKED_STATUS
     end
@@ -53,6 +58,18 @@ module Archaeo
       client_error? || server_error?
     end
 
+    def age
+      Time.now - @timestamp.to_time
+    end
+
+    def older_than?(seconds)
+      age > seconds
+    end
+
+    def newer_than?(seconds)
+      age <= seconds
+    end
+
     def fetch(client: HttpClient.new, identity: false)
       Fetcher.new(client: client).fetch(
         original_url, timestamp: @timestamp, identity: identity
@@ -102,5 +119,10 @@ module Archaeo
     def hash
       to_a.hash
     end
+
+    def inspect
+      "#<#{self.class.name} #{timestamp} " \
+        "#{original_url} status=#{status_code}>"
+    end
   end
 end

data/lib/archaeo/timestamp.rb

@@ -17,6 +17,7 @@ module Archaeo
     def initialize(year:, month: 1, day: 1,
                    hour: 0, minute: 0, second: 0)
       @to_time = Time.utc(year, month, day, hour, minute, second)
+      freeze
     end
 
     def self.parse(string)
@@ -129,5 +130,18 @@ module Archaeo
     def second
       @to_time.sec
     end
+
+    def to_h
+      { year: year, month: month, day: day,
+        hour: hour, minute: minute, second: second }
+    end
+
+    def to_a
+      [year, month, day, hour, minute, second]
+    end
+
+    def inspect
+      "#<#{self.class.name} #{self}>"
+    end
   end
 end

data/lib/archaeo/url_normalizer.rb

@@ -49,7 +49,8 @@ module Archaeo
       url = strip_whitespace(url)
       url = strip_surrounding_quotes(url)
       url = fix_double_percent_encoding(url)
-      normalize_percent_encoding(url)
+      url = normalize_percent_encoding(url)
+      remove_default_port(url)
     end
 
     def strip_whitespace(url)
@@ -69,5 +70,10 @@ module Archaeo
     def normalize_percent_encoding(url)
       url.gsub(/%[0-9a-f]{2}/i, &:upcase)
     end
+
+    def remove_default_port(url)
+      url.sub(%r{(https://[^/:]+):443(?=/|$)}, '\1')
+        .sub(%r{(http://[^/:]+):80(?=/|$)}, '\1')
+    end
   end
 end

data/lib/archaeo/url_rewriter.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "nokogiri"
+
 module Archaeo
   # Rewrites Wayback Machine archive URLs to local file paths.
   #
@@ -7,6 +9,8 @@ module Archaeo
   # browsing. Converts absolute archive URLs into relative paths
   # rooted at a configurable local directory.
   class UrlRewriter
+    URL_ATTRS = %w[src href data-src poster].freeze
+
     def initialize(archive_prefix, local_prefix)
       @archive_prefix = archive_prefix.to_s
       @local_prefix = local_prefix.to_s
@@ -18,5 +22,47 @@ module Archaeo
       relative = url.sub(@archive_prefix, "")
       File.join(@local_prefix, relative)
     end
+
+    def rewrite_batch(urls)
+      urls.map { |url| rewrite(url) }
+    end
+
+    def rewrite_html(html_content)
+      doc = Nokogiri::HTML(html_content)
+      rewrite_url_attrs(doc)
+      rewrite_srcset_attrs(doc)
+      doc.to_html
+    end
+
+    def rewrite_url_attrs(doc)
+      URL_ATTRS.each do |attr|
+        doc.css("[#{attr}]").each do |el|
+          original = el[attr]
+          next unless original&.start_with?(@archive_prefix)
+
+          el[attr] = rewrite(original)
+        end
+      end
+    end
+
+    def rewrite_srcset_attrs(doc)
+      doc.css("[srcset]").each do |el|
+        el["srcset"] = rewrite_srcset(el["srcset"])
+      end
+    end
+
+    private
+
+    def rewrite_srcset(srcset)
+      return srcset unless srcset
+
+      srcset.split(",").map do |entry|
+        parts = entry.strip.split(/\s+/, 2)
+        url = parts[0]
+        descriptor = parts[1]
+        rewritten = url.start_with?(@archive_prefix) ? rewrite(url) : url
+        descriptor ? "#{rewritten} #{descriptor}" : rewritten
+      end.join(", ")
+    end
   end
 end

data/lib/archaeo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Archaeo
-  VERSION = "0.2.5"
+  VERSION = "0.2.6"
 end

data/lib/archaeo.rb

@@ -14,6 +14,7 @@ module Archaeo
   class ArchiveNotAvailable < Error; end
   class InvalidResponse < Error; end
   class SaveFailed < Error; end
+  class IntegrityError < Error; end
 
   autoload :Timestamp, "archaeo/timestamp"
   autoload :ArchiveUrl, "archaeo/archive_url"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: archaeo
 version: !ruby/object:Gem::Version
-  version: 0.2.5
+  version: 0.2.6
 platform: ruby
 authors:
 - Ribose Inc.