archaeo 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: afb71bf4cd68f3d2aa80cd7169fba891b42975dd25ee3c35cc2323663a04dec7
-  data.tar.gz: 566cfaf6758c96c09e1d718a9fc8a576d2b3800dee57456ff5d107a12eaf692b
+  metadata.gz: 721131f1413aaacb26685abe006fdd243e3ef53e1d5f23764b2774717aae93ec
+  data.tar.gz: f3c90969cf684e06a6cdd1e0025a605141e1ee543430488fbf960e787ca1ba7d
 SHA512:
-  metadata.gz: efd13cd5d167bc2ac72fc278f45520808e79bf64f4bf2253161806efccbe9c8e16991be8e6171757e3c6a3b773e3c82c3bdf4c92562c276e1568d7098f6ade1f
-  data.tar.gz: 0f8c8e91959c79b5ff2b0d2a868e08e86dee8675965696eba583f408940ffa5fd63caa87bb0567e916baf4a4c120f8b81ddf546f748802cfc9e07635dfa52e3a
+  metadata.gz: 494ba22650c56df4a2ae119f0b6965679441bf988f013fe22f54c8c02e285d9df6ade6db4c2574ec23ba3e95f917e541e96dbd79a5b6deae178d7b6eaa5fd1a7
+  data.tar.gz: cbaf296d51ecae3ad77eee66100f6ca6aa40d0ddf0abd3b6c73b7c61b5cf92131b991437c28537a4db97de0182b7a9133df54ad07c7017d6f613760bcadf3cca

data/lib/archaeo/asset_extractor.rb

@@ -32,6 +32,9 @@ module Archaeo
       @doc.css('link[rel="stylesheet"]').each do |el|
         list.add(resolve(el["href"]), type: :css)
       end
+      @doc.css('link[rel="icon"], link[rel="shortcut icon"]').each do |el|
+        list.add(resolve(el["href"]), type: :image)
+      end
     end
 
     def extract_js(list)

data/lib/archaeo/availability_api.rb

@@ -42,6 +42,11 @@ module Archaeo
 
     def parse_response(response, url)
       unless response.status == 200
+        if response.status == 503
+          raise RateLimitError,
+                "Availability API rate limited (HTTP 503)"
+        end
+
         raise InvalidResponse,
               "Availability API returned HTTP #{response.status}"
       end

data/lib/archaeo/bulk_downloader.rb

@@ -9,9 +9,11 @@ module Archaeo
   # and saves content to disk. Progress is tracked in a state file
   # for interrupted download recovery.
   class BulkDownloader
-    def initialize(client: HttpClient.new, output_dir: "archive")
+    def initialize(client: HttpClient.new, output_dir: "archive",
+                   cdx_api: nil)
       @client = client
       @output_dir = output_dir
+      @cdx_api = cdx_api
     end
 
     def download(url, from: nil, to: nil, resume: false)
@@ -35,7 +37,7 @@ module Archaeo
     private
 
     def fetch_snapshots(url, from:, to:)
-      cdx = CdxApi.new(client: @client)
+      cdx = @cdx_api || CdxApi.new(client: @client)
       options = {}
       options[:from] = from if from
       options[:to] = to if to
@@ -53,6 +55,27 @@ module Archaeo
       File.binwrite(filename, page.content)
     end
 
+    EXTENSION_MAP = {
+      "text/html" => ".html",
+      "text/css" => ".css",
+      "application/javascript" => ".js",
+      "application/json" => ".json",
+      "application/pdf" => ".pdf",
+      "image/png" => ".png",
+      "image/jpeg" => ".jpg",
+      "image/gif" => ".gif",
+      "image/svg+xml" => ".svg",
+      "image/webp" => ".webp",
+      "font/woff2" => ".woff2",
+      "font/woff" => ".woff",
+      "video/mp4" => ".mp4",
+      "audio/mpeg" => ".mp3",
+    }.freeze
+
+    def extension_for(snapshot)
+      EXTENSION_MAP[snapshot.mimetype] || ".bin"
+    end
+
     def build_filename(snapshot)
       ts = snapshot.timestamp.to_s
       safe_path = snapshot.original_url
@@ -62,7 +85,7 @@ module Archaeo
       safe_path = safe_path[0..-2] if safe_path.end_with?(File::SEPARATOR)
       safe_path = "#{safe_path}index" if safe_path.empty?
 
-      File.join(@output_dir, safe_path, "#{ts}.html")
+      File.join(@output_dir, safe_path, "#{ts}#{extension_for(snapshot)}")
     end
   end
 end

data/lib/archaeo/cdx_filter.rb

@@ -55,6 +55,10 @@ module Archaeo
       new("original:#{pattern}")
     end
 
+    def self.by_urlkey(pattern)
+      new("urlkey:#{pattern}")
+    end
+
     private
 
     def validate!

data/lib/archaeo/cli.rb

@@ -119,6 +119,18 @@ module Archaeo
       end
     end
 
+    desc "num_pages URL",
+         "Show number of CDX result pages for a URL"
+    def num_pages(url)
+      puts CdxApi.new.num_pages(url)
+    rescue RateLimitError => e
+      warn "Error: #{e.message}"
+      exit 1
+    rescue Error => e
+      warn "Error: #{e.message}"
+      exit 1
+    end
+
     CDX_OPTION_MAP = {
       from: :from,
       to: :to,

data/lib/archaeo/http_client.rb

@@ -70,7 +70,13 @@ module Archaeo
     def get(url, headers: {})
       merged = default_headers.merge(headers)
       uri = URI(url)
-      attempt_with_retries(uri, merged)
+      attempt_with_retries(uri, merged, Net::HTTP::Get)
+    end
+
+    def head(url, headers: {})
+      merged = default_headers.merge(headers)
+      uri = URI(url)
+      attempt_with_retries(uri, merged, Net::HTTP::Head)
     end
 
     def shutdown
@@ -128,10 +134,10 @@ module Archaeo
       end
     end
 
-    def attempt_with_retries(uri, headers)
+    def attempt_with_retries(uri, headers, request_class)
       retries = 0
       begin
-        execute_with_connection(uri, headers)
+        execute_with_connection(uri, headers, request_class)
       rescue *TRANSIENT_ERRORS => e
         retries += 1
         raise_if_exhausted(retries, e)
@@ -148,9 +154,9 @@ module Archaeo
             "Failed after #{retries} retries: #{error.message}"
     end
 
-    def execute_with_connection(uri, headers)
+    def execute_with_connection(uri, headers, request_class)
       http = connection_for(uri)
-      request = Net::HTTP::Get.new(uri)
+      request = request_class.new(uri)
       headers.each { |k, v| request[k] = v }
       raw = http.request(request)
       build_response(raw)

data/lib/archaeo/page.rb

@@ -26,6 +26,10 @@ module Archaeo
       @content ||= transcode(@raw_content)
     end
 
+    def size
+      content.length
+    end
+
     def encoding
       @encoding ||= detect_encoding
     end

data/lib/archaeo/snapshot.rb

@@ -33,11 +33,27 @@ module Archaeo
       @status_code == BLOCKED_STATUS
     end
 
+    def success?
+      @status_code == 200
+    end
+
     def to_a
       [@urlkey, @timestamp, @original_url, @mimetype,
        @status_code, @digest, @length]
     end
 
+    def to_h
+      {
+        urlkey: @urlkey,
+        timestamp: @timestamp,
+        original_url: @original_url,
+        mimetype: @mimetype,
+        status_code: @status_code,
+        digest: @digest,
+        length: @length,
+      }
+    end
+
     def ==(other)
       other.is_a?(self.class) && to_a == other.to_a
     end

data/lib/archaeo/timestamp.rb

@@ -62,6 +62,14 @@ module Archaeo
       @to_time.strftime(FORMAT)
     end
 
+    def to_date
+      Date.new(year, month, day)
+    end
+
+    def to_i
+      @to_time.to_i
+    end
+
     def <=>(other)
       return nil unless other.is_a?(self.class)
 

data/lib/archaeo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Archaeo
-  VERSION = "0.2.3"
+  VERSION = "0.2.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: archaeo
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-05-09 00:00:00.000000000 Z
+date: 2026-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: csv