RubyGems - archaeo - Versions diffs - 0.1.0 - Mend

archaeo 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (25) hide show

checksums.yaml +7 -0
data/CHANGELOG.md +5 -0
data/CODE_OF_CONDUCT.md +10 -0
data/README.adoc +186 -0
data/Rakefile +12 -0
data/archaeo.gemspec +38 -0
data/bin/console +11 -0
data/bin/setup +8 -0
data/exe/archaeo +6 -0
data/lib/archaeo/archive_url.rb +54 -0
data/lib/archaeo/availability_api.rb +74 -0
data/lib/archaeo/availability_result.rb +22 -0
data/lib/archaeo/cdx_api.rb +162 -0
data/lib/archaeo/cli.rb +94 -0
data/lib/archaeo/fetcher.rb +62 -0
data/lib/archaeo/http_client.rb +137 -0
data/lib/archaeo/page.rb +22 -0
data/lib/archaeo/save_api.rb +112 -0
data/lib/archaeo/save_result.rb +21 -0
data/lib/archaeo/snapshot.rb +40 -0
data/lib/archaeo/timestamp.rb +103 -0
data/lib/archaeo/version.rb +5 -0
data/lib/archaeo.rb +30 -0
data/sig/archaeo.rbs +241 -0
metadata +84 -0

checksums.yaml ADDED Viewed

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 65cb8ec1434b72774ed3a1d49ac87920bebb549cc5a4aebb0966b8d110d740ba
+  data.tar.gz: a10b0bf2b8555d3a259c8ec02364e3031697189a784411ab54c4a1bfd17ab402
+SHA512:
+  metadata.gz: 74eac73369d611a491152f7018d63b8fe3b46f8154a374d43d62f3ca023e1837dae666b98c795c211add5c6282fffe70576e581165aecd6644ecbccd15efe623
+  data.tar.gz: 03dd2e1ea518ef34a2b2c427c91a9a562aa328aced3b8cad4910fb634ba8e723f1cfb50bd57b247403a060c67f2f2f4638ad9e4293abb76e6225a731bb7493fd

data/CHANGELOG.md ADDED Viewed

@@ -0,0 +1,5 @@
+## [Unreleased]
+## [0.1.0] - 2026-05-09
+- Initial release

data/CODE_OF_CONDUCT.md ADDED Viewed

@@ -0,0 +1,10 @@
+# Code of Conduct
+"archaeo" follows [The Ruby Community Conduct Guideline](https://www.ruby-lang.org/en/conduct) in all "collaborative space", which is defined as community communications channels (such as mailing lists, submitted patches, commit comments, etc.):
+* Participants will be tolerant of opposing views.
+* Participants must ensure that their language and actions are free of personal attacks and disparaging personal remarks.
+* When interpreting the words and actions of others, participants should always assume good intentions.
+* Behaviour which can be reasonably considered harassment will not be tolerated.
+If you have any concerns about behaviour within this project, please contact us at ["ronald.tse@ribose.com"](mailto:"ronald.tse@ribose.com").

data/README.adoc ADDED Viewed

@@ -0,0 +1,186 @@
+= Archaeo
+== Purpose
+Archaeo is a Ruby client for the Internet Archive's https://web.archive.org[Wayback Machine] APIs.
+It provides a model-driven interface for querying archived snapshots, checking availability, saving URLs, and fetching archived content.
+== Installation
+[source,bash]
+----
+gem install archaeo
+----
+Or add to your Gemfile:
+[source,ruby]
+----
+gem "archaeo"
+----
+== Quick Start
+[source,ruby]
+----
+require "archaeo"
+----
+=== Query Snapshots (CDX API)
+[source,ruby]
+----
+cdx = Archaeo::CdxApi.new
+# Enumerate all snapshots
+cdx.snapshots("example.com").each do |snapshot|
+  puts snapshot.timestamp
+  puts snapshot.original_url
+  puts snapshot.archive_url
+end
+# Find specific snapshots
+oldest = cdx.oldest("example.com")
+newest = cdx.newest("example.com")
+near   = cdx.near("example.com", timestamp: "20220101")
+# Filter by time
+before = cdx.before("example.com", timestamp: "20220101")
+after  = cdx.after("example.com", timestamp: "20220101")
+----
+=== Check Availability
+[source,ruby]
+----
+api = Archaeo::AvailabilityApi.new
+result = api.near("example.com")
+result.available?   # => true/false
+result.archive_url  # => "https://web.archive.org/web/..."
+result.timestamp    # => Archaeo::Timestamp
+api.available?("example.com")  # => true/false
+----
+=== Save a URL (SavePageNow)
+[source,ruby]
+----
+save = Archaeo::SaveApi.new
+result = save.save("https://example.com/")
+result.archive_url  # => "https://web.archive.org/web/..."
+result.timestamp    # => Archaeo::Timestamp
+result.cached?      # => true if already archived
+----
+=== Fetch Archived Content
+[source,ruby]
+----
+fetcher = Archaeo::Fetcher.new
+page = fetcher.fetch("https://example.com/",
+                     timestamp: "20220615000000")
+page.content        # => "<html>...</html>"
+page.content_type   # => "text/html"
+page.status_code    # => 200
+page.archive_url    # => full archive URL
+# Raw (identity) mode -- no Wayback Machine rewriting
+page = fetcher.fetch("https://example.com/",
+                     timestamp: "20220615000000",
+                     identity: true)
+----
+=== Timestamps
+[source,ruby]
+----
+# Create from components
+ts = Archaeo::Timestamp.new(year: 2022, month: 6, day: 15)
+# Parse from Wayback format
+ts = Archaeo::Timestamp.parse("20220615120000")
+# From Time object
+ts = Archaeo::Timestamp.from_time(Time.now)
+# Current time
+ts = Archaeo::Timestamp.now
+# Format as 14-digit string
+ts.to_s  # => "20220615000000"
+# Comparison
+ts1 < ts2   # => true/false
+----
+=== Command-Line Interface
+[source,bash]
+----
+# List snapshots
+archaeo snapshots example.com
+# Find closest snapshot
+archaeo near example.com 20220101
+# Check availability
+archaeo available example.com
+# Save a URL
+archaeo save https://example.com/
+# Fetch archived content
+archaeo fetch https://example.com/ 20220615120000
+# Fetch raw (identity) content
+archaeo fetch --identity https://example.com/ 20220615120000
+----
+== Architecture
+Archaeo follows a model-driven, OOP design:
+[cols="1,2,1"]
+|===
+| Layer | Classes | Purpose
+| *Models*
+| `Timestamp`, `ArchiveUrl`, `Snapshot`, `Page`, `SaveResult`, `AvailabilityResult`
+| Domain value objects
+| *APIs*
+| `CdxApi`, `AvailabilityApi`, `SaveApi`
+| Query and mutate the archive
+| *Operations*
+| `Fetcher`
+| Download archived content
+| *Infrastructure*
+| `HttpClient`
+| HTTP transport with retries and gzip
+|===
+All API classes accept an `HttpClient` via dependency injection for testability.
+== Development
+[source,bash]
+----
+bundle install
+bundle exec rspec
+bundle exec rubocop
+----
+== Contributing
+Bug reports and pull requests are welcome on GitHub at https://github.com/riboseinc/archaeo[].
+== License
+MIT License.
+See link:LICENSE[LICENSE] for details.

data/Rakefile ADDED Viewed

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+RSpec::Core::RakeTask.new(:spec)
+require "rubocop/rake_task"
+RuboCop::RakeTask.new
+task default: %i[spec rubocop]

data/archaeo.gemspec ADDED Viewed

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+require_relative "lib/archaeo/version"
+Gem::Specification.new do |spec|
+  spec.name = "archaeo"
+  spec.version = Archaeo::VERSION
+  spec.authors = ["Ribose Inc."]
+  spec.email = ["open.source@ribose.com"]
+  spec.summary = "Ruby client for the Internet Archive Wayback Machine APIs"
+  spec.description = "Archaeo provides a Ruby interface to query, fetch, " \
+                     "and save archived web content via the Wayback Machine " \
+                     "CDX Server API, Availability API, SavePageNow API, " \
+                     "and content fetching."
+  spec.homepage = "https://github.com/riboseinc/archaeo"
+  spec.required_ruby_version = ">= 3.0.0"
+  spec.license = "MIT"
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["changelog_uri"] =
+    "#{spec.homepage}/blob/main/CHANGELOG.md"
+  spec.metadata["rubygems_mfa_required"] = "true"
+  spec.files = IO.popen(%w[git ls-files -z], chdir: __dir__,
+                                             err: IO::NULL) do |ls|
+    ls.readlines("\x0", chomp: true).reject do |f|
+      f == __FILE__ ||
+        f.start_with?(*%w[Gemfile .gitignore .rspec spec/ .github/ .rubocop])
+    end
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+  spec.add_dependency "thor", "~> 1.3"
+end

data/bin/console ADDED Viewed

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+require "bundler/setup"
+require "archaeo"
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+require "irb"
+IRB.start(__FILE__)

data/bin/setup ADDED Viewed

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+bundle install
+# Do any other automated setup that you need to do here

data/exe/archaeo ADDED Viewed

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+require "archaeo/cli"
+Archaeo::Cli.start(ARGV)

data/lib/archaeo/archive_url.rb ADDED Viewed

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+module Archaeo
+  # Model representing a Wayback Machine archive URL.
+  #
+  # Encapsulates URL construction and parsing for archive.org URLs,
+  # supporting both normal and identity (raw) modes.
+  class ArchiveUrl
+    BASE = "https://web.archive.org/web"
+    TIMESTAMP_RE = %r{web\.archive\.org/web/(\d{14})}
+    attr_reader :original_url, :timestamp
+    def initialize(original_url, timestamp:, identity: false)
+      @original_url = original_url.to_s
+      @timestamp = Timestamp.coerce(timestamp)
+      @identity = identity
+    end
+    def self.parse(string)
+      match = string.match(TIMESTAMP_RE)
+      unless match
+        raise ArgumentError,
+              "Not a valid archive URL: #{string}"
+      end
+      ts = Timestamp.parse(match[1])
+      identity = string.include?("#{match[1]}id_/")
+      rest = extract_original_url(string, match[1], identity)
+      new(rest, timestamp: ts, identity: identity)
+    end
+    def identity?
+      @identity
+    end
+    def to_s
+      suffix = identity? ? "id_" : ""
+      "#{BASE}/#{@timestamp}#{suffix}/#{@original_url}"
+    end
+    def self.extract_original_url(string, ts_str, identity)
+      marker = identity ? "#{ts_str}id_/" : "#{ts_str}/"
+      idx = string.index(marker)
+      return "" unless idx
+      string[(idx + marker.length)..]
+    end
+    private_class_method :extract_original_url
+  end
+end

data/lib/archaeo/availability_api.rb ADDED Viewed

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+require "json"
+require "uri"
+module Archaeo
+  # Client for the Wayback Machine Availability API.
+  #
+  # Check whether a URL has been archived and retrieve the closest
+  # available snapshot for a given point in time.
+  class AvailabilityApi
+    ENDPOINT = "https://archive.org/wayback/available"
+    def initialize(client: HttpClient.new)
+      @client = client
+    end
+    def near(url, timestamp: nil)
+      params = { "url" => url }
+      params["timestamp"] = timestamp.to_s if timestamp
+      response = @client.get(
+        "#{ENDPOINT}?#{URI.encode_www_form(params)}",
+      )
+      parse_response(response, url)
+    end
+    def oldest(url)
+      near(url, timestamp: Timestamp.new(year: 1994, month: 1, day: 1))
+    end
+    def newest(url)
+      near(url, timestamp: Timestamp.now)
+    end
+    def available?(url)
+      near(url).available?
+    end
+    private
+    def parse_response(response, url)
+      unless response.status == 200
+        raise InvalidResponse,
+              "Availability API returned HTTP #{response.status}"
+      end
+      json = JSON.parse(response.body)
+      snapshots = json["archived_snapshots"]
+      return unavailable(url) if snapshots.nil? || snapshots.empty?
+      closest = snapshots["closest"]
+      return unavailable(url) if closest.nil?
+      build_result(closest, url)
+    end
+    def unavailable(url)
+      AvailabilityResult.new(url: url, available: false)
+    end
+    def build_result(closest, url)
+      archive_url = closest["url"].to_s.sub(%r{^http://}, "https://")
+      ts = Timestamp.parse(closest["timestamp"])
+      AvailabilityResult.new(
+        url: url,
+        available: closest["status"].to_s == "200",
+        archive_url: archive_url,
+        timestamp: ts,
+      )
+    end
+  end
+end

data/lib/archaeo/availability_result.rb ADDED Viewed

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+module Archaeo
+  # Model representing the result of an availability query.
+  #
+  # Indicates whether a URL is archived and, if so, provides
+  # the closest snapshot's archive URL and timestamp.
+  class AvailabilityResult
+    attr_reader :url, :archive_url, :timestamp
+    def initialize(url:, available:, archive_url: nil, timestamp: nil)
+      @url = url
+      @available = available
+      @archive_url = archive_url
+      @timestamp = timestamp
+    end
+    def available?
+      @available
+    end
+  end
+end

data/lib/archaeo/cdx_api.rb ADDED Viewed

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+require "json"
+require "uri"
+module Archaeo
+  # Client for the Wayback Machine CDX Server API.
+  #
+  # Query archived snapshots by URL, timestamp range, filters,
+  # and more. Returns Snapshot objects for each matching CDX record.
+  class CdxApi
+    ENDPOINT = "https://web.archive.org/cdx/search/cdx"
+    ALL_FIELDS = %w[
+      urlkey timestamp original
+      mimetype statuscode digest length
+    ].freeze
+    MATCH_TYPES = %w[exact prefix host domain].freeze
+    SORT_ORDERS = %w[default closest reverse].freeze
+    DEFAULT_LIMIT = 25_000
+    SCALAR_PARAMS = {
+      from: "from",
+      to: "to",
+      match_type: "matchType",
+      sort: "sort",
+      limit: "limit",
+      closest: "closest",
+    }.freeze
+    def initialize(client: HttpClient.new)
+      @client = client
+    end
+    def snapshots(url, **options)
+      validate_options!(options)
+      Enumerator.new do |yielder|
+        fetch_snapshots(url, options, yielder)
+      end
+    end
+    def near(url, timestamp:)
+      ts = Timestamp.coerce(timestamp)
+      result = snapshots(url, sort: "closest",
+                              closest: ts.to_s, limit: 1).first
+      result || raise(NoSnapshotFound,
+                      "No snapshot found near #{ts} for #{url}")
+    end
+    def oldest(url)
+      near(url, timestamp: Timestamp.new(year: 1994, month: 1, day: 1))
+    end
+    def newest(url)
+      near(url, timestamp: Timestamp.now)
+    end
+    def before(url, timestamp:)
+      ts = Timestamp.coerce(timestamp)
+      snapshots(url, sort: "closest", closest: ts.to_s).each do |snap|
+        return snap if snap.timestamp < ts
+      end
+      raise NoSnapshotFound,
+            "No snapshot found before #{ts} for #{url}"
+    end
+    def after(url, timestamp:)
+      ts = Timestamp.coerce(timestamp)
+      snapshots(url, sort: "closest", closest: ts.to_s).each do |snap|
+        return snap if snap.timestamp > ts
+      end
+      raise NoSnapshotFound,
+            "No snapshot found after #{ts} for #{url}"
+    end
+    private
+    def fetch_snapshots(url, options, yielder)
+      params = build_params(url, options)
+      response = @client.get(
+        "#{ENDPOINT}?#{URI.encode_www_form(params)}",
+      )
+      unless response.status == 200
+        raise Error, "CDX API returned HTTP #{response.status}"
+      end
+      return if response.body.nil? || response.body.strip.empty?
+      parse_cdx_json(response.body, yielder)
+    end
+    def validate_options!(options)
+      validate_match_type!(options[:match_type])
+      validate_sort!(options[:sort])
+    end
+    def validate_match_type!(type)
+      return if type.nil? || MATCH_TYPES.include?(type.to_s)
+      raise ArgumentError,
+            "Invalid match_type: #{type}. " \
+            "Use: #{MATCH_TYPES.join(', ')}"
+    end
+    def validate_sort!(sort)
+      return if sort.nil? || SORT_ORDERS.include?(sort.to_s)
+      raise ArgumentError,
+            "Invalid sort: #{sort}. Use: #{SORT_ORDERS.join(', ')}"
+    end
+    def build_params(url, options)
+      {
+        "url" => url,
+        "output" => "json",
+        "fl" => ALL_FIELDS.join(","),
+        "gzip" => options.fetch(:gzip, true) ? "true" : "false",
+      }.tap do |params|
+        merge_scalar_params!(params, options)
+        merge_array_params!(params, options[:filters], "filter")
+        merge_array_params!(params, options[:collapse], "collapse")
+      end
+    end
+    def merge_scalar_params!(params, options)
+      SCALAR_PARAMS.each do |key, api_key|
+        value = options[key]
+        params[api_key] = value.to_s if value
+      end
+    end
+    def merge_array_params!(params, values, prefix)
+      Array(values).each_with_index do |v, i|
+        params["#{prefix}#{i}"] = v
+      end
+    end
+    def parse_cdx_json(body, yielder)
+      json = JSON.parse(body)
+      return unless json.is_a?(Array) && json.length > 1
+      header, *rows = json
+      field_map = header.each_with_index.to_h
+      rows.each { |row| yielder << build_snapshot(field_map, row) }
+    end
+    def build_snapshot(field_map, row)
+      fetch = ->(f) { row[field_map[f]] if field_map[f] }
+      Snapshot.new(
+        urlkey: fetch.call("urlkey"),
+        timestamp: fetch.call("timestamp"),
+        original_url: fetch.call("original"),
+        mimetype: fetch.call("mimetype"),
+        status_code: fetch.call("statuscode"),
+        digest: fetch.call("digest"),
+        length: fetch.call("length"),
+      )
+    end
+  end
+end

data/lib/archaeo/cli.rb ADDED Viewed

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+require "thor"
+module Archaeo
+  # Command-line interface powered by Thor.
+  class Cli < Thor
+    desc "snapshots URL", "List archived snapshots for a URL"
+    option :from, desc: "Start timestamp (YYYYMMDDHHmmss)"
+    option :to, desc: "End timestamp (YYYYMMDDHHmmss)"
+    option :match_type,
+           desc: "Match type (exact, prefix, host, domain)"
+    option :filter, type: :array, desc: "CDX filter expressions"
+    option :collapse, type: :array, desc: "CDX collapse fields"
+    option :sort, desc: "Sort order (default, closest, reverse)"
+    option :limit, type: :numeric, desc: "Max snapshots to return"
+    def snapshots(url)
+      cdx = CdxApi.new
+      opts = build_cdx_options(options)
+      cdx.snapshots(url, **opts).each do |snap|
+        puts "#{snap.timestamp}  #{snap.status_code}  " \
+             "#{snap.original_url}"
+      end
+    end
+    desc "near URL TIMESTAMP",
+         "Find the snapshot closest to a timestamp"
+    def near(url, timestamp)
+      snap = CdxApi.new.near(url, timestamp: timestamp)
+      puts snap.archive_url
+    end
+    desc "oldest URL", "Find the oldest snapshot of a URL"
+    def oldest(url)
+      snap = CdxApi.new.oldest(url)
+      puts snap.archive_url
+    end
+    desc "newest URL", "Find the newest snapshot of a URL"
+    def newest(url)
+      snap = CdxApi.new.newest(url)
+      puts snap.archive_url
+    end
+    desc "available URL", "Check if a URL is archived"
+    def available(url)
+      result = AvailabilityApi.new.near(url)
+      if result.available?
+        puts "Available: #{result.archive_url}"
+      else
+        puts "Not available"
+        exit 1
+      end
+    end
+    desc "save URL", "Save a URL to the Wayback Machine"
+    def save(url)
+      result = SaveApi.new.save(url)
+      label = result.cached? ? "Cached" : "Saved"
+      puts "#{label}: #{result.archive_url}"
+    end
+    desc "fetch URL TIMESTAMP",
+         "Fetch archived content for a URL at a timestamp"
+    option :identity, type: :boolean, default: false,
+                      desc: "Fetch raw (identity) content"
+    def fetch(url, timestamp)
+      page = Fetcher.new.fetch(
+        url, timestamp: timestamp,
+             identity: options[:identity]
+      )
+      $stdout.write(page.content)
+    end
+    CDX_OPTION_MAP = {
+      from: :from,
+      to: :to,
+      match_type: :match_type,
+      filter: :filters,
+      collapse: :collapse,
+      sort: :sort,
+      limit: :limit,
+    }.freeze
+    private
+    def build_cdx_options(opts)
+      CDX_OPTION_MAP.each_with_object({}) do |(cli_key, api_key), result|
+        value = opts[cli_key]
+        result[api_key] = value if value
+      end
+    end
+  end
+end