archaeo 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b82c7c8f2e568697056587556ac01bee92f51376af390aa26ed9c76221200d8
-  data.tar.gz: 411cf04b07219a02e5b049120bb74528dc17cf5fcb8424acd3c7c5ee69659bea
+  metadata.gz: cac7a475384c04aaa8a1879a207ac2bb5fad40347f21142de904cc78f5525717
+  data.tar.gz: 4282ca2795d7d8baefd750e0c283302ba5c8138d3105fcc94b66162d2671dceb
 SHA512:
-  metadata.gz: ec87c66817c1ce3f318e45d5b26221ca44c1958dfa9009fbe1541a1b3bbdb8e4585f6baf66e73c6a839a10d938e189b61dbea43afccf74a6f55476ad8719e18e
-  data.tar.gz: 0c33454fa368f009a7383ebc8a6a79f3638615740bd312eaab568c21d90cb20a0c4c6615324b20634f00614021016cf6d464910679fcf61fdf8d663097935cd7
+  metadata.gz: 4e1802542e03ca5f467d383897297593dc3e20232774d71c1fca57a56b8610d6b3e0985d8692554c7a7e4532604f66088c50782433cee3730276e8ac1c4de9e4
+  data.tar.gz: d763469fabd810b6c81deca4f2463982ec1e92016a029edf33871ea62e105c231ed43eb7dab0524a374a5782d0ab3cd2e8f53f9a6e0a01f61c5cfdcbcb3a3724

data/README.adoc

@@ -4,7 +4,7 @@
 
 Archaeo is a Ruby client for the Internet Archive's https://web.archive.org[Wayback Machine] APIs.
 
-It provides a model-driven interface for querying archived snapshots, checking availability, saving URLs, and fetching archived content.
+It provides a model-driven interface for querying archived snapshots, checking availability, saving URLs, fetching archived content, and bulk downloading with resume support.
 
 == Installation
 
@@ -33,7 +33,7 @@ require "archaeo"
 ----
 cdx = Archaeo::CdxApi.new
 
-# Enumerate all snapshots
+# Enumerate all snapshots (auto-paginates via resume key)
 cdx.snapshots("example.com").each do |snapshot|
   puts snapshot.timestamp
   puts snapshot.original_url
@@ -48,6 +48,23 @@ near   = cdx.near("example.com", timestamp: "20220101")
 # Filter by time
 before = cdx.before("example.com", timestamp: "20220101")
 after  = cdx.after("example.com", timestamp: "20220101")
+
+# Filter by status code, mimetype, or URL pattern
+cdx.snapshots("example.com",
+  filters: [Archaeo::CdxFilter.by_status(200)],
+  collapse: ["digest"],
+  match_type: "domain",
+  sort: "reverse",
+)
+
+# Page-based pagination
+cdx.snapshots("example.com", page: 0)
+
+# Count pages
+cdx.num_pages("example.com")
+
+# Discover all known URLs for a domain
+cdx.known_urls("example.com")
 ----
 
 === Check Availability
@@ -94,6 +111,65 @@ page = fetcher.fetch("https://example.com/",
                      identity: true)
 ----
 
+=== Fetch Page with Assets
+
+[source,ruby]
+----
+fetcher = Archaeo::Fetcher.new
+bundle = fetcher.fetch_page_with_assets("https://example.com/",
+                                        timestamp: "20220615000000")
+
+bundle.page        # => Archaeo::Page
+bundle.assets      # => Archaeo::AssetList
+bundle.assets.css  # => ["https://example.com/style.css", ...]
+bundle.assets.js   # => ["https://example.com/app.js", ...]
+bundle.assets.images
+bundle.assets.fonts
+bundle.assets.media
+----
+
+=== Bulk Download with Resume
+
+[source,ruby]
+----
+downloader = Archaeo::BulkDownloader.new(output_dir: "archive")
+downloader.download("example.com") do |current, total, snapshot|
+  puts "[#{current}/#{total}] #{snapshot.original_url}"
+end
+
+# Resume interrupted download
+downloader.download("example.com", resume: true)
+
+# Filter by date range
+downloader.download("example.com",
+                    from: "20220101", to: "20221231")
+----
+
+=== URL Normalization
+
+[source,ruby]
+----
+Archaeo::UrlNormalizer.normalize("  https://example.com/  ")
+# => "https://example.com/"
+
+Archaeo::UrlNormalizer.normalize('"https://example.com/%252F"')
+# => "https://example.com/%2F"
+
+Archaeo::UrlNormalizer.with_scheme("example.com")
+# => "https://example.com"
+----
+
+=== CDX Filters
+
+[source,ruby]
+----
+# Build validated filter expressions
+Archaeo::CdxFilter.by_status(200)           # => "statuscode:200"
+Archaeo::CdxFilter.excluding_status(404)    # => "!statuscode:404"
+Archaeo::CdxFilter.by_mimetype("text/html") # => "mimetype:text/html"
+Archaeo::CdxFilter.by_url("example.com")    # => "original:example.com"
+----
+
 === Timestamps
 
 [source,ruby]
@@ -121,8 +197,13 @@ ts1 < ts2   # => true/false
 
 [source,bash]
 ----
-# List snapshots
+# Show version
+archaeo --version
+
+# List snapshots (table, json, or csv format)
 archaeo snapshots example.com
+archaeo snapshots --format json example.com
+archaeo snapshots --format csv --from 20220101 --to 20221231 example.com
 
 # Find closest snapshot
 archaeo near example.com 20220101
@@ -136,8 +217,40 @@ archaeo save https://example.com/
 # Fetch archived content
 archaeo fetch https://example.com/ 20220615120000
 
+# Fetch and save to file
+archaeo fetch --output page.html https://example.com/ 20220615120000
+
 # Fetch raw (identity) content
 archaeo fetch --identity https://example.com/ 20220615120000
+
+# Download all snapshots
+archaeo download example.com --output ./archive
+
+# Resume interrupted download
+archaeo download example.com --resume
+
+# Discover all known URLs for a domain
+archaeo known_urls example.com
+----
+
+=== Error Handling
+
+[source,ruby]
+----
+# Blocked site (robots.txt)
+Archaeo::BlockedSiteError
+
+# No snapshot found
+Archaeo::NoSnapshotFound
+
+# Rate limited by Wayback Machine
+Archaeo::RateLimitError
+
+# Maximum retries exceeded
+Archaeo::MaximumRetriesExceeded
+
+# SavePageNow session limit
+Archaeo::SaveFailed
 ----
 
 == Architecture
@@ -149,20 +262,28 @@ Archaeo follows a model-driven, OOP design:
 | Layer | Classes | Purpose
 
 | *Models*
-| `Timestamp`, `ArchiveUrl`, `Snapshot`, `Page`, `SaveResult`, `AvailabilityResult`
+| `Timestamp`, `ArchiveUrl`, `Snapshot`, `Page`, `PageBundle`, `SaveResult`, `AvailabilityResult`
 | Domain value objects
 
+| *URL Processing*
+| `UrlNormalizer`, `CdxFilter`, `UrlRewriter`
+| URL sanitization, filtering, and rewriting
+
+| *Asset Extraction*
+| `AssetExtractor`, `AssetList`
+| Parse HTML for resource URLs
+
 | *APIs*
 | `CdxApi`, `AvailabilityApi`, `SaveApi`
 | Query and mutate the archive
 
 | *Operations*
-| `Fetcher`
-| Download archived content
+| `Fetcher`, `BulkDownloader`, `DownloadState`
+| Download content with resume support
 
 | *Infrastructure*
 | `HttpClient`
-| HTTP transport with retries and gzip
+| HTTP transport with retries, gzip, connection pooling
 |===
 
 All API classes accept an `HttpClient` via dependency injection for testability.

data/lib/archaeo/cli.rb

@@ -158,13 +158,14 @@ module Archaeo
     end
 
     def output_csv(snaps)
-      puts CSV.generate do |csv|
+      csv = CSV.generate do |csv|
         csv << %w[timestamp status_code url archive_url]
         snaps.each do |snap|
           csv << [snap.timestamp.to_s, snap.status_code,
                   snap.original_url, snap.archive_url]
         end
       end
+      puts csv
     end
 
     def write_output(path, content)

data/lib/archaeo/url_normalizer.rb

@@ -22,6 +22,23 @@ module Archaeo
       normalized.match?(%r{\A[a-z][a-z0-9+\-.]*://}) ? normalized : "https://#{normalized}"
     end
 
+    VALID_URL_RE = %r{\A([a-z][a-z0-9+\-.]*://)?[^\s]+\z}
+
+    def self.valid?(url)
+      normalized = normalize(url)
+      return false if normalized.empty?
+
+      normalized.match?(VALID_URL_RE)
+    end
+
+    def self.validate!(url)
+      normalized = normalize(url)
+      raise ArgumentError, "URL cannot be empty" if normalized.empty?
+      raise ArgumentError, "Invalid URL: #{url}" unless valid?(url)
+
+      normalized
+    end
+
     def to_s
       @normalized
     end

data/lib/archaeo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Archaeo
-  VERSION = "0.2.1"
+  VERSION = "0.2.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: archaeo
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Ribose Inc.