@hraness/sponge 0.6.6 → 0.6.8

package/README.md

@@ -41,6 +41,22 @@ websites.
 `doctor` reports runtime integrity and pending cleanup. Private-network capture
 requires `--allow-private-network` with `--wait` or `run --drain`.
 
+### Dead, blocked and paywalled pages
+
+`sponge status` counts saved web articles under 200 words (social posts aside),
+and `list --short` names them: usually a paywall stub, an error page or a page
+that needed scripts. `refresh ITEM --archive` (or `add URL --archive`) captures
+the newest Internet Archive copy instead, found through the Wayback Machine's CDX
+index, and fetches the raw saved bytes (`id_`) without the replay toolbar. The
+item keeps its original URL; the capture's `source_url` names the snapshot. Only
+`--archive` sends the URL to archive.org. No copy fails with `unavailable` and
+leaves the saved revision alone.
+
+archive.today (archive.ph, archive.is) has no API, answers automated clients with
+a CAPTCHA, and gives wrong DNS answers to some public resolvers, so Sponge does not
+fetch from it. To use one of its copies, open it in a browser and save it as a PDF;
+then `sponge add FILE.pdf` keeps it.
+
 Images use `sponge-asset:SHA256` references. `asset ITEM SHA256 > image.png`
 retrieves verified bytes offline; `asset ITEM SHA256 --json` returns its metadata.
 Unresolved remote images become unavailable markers, and captures retain partial

package/dist/src/app/archive.d.ts

@@ -0,0 +1,15 @@
+import type { ErrorEnvelope } from '../../contracts/index.js';
+import { type SafeFetchOptions, type SafeFetchResult } from '../capture/network.js';
+type Fetch = (url: URL, options: SafeFetchOptions) => Promise<SafeFetchResult>;
+/** A saved copy of a page: the raw bytes the archive stored (`id_`), without its replay toolbar or rewritten links. */
+export interface Snapshot {
+    url: string;
+    timestamp: string;
+}
+/**
+ * The newest Internet Archive copy of `url` that the archive saw answer 200, found through its CDX index. This
+ * sends the URL to archive.org, so only an explicit `--archive` asks. No copy is `unavailable`; a busy index is
+ * `rate-limited` and retryable.
+ */
+export declare function waybackSnapshot(url: string, requestId: string, fetch?: Fetch): Promise<Snapshot | ErrorEnvelope>;
+export {};

package/dist/src/app/archive.js

@@ -0,0 +1,51 @@
+import { failure } from '../acquisition/errors.js';
+import { FetchFailure, safeFetch } from '../capture/network.js';
+import { DEFAULT_USER_AGENT } from '../capture/types.js';
+/**
+ * The newest Internet Archive copy of `url` that the archive saw answer 200, found through its CDX index. This
+ * sends the URL to archive.org, so only an explicit `--archive` asks. No copy is `unavailable`; a busy index is
+ * `rate-limited` and retryable.
+ */
+export async function waybackSnapshot(url, requestId, fetch = safeFetch) {
+    const query = new URL('https://web.archive.org/cdx/search/cdx');
+    for (const [key, value] of Object.entries({
+        url,
+        output: 'json',
+        filter: 'statuscode:200',
+        limit: '-1',
+        fl: 'timestamp,original',
+    }))
+        query.searchParams.set(key, value);
+    let rows;
+    try {
+        const response = await fetch(query, {
+            timeoutMs: 30_000,
+            maxBytes: 64 * 1024,
+            allowPrivateNetwork: false,
+            userAgent: DEFAULT_USER_AGENT,
+            accept: 'application/json',
+            retries: 2,
+        });
+        const text = new TextDecoder().decode(response.bytes).trim();
+        rows = text ? JSON.parse(text) : [];
+    }
+    catch (error) {
+        if (error instanceof FetchFailure && error.status === 429)
+            return failure(requestId, 'rate-limited', 'acquisition', true);
+        return failure(requestId, 'unavailable', 'acquisition', true);
+    }
+    // The first row names the fields; the last is the newest capture.
+    const row = Array.isArray(rows) && rows.length > 1 ? rows.at(-1) : undefined;
+    const [timestamp, original] = Array.isArray(row) ? row : [];
+    if (typeof timestamp !== 'string' || !/^\d{14}$/.test(timestamp) || typeof original !== 'string')
+        return failure(requestId, 'unavailable');
+    try {
+        const page = new URL(original);
+        if (!['http:', 'https:'].includes(page.protocol))
+            return failure(requestId, 'unavailable');
+        return { url: `https://web.archive.org/web/${timestamp}id_/${page.href}`, timestamp };
+    }
+    catch {
+        return failure(requestId, 'unavailable');
+    }
+}

package/dist/src/app/article.d.ts

@@ -2,6 +2,7 @@ import type { ErrorEnvelope, JobCheckpoint } from '../../contracts/index.js';
 import { type CaptureRequest, type CaptureOutcome, type CaptureConnection } from '../acquisition/index.js';
 import type { LibraryStore, AttemptToken } from '../store/index.js';
 import { type PdfCaptureRequest, type CaptureOutcome as PdfCaptureOutcome } from '../pdf/capture.js';
+import { waybackSnapshot } from './archive.js';
 interface ArticleResult {
     ok: true;
     itemId: string;
@@ -19,6 +20,7 @@ export interface ArticleOptions {
     allowPrivateNetwork?: boolean;
     capture?: (request: CaptureRequest) => Promise<CaptureOutcome>;
     capturePdf?: (request: PdfCaptureRequest) => Promise<PdfCaptureOutcome>;
+    snapshot?: typeof waybackSnapshot;
 }
 /** One attempt. Durable queue policy and process recovery live above this function. */
 export declare function ingestArticle(store: LibraryStore, token: AttemptToken, options: ArticleOptions): Promise<ArticleOutcome>;

package/dist/src/app/article.js

@@ -5,10 +5,11 @@ import { runAttempt, classifyErrors } from '../ingest/attempt.js';
 import { sha256 } from '../util/hash.js';
 import { bundleBytes, bundleText } from './bundle-files.js';
 import { selectedMetadata } from '../metadata/article.js';
-import { localizeMarkdown } from './markdown.js';
+import { localizeMarkdown, proseWords } from './markdown.js';
 import { SpongeError } from '../errors.js';
 import { capturePdf } from '../pdf/capture.js';
 import { storePdfCapture } from '../pdf/handler.js';
+import { waybackSnapshot } from './archive.js';
 const classify = classifyErrors([
     'integrity',
     'limit-exceeded',
@@ -23,12 +24,19 @@ export function ingestArticle(store, token, options) {
         const source = store.queue.source(token.jobId);
         if (!source || source.kind !== 'article')
             throw new SpongeError('invalid-input');
+        // An archived copy is public: it is fetched directly, never through a signed-in browser.
+        const snapshot = source.archive
+            ? await (options.snapshot ?? waybackSnapshot)(source.locator, token.attemptId)
+            : null;
+        if (snapshot && 'ok' in snapshot)
+            return snapshot;
+        const fetched = snapshot?.url ?? source.locator;
         const result = await (options.capture ?? captureArticle)({
             runProcess: options.runProcess,
             stagingRoot: join(store.root, 'staging'),
-            sourceUrl: source.locator,
+            sourceUrl: fetched,
             requestId: token.attemptId,
-            connection: options.connection,
+            connection: snapshot ? undefined : options.connection,
             signal: options.signal,
             allowPrivateNetwork: options.allowPrivateNetwork,
         });
@@ -41,12 +49,12 @@ export function ingestArticle(store, token, options) {
         await options.checkpoint?.('extracting');
         const capture = result.capture;
         validateCapture(capture);
-        if (capture.kind !== 'article' || capture.requestId !== token.attemptId || capture.sourceUrl !== source.locator)
+        if (capture.kind !== 'article' || capture.requestId !== token.attemptId || capture.sourceUrl !== fetched)
             throw new SpongeError('integrity');
         const original = await bundleText(result.bundleRoot, capture.markdown);
         const selectedTitle = selectedMetadata(capture.metadata, 'title')?.value;
         const title = typeof selectedTitle === 'string' ? selectedTitle : undefined;
-        const paperUrl = proseWords(original) < landingPageWords ? fullTextPdf(capture.metadata, source.locator) : undefined;
+        const paperUrl = proseWords(original) < landingPageWords ? fullTextPdf(capture.metadata, fetched) : undefined;
         if (paperUrl) {
             const paper = await (options.capturePdf ?? capturePdf)({
                 runProcess: options.runProcess,
@@ -115,11 +123,6 @@ export function ingestArticle(store, token, options) {
 }
 /** Prose words under which a page that names its paper's PDF is only a landing page: a title, authors and abstract. */
 const landingPageWords = 1000;
-function proseWords(markdown) {
-    const body = markdown.startsWith('---\n') ? markdown.slice(Math.max(0, markdown.indexOf('\n---', 4))) : markdown;
-    const prose = body.replace(/!\[[^\]]*\]\([^)]*\)/g, ' ').replace(/\[([^\]]*)\]\([^)]*\)/g, '$1');
-    return (prose.match(/\p{L}[\p{L}\p{M}'’-]*/gu) ?? []).length;
-}
 /**
  * The full text a scholarly landing page declares in `citation_pdf_url` (arXiv, NBER, OSF and most journals), when
  * it is served from the page's own site or a subdomain relation of it; other sites are never fetched on its word.

package/dist/src/app/markdown.d.ts

@@ -4,3 +4,5 @@ export declare function localizeMarkdown(markdown: string, assets: Asset[]): {
     markdown: string;
     missingImages: boolean;
 };
+/** Words with letters past the frontmatter, images dropped and link text kept: how much a capture says. */
+export declare function proseWords(markdown: string): number;

package/dist/src/app/markdown.js

@@ -100,3 +100,9 @@ export function localizeMarkdown(markdown, assets) {
         markdown = markdown.slice(0, patch.start) + patch.value + markdown.slice(patch.end);
     return { markdown, missingImages };
 }
+/** Words with letters past the frontmatter, images dropped and link text kept: how much a capture says. */
+export function proseWords(markdown) {
+    const body = markdown.startsWith('---\n') ? markdown.slice(Math.max(0, markdown.indexOf('\n---', 4))) : markdown;
+    const prose = body.replace(/!\[[^\]]*\]\([^)]*\)/g, ' ').replace(/\[([^\]]*)\]\([^)]*\)/g, '$1');
+    return (prose.match(/\p{L}[\p{L}\p{M}'’-]*/gu) ?? []).length;
+}

package/dist/src/capture/network.d.ts

@@ -0,0 +1,56 @@
+export type FetchFailureCode = 'invalid-url' | 'private-network' | 'dns' | 'timeout' | 'http' | 'redirect' | 'too-large' | 'network';
+export declare class FetchFailure extends Error {
+    readonly code: FetchFailureCode;
+    readonly status: number | null;
+    constructor(code: FetchFailureCode, message: string, options?: ErrorOptions & {
+        status?: number;
+    });
+}
+export interface SafeFetchOptions {
+    timeoutMs: number;
+    maxBytes: number;
+    allowPrivateNetwork: boolean;
+    userAgent: string;
+    referer?: string;
+    accept?: string;
+    retries?: number;
+    maxRedirects?: number;
+}
+export interface SafeFetchResult {
+    bytes: Uint8Array;
+    finalUrl: URL;
+    status: number;
+    contentType: string | null;
+}
+interface ResolvedAddress {
+    address: string;
+    family: 4 | 6;
+}
+export interface PinnedRequest {
+    url: URL;
+    address: ResolvedAddress;
+    headers: Headers;
+    signal: AbortSignal;
+}
+export interface PinnedResponse {
+    status: number;
+    headers: Headers;
+    body: AsyncIterable<unknown> | null;
+    cancel: () => void;
+}
+export interface SafeFetchDependencies {
+    resolveHostname: (hostname: string) => Promise<readonly ResolvedAddress[]>;
+    transport: (request: PinnedRequest) => Promise<PinnedResponse>;
+    /** Test seam. Production callers use the system interface snapshot. */
+    localAddresses: () => readonly string[];
+}
+/** True for loopback, link-local, private, reserved, multicast and unspecified addresses. */
+export declare function isPrivateAddress(address: string): boolean;
+export declare function isPrivateHostname(hostname: string): boolean;
+/** Refuse a private, reserved or host-assigned target unless explicitly allowed. */
+export declare function assertPublicUrl(url: URL, allowPrivateNetwork: boolean, timeoutMs?: number): Promise<void>;
+/** A bounded fetcher: every hop is resolved, validated and pinned; redirects are re-checked; bodies are capped. */
+export declare function createSafeFetch(dependencies?: Partial<SafeFetchDependencies>): (url: URL, options: SafeFetchOptions) => Promise<SafeFetchResult>;
+export declare const safeFetch: (url: URL, options: SafeFetchOptions) => Promise<SafeFetchResult>;
+export declare function decodeBytes(bytes: Uint8Array, contentType: string | null): string;
+export {};

package/dist/src/capture/network.js

@@ -0,0 +1,453 @@
+// Ported from Wordcell's clip/network.ts (MIT, see native/NOTICES-capture.txt). Cookie forwarding, connection
+// pooling and manual redirects are dropped: Sponge captures only public pages over a pinned, bounded fetch.
+import { lookup } from 'node:dns/promises';
+import { request as requestHttp } from 'node:http';
+import { request as requestHttps } from 'node:https';
+import { isIP } from 'node:net';
+import { networkInterfaces } from 'node:os';
+import { setTimeout as sleep } from 'node:timers/promises';
+export class FetchFailure extends Error {
+    code;
+    status;
+    constructor(code, message, options) {
+        super(message, options);
+        this.name = 'FetchFailure';
+        this.code = code;
+        this.status = options?.status ?? null;
+    }
+}
+const privateSuffixes = ['.localhost', '.local', '.internal', '.home', '.lan'];
+const retryableStatuses = new Set([408, 425, 429, 500, 502, 503, 504]);
+const normalizeHostname = (hostname) => (hostname.startsWith('[') && hostname.endsWith(']') ? hostname.slice(1, -1) : hostname)
+    .toLowerCase()
+    .replace(/\.$/, '');
+function parseIpv4(address) {
+    const pieces = address.split('.');
+    if (pieces.length !== 4)
+        return null;
+    const numbers = pieces.map(Number);
+    return numbers.every((piece) => Number.isInteger(piece) && piece >= 0 && piece <= 255) ? numbers : null;
+}
+function parseIpv6(address) {
+    let value = address;
+    const separator = value.lastIndexOf(':');
+    const tail = separator >= 0 ? value.slice(separator + 1) : value;
+    if (tail.includes('.')) {
+        const ipv4 = parseIpv4(tail);
+        if (ipv4 === null || separator < 0)
+            return null;
+        value = `${value.slice(0, separator)}:${((ipv4[0] << 8) | ipv4[1]).toString(16)}:${((ipv4[2] << 8) | ipv4[3]).toString(16)}`;
+    }
+    const compression = value.indexOf('::');
+    if (compression !== -1 && compression !== value.lastIndexOf('::'))
+        return null;
+    const leftText = compression === -1 ? value : value.slice(0, compression);
+    const rightText = compression === -1 ? '' : value.slice(compression + 2);
+    const left = leftText === '' ? [] : leftText.split(':');
+    const right = rightText === '' ? [] : rightText.split(':');
+    if ([...left, ...right].some((part) => !/^[0-9a-f]{1,4}$/i.test(part)))
+        return null;
+    const missing = 8 - left.length - right.length;
+    if ((compression === -1 && missing !== 0) || (compression !== -1 && missing < 1))
+        return null;
+    return [
+        ...left.map((part) => Number.parseInt(part, 16)),
+        ...Array.from({ length: missing }, () => 0),
+        ...right.map((part) => Number.parseInt(part, 16)),
+    ];
+}
+function withoutScope(address) {
+    const normalized = normalizeHostname(address);
+    const scope = normalized.indexOf('%');
+    return scope === -1 ? normalized : normalized.slice(0, scope);
+}
+/** The IPv4 address an IPv4-mapped or IPv4-compatible IPv6 address reaches, if any. */
+function embeddedIpv4(groups) {
+    const compatible = groups.slice(0, 6).every((group) => group === 0);
+    const mapped = groups.slice(0, 5).every((group) => group === 0) && groups[5] === 0xffff;
+    if (!compatible && !mapped)
+        return null;
+    const high = groups[6], low = groups[7];
+    return `${high >>> 8}.${high & 0xff}.${low >>> 8}.${low & 0xff}`;
+}
+/** Syntax-independent keys for one IP literal; an embedded IPv4 address also gets its IPv4 key. */
+function addressKeys(address) {
+    const normalized = withoutScope(address);
+    const version = isIP(normalized);
+    if (version === 4) {
+        const pieces = parseIpv4(normalized);
+        return pieces === null ? [] : [`4:${pieces.join('.')}`];
+    }
+    if (version !== 6)
+        return [];
+    const groups = parseIpv6(normalized);
+    if (groups === null)
+        return [];
+    const keys = [`6:${groups.map((group) => group.toString(16).padStart(4, '0')).join(':')}`];
+    const ipv4 = embeddedIpv4(groups);
+    if (ipv4 !== null)
+        keys.push(`4:${ipv4}`);
+    return keys;
+}
+function systemLocalAddresses() {
+    try {
+        return Object.values(networkInterfaces()).flatMap((records) => (records ?? []).map((record) => record.address));
+    }
+    catch (error) {
+        throw new FetchFailure('network', 'could not inspect local network interfaces', { cause: error });
+    }
+}
+function localKeys(provider) {
+    let addresses;
+    try {
+        addresses = provider();
+    }
+    catch (error) {
+        if (error instanceof FetchFailure)
+            throw error;
+        throw new FetchFailure('network', 'could not inspect local network interfaces', { cause: error });
+    }
+    return new Set(addresses.flatMap(addressKeys));
+}
+const isLocalAddress = (address, keys) => addressKeys(address).some((key) => keys.has(key));
+/** True for loopback, link-local, private, reserved, multicast and unspecified addresses. */
+export function isPrivateAddress(address) {
+    const normalized = withoutScope(address);
+    const version = isIP(normalized);
+    if (version === 4) {
+        const pieces = parseIpv4(normalized);
+        if (pieces === null)
+            return true;
+        const [a, b, c] = pieces;
+        return (a === 0 ||
+            a === 10 ||
+            a === 127 ||
+            (a === 100 && b >= 64 && b <= 127) ||
+            (a === 169 && b === 254) ||
+            (a === 172 && b >= 16 && b <= 31) ||
+            (a === 192 && (b === 0 || b === 168)) ||
+            (a === 198 && (b === 18 || b === 19)) ||
+            (a === 198 && b === 51 && c === 100) ||
+            (a === 203 && b === 0 && c === 113) ||
+            a >= 224);
+    }
+    if (version === 6) {
+        const groups = parseIpv6(normalized);
+        if (groups === null)
+            return true;
+        const ipv4 = embeddedIpv4(groups);
+        if (ipv4 !== null)
+            return isPrivateAddress(ipv4);
+        const [first, second] = groups;
+        return ((first & 0xfe00) === 0xfc00 ||
+            (first & 0xffc0) === 0xfe80 ||
+            (first & 0xffc0) === 0xfec0 ||
+            (first & 0xff00) === 0xff00 ||
+            first === 0x0064 ||
+            first === 0x0100 ||
+            (first === 0x2001 && second <= 0x01ff) ||
+            (first === 0x2001 && second === 0x0db8) ||
+            first === 0x2002 ||
+            first === 0x3ffe ||
+            (first === 0x3fff && (second & 0xf000) === 0) ||
+            first === 0x5f00);
+    }
+    return true;
+}
+export function isPrivateHostname(hostname) {
+    const normalized = normalizeHostname(hostname);
+    if (normalized === 'localhost' || normalized === 'localhost.localdomain')
+        return true;
+    if (privateSuffixes.some((suffix) => normalized.endsWith(suffix)))
+        return true;
+    return isIP(normalized) !== 0 && isPrivateAddress(normalized);
+}
+async function systemResolve(hostname) {
+    const answers = await lookup(hostname, { all: true, verbatim: true });
+    return answers.flatMap((answer) => answer.family === 4 || answer.family === 6 ? [{ address: answer.address, family: answer.family }] : []);
+}
+async function beforeDeadline(promise, deadline, message) {
+    const remaining = deadline - Date.now();
+    if (remaining <= 0)
+        throw new FetchFailure('timeout', message);
+    let timer;
+    try {
+        return await Promise.race([
+            promise,
+            new Promise((_, reject) => {
+                timer = setTimeout(() => reject(new FetchFailure('timeout', message)), remaining);
+            }),
+        ]);
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+async function resolveTarget(url, allowPrivateNetwork, dependencies, deadline, timeoutMs) {
+    if (url.protocol !== 'http:' && url.protocol !== 'https:')
+        throw new FetchFailure('invalid-url', `unsupported URL protocol: ${url.protocol}`);
+    if (url.username !== '' || url.password !== '')
+        throw new FetchFailure('invalid-url', 'credential-bearing URLs are not accepted');
+    const hostname = normalizeHostname(url.hostname);
+    if (!allowPrivateNetwork && isPrivateHostname(hostname))
+        throw new FetchFailure('private-network', `private-network URL: ${url.origin}`);
+    const local = allowPrivateNetwork ? new Set() : localKeys(dependencies.localAddresses);
+    const literal = isIP(hostname);
+    if (literal === 4 || literal === 6) {
+        if (isLocalAddress(hostname, local))
+            throw new FetchFailure('private-network', `address ${hostname} is assigned to a local interface`);
+        return [{ address: hostname, family: literal }];
+    }
+    let answers;
+    try {
+        answers = await beforeDeadline(dependencies.resolveHostname(hostname), deadline, `request timed out after ${timeoutMs}ms while resolving ${hostname}`);
+    }
+    catch (error) {
+        if (error instanceof FetchFailure)
+            throw error;
+        throw new FetchFailure('dns', `could not resolve ${hostname}`, { cause: error });
+    }
+    if (answers.length === 0)
+        throw new FetchFailure('dns', `could not resolve ${hostname}`);
+    // DNS can take most of the deadline; an interface that appeared meanwhile must not become a local hop.
+    if (!allowPrivateNetwork)
+        for (const key of localKeys(dependencies.localAddresses))
+            local.add(key);
+    const unique = new Map();
+    for (const answer of answers) {
+        const address = normalizeHostname(answer.address);
+        const family = isIP(address);
+        if ((family !== 4 && family !== 6) || family !== answer.family)
+            throw new FetchFailure('dns', `${hostname} returned an invalid DNS answer`);
+        if (!allowPrivateNetwork && (isPrivateAddress(address) || isLocalAddress(address, local)))
+            throw new FetchFailure('private-network', `${hostname} resolves to a private or local address`);
+        unique.set(`${family}:${address}`, { address, family });
+    }
+    return [...unique.values()];
+}
+/** Refuse a private, reserved or host-assigned target unless explicitly allowed. */
+export async function assertPublicUrl(url, allowPrivateNetwork, timeoutMs = 30_000) {
+    await resolveTarget(url, allowPrivateNetwork, { resolveHostname: systemResolve, localAddresses: systemLocalAddresses }, Date.now() + timeoutMs, timeoutMs);
+}
+/** A lookup callback that can only answer with the address already validated. */
+const pinnedLookup = ({ address, family }) => (_hostname, options, callback) => queueMicrotask(() => options.all === true ? callback(null, [{ address, family }]) : callback(null, address, family));
+function responseHeaders(headers) {
+    const result = new Headers();
+    for (const [name, value] of Object.entries(headers)) {
+        if (typeof value === 'string')
+            result.append(name, value);
+        else if (Array.isArray(value))
+            for (const item of value)
+                result.append(name, item);
+    }
+    return result;
+}
+/** One HTTP(S) request through one validated address. No DNS, redirects, retries or body accumulation. */
+function requestPinned(request) {
+    const hostname = normalizeHostname(request.url.hostname);
+    if (isIP(request.address.address) !== request.address.family)
+        return Promise.reject(new Error('pinned address does not match its family'));
+    const literal = isIP(hostname);
+    if (literal !== 0 &&
+        (literal !== request.address.family || addressKeys(hostname)[0] !== addressKeys(request.address.address)[0]))
+        return Promise.reject(new Error('IP-literal hostname does not match the pinned address'));
+    const headers = {};
+    request.headers.forEach((value, name) => (headers[name] = value));
+    const options = {
+        protocol: request.url.protocol,
+        hostname,
+        method: 'GET',
+        path: `${request.url.pathname}${request.url.search}`,
+        headers,
+        lookup: pinnedLookup(request.address),
+        family: request.address.family,
+        signal: request.signal,
+        agent: false,
+        ...(request.url.port === '' ? {} : { port: request.url.port }),
+    };
+    return new Promise((resolve, reject) => {
+        const onResponse = (response) => {
+            if (response.statusCode === undefined) {
+                response.destroy();
+                reject(new Error('HTTP response omitted a status code'));
+                return;
+            }
+            resolve({
+                status: response.statusCode,
+                headers: responseHeaders(response.headers),
+                body: response,
+                cancel: () => response.destroy(),
+            });
+        };
+        const client = request.url.protocol === 'https:'
+            ? requestHttps({ ...options, ...(literal === 0 ? { servername: hostname } : {}) }, onResponse)
+            : requestHttp(options, onResponse);
+        // A handshake error can be followed by a socket teardown error; keep the listener for the request lifetime.
+        client.on('error', reject);
+        client.end();
+    });
+}
+function retryDelay(response, attempt) {
+    const header = response?.headers.get('retry-after')?.trim();
+    if (header) {
+        const seconds = Number(header);
+        if (Number.isFinite(seconds) && seconds >= 0)
+            return Math.min(seconds * 1000, 5000);
+        const date = Date.parse(header);
+        if (!Number.isNaN(date))
+            return Math.min(Math.max(date - Date.now(), 0), 5000);
+    }
+    return Math.min(250 * 2 ** attempt, 2000);
+}
+async function readBounded(response, maxBytes) {
+    const declared = Number(response.headers.get('content-length') ?? NaN);
+    if (Number.isFinite(declared) && declared > maxBytes) {
+        response.cancel();
+        throw new FetchFailure('too-large', `response declares ${declared} bytes; limit is ${maxBytes}`);
+    }
+    if (response.body === null)
+        return new Uint8Array();
+    const chunks = [];
+    let total = 0;
+    for await (const chunk of response.body) {
+        if (!(chunk instanceof Uint8Array)) {
+            response.cancel();
+            throw new FetchFailure('network', 'response body yielded a non-byte chunk');
+        }
+        total += chunk.byteLength;
+        if (total > maxBytes) {
+            response.cancel();
+            throw new FetchFailure('too-large', `response exceeded ${maxBytes} bytes`);
+        }
+        chunks.push(chunk);
+    }
+    return Buffer.concat(chunks, total);
+}
+function requestHeaders(current, options) {
+    const headers = new Headers({
+        Accept: options.accept ?? 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.5',
+        'Accept-Encoding': 'identity',
+        'User-Agent': options.userAgent,
+    });
+    if (options.referer !== undefined) {
+        try {
+            const referer = new URL(options.referer);
+            headers.set('Referer', referer.origin === current.origin ? referer.href : `${referer.origin}/`);
+        }
+        catch {
+            // An invalid optional referrer is dropped rather than forwarded.
+        }
+    }
+    return headers;
+}
+/** A bounded fetcher: every hop is resolved, validated and pinned; redirects are re-checked; bodies are capped. */
+export function createSafeFetch(dependencies = {}) {
+    const resolve = {
+        resolveHostname: dependencies.resolveHostname ?? systemResolve,
+        localAddresses: dependencies.localAddresses ?? systemLocalAddresses,
+    };
+    const transport = dependencies.transport ?? requestPinned;
+    return async (url, options) => {
+        const maxRedirects = options.maxRedirects ?? 8, retries = options.retries ?? 2, deadline = Date.now() + options.timeoutMs;
+        let current = new URL(url);
+        for (let redirects = 0; redirects <= maxRedirects; redirects += 1) {
+            const timedOut = `request timed out after ${options.timeoutMs}ms: ${current.href}`;
+            const addresses = await resolveTarget(current, options.allowPrivateNetwork, resolve, deadline, options.timeoutMs);
+            let response = null, lastError, controller = null, timer;
+            for (let attempt = 0; attempt <= retries; attempt += 1) {
+                const remaining = deadline - Date.now();
+                if (remaining <= 0)
+                    throw new FetchFailure('timeout', timedOut);
+                const attemptController = new AbortController();
+                const attemptTimer = setTimeout(() => attemptController.abort(), remaining);
+                try {
+                    response = await beforeDeadline(transport({
+                        url: new URL(current),
+                        address: addresses[attempt % addresses.length],
+                        headers: requestHeaders(current, options),
+                        signal: attemptController.signal,
+                    }), deadline, timedOut);
+                    if (!retryableStatuses.has(response.status) || attempt === retries) {
+                        controller = attemptController;
+                        timer = attemptTimer;
+                        break;
+                    }
+                    attemptController.abort();
+                    response.cancel();
+                    clearTimeout(attemptTimer);
+                }
+                catch (error) {
+                    lastError = error;
+                    // The deadline race and the abort timer are separate callbacks; abort so a pending transport frees its socket.
+                    attemptController.abort();
+                    clearTimeout(attemptTimer);
+                    if (error instanceof FetchFailure && error.code === 'timeout')
+                        throw error;
+                    if (attempt === retries) {
+                        if (attemptController.signal.aborted && Date.now() >= deadline)
+                            throw new FetchFailure('timeout', timedOut, { cause: error });
+                        if (error instanceof FetchFailure)
+                            throw error;
+                        throw new FetchFailure('network', `request failed: ${current.href}`, { cause: error });
+                    }
+                }
+                const delay = Math.min(retryDelay(response, attempt), Math.max(0, deadline - Date.now()));
+                if (delay > 0)
+                    await sleep(delay);
+            }
+            if (response === null)
+                throw new FetchFailure('network', `request failed: ${current.href}`, { cause: lastError });
+            if (response.status >= 300 && response.status < 400) {
+                try {
+                    const location = response.headers.get('location');
+                    if (location === null)
+                        throw new FetchFailure('redirect', `HTTP ${response.status} omitted Location`);
+                    if (redirects === maxRedirects)
+                        throw new FetchFailure('redirect', `more than ${maxRedirects} redirects`);
+                    try {
+                        current = new URL(location, current);
+                    }
+                    catch (error) {
+                        throw new FetchFailure('redirect', 'invalid redirect target', { cause: error });
+                    }
+                    continue;
+                }
+                finally {
+                    controller?.abort();
+                    response.cancel();
+                    clearTimeout(timer);
+                }
+            }
+            if (response.status < 200 || response.status >= 300) {
+                clearTimeout(timer);
+                controller?.abort();
+                response.cancel();
+                throw new FetchFailure('http', `HTTP ${response.status} for ${current.href}`, { status: response.status });
+            }
+            const bodyTimedOut = `response body timed out after ${options.timeoutMs}ms: ${current.href}`;
+            try {
+                const bytes = await beforeDeadline(readBounded(response, options.maxBytes), deadline, bodyTimedOut);
+                return { bytes, finalUrl: current, status: response.status, contentType: response.headers.get('content-type') };
+            }
+            catch (error) {
+                if (controller?.signal.aborted || (error instanceof FetchFailure && error.code === 'timeout')) {
+                    response.cancel();
+                    throw new FetchFailure('timeout', bodyTimedOut, { cause: error });
+                }
+                if (error instanceof FetchFailure)
+                    throw error;
+                throw new FetchFailure('network', `response body failed: ${current.href}`, { cause: error });
+            }
+            finally {
+                clearTimeout(timer);
+            }
+        }
+        throw new FetchFailure('redirect', `more than ${maxRedirects} redirects`);
+    };
+}
+export const safeFetch = createSafeFetch();
+export function decodeBytes(bytes, contentType) {
+    const charset = /charset\s*=\s*["']?([^;"'\s]+)/i.exec(contentType ?? '')?.[1]?.toLowerCase();
+    const encoding = charset === 'iso-8859-1' || charset === 'windows-1252' ? 'windows-1252' : 'utf-8';
+    return new TextDecoder(encoding, { fatal: false }).decode(bytes);
+}

package/dist/src/capture/types.d.ts

@@ -0,0 +1,26 @@
+export type CaptureScope = 'auto' | 'page' | 'thread' | 'comments';
+export type AcquisitionMethod = 'http' | 'hacker-news-api' | 'bluesky-api' | 'reddit-json' | 'browser-cdp';
+export type AcquiredPage = {
+    readonly body: string;
+    readonly contentType: string | null;
+    readonly finalUrl: URL;
+    readonly method: AcquisitionMethod;
+    readonly warnings: readonly string[];
+    readonly browserTitle?: string;
+    readonly contentTruncated?: boolean;
+    readonly renderedText?: string;
+    readonly renderedTextTruncated?: boolean;
+    readonly renderedTextByteLimit?: number;
+};
+/** The bounded knobs one capture runs under; every lane reads the same values. */
+export type CaptureOptions = {
+    readonly url: URL;
+    readonly scope: CaptureScope;
+    readonly timeoutMs: number;
+    readonly maxItems: number;
+    readonly maxDepth: number;
+    readonly maxHtmlBytes: number;
+    readonly allowPrivateNetwork: boolean;
+    readonly userAgent: string;
+};
+export declare const DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";

package/dist/src/capture/types.js

@@ -0,0 +1 @@
+export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';

package/dist/src/cli/arguments.js

@@ -8,8 +8,10 @@ const globals = {
     help: 'boolean',
 };
 /** A usage error names its command so the CLI can print that command's synopsis. */
-const usage = (command) => {
-    throw new SpongeError('invalid-input', { detail: command ? { command } : {} });
+const usage = (command, unknown) => {
+    throw new SpongeError('invalid-input', {
+        detail: { usage: true, ...(command ? { command } : {}), ...(unknown ? { unknown } : {}) },
+    });
 };
 /** No shell grammar, implicit option coercion, duplicate flags, or unbounded argv. Commands and their rules live in the registry. */
 export function parseArguments(argv) {
@@ -56,7 +58,7 @@ export function parseArguments(argv) {
     if (!command && flags.help)
         return { command: 'help', positional: [], flags: {}, json: false, debug: flags.debug === true };
     if (!command || !spec)
-        return usage();
+        return usage(undefined, command);
     if (flags.help)
         return { command: 'help', positional: [spec.name], flags: {}, json: false, debug: flags.debug === true };
     if (typeof spec.positional === 'number' ? positional.length !== spec.positional : !spec.positional(positional))

package/dist/src/cli/commands.js

@@ -4,11 +4,11 @@ import { connectionFor, readConnections } from './config.js';
 import { pdfSource } from '../pdf/index.js';
 import { articleSource, inferKind, mediaSource } from '../app/source.js';
 import { pdfToolsAvailable } from '../pdf/tools.js';
-import { queueAction } from './queue.js';
+import { queueAction, queueText } from './queue.js';
 import { portabilityAction } from './portability.js';
 import { restoreLibrary } from '../portability/backup.js';
 import { workerAction, serveWorker, hasDueAcquisition, workerDoctor } from './worker.js';
-import { libraryStatus, statusText } from './status.js';
+import { libraryStatus, shortCaptures, statusText } from './status.js';
 import { nativeSetupPlan, provisionNative, inspectNative, optionalNative } from '../models/runtime.js';
 import { digestModel, digestSetupPlan, inspectDigest, provisionDigest } from '../models/digest.js';
 import { withDigestServer } from '../digest/server.js';
@@ -28,6 +28,11 @@ import { minAge, publishSite } from '../publish/index.js';
 import { SpongeError, codeOf, publicCode, exitFor, isSpongeCode } from '../errors.js';
 const abs = (v) => typeof v === 'string' && isAbsolute(v);
 const media = { language: 'value', 'force-local': 'boolean', 'allow-private-network': 'boolean' };
+/** Only a public web article or PDF has an Internet Archive copy; a signed-in page's copy would not be the reader's. */
+function archivable(kind, locator, connection) {
+    if (kind === 'media' || connection || !/^https?:/.test(locator))
+        throw new SpongeError('unsupported');
+}
 const lines = (items) => items.map((item) => `${item.itemId}\t${item.title}`);
 /** The failure of a job that did not run here: its state, or its recorded reason. */
 function jobFailure(store, jobId) {
@@ -169,9 +174,12 @@ export const commands = [
     {
         name: 'add',
         summary: 'Save an article, PDF or media source',
-        usage: [['add URL_OR_PDF [--kind article|pdf|media] [--connection ID] [--wait]', '']],
+        usage: [
+            ['add URL_OR_PDF [--kind article|pdf|media] [--connection ID] [--wait]', ''],
+            ['add URL --archive [--wait]', 'Capture the newest Internet Archive copy of a dead or paywalled page'],
+        ],
         positional: 1,
-        flags: { connection: 'value', kind: 'value', wait: 'boolean', ...media },
+        flags: { connection: 'value', kind: 'value', wait: 'boolean', archive: 'boolean', ...media },
         library: 'open',
         handler: async (c) => {
             const { store, args } = c, connections = await readConnections(store.root);
@@ -180,7 +188,15 @@ export const commands = [
                 throw new SpongeError('invalid-input');
             const input = args.positional[0], kind = (args.flags.kind ?? inferKind(input));
             const source = (kind === 'pdf' ? pdfSource : kind === 'media' ? mediaSource : articleSource)(input, connection?.realm ?? 'public');
-            const submission = store.queue.submit({ kind, ...source, idempotencyKey: 'save:' + source.sourceKey }), { jobId, itemId } = submission;
+            const archive = args.flags.archive === true;
+            if (archive)
+                archivable(kind, source.locator, connection);
+            const submission = store.queue.submit({
+                kind,
+                ...source,
+                ...(archive ? { intent: 'archive' } : {}),
+                idempotencyKey: (archive ? 'archive:' : 'save:') + source.sourceKey,
+            }), { jobId, itemId } = submission;
             const job = store.queue.list().find((j) => j.id === jobId);
             if (!args.flags.wait) {
                 c.emit({ ...submission, state: job?.state ?? 'queued' }, `${itemId}\t${job?.state ?? 'queued'}`);
@@ -234,18 +250,27 @@ export const commands = [
         flags: {},
         library: 'open',
         handler: async (c) => {
-            c.emit(await queueAction(c.store, c.args.positional[0], c.args.positional[1]));
+            const action = c.args.positional[0], result = await queueAction(c.store, action, c.args.positional[1]);
+            c.emit(result, queueText(c.store, action, result));
             return 0;
         },
     },
     {
         name: 'list',
         summary: 'List saved and queued items',
-        usage: [['list', 'List saved and queued items']],
+        usage: [
+            ['list', 'List saved and queued items'],
+            ['list --short', 'Web articles saved with under 200 words: paywalls, error pages, script-only pages'],
+        ],
         positional: 0,
-        flags: {},
+        flags: { short: 'boolean' },
         library: 'open',
         handler: (c) => {
+            if (c.args.flags.short) {
+                const items = shortCaptures(c.store);
+                c.emit({ items }, items.map((item) => `${item.itemId}\t${item.words}\t${item.title}`).join('\n'));
+                return 0;
+            }
             const items = c.store.list();
             c.emit({ items }, lines(items).join('\n'));
             return 0;
@@ -328,18 +353,24 @@ export const commands = [
     {
         name: 'refresh',
         summary: 'Capture a new revision',
-        usage: [['refresh ITEM [--wait]', 'Capture a new revision']],
+        usage: [
+            ['refresh ITEM [--wait]', 'Capture a new revision'],
+            ['refresh ITEM --archive [--wait]', 'Capture the newest Internet Archive copy instead of the live page'],
+        ],
         positional: 1,
-        flags: { wait: 'boolean', ...media },
+        flags: { wait: 'boolean', archive: 'boolean', ...media },
         library: 'open',
         handler: async (c) => {
             const { store, args } = c, connections = await readConnections(store.root), previous = store.queue.list().find((j) => j.item_id === args.positional[0]), source = previous && store.queue.source(String(previous.id));
             if (!source || !['article', 'pdf', 'media'].includes(source.kind))
                 throw new SpongeError('unavailable');
+            const { archive: _, ...rest } = source, kind = source.kind;
+            if (args.flags.archive)
+                archivable(kind, source.locator);
             const submission = store.queue.submit({
-                ...source,
-                kind: source.kind,
-                intent: 'refresh',
+                ...rest,
+                kind,
+                intent: args.flags.archive ? 'archive' : 'refresh',
                 idempotencyKey: randomUUID(),
             });
             if (!args.flags.wait) {

package/dist/src/cli/main.js

@@ -4,20 +4,43 @@ import { lstat } from 'node:fs/promises';
 import { randomUUID } from 'node:crypto';
 import { terminalSafe } from './terminal.js';
 import { parseArguments } from './arguments.js';
-import { command, usageOf } from './commands.js';
+import { command, commands, usageOf } from './commands.js';
 import { openLibrary } from '../store/index.js';
 import { failure } from '../acquisition/errors.js';
 import { SpongeError, codeOf, publicCode, exitFor, hints, detailHints, debugChain, } from '../errors.js';
+/** Names the registered command closest to a mistyped one, never the typed text, which may hold a URL or a secret. */
+function unknownCommandHint(typed) {
+    const distance = (a, b) => {
+        let row = Array.from({ length: b.length + 1 }, (_, j) => j);
+        for (let i = 1; i <= a.length; i++) {
+            const next = [i];
+            for (let j = 1; j <= b.length; j++)
+                next[j] = Math.min(row[j] + 1, next[j - 1] + 1, row[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1));
+            row = next;
+        }
+        return row[b.length];
+    };
+    const [best] = commands
+        .map((c) => ({ name: c.name, d: distance(typed.toLowerCase().slice(0, 32), c.name) }))
+        .filter(({ d }) => d <= 2)
+        .sort((a, b) => a.d - b.d);
+    return `Unknown command.${best ? ` Did you mean \`sponge ${best.name}\`?` : ''} Run \`sponge help\` for the list.`;
+}
 /** Failures print one categorical JSON line: the public code, its specific `reason` when narrower, and a fixed hint.
  * Causes, details, paths and upstream messages appear only with --debug or SPONGE_DEBUG=1. */
 function report(error, requestId, debug) {
-    const reason = codeOf(error), code = publicCode(reason), issues = error?.issues, usage = error?.detail?.command;
+    const reason = codeOf(error), code = publicCode(reason), issues = error?.issues, detail = error?.detail, usage = detail?.command, unknown = detail?.unknown;
     if (Array.isArray(issues))
         process.stderr.write(JSON.stringify({ issues }) + '\n');
     const hint = code === 'invalid-input' && typeof usage === 'string' && command(usage)
         ? `Usage: ${usageOf(usage).replace(/\n/g, ' | ')}`
-        : (detailHints[reason] ?? hints[code]);
-    process.stderr.write(JSON.stringify({ ...failure(requestId, code), ...(reason !== code ? { reason } : {}), hint }) + '\n');
+        : code === 'invalid-input' && typeof unknown === 'string'
+            ? unknownCommandHint(unknown)
+            : (detailHints[reason] ?? hints[code]);
+    // A usage error happens before any acquisition starts, so it names no phase.
+    const { phase, ...envelope } = failure(requestId, code);
+    process.stderr.write(JSON.stringify({ ...envelope, ...(detail?.usage ? {} : { phase }), ...(reason !== code ? { reason } : {}), hint }) +
+        '\n');
     if (debug)
         process.stderr.write(debugChain(error));
     return exitFor(code);

package/dist/src/cli/queue.d.ts

@@ -1,2 +1,4 @@
 import type { LibraryStore } from '../store/index.js';
 export declare function queueAction(store: LibraryStore, action: 'status' | 'cancel' | 'retry' | 'recover', jobId?: string): Promise<Record<string, unknown>>;
+/** `sponge queue` without --json: open jobs with the IDs that cancel and retry take, then what was done. */
+export declare function queueText(store: LibraryStore, action: 'status' | 'cancel' | 'retry' | 'recover', result: Record<string, unknown>): string;

package/dist/src/cli/queue.js

@@ -1,5 +1,6 @@
 import { recoverWorker } from '../worker/custody.js';
 import { SpongeError } from '../errors.js';
+import { openJobs } from './status.js';
 export async function queueAction(store, action, jobId) {
     if (action === 'status') {
         if (jobId)
@@ -25,3 +26,24 @@ export async function queueAction(store, action, jobId) {
         throw new SpongeError('invalid-input');
     return { jobId, action };
 }
+/** `sponge queue` without --json: open jobs with the IDs that cancel and retry take, then what was done. */
+export function queueText(store, action, result) {
+    if (action === 'cancel')
+        return `Cancelled ${String(result.jobId)}`;
+    if (action === 'retry')
+        return `Queued ${String(result.jobId)} to run again`;
+    if (action === 'recover')
+        return result.recovered
+            ? `Recovered the stopped worker; ${String(result.attempts)} unfinished attempts released`
+            : 'Nothing to recover';
+    const jobs = result.jobs, open = openJobs(store), titles = new Map(store.list().map((item) => [item.itemId, item.title])), lines = open.length ? open.map((job) => jobLine(job, titles.get(job.item_id))) : ['Nothing waiting'];
+    if (jobs.length > open.length)
+        lines.push(`${jobs.length - open.length} settled or replaced jobs hidden; --json lists every job`);
+    if (result.pendingCleanup)
+        lines.push(`Pending cleanup: ${String(result.pendingCleanup)}; run sponge queue recover`);
+    return lines.join('\n');
+}
+function jobLine(job, title) {
+    const reason = job.reason && job.reason !== job.state ? ` (${String(job.reason)})` : '';
+    return `${String(job.id)}  ${String(job.operation)} ${String(job.state)}${reason}  ${String(job.created_at).slice(0, 10)}${title ? `  ${title}` : ''}`;
+}

package/dist/src/cli/status.d.ts

@@ -1,4 +1,16 @@
 import type { LibraryStore } from '../store/index.js';
+import type { Row } from '../store/context.js';
+/**
+ * Jobs still waiting, running or needing the reader, oldest first. Settled jobs are history, and only each target's
+ * newest job counts: a failure that a later capture of the item (or index job) fixed is history too.
+ */
+export declare function openJobs(store: LibraryStore): Row[];
+/** Saved web articles under `shortWords` prose words, other than social posts: candidates for `refresh --archive`. */
+export declare function shortCaptures(store: LibraryStore): {
+    itemId: string;
+    title: string;
+    words: number;
+}[];
 /** What the library holds, what is waiting or failed, and what to run next: one screen for `sponge status`. */
 export declare function libraryStatus(store: LibraryStore): {
     items: {
@@ -16,10 +28,11 @@ export declare function libraryStatus(store: LibraryStore): {
         draft: number;
         none: number;
     };
+    short: number;
     jobs: {
         operation: string;
         state: string;
-        reason: string | null;
+        reason: string;
     }[];
     pendingCleanup: number;
 };

package/dist/src/cli/status.js

@@ -1,3 +1,40 @@
+import { proseWords } from '../app/markdown.js';
+/**
+ * Jobs still waiting, running or needing the reader, oldest first. Settled jobs are history, and only each target's
+ * newest job counts: a failure that a later capture of the item (or index job) fixed is history too.
+ */
+export function openJobs(store) {
+    const newest = new Map();
+    for (const job of store.queue.list())
+        newest.set(job.operation === 'index' ? 'index' : `${job.operation} ${job.item_id}`, job);
+    return [...newest.values()].filter((job) => !['succeeded', 'cancelled'].includes(String(job.state)));
+}
+/** Prose words under which a saved article is likely a paywall stub, an error page or a page that needed scripts. */
+const shortWords = 200;
+// Posts are short by nature; their length says nothing about whether the capture worked.
+const postHosts = new Set(['x.com', 'twitter.com', 'mobile.twitter.com', 'bsky.app', 'threads.net']);
+/** Saved web articles under `shortWords` prose words, other than social posts: candidates for `refresh --archive`. */
+export function shortCaptures(store) {
+    const short = [];
+    for (const item of store.list()) {
+        if (item.kind !== 'article' || !item.revisionId)
+            continue;
+        const hosts = store.identity(item.itemId).uris.map(({ uri }) => {
+            try {
+                return new URL(uri).hostname.replace(/^www\./, '');
+            }
+            catch {
+                return '';
+            }
+        });
+        if (!hosts.some((host) => host) || hosts.some((host) => postHosts.has(host)))
+            continue;
+        const read = store.read(item.itemId), words = read ? proseWords(read.markdown) : 0;
+        if (read && words < shortWords)
+            short.push({ itemId: item.itemId, title: read.title, words });
+    }
+    return short;
+}
 const plural = (n, one, many = one + 's') => `${n} ${n === 1 ? one : many}`;
 /** What the library holds, what is waiting or failed, and what to run next: one screen for `sponge status`. */
 export function libraryStatus(store) {
@@ -7,14 +44,7 @@ export function libraryStatus(store) {
         const digest = store.derivations.current('digest', item.itemId), gist = digest?.review.gist;
         digests[!digest ? 'none' : gist === 'accepted' || gist === 'rejected' ? gist : 'draft']++;
     }
-    // Settled jobs are history; what is left is waiting, running or needs the reader.
-    // Only each target's newest job counts: a failure that a later capture of the item (or index job) fixed is history.
-    const newest = new Map();
-    for (const job of [...store.queue.list()].sort((a, b) => String(a.created_at).localeCompare(String(b.created_at))))
-        newest.set(job.operation === 'index' ? 'index' : `${job.operation} ${job.item_id}`, job);
-    const jobs = [...newest.values()]
-        .filter((job) => !['succeeded', 'cancelled'].includes(String(job.state)))
-        .map((job) => ({
+    const jobs = openJobs(store).map((job) => ({
         operation: String(job.operation),
         state: String(job.state),
         reason: job.reason ?? null,
@@ -26,6 +56,7 @@ export function libraryStatus(store) {
             ? { embedded: generation.completed, expected: generation.expected, pending: generation.pending }
             : null,
         digests: { ...digests, auto: store.digests.settings().auto },
+        short: shortCaptures(store).length,
         jobs,
         pendingCleanup: store.queue.pendingCleanup(),
     };
@@ -51,6 +82,11 @@ export function statusText(s, worker) {
             : ''}`,
         `Search index: ${s.index ? `${s.index.embedded} of ${s.index.expected} chunks and images embedded${s.index.pending ? `; run sponge index run or let the worker` : ''}` : 'none; run sponge setup --embedding, then sponge index create'}`,
         `Digests: ${drafted} of ${total} items (${d.accepted} accepted, ${d.rejected} rejected, ${d.draft} unreviewed); auto-digest ${d.auto ? 'on' : 'off'}`,
+        ...(s.short
+            ? [
+                `Short captures: ${plural(s.short, 'article')} under ${shortWords} words; see sponge list --short, then try sponge refresh ITEM --archive`,
+            ]
+            : []),
         `Queue: ${s.jobs.length
             ? Object.entries(counts)
                 .map(([key, n]) => `${n} ${key}`)

package/dist/src/pdf/handler.d.ts

@@ -2,6 +2,7 @@ import type { CaptureResult, ErrorEnvelope, JobCheckpoint } from '../../contract
 import { type CaptureConnection } from '../acquisition/index.js';
 import { type PdfCaptureRequest as CaptureRequest, type CaptureOutcome } from './capture.js';
 import type { LibraryStore, AttemptToken } from '../store/index.js';
+import { waybackSnapshot } from '../app/archive.js';
 interface PdfResult {
     ok: true;
     itemId: string;
@@ -18,6 +19,7 @@ export interface PdfOptions {
     signal?: AbortSignal;
     allowPrivateNetwork?: boolean;
     capture?: (request: CaptureRequest) => Promise<CaptureOutcome>;
+    snapshot?: typeof waybackSnapshot;
 }
 /** One attempt. Durable queue policy and process recovery live above this function. */
 export declare function ingestPdf(store: LibraryStore, token: AttemptToken, options: PdfOptions): Promise<PdfOutcome>;

package/dist/src/pdf/handler.js

@@ -7,6 +7,7 @@ import { bundleBytes, bundleText } from '../app/bundle-files.js';
 import { localizeMarkdown } from '../app/markdown.js';
 import { pdfTitle } from './title.js';
 import { SpongeError } from '../errors.js';
+import { waybackSnapshot } from '../app/archive.js';
 const classify = classifyErrors([
     'integrity',
     'limit-exceeded',
@@ -21,10 +22,16 @@ export function ingestPdf(store, token, options) {
         const source = store.queue.source(token.jobId);
         if (!source || source.kind !== 'pdf')
             throw new SpongeError('invalid-input');
+        const snapshot = source.archive
+            ? await (options.snapshot ?? waybackSnapshot)(source.locator, token.attemptId)
+            : null;
+        if (snapshot && 'ok' in snapshot)
+            return snapshot;
+        const fetched = snapshot?.url ?? source.locator;
         const result = await (options.capture ?? capturePdf)({
             runProcess: options.runProcess,
             stagingRoot: join(store.root, 'staging'),
-            sourceUrl: source.locator,
+            sourceUrl: fetched,
             requestId: token.attemptId,
             connection: options.connection,
             signal: options.signal,
@@ -37,7 +44,7 @@ export function ingestPdf(store, token, options) {
         }
         attempt.hold(result);
         await options.checkpoint?.('extracting');
-        const pdf = await storePdfCapture(store, token, result, source.locator, options.signal);
+        const pdf = await storePdfCapture(store, token, result, fetched, options.signal);
         // Representation algorithm participates in identity: later extraction changes cannot reuse an old payload.
         const bundleHash = sha256(JSON.stringify(['pdf-local-markdown-v1', pdf.capture.manifestSha256]));
         await options.checkpoint?.('promoting');

package/dist/src/store/queue.d.ts

@@ -44,13 +44,17 @@ export declare class Queue {
     submit(input: SourceSubmission): Submission;
     /** Jobs of every kind, oldest first. */
     list(): Row[];
-    /** Acquisition data for a capture job. A local file's locator is its path on this device (`local_sources`), never the synced form. */
+    /**
+     * Acquisition data for a capture job. A local file's locator is its path on this device (`local_sources`), never the
+     * synced form. `archive` is set when any submission the job serves asked for the Internet Archive copy.
+     */
     source(jobId: string): {
         itemId: string;
         kind: string;
         sourceKey: string;
         locator: string;
         realm: string;
+        archive: boolean;
     } | null;
     claimNext(token: OwnerToken): AttemptToken | null;
     claim(token: OwnerToken, jobId: string): AttemptToken;

package/dist/src/store/queue.js

@@ -163,7 +163,7 @@ export class Queue {
         const realm = input.realm ?? '';
         if (realm)
             text(realm, 256);
-        if (input.intent !== undefined && !['save', 'refresh'].includes(input.intent))
+        if (input.intent !== undefined && !['save', 'refresh', 'archive'].includes(input.intent))
             throw new SpongeError('invalid-input');
         const db = this.c.db;
         return transaction(db, () => {
@@ -224,13 +224,16 @@ export class Queue {
             .flatMap((kind) => kind.list())
             .sort((a, b) => String(a.created_at).localeCompare(String(b.created_at)) || String(a.id).localeCompare(String(b.id)));
     }
-    /** Acquisition data for a capture job. A local file's locator is its path on this device (`local_sources`), never the synced form. */
+    /**
+     * Acquisition data for a capture job. A local file's locator is its path on this device (`local_sources`), never the
+     * synced form. `archive` is set when any submission the job serves asked for the Internet Archive copy.
+     */
     source(jobId) {
         text(jobId, 256);
         const row = this.c.db
-            .prepare('SELECT i.id itemId,i.kind,r.source_key sourceKey,COALESCE(l.locator,r.locator) locator,r.realm_key realm FROM jobs j JOIN items i ON i.id=j.item_id JOIN source_refs r ON r.item_id=i.id LEFT JOIN local_sources l ON l.item_id=i.id WHERE j.id=? AND i.deleted_at IS NULL ORDER BY r.id LIMIT 1')
+            .prepare("SELECT i.id itemId,i.kind,r.source_key sourceKey,COALESCE(l.locator,r.locator) locator,r.realm_key realm,EXISTS(SELECT 1 FROM submissions s WHERE s.job_id=j.id AND s.intent='archive') archive FROM jobs j JOIN items i ON i.id=j.item_id JOIN source_refs r ON r.item_id=i.id LEFT JOIN local_sources l ON l.item_id=i.id WHERE j.id=? AND i.deleted_at IS NULL ORDER BY r.id LIMIT 1")
             .get(jobId);
-        return row ?? null;
+        return row ? { ...row, archive: row.archive === 1 } : null;
     }
     claimNext(token) {
         checkOwner(this.c.db, token);

package/dist/src/store/types.d.ts

@@ -27,7 +27,8 @@ export interface SourceSubmission {
     realm?: string;
     kind: ItemKind;
     idempotencyKey: string;
-    intent?: 'save' | 'refresh';
+    /** `archive` captures the source's newest Internet Archive copy instead of the live page. */
+    intent?: 'save' | 'refresh' | 'archive';
 }
 export interface Submission {
     itemId: string;

package/native/runtime-manifest.json

@@ -33,16 +33,16 @@
     }
   ],
   "build": {
-    "asrSourceSha256": "94c7527e57b87337bb90f4c3a36a305eae6ddb16d558226f40f17c16bb31587f",
+    "asrSourceSha256": "995c66f8bf9b9c724f44266e686b561171bdfe5aff063e758d2f2f5bc9523789",
     "clangVersion": "Apple clang version 21.0.0 (clang-2100.1.1.101)",
-    "embeddingSourceSha256": "762fc9f018b12a8fb95e2d7349b2756622b1a40c8bd4cf2986fc290370a058a4",
-    "mediaSourceSha256": "3945d6a5bf8606c91270a2f837e805aa911ddf40baf701f3b5bb98852c4627fd",
+    "embeddingSourceSha256": "8c8719739c48a174ea1bdc77862245534c6b963a4a824e82f9f5e27d7b24705c",
+    "mediaSourceSha256": "9d30fb7e66b79e6b584f564b7dc14c822a7a955068d9084ca3606f9d62a49c9a",
     "rustcVersion": "rustc 1.97.1 (8bab26f4f 2026-07-14)",
     "swiftVersion": "Apple Swift version 6.3.3 (swiftlang-6.3.3.1.3 clang-2100.1.1.101)"
   },
   "minimumMacOS": "26.0",
-  "packageVersion": "0.6.6",
+  "packageVersion": "0.6.8",
   "platform": "darwin",
   "schemaVersion": 1,
-  "sourceCommit": "516ab78f62c7bec19e6fe8e44bcf23ca8ebd8fb3"
+  "sourceCommit": "7003c66883fa24e04ec3b961b13207a813919025"
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hraness/sponge",
-  "version": "0.6.6",
+  "version": "0.6.8",
   "description": "A local library for captured articles and verified offline assets.",
   "repository": {
     "type": "git",