@hraness/sponge 0.6.7 → 0.6.9

package/dist/src/cli/commands.js

@@ -8,7 +8,7 @@ import { queueAction, queueText } from './queue.js';
 import { portabilityAction } from './portability.js';
 import { restoreLibrary } from '../portability/backup.js';
 import { workerAction, serveWorker, hasDueAcquisition, workerDoctor } from './worker.js';
-import { libraryStatus, statusText } from './status.js';
+import { libraryStatus, shortCaptures, statusText } from './status.js';
 import { nativeSetupPlan, provisionNative, inspectNative, optionalNative } from '../models/runtime.js';
 import { digestModel, digestSetupPlan, inspectDigest, provisionDigest } from '../models/digest.js';
 import { withDigestServer } from '../digest/server.js';
@@ -24,10 +24,31 @@ import { createDigest, undigestible } from '../digest/pipeline.js';
 import { createQueryEmbedder, createTextEmbedder } from '../search/query.js';
 import { runtimeGcAction, runtimeGcText, setupRuntimeGc } from './runtimes.js';
 import { drainLibrary } from '../worker/basic.js';
+import { browserStatus, removeBrowser, setupBrowser } from '../browser/setup.js';
 import { minAge, publishSite } from '../publish/index.js';
 import { SpongeError, codeOf, publicCode, exitFor, isSpongeCode } from '../errors.js';
 const abs = (v) => typeof v === 'string' && isAbsolute(v);
 const media = { language: 'value', 'force-local': 'boolean', 'allow-private-network': 'boolean' };
+/** Only a public web article or PDF has an Internet Archive copy; a signed-in page's copy would not be the reader's. */
+function archivable(kind, locator, connection) {
+    if (kind === 'media' || connection || !/^https?:/.test(locator))
+        throw new SpongeError('unsupported');
+}
+/** The extension captures web articles; PDFs and media keep their own fetchers. */
+function browsable(kind, locator, connection) {
+    if (kind !== 'article' || connection || !/^https?:/.test(locator))
+        throw new SpongeError('unsupported');
+}
+/** `--archive` or `--browser`: at most one route. */
+function route(flags, kind, locator, connection) {
+    if (flags.archive && flags.browser)
+        throw new SpongeError('invalid-input');
+    if (flags.archive)
+        return (archivable(kind, locator, connection), 'archive');
+    if (flags.browser)
+        return (browsable(kind, locator, connection), 'browser');
+    return undefined;
+}
 const lines = (items) => items.map((item) => `${item.itemId}\t${item.title}`);
 /** The failure of a job that did not run here: its state, or its recorded reason. */
 function jobFailure(store, jobId) {
@@ -78,6 +99,10 @@ async function drain(c, jobId, connections) {
     }
     const failed = results.find((result) => !result.ok);
     if (failed && !failed.ok) {
+        // A narrower recorded reason, such as browser-offline, prints with its own hint.
+        const reason = jobId && store.queue.list().find((j) => j.id === jobId)?.reason;
+        if (isSpongeCode(reason) && reason !== failed.code && publicCode(reason) === failed.code)
+            throw jobFailure(store, jobId);
         process.stderr.write(JSON.stringify(failed) + '\n');
         return exitFor(failed.code);
     }
@@ -169,9 +194,13 @@ export const commands = [
     {
         name: 'add',
         summary: 'Save an article, PDF or media source',
-        usage: [['add URL_OR_PDF [--kind article|pdf|media] [--connection ID] [--wait]', '']],
+        usage: [
+            ['add URL_OR_PDF [--kind article|pdf|media] [--connection ID] [--wait]', ''],
+            ['add URL --archive [--wait]', 'Capture the newest Internet Archive copy of a dead or paywalled page'],
+            ['add URL --browser [--wait]', 'Capture through your browser and its sign-ins (sponge browser setup)'],
+        ],
         positional: 1,
-        flags: { connection: 'value', kind: 'value', wait: 'boolean', ...media },
+        flags: { connection: 'value', kind: 'value', wait: 'boolean', archive: 'boolean', browser: 'boolean', ...media },
         library: 'open',
         handler: async (c) => {
             const { store, args } = c, connections = await readConnections(store.root);
@@ -180,7 +209,13 @@ export const commands = [
                 throw new SpongeError('invalid-input');
             const input = args.positional[0], kind = (args.flags.kind ?? inferKind(input));
             const source = (kind === 'pdf' ? pdfSource : kind === 'media' ? mediaSource : articleSource)(input, connection?.realm ?? 'public');
-            const submission = store.queue.submit({ kind, ...source, idempotencyKey: 'save:' + source.sourceKey }), { jobId, itemId } = submission;
+            const via = route(args.flags, kind, source.locator, connection);
+            const submission = store.queue.submit({
+                kind,
+                ...source,
+                ...(via ? { intent: via } : {}),
+                idempotencyKey: `${via ?? 'save'}:${source.sourceKey}`,
+            }), { jobId, itemId } = submission;
             const job = store.queue.list().find((j) => j.id === jobId);
             if (!args.flags.wait) {
                 c.emit({ ...submission, state: job?.state ?? 'queued' }, `${itemId}\t${job?.state ?? 'queued'}`);
@@ -242,11 +277,19 @@ export const commands = [
     {
         name: 'list',
         summary: 'List saved and queued items',
-        usage: [['list', 'List saved and queued items']],
+        usage: [
+            ['list', 'List saved and queued items'],
+            ['list --short', 'Web articles saved with under 200 words: paywalls, error pages, script-only pages'],
+        ],
         positional: 0,
-        flags: {},
+        flags: { short: 'boolean' },
         library: 'open',
         handler: (c) => {
+            if (c.args.flags.short) {
+                const items = shortCaptures(c.store);
+                c.emit({ items }, items.map((item) => `${item.itemId}\t${item.words}\t${item.title}`).join('\n'));
+                return 0;
+            }
             const items = c.store.list();
             c.emit({ items }, lines(items).join('\n'));
             return 0;
@@ -329,18 +372,24 @@ export const commands = [
     {
         name: 'refresh',
         summary: 'Capture a new revision',
-        usage: [['refresh ITEM [--wait]', 'Capture a new revision']],
+        usage: [
+            ['refresh ITEM [--wait]', 'Capture a new revision'],
+            ['refresh ITEM --archive [--wait]', 'Capture the newest Internet Archive copy instead of the live page'],
+            ['refresh ITEM --browser [--wait]', 'Capture through your browser and its sign-ins'],
+        ],
         positional: 1,
-        flags: { wait: 'boolean', ...media },
+        flags: { wait: 'boolean', archive: 'boolean', browser: 'boolean', ...media },
         library: 'open',
         handler: async (c) => {
             const { store, args } = c, connections = await readConnections(store.root), previous = store.queue.list().find((j) => j.item_id === args.positional[0]), source = previous && store.queue.source(String(previous.id));
             if (!source || !['article', 'pdf', 'media'].includes(source.kind))
                 throw new SpongeError('unavailable');
+            const { via: _, ...rest } = source, kind = source.kind;
+            const via = route(args.flags, kind, source.locator, source.realm.startsWith('connection:') || undefined);
             const submission = store.queue.submit({
-                ...source,
-                kind: source.kind,
-                intent: 'refresh',
+                ...rest,
+                kind,
+                intent: via ?? 'refresh',
                 idempotencyKey: randomUUID(),
             });
             if (!args.flags.wait) {
@@ -433,10 +482,17 @@ export const commands = [
             catch (error) {
                 digest = publicCode(error, 'runtime-missing');
             }
+            const browser = await browserStatus(store.root), browserState = browser.connected
+                ? browser.background
+                    ? 'connected'
+                    : 'connected, background captures off'
+                : browser.registered.length
+                    ? 'not connected'
+                    : 'not set up';
             const cleanup = store.queue.pendingCleanup(), worker = await workerDoctor(store, c.version), workerBroken = !['ready', 'not-installed', 'unsupported'].includes(worker.worker);
             // Each runtime that is not ready names the command that installs it.
             const line = (label, state, fix) => `${label}: ${state}${state === 'runtime-missing' ? ` (${fix})` : ''}`;
-            c.emit({ library: 'ready', capture, pdf, ...states, digest, pendingCleanup: cleanup, worker }, [
+            c.emit({ library: 'ready', capture, pdf, ...states, digest, browser: browserState, pendingCleanup: cleanup, worker }, [
                 'Library: ready',
                 `Capture: ${capture}`,
                 `PDF: ${pdf === 'ready' ? 'ready' : 'Poppler not found (brew install poppler)'}`,
@@ -444,6 +500,7 @@ export const commands = [
                 line('Speech-to-text', states.asr, 'sponge setup --asr'),
                 line('Embedding', states.embedding, 'sponge setup --embedding'),
                 line('Digest model', digest, 'sponge setup --digest'),
+                `Browser: ${browserState}${browserState === 'not set up' ? ' (optional: sponge browser setup)' : ''}`,
                 `Pending cleanup: ${cleanup}${cleanup ? ' (sponge queue recover)' : ''}`,
                 `Worker: ${worker.worker}${Array.isArray(worker.drift) ? ` (${worker.drift.join(', ')})` : ''}${worker.repair ? `\n  ${worker.repair}` : ''}`,
             ].join('\n'));
@@ -562,6 +619,51 @@ export const commands = [
             return 0;
         },
     },
+    {
+        name: 'browser',
+        summary: 'Capture through your own browser and its sign-ins',
+        usage: [
+            ['browser setup [--hosts-dir DIR]', 'Connect the Sponge extension in Chrome and other Chromium browsers'],
+            ['browser status', 'Show which browsers are set up and whether the extension is connected'],
+            ['browser remove', 'Disconnect Sponge from every browser'],
+        ],
+        positional: 1,
+        flags: { 'hosts-dir': 'value' },
+        library: 'open',
+        check: (p) => ['setup', 'status', 'remove'].includes(p[0]),
+        handler: async (c) => {
+            const { store, args } = c, hostsDir = args.flags['hosts-dir'];
+            if (hostsDir !== undefined && !isAbsolute(hostsDir))
+                throw new SpongeError('invalid-input');
+            const options = hostsDir ? { hostsDirs: [hostsDir] } : {};
+            if (args.positional[0] === 'setup') {
+                const result = await setupBrowser(store.root, options);
+                c.emit(result, [
+                    result.registered.length
+                        ? `Registered with: ${result.registered.join(', ')}`
+                        : 'No Chromium browser found. Install Chrome, or pass --hosts-dir.',
+                    'Then load the extension once:',
+                    '  1. Open chrome://extensions and turn on Developer mode',
+                    `  2. Click "Load unpacked" and choose ${result.extension}`,
+                    '  3. Pin Sponge, then check `sponge browser status`',
+                ].join('\n'));
+                return result.registered.length ? 0 : 4;
+            }
+            if (args.positional[0] === 'remove') {
+                const result = await removeBrowser(options);
+                c.emit(result, result.removed.length ? `Removed from: ${result.removed.join(', ')}` : 'Nothing to remove');
+                return 0;
+            }
+            const status = await browserStatus(store.root, options);
+            c.emit(status, [
+                `Registered: ${status.registered.join(', ') || 'none (sponge browser setup)'}`,
+                `Library: ${status.bound ? 'this one' : 'another library (sponge browser setup to switch)'}`,
+                `Extension: ${status.connected ? `connected${status.extension ? ` (${status.extension})` : ''}` : 'not connected'}`,
+                ...(status.connected ? [`Background captures: ${status.background ? 'on' : 'off'}`] : []),
+            ].join('\n'));
+            return status.connected ? 0 : 4;
+        },
+    },
     {
         name: 'import',
         summary: 'Import existing captures',

package/dist/src/cli/status.d.ts

@@ -5,6 +5,12 @@ import type { Row } from '../store/context.js';
  * newest job counts: a failure that a later capture of the item (or index job) fixed is history too.
  */
 export declare function openJobs(store: LibraryStore): Row[];
+/** Saved web articles under `shortWords` prose words, other than social posts: candidates for `refresh --archive`. */
+export declare function shortCaptures(store: LibraryStore): {
+    itemId: string;
+    title: string;
+    words: number;
+}[];
 /** What the library holds, what is waiting or failed, and what to run next: one screen for `sponge status`. */
 export declare function libraryStatus(store: LibraryStore): {
     items: {
@@ -22,6 +28,7 @@ export declare function libraryStatus(store: LibraryStore): {
         draft: number;
         none: number;
     };
+    short: number;
     jobs: {
         operation: string;
         state: string;

package/dist/src/cli/status.js

@@ -1,3 +1,4 @@
+import { proseWords } from '../app/markdown.js';
 /**
  * Jobs still waiting, running or needing the reader, oldest first. Settled jobs are history, and only each target's
  * newest job counts: a failure that a later capture of the item (or index job) fixed is history too.
@@ -8,6 +9,32 @@ export function openJobs(store) {
         newest.set(job.operation === 'index' ? 'index' : `${job.operation} ${job.item_id}`, job);
     return [...newest.values()].filter((job) => !['succeeded', 'cancelled'].includes(String(job.state)));
 }
+/** Prose words under which a saved article is likely a paywall stub, an error page or a page that needed scripts. */
+const shortWords = 200;
+// Posts are short by nature; their length says nothing about whether the capture worked.
+const postHosts = new Set(['x.com', 'twitter.com', 'mobile.twitter.com', 'bsky.app', 'threads.net']);
+/** Saved web articles under `shortWords` prose words, other than social posts: candidates for `refresh --archive`. */
+export function shortCaptures(store) {
+    const short = [];
+    for (const item of store.list()) {
+        if (item.kind !== 'article' || !item.revisionId)
+            continue;
+        const hosts = store.identity(item.itemId).uris.map(({ uri }) => {
+            try {
+                return new URL(uri).hostname.replace(/^www\./, '');
+            }
+            catch {
+                return '';
+            }
+        });
+        if (!hosts.some((host) => host) || hosts.some((host) => postHosts.has(host)))
+            continue;
+        const read = store.read(item.itemId), words = read ? proseWords(read.markdown) : 0;
+        if (read && words < shortWords)
+            short.push({ itemId: item.itemId, title: read.title, words });
+    }
+    return short;
+}
 const plural = (n, one, many = one + 's') => `${n} ${n === 1 ? one : many}`;
 /** What the library holds, what is waiting or failed, and what to run next: one screen for `sponge status`. */
 export function libraryStatus(store) {
@@ -29,6 +56,7 @@ export function libraryStatus(store) {
             ? { embedded: generation.completed, expected: generation.expected, pending: generation.pending }
             : null,
         digests: { ...digests, auto: store.digests.settings().auto },
+        short: shortCaptures(store).length,
         jobs,
         pendingCleanup: store.queue.pendingCleanup(),
     };
@@ -54,6 +82,11 @@ export function statusText(s, worker) {
             : ''}`,
         `Search index: ${s.index ? `${s.index.embedded} of ${s.index.expected} chunks and images embedded${s.index.pending ? `; run sponge index run or let the worker` : ''}` : 'none; run sponge setup --embedding, then sponge index create'}`,
         `Digests: ${drafted} of ${total} items (${d.accepted} accepted, ${d.rejected} rejected, ${d.draft} unreviewed); auto-digest ${d.auto ? 'on' : 'off'}`,
+        ...(s.short
+            ? [
+                `Short captures: ${plural(s.short, 'article')} under ${shortWords} words; see sponge list --short, then try sponge refresh ITEM --archive`,
+            ]
+            : []),
         `Queue: ${s.jobs.length
             ? Object.entries(counts)
                 .map(([key, n]) => `${n} ${key}`)

package/dist/src/errors.d.ts

@@ -23,6 +23,7 @@ declare const parents: {
     readonly 'parent-missing': "invalid-input";
     readonly 'target-exists': "invalid-input";
     readonly 'connection-retired': "unsupported";
+    readonly 'browser-offline': "unavailable";
 };
 export type DetailCode = keyof typeof parents;
 export type SpongeCode = ErrorCode | DetailCode;

package/dist/src/errors.js

@@ -37,6 +37,7 @@ const parents = {
     'parent-missing': 'invalid-input',
     'target-exists': 'invalid-input',
     'connection-retired': 'unsupported',
+    'browser-offline': 'unavailable',
 };
 export const indexCodes = [
     'invalid-index',
@@ -88,6 +89,7 @@ export const detailHints = {
     'model-unavailable': 'The model server did not answer or returned an error. Check that it is still running and has free GPU memory (another loaded model can exhaust it), then retry.',
     'source-too-thin': 'The capture holds too little text to digest: under 20 words of prose, or only an archive’s metadata. Refresh the item when a fuller capture is possible.',
     'connection-retired': 'Saved GhostGet account connections were retired in Sponge 0.5.0. Attach a signed-in browser over CDP and add it to connections.json instead.',
+    'browser-offline': 'No browser has the Sponge extension connected, or its background captures are off. Open Chrome with the extension on, check `sponge browser status`, then retry.',
     'model-busy': 'Another llama-server is running, and two loaded models can exhaust GPU memory. Wait for it to exit, then retry.',
 };
 /** The one error type. `message` is the code, so bare `String(error)` never carries detail, paths or causes. */

package/dist/src/pdf/handler.d.ts

@@ -2,6 +2,7 @@ import type { CaptureResult, ErrorEnvelope, JobCheckpoint } from '../../contract
 import { type CaptureConnection } from '../acquisition/index.js';
 import { type PdfCaptureRequest as CaptureRequest, type CaptureOutcome } from './capture.js';
 import type { LibraryStore, AttemptToken } from '../store/index.js';
+import { waybackSnapshot } from '../app/archive.js';
 interface PdfResult {
     ok: true;
     itemId: string;
@@ -18,6 +19,7 @@ export interface PdfOptions {
     signal?: AbortSignal;
     allowPrivateNetwork?: boolean;
     capture?: (request: CaptureRequest) => Promise<CaptureOutcome>;
+    snapshot?: typeof waybackSnapshot;
 }
 /** One attempt. Durable queue policy and process recovery live above this function. */
 export declare function ingestPdf(store: LibraryStore, token: AttemptToken, options: PdfOptions): Promise<PdfOutcome>;

package/dist/src/pdf/handler.js

@@ -7,6 +7,7 @@ import { bundleBytes, bundleText } from '../app/bundle-files.js';
 import { localizeMarkdown } from '../app/markdown.js';
 import { pdfTitle } from './title.js';
 import { SpongeError } from '../errors.js';
+import { waybackSnapshot } from '../app/archive.js';
 const classify = classifyErrors([
     'integrity',
     'limit-exceeded',
@@ -21,10 +22,14 @@ export function ingestPdf(store, token, options) {
         const source = store.queue.source(token.jobId);
         if (!source || source.kind !== 'pdf')
             throw new SpongeError('invalid-input');
+        const snapshot = source.via === 'archive' ? await (options.snapshot ?? waybackSnapshot)(source.locator, token.attemptId) : null;
+        if (snapshot && 'ok' in snapshot)
+            return snapshot;
+        const fetched = snapshot?.url ?? source.locator;
         const result = await (options.capture ?? capturePdf)({
             runProcess: options.runProcess,
             stagingRoot: join(store.root, 'staging'),
-            sourceUrl: source.locator,
+            sourceUrl: fetched,
             requestId: token.attemptId,
             connection: options.connection,
             signal: options.signal,
@@ -37,7 +42,7 @@ export function ingestPdf(store, token, options) {
         }
         attempt.hold(result);
         await options.checkpoint?.('extracting');
-        const pdf = await storePdfCapture(store, token, result, source.locator, options.signal);
+        const pdf = await storePdfCapture(store, token, result, fetched, options.signal);
         // Representation algorithm participates in identity: later extraction changes cannot reuse an old payload.
         const bundleHash = sha256(JSON.stringify(['pdf-local-markdown-v1', pdf.capture.manifestSha256]));
         await options.checkpoint?.('promoting');

package/dist/src/store/queue.d.ts

@@ -44,13 +44,17 @@ export declare class Queue {
     submit(input: SourceSubmission): Submission;
     /** Jobs of every kind, oldest first. */
     list(): Row[];
-    /** Acquisition data for a capture job. A local file's locator is its path on this device (`local_sources`), never the synced form. */
+    /**
+     * Acquisition data for a capture job. A local file's locator is its path on this device (`local_sources`), never the
+     * synced form. `via` names the route the job's newest archive or browser submission asked for.
+     */
     source(jobId: string): {
         itemId: string;
         kind: string;
         sourceKey: string;
         locator: string;
         realm: string;
+        via?: 'archive' | 'browser';
     } | null;
     claimNext(token: OwnerToken): AttemptToken | null;
     claim(token: OwnerToken, jobId: string): AttemptToken;

package/dist/src/store/queue.js

@@ -163,7 +163,7 @@ export class Queue {
         const realm = input.realm ?? '';
         if (realm)
             text(realm, 256);
-        if (input.intent !== undefined && !['save', 'refresh'].includes(input.intent))
+        if (input.intent !== undefined && !['save', 'refresh', 'archive', 'browser'].includes(input.intent))
             throw new SpongeError('invalid-input');
         const db = this.c.db;
         return transaction(db, () => {
@@ -224,13 +224,19 @@ export class Queue {
             .flatMap((kind) => kind.list())
             .sort((a, b) => String(a.created_at).localeCompare(String(b.created_at)) || String(a.id).localeCompare(String(b.id)));
     }
-    /** Acquisition data for a capture job. A local file's locator is its path on this device (`local_sources`), never the synced form. */
+    /**
+     * Acquisition data for a capture job. A local file's locator is its path on this device (`local_sources`), never the
+     * synced form. `via` names the route the job's newest archive or browser submission asked for.
+     */
     source(jobId) {
         text(jobId, 256);
         const row = this.c.db
-            .prepare('SELECT i.id itemId,i.kind,r.source_key sourceKey,COALESCE(l.locator,r.locator) locator,r.realm_key realm FROM jobs j JOIN items i ON i.id=j.item_id JOIN source_refs r ON r.item_id=i.id LEFT JOIN local_sources l ON l.item_id=i.id WHERE j.id=? AND i.deleted_at IS NULL ORDER BY r.id LIMIT 1')
+            .prepare("SELECT i.id itemId,i.kind,r.source_key sourceKey,COALESCE(l.locator,r.locator) locator,r.realm_key realm,(SELECT s.intent FROM submissions s WHERE s.job_id=j.id AND s.intent IN ('archive','browser') ORDER BY s.created_at DESC,s.rowid DESC LIMIT 1) via FROM jobs j JOIN items i ON i.id=j.item_id JOIN source_refs r ON r.item_id=i.id LEFT JOIN local_sources l ON l.item_id=i.id WHERE j.id=? AND i.deleted_at IS NULL ORDER BY r.id LIMIT 1")
             .get(jobId);
-        return row ?? null;
+        if (!row)
+            return null;
+        const { via, ...source } = row;
+        return via ? { ...source, via } : source;
     }
     claimNext(token) {
         checkOwner(this.c.db, token);

package/dist/src/store/types.d.ts

@@ -27,7 +27,11 @@ export interface SourceSubmission {
     realm?: string;
     kind: ItemKind;
     idempotencyKey: string;
-    intent?: 'save' | 'refresh';
+    /**
+     * `archive` captures the source's newest Internet Archive copy instead of the live page; `browser` captures it
+     * through the reader's own browser and sign-ins.
+     */
+    intent?: 'save' | 'refresh' | 'archive' | 'browser';
 }
 export interface Submission {
     itemId: string;

package/dist/src/worker/basic.js

@@ -4,8 +4,10 @@ import { ingestArticle } from '../app/article.js';
 import { ingestPdf } from '../pdf/index.js';
 import { ingestMedia } from '../media/handler.js';
 import { drainQueue } from './engine.js';
+import { askBridge, bridgeSocket } from '../browser/bridge.js';
 /** Content handlers over the durable sequential worker and its process supervisor. */
 export async function drainLibrary(store, options) {
+    const browser = options.browser ?? bridgeSocket(store.root);
     const handler = async (context) => {
         const source = store.queue.source(context.attempt.jobId);
         let connection;
@@ -18,6 +20,18 @@ export async function drainLibrary(store, options) {
             store.queue.settle(context.attempt);
             return { ...failure(context.attempt.attemptId, publicCode(code), 'acquisition'), cleanup: 'settled' };
         }
+        if (source?.via === 'browser') {
+            // Asked once up front, so an absent browser is a clear reason rather than an empty capture.
+            const status = await askBridge(browser, { op: 'status' }, { timeoutMs: 5000, maxBytes: 4096 });
+            if (!status.ok || !('background' in status) || !status.background) {
+                store.queue.fail(context.attempt, 'browser-offline', { retryable: true });
+                store.queue.settle(context.attempt);
+                return {
+                    ...failure(context.attempt.attemptId, 'unavailable', 'acquisition', true),
+                    cleanup: 'settled',
+                };
+            }
+        }
         if (source?.kind === 'media') {
             const { capture: _articleCapture, ...mediaOptions } = options;
             return ingestMedia(store, context.attempt, {
@@ -30,6 +44,7 @@ export async function drainLibrary(store, options) {
         }
         return (source?.kind === 'pdf' ? ingestPdf : ingestArticle)(store, context.attempt, {
             ...options,
+            browser,
             connection,
             signal: context.signal,
             runProcess: context.runProcess,

package/extension/background.js

@@ -0,0 +1,180 @@
+// The Sponge extension. Sponge asks over native messaging for a page; this worker loads it with the reader's own
+// sign-ins and answers with the page alone. Cookies and passwords never leave the browser.
+const host = 'com.hraness.sponge';
+const version = chrome.runtime.getManifest().version;
+/** Sponge holds a page to 16 MiB of HTML. */
+const maxHtml = 16 * 1024 * 1024;
+let port = null;
+let connectedAt = 0;
+let retryMs = 1000;
+const saves = new Map();
+
+async function backgroundAllowed() {
+  const { background } = await chrome.storage.local.get('background');
+  return background !== false;
+}
+
+async function connect() {
+  if (port) return;
+  try {
+    port = chrome.runtime.connectNative(host);
+  } catch {
+    port = null;
+    return reconnectLater();
+  }
+  connectedAt = Date.now();
+  port.onMessage.addListener(onHostMessage);
+  port.onDisconnect.addListener(() => {
+    void chrome.runtime.lastError;
+    port = null;
+    for (const resolve of saves.values()) resolve({ ok: false, code: 'offline' });
+    saves.clear();
+    // A host that ran for a while left normally; one that never started (not set up) backs off to a minute.
+    if (Date.now() - connectedAt > 10_000) retryMs = 1000;
+    reconnectLater();
+  });
+  port.postMessage({ type: 'hello', version, background: await backgroundAllowed() });
+}
+
+function reconnectLater() {
+  setTimeout(connect, retryMs);
+  retryMs = Math.min(retryMs * 2, 60_000);
+}
+
+function onHostMessage(message) {
+  if (message?.type === 'saved') {
+    saves.get(message.requestId)?.(message);
+    saves.delete(message.requestId);
+    return;
+  }
+  if (typeof message?.id !== 'string') return;
+  const run = message.op === 'fetch' ? fetchPage : message.op === 'render' ? renderPage : null;
+  if (!run) return port?.postMessage({ id: message.id, ok: false, code: 'failed' });
+  backgroundAllowed()
+    .then((allowed) => (allowed ? run(message.url, message.timeoutMs) : { ok: false, code: 'disabled' }))
+    .catch(() => ({ ok: false, code: 'failed' }))
+    .then((answer) => port?.postMessage({ id: message.id, ...answer }));
+}
+
+/** The page's HTML as the reader's browser would fetch it, with their cookies. */
+async function fetchPage(url, timeoutMs) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const response = await fetch(url, { credentials: 'include', cache: 'no-store', signal: controller.signal });
+    if (!response.ok) return { ok: false, code: 'failed', status: response.status };
+    if (!/^(text\/html|application\/xhtml\+xml)\b/i.test(response.headers.get('content-type') ?? ''))
+      return { ok: false, code: 'not-html' };
+    const html = await response.text();
+    if (html.length > maxHtml) return { ok: false, code: 'too-large' };
+    return { ok: true, html, finalUrl: response.url };
+  } catch {
+    return { ok: false, code: controller.signal.aborted ? 'timeout' : 'failed' };
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/** The page after its scripts run, in a muted background tab that closes when done. */
+async function renderPage(url, timeoutMs) {
+  const deadline = Date.now() + timeoutMs;
+  let tab;
+  try {
+    tab = await chrome.tabs.create({ url, active: false });
+  } catch {
+    // No window to open a tab in: a minimized one serves instead.
+    const window = await chrome.windows.create({ url, focused: false, state: 'minimized' });
+    tab = window.tabs[0];
+  }
+  try {
+    await chrome.tabs.update(tab.id, { muted: true });
+    if (!(await loaded(tab.id, deadline))) return { ok: false, code: 'timeout' };
+    const [result] = await chrome.scripting.executeScript({ target: { tabId: tab.id }, func: snapshot });
+    const page = result?.result;
+    if (!page || typeof page.html !== 'string') return { ok: false, code: 'failed' };
+    if (page.html.length > maxHtml) return { ok: false, code: 'too-large' };
+    return { ok: true, html: page.html, finalUrl: page.url, title: page.title };
+  } catch {
+    return { ok: false, code: 'failed' };
+  } finally {
+    await chrome.tabs.remove(tab.id).catch(() => undefined);
+  }
+}
+
+/** A new tab first reports its blank start page as complete; only the requested page counts. */
+const navigated = (tab) => tab.status === 'complete' && !tab.pendingUrl && /^https?:/.test(tab.url ?? '');
+
+function loaded(tabId, deadline) {
+  return new Promise((resolve) => {
+    const done = (value) => {
+      clearTimeout(timer);
+      clearInterval(poll);
+      chrome.tabs.onUpdated.removeListener(listen);
+      resolve(value);
+    };
+    const check = () =>
+      chrome.tabs.get(tabId).then(
+        (tab) => navigated(tab) && done(true),
+        () => done(false),
+      );
+    const listen = (id, _change, tab) => id === tabId && navigated(tab) && done(true);
+    const timer = setTimeout(() => done(false), Math.max(0, deadline - Date.now()));
+    // Update events can be missed while a fresh browser settles, so the tab is also checked every half second.
+    const poll = setInterval(check, 500);
+    chrome.tabs.onUpdated.addListener(listen);
+    check();
+  });
+}
+
+/** Runs in the page: scrolls once through it so lazy text loads, then returns the document. */
+async function snapshot() {
+  const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+  await pause(1000);
+  for (let step = 0; step < 10 && innerHeight + scrollY < document.body.scrollHeight; step++) {
+    scrollBy(0, innerHeight);
+    await pause(200);
+  }
+  scrollTo(0, 0);
+  return { html: '<!doctype html>\n' + document.documentElement.outerHTML, url: location.href, title: document.title };
+}
+
+/** Asks Sponge to save a page; the Sponge worker then captures it through this browser. */
+function save(url) {
+  if (!port) return Promise.resolve({ ok: false, code: 'offline' });
+  const requestId = crypto.randomUUID();
+  return new Promise((resolve) => {
+    saves.set(requestId, resolve);
+    setTimeout(() => saves.delete(requestId) && resolve({ ok: false, code: 'timeout' }), 15_000);
+    port.postMessage({ type: 'save', requestId, url });
+  });
+}
+
+chrome.runtime.onMessage.addListener((message, _sender, reply) => {
+  if (message?.type === 'status') {
+    backgroundAllowed().then((background) => reply({ connected: port !== null, background }));
+    return true;
+  }
+  if (message?.type === 'set-background') {
+    chrome.storage.local
+      .set({ background: message.value === true })
+      .then(() => port?.postMessage({ type: 'settings', version, background: message.value === true }))
+      .then(() => reply({ ok: true }));
+    return true;
+  }
+  if (message?.type === 'save') {
+    save(message.url).then(reply);
+    return true;
+  }
+  return false;
+});
+
+chrome.runtime.onInstalled.addListener(() => {
+  chrome.contextMenus.create({ id: 'save-link', title: 'Save link to Sponge', contexts: ['link'] });
+  chrome.contextMenus.create({ id: 'save-page', title: 'Save page to Sponge', contexts: ['page'] });
+});
+chrome.contextMenus.onClicked.addListener((info, tab) => {
+  const url = info.menuItemId === 'save-link' ? info.linkUrl : tab?.url;
+  if (url) void save(url);
+});
+chrome.runtime.onStartup.addListener(connect);
+void connect();