@hraness/sponge 0.5.2 → 0.6.1

package/dist/src/cli/commands.js

@@ -1,9 +1,9 @@
-import { join, isAbsolute } from 'node:path';
+import { isAbsolute } from 'node:path';
 import { randomUUID } from 'node:crypto';
-import { readConnections } from './config.js';
+import { connectionFor, readConnections } from './config.js';
 import { pdfSource } from '../pdf/index.js';
 import { articleSource, inferKind, mediaSource } from '../app/source.js';
-import { defaultCapturePins, inspectCaptureRuntime, provisionCapture, captureSetupPlan } from '../acquisition/index.js';
+import { pdfToolsAvailable } from '../pdf/tools.js';
 import { queueAction } from './queue.js';
 import { portabilityAction } from './portability.js';
 import { restoreLibrary } from '../portability/backup.js';
@@ -28,15 +28,6 @@ import { SpongeError, codeOf, publicCode, exitFor, isSpongeCode } from '../error
 const abs = (v) => typeof v === 'string' && isAbsolute(v);
 const media = { language: 'value', 'force-local': 'boolean', 'allow-private-network': 'boolean' };
 const lines = (items) => items.map((item) => `${item.itemId}\t${item.title}`);
-/** A required capture runtime, as the categorical code of its inspection failure. */
-async function captureRuntime(c) {
-    try {
-        return await inspectCaptureRuntime(c.captureRoot, defaultCapturePins);
-    }
-    catch (error) {
-        throw new SpongeError(publicCode(error, 'runtime-missing'), { cause: error });
-    }
-}
 /** The failure of a job that did not run here: its state, or its recorded reason. */
 function jobFailure(store, jobId) {
     const job = store.queue.list().find((j) => j.id === jobId), reason = job?.reason;
@@ -50,16 +41,7 @@ function jobFailure(store, jobId) {
                     ? reason
                     : 'unclassified');
 }
-const resolveConnection = (connections) => (realm) => {
-    if (realm === 'public')
-        return undefined;
-    const connection = connections.find((c) => c.realm === realm);
-    if (!connection)
-        throw new SpongeError('repair-auth');
-    if (!connection.capture)
-        throw new SpongeError('connection-retired');
-    return connection.capture;
-};
+const resolveConnection = (connections) => (realm) => connectionFor(connections, realm);
 /** add/refresh with --wait and run --drain: drain acquisition (one job when given), then queued indexes for run. */
 async function drain(c, jobId, connections) {
     const { store, args, signal } = c;
@@ -69,13 +51,11 @@ async function drain(c, jobId, connections) {
         if (job && job.state !== 'queued')
             throw jobFailure(store, jobId);
     }
-    const runtime = await captureRuntime(c);
     const results = await drainLibrary(store, {
         asrRuntime: await optionalNative('asr', c.nativeRoot, c.version),
         mediaRuntime: await optionalNative('media', c.nativeRoot, c.version),
         language: args.flags.language,
         forceLocal: args.flags['force-local'] === true,
-        runtime,
         jobId,
         signal,
         allowPrivateNetwork: args.flags['allow-private-network'] === true,
@@ -175,14 +155,7 @@ export const commands = [
                 await installNative('media').catch((error) => {
                     media = codeOf(error);
                 });
-            const plan = captureSetupPlan(defaultCapturePins);
-            process.stderr.write(`Capture setup: ${plan.downloadBytes} pinned artifact bytes, plus frozen dependency downloads. Temporary installation limit: ${plan.stagingBudgetBytes} bytes; retained runtime limit: ${plan.installationBudgetBytes} bytes; dependency installation deadline: ${plan.installationDeadlineMs / 60000} minutes. ${plan.requiredFreeBytes} free bytes required.\n`);
-            await provisionCapture({
-                runtimeRoot: join(c.store.root, 'runtime', 'capture'),
-                pins: defaultCapturePins,
-                signal: c.signal,
-            });
-            // Articles and PDFs run anywhere; the media helper is a native release asset and is best-effort.
+            // Articles and PDFs are built in; the media helper is a native release asset and is best-effort.
             const gc = await setupRuntimeGc(c.store, c.version), ready = media === 'ready'
                 ? 'Capture ready'
                 : media === 'unsupported'
@@ -438,13 +411,8 @@ export const commands = [
         library: 'open',
         handler: async (c) => {
             const { store } = c;
-            let capture = 'ready';
-            try {
-                await inspectCaptureRuntime(c.captureRoot, defaultCapturePins);
-            }
-            catch (error) {
-                capture = publicCode(error, 'runtime-missing');
-            }
+            // Article capture is built in; PDFs need the local Poppler tools.
+            const capture = 'ready', pdf = pdfToolsAvailable() ? 'ready' : 'runtime-missing';
             const states = {};
             for (const kind of ['asr', 'embedding', 'media']) {
                 try {
@@ -456,8 +424,8 @@ export const commands = [
                 }
             }
             const cleanup = store.queue.pendingCleanup(), worker = await workerDoctor(store, c.version), workerBroken = !['ready', 'not-installed', 'unsupported'].includes(worker.worker);
-            c.emit({ library: 'ready', capture, ...states, pendingCleanup: cleanup, worker }, `Library: ready\nCapture: ${capture}\nASR: ${states.asr}\nEmbedding: ${states.embedding}\nPending cleanup: ${cleanup}\nWorker: ${worker.worker}${Array.isArray(worker.drift) ? ` (${worker.drift.join(', ')})` : ''}${worker.repair ? `\n  ${worker.repair}` : ''}`);
-            return cleanup || capture !== 'ready' || workerBroken ? 4 : 0;
+            c.emit({ library: 'ready', capture, pdf, ...states, pendingCleanup: cleanup, worker }, `Library: ready\nCapture: ${capture}\nPDF: ${pdf === 'ready' ? 'ready' : 'Poppler not found (brew install poppler)'}\nASR: ${states.asr}\nEmbedding: ${states.embedding}\nPending cleanup: ${cleanup}\nWorker: ${worker.worker}${Array.isArray(worker.drift) ? ` (${worker.drift.join(', ')})` : ''}${worker.repair ? `\n  ${worker.repair}` : ''}`);
+            return cleanup || workerBroken ? 4 : 0;
         },
     },
     {
@@ -554,7 +522,6 @@ export const commands = [
                 signal: c.signal,
                 language: args.flags.language,
                 forceLocal: args.flags['force-local'] === true,
-                resolveCaptureRuntime: () => inspectCaptureRuntime(c.captureRoot, defaultCapturePins),
                 resolveConnection: resolveConnection(connections),
             }, c.version));
             return 0;
@@ -577,10 +544,8 @@ export const commands = [
             const { store, args, signal } = c, dryRun = args.flags.apply !== true, offset = Number(args.flags.offset ?? 0);
             if (args.positional[0] === 'jungle' && args.positional.length === 2) {
                 // A dry run reads Jungle and the library only; applying reverifies every capture with the released verifier.
-                const runtime = dryRun ? undefined : await captureRuntime(c);
                 const report = await jungleAction(store, args.positional[1], {
                     dryRun,
-                    runtime,
                     limit: Number(args.flags.limit ?? 1000),
                     offset,
                     signal,
@@ -592,8 +557,7 @@ export const commands = [
                         ? 3
                         : 0;
             }
-            const runtime = await captureRuntime(c), report = await importAction(store, args.positional[0], {
-                runtime,
+            const report = await importAction(store, args.positional[0], {
                 dryRun,
                 limit: Number(args.flags.limit ?? 20),
                 offset,

package/dist/src/cli/config.d.ts

@@ -5,5 +5,10 @@ export interface Connection {
     /** Absent for a retired GhostGet account connection: it still names its items' realm, but cannot capture. */
     capture?: CaptureConnection;
 }
+/**
+ * The capture connection for a source realm. Public and imported realms capture without one: an import realm
+ * records where an item came from, not an account, so refreshing an imported web page fetches it publicly.
+ */
+export declare function connectionFor(connections: Connection[], realm: string): CaptureConnection | undefined;
 /** Local references only. Tokens, cookies and passwords never belong in this file. */
 export declare function readConnections(root: string): Promise<Connection[]>;

package/dist/src/cli/config.js

@@ -3,6 +3,20 @@ import { readConfined } from '../acquisition/files.js';
 import { join } from 'node:path';
 import { sha256 } from '../util/hash.js';
 import { SpongeError } from '../errors.js';
+/**
+ * The capture connection for a source realm. Public and imported realms capture without one: an import realm
+ * records where an item came from, not an account, so refreshing an imported web page fetches it publicly.
+ */
+export function connectionFor(connections, realm) {
+    if (realm === 'public' || realm.startsWith('import:'))
+        return undefined;
+    const connection = connections.find((c) => c.realm === realm);
+    if (!connection)
+        throw new SpongeError('repair-auth');
+    if (!connection.capture)
+        throw new SpongeError('connection-retired');
+    return connection.capture;
+}
 /** Local references only. Tokens, cookies and passwords never belong in this file. */
 export async function readConnections(root) {
     const path = join(root, 'connections.json');

package/dist/src/cli/import.d.ts

@@ -1,13 +1,11 @@
 import type { LibraryStore } from '../store/index.js';
-import type { CaptureRuntime } from '../acquisition/index.js';
 import { type ImportOptions, type ImportReport } from '../importers/apply.js';
 import { type JungleImportReport } from '../importers/jungle.js';
 /** A source manifest is explicit; default execution produces a private dry-run report. */
 export declare function importAction(store: LibraryStore, path: string, options: ImportOptions): Promise<ImportReport>;
-/** Jungle is read in place; only an applied import needs the capture runtime, to reverify each bundle. */
+/** Jungle is read in place; only an applied import reverifies each bundle. */
 export declare function jungleAction(store: LibraryStore, path: string, options: {
     dryRun: boolean;
-    runtime?: CaptureRuntime;
     limit: number;
     offset: number;
     signal?: AbortSignal;

package/dist/src/cli/import.js

@@ -1,18 +1,18 @@
 import { importManifest } from '../importers/apply.js';
-import { releasedBundleVerifier } from '../importers/inspect.js';
+import { ownedBundleVerifier } from '../importers/inspect.js';
 import { importJungle } from '../importers/jungle.js';
 /** A source manifest is explicit; default execution produces a private dry-run report. */
 export function importAction(store, path, options) {
     return importManifest(store, path, options);
 }
-/** Jungle is read in place; only an applied import needs the capture runtime, to reverify each bundle. */
+/** Jungle is read in place; only an applied import reverifies each bundle. */
 export function jungleAction(store, path, options) {
     return importJungle(store, path, {
         dryRun: options.dryRun,
         limit: options.limit,
         offset: options.offset,
         ...(options.signal ? { signal: options.signal } : {}),
-        ...(options.runtime ? { verify: releasedBundleVerifier(options.runtime, store.root, options.signal) } : {}),
+        ...(options.dryRun ? {} : { verify: ownedBundleVerifier(store.root, options.signal) }),
     });
 }
 const tally = (m) => Object.entries(m)

package/dist/src/cli/main.js

@@ -6,7 +6,6 @@ import { terminalSafe } from './terminal.js';
 import { parseArguments } from './arguments.js';
 import { command, usageOf } from './commands.js';
 import { openLibrary } from '../store/index.js';
-import { defaultCapturePins } from '../acquisition/index.js';
 import { failure } from '../acquisition/errors.js';
 import { SpongeError, codeOf, publicCode, exitFor, hints, detailHints, debugChain, } from '../errors.js';
 /** Failures print one categorical JSON line: the public code, its specific `reason` when narrower, and a fixed hint.
@@ -65,7 +64,6 @@ export async function main(argv, version) {
         return await spec.handler({
             ...context,
             store,
-            captureRoot: join(store.root, 'runtime', 'capture', defaultCapturePins.id),
             nativeRoot: join(store.root, 'runtime', 'models'),
         });
     }

package/dist/src/cli/runtimes.js

@@ -1,10 +1,9 @@
-import { defaultCapturePins } from '../acquisition/index.js';
 import { loadNativeCatalog, nativeModelPackage } from '../models/catalog.js';
 import { modelSetupPlan } from '../models/artifacts.js';
 import { digestFingerprints } from '../models/digest.js';
 import { collectRuntimes } from '../worker/runtime-gc.js';
 import { codeOf } from '../errors.js';
-/** References of this installed Sponge: its capture pin, its catalog's model packages and every digest pin. */
+/** References of this installed Sponge: its catalog's model packages and every digest pin. Capture is built in, so every capture runtime is superseded. */
 async function installedReferences(version) {
     let models = null;
     try {
@@ -19,7 +18,7 @@ async function installedReferences(version) {
     catch {
         /* Without the catalog no model directory is provably superseded. */
     }
-    return { captureIds: [defaultCapturePins.id], models };
+    return { captureIds: [], models };
 }
 export async function runtimeGcAction(store, version, dryRun) {
     return collectRuntimes(store, { ...(await installedReferences(version)), dryRun });

package/dist/src/cli/worker.d.ts

@@ -12,10 +12,7 @@ export declare function workerDoctor(store: LibraryStore, version: string, envir
     lastExit?: unknown;
 }>;
 type DrainOptions = Parameters<typeof drainLibrary>[1];
-export type ServiceDrainOptions = Omit<DrainOptions, 'runtime'> & {
-    runtime?: DrainOptions['runtime'];
-    resolveCaptureRuntime?: () => Promise<DrainOptions['runtime']>;
-};
+export type ServiceDrainOptions = DrainOptions;
 /** Dispatch only work that claimNext can admit; delayed retries retain their due time. */
 export declare function hasDueAcquisition(store: LibraryStore): boolean;
 export declare function drainServiceQueues(store: LibraryStore, options: ServiceDrainOptions, version: string, signal: AbortSignal): Promise<readonly unknown[]>;

package/dist/src/cli/worker.js

@@ -60,16 +60,12 @@ export async function drainServiceQueues(store, options, version, signal) {
     let acquisitionError;
     if (hasDueAcquisition(store)) {
         try {
-            const runtime = options.runtime ?? (await options.resolveCaptureRuntime?.());
-            if (!runtime)
-                throw new SpongeError('runtime-missing');
             const models = join(store.root, 'runtime', 'models');
             // Media jobs report a missing helper themselves; articles and PDFs still drain.
             const asrRuntime = options.asrRuntime ?? (await optionalNative('asr', models, version));
             const mediaRuntime = options.mediaRuntime ?? (await optionalNative('media', models, version));
             results.push(...(await drainLibrary(store, {
                 ...options,
-                runtime,
                 asrRuntime,
                 mediaRuntime,
                 signal,

package/dist/src/importers/apply.d.ts

@@ -1,9 +1,7 @@
 import type { LibraryStore } from '../store/index.js';
 import { type ImportEntry } from './manifest.js';
 import { type BundleVerifier } from './inspect.js';
-import type { CaptureRuntime } from '../acquisition/runtime.js';
 export interface ImportOptions {
-    runtime: CaptureRuntime;
     dryRun?: boolean;
     limit?: number;
     offset?: number;

package/dist/src/importers/apply.js

@@ -5,7 +5,7 @@ import { localizeMarkdown } from '../app/markdown.js';
 import { selectedMetadata } from '../metadata/article.js';
 import { drainQueue } from '../worker/engine.js';
 import { readImportManifest, importBundleRoot } from './manifest.js';
-import { inspectImportBundle, releasedBundleVerifier } from './inspect.js';
+import { inspectImportBundle, ownedBundleVerifier } from './inspect.js';
 import { SpongeError, publicCode } from '../errors.js';
 const errorCode = (error) => {
     const code = publicCode(error);
@@ -125,7 +125,7 @@ export async function importManifest(store, path, options) {
         entries: [],
         nextOffset: offset + limit < manifest.entries.length ? offset + limit : null,
     };
-    const verify = options.verify ?? releasedBundleVerifier(options.runtime, store.root, options.signal);
+    const verify = options.verify ?? ownedBundleVerifier(store.root, options.signal);
     for (const entry of manifest.entries.slice(offset, offset + limit)) {
         if (options.signal?.aborted)
             throw new SpongeError('cancelled');

package/dist/src/importers/inspect.d.ts

@@ -1,5 +1,4 @@
 import type { Asset, MetadataClaim } from '../../contracts/index.js';
-import { type CaptureRuntime } from '../acquisition/runtime.js';
 export interface ImportedBundle {
     root: string;
     kind: 'article' | 'pdf';
@@ -16,6 +15,7 @@ export interface ImportedBundle {
     importHash: string;
 }
 export type BundleVerifier = (root: string, kind: 'article' | 'pdf') => Promise<unknown>;
-export declare function releasedBundleVerifier(runtime: CaptureRuntime, libraryRoot: string, signal?: AbortSignal): BundleVerifier;
+/** Bundles are read by the owned child with networking denied; its report carries no document text. */
+export declare function ownedBundleVerifier(libraryRoot: string, signal?: AbortSignal): BundleVerifier;
 /** An import-time digest never upgrades a missing historical document digest. */
 export declare function inspectImportBundle(root: string, verify: BundleVerifier): Promise<ImportedBundle>;

package/dist/src/importers/inspect.js

@@ -1,4 +1,5 @@
-import { fileURLToPath, pathToFileURL } from 'node:url';
+import { writeFile } from 'node:fs/promises';
+import { pathToFileURL } from 'node:url';
 import { join } from 'node:path';
 import { limits } from '../../contracts/index.js';
 import { validateAsset } from '../../contracts/validate.js';
@@ -6,32 +7,32 @@ import { object, metadataFromMarkdown } from '../acquisition/normalize.js';
 import { readConfined } from '../acquisition/files.js';
 import { sha256 as hash } from '../util/hash.js';
 import { withImportVerification } from './custody.js';
-import { inspectCaptureRuntime } from '../acquisition/runtime.js';
+import { workerArguments } from '../acquisition/child.js';
 import { articleMetadata } from '../metadata/article.js';
 import { bundleBytes } from '../app/bundle-files.js';
 import { SpongeError } from '../errors.js';
-export function releasedBundleVerifier(runtime, libraryRoot, signal) {
-    return async (root, kind) => {
-        const checked = await inspectCaptureRuntime(runtime.root, runtime.pins), helper = fileURLToPath(new URL(import.meta.url.endsWith('.ts') ? './verify-helper.ts' : './verify-helper.js', import.meta.url));
-        return withImportVerification(libraryRoot, async ({ scratchRoot, runProcess }) => {
-            const args = [helper, checked.root, root, kind];
-            const result = await runProcess(process.platform === 'darwin' ? '/usr/bin/sandbox-exec' : checked.bun, process.platform === 'darwin'
-                ? ['-p', '(version 1) (allow default) (deny network*)', checked.bun, ...args]
-                : args, {
-                env: { PATH: '/usr/bin:/bin', HOME: checked.root, TMPDIR: scratchRoot, LANG: 'en_US.UTF-8' },
-                deadlineMs: 45000,
-                maxOutputBytes: 65536,
-                signal,
-            });
-            if (result.cleanup !== 'settled')
-                throw new SpongeError('cleanup-required');
-            if (result.stopped)
-                throw new SpongeError(result.stopped);
-            if (result.code !== 0)
-                throw new SpongeError('unsupported');
-            return JSON.parse(result.stdout);
+/** Bundles are read by the owned child with networking denied; its report carries no document text. */
+export function ownedBundleVerifier(libraryRoot, signal) {
+    return (bundle, format) => withImportVerification(libraryRoot, async ({ scratchRoot, runProcess }) => {
+        const requestFile = join(scratchRoot, 'request.json'), request = { schemaVersion: 1, kind: 'verify', format, bundle };
+        await writeFile(requestFile, JSON.stringify(request), { mode: 0o600 });
+        const args = [...workerArguments(), requestFile];
+        const result = await runProcess(process.platform === 'darwin' ? '/usr/bin/sandbox-exec' : process.execPath, process.platform === 'darwin'
+            ? ['-p', '(version 1) (allow default) (deny network*)', process.execPath, ...args]
+            : args, {
+            env: { PATH: '/usr/bin:/bin', HOME: scratchRoot, TMPDIR: scratchRoot, LANG: 'en_US.UTF-8' },
+            deadlineMs: 45000,
+            maxOutputBytes: 65536,
+            signal,
         });
-    };
+        if (result.cleanup !== 'settled')
+            throw new SpongeError('cleanup-required');
+        if (result.stopped)
+            throw new SpongeError(result.stopped);
+        if (result.code !== 0)
+            throw new SpongeError('unsupported');
+        return JSON.parse(result.stdout);
+    });
 }
 function imageType(bytes) {
     if (bytes[0] === 255 && bytes[1] === 216)

package/dist/src/metadata/article.d.ts

@@ -1,5 +1,4 @@
 import type { MetadataClaim, MetadataWarning } from '../../contracts/index.js';
-/** Consume bounded observed Wordcell evidence without fetching linked JSON-LD contexts. */
 export declare function articleMetadata(input: unknown, observedAt: string): MetadataClaim[];
 export declare function selectedMetadata(claims: readonly MetadataClaim[], field: string): MetadataClaim | undefined;
 export declare function articleMetadataWarnings(input: unknown): MetadataWarning[];

package/dist/src/metadata/article.js

@@ -6,11 +6,12 @@ function object(v) {
         throw new SpongeError('integrity');
     return v;
 }
-/** Consume bounded observed Wordcell evidence without fetching linked JSON-LD contexts. */
+/** Consume bounded observed page evidence without fetching linked JSON-LD contexts. Imported Wordcell bundles keep their own version. */
+const extractorVersions = new Set(['sponge-metadata-v1', 'wordcell-metadata-v1']);
 export function articleMetadata(input, observedAt) {
     const envelope = object(input);
     articleMetadataWarnings(input);
-    if (envelope.schemaVersion !== 1 || envelope.extractorVersion !== 'wordcell-metadata-v1')
+    if (envelope.schemaVersion !== 1 || !extractorVersions.has(String(envelope.extractorVersion)))
         throw new SpongeError('schema-newer');
     if (!Array.isArray(envelope.claims) ||
         envelope.claims.length > limits.metadataClaims ||

package/dist/src/pdf/capture.d.ts

@@ -1,26 +1,17 @@
-import type { CaptureResult, ErrorEnvelope } from '../../contracts/index.js';
-import { runOwned } from '../acquisition/process.js';
-import { type CaptureRuntime } from '../acquisition/runtime.js';
-type CaptureConnection = {
-    kind: 'cdp';
-    endpoint: string;
-};
+import type { runOwned } from '../acquisition/process.js';
+import { type CaptureOutcome } from '../acquisition/child.js';
+export type { CaptureOutcome } from '../acquisition/child.js';
 export interface PdfCaptureRequest {
-    runtime: CaptureRuntime;
     stagingRoot: string;
     sourceUrl: string;
     requestId: string;
-    connection?: CaptureConnection;
+    connection?: {
+        kind: 'cdp';
+        endpoint: string;
+    };
     signal?: AbortSignal;
     deadlineMs?: number;
     allowPrivateNetwork?: boolean;
     runProcess?: typeof runOwned;
 }
-export type CaptureOutcome = {
-    ok: true;
-    capture: CaptureResult;
-    bundleRoot: string;
-    dispose: () => Promise<void>;
-} | ErrorEnvelope;
 export declare function capturePdf(request: PdfCaptureRequest): Promise<CaptureOutcome>;
-export {};