@hraness/sponge 0.5.3 → 0.6.1

package/dist/src/pdf/capture.js

@@ -1,212 +1,58 @@
-import { mkdir, mkdtemp, writeFile, statfs, realpath } from 'node:fs/promises';
 import { join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { limits } from '../../contracts/index.js';
-import { privateRoot, physicalBytes } from '../acquisition/files.js';
-import { runOwned, isolatedEnvironment } from '../acquisition/process.js';
-import { inspectCaptureRuntime } from '../acquisition/runtime.js';
-import { settleCaptureRoot } from '../acquisition/cleanup.js';
+import { superviseChild } from '../acquisition/child.js';
 import { failure } from '../acquisition/errors.js';
-import { object } from '../acquisition/normalize.js';
 import { normalizePdfBundle } from './normalize.js';
 import { copyPdfInput } from './input.js';
-import { pdfFailure } from './errors.js';
-import { SpongeError, publicCode, codeOf } from '../errors.js';
-export async function capturePdf(request) {
-    const { requestId, signal } = request;
-    if (!/^[a-zA-Z0-9_-]{1,128}$/.test(requestId))
-        return failure('invalid-request', 'invalid-input');
-    let url;
-    try {
-        url = new URL(request.sourceUrl);
-        if (!['http:', 'https:', 'file:'].includes(url.protocol) ||
-            url.username ||
-            url.password ||
-            request.sourceUrl.length > 16384)
-            throw Error();
-    }
-    catch {
-        return failure(requestId, 'invalid-input');
-    }
-    if (signal?.aborted)
-        return failure(requestId, 'cancelled');
-    let runtime;
+function sourceOf(sourceUrl) {
     try {
-        runtime = await inspectCaptureRuntime(request.runtime.root, request.runtime.pins);
-    }
-    catch (error) {
-        return failure(requestId, publicCode(error, 'runtime-missing'), 'setup');
-    }
-    let base;
-    try {
-        base = await privateRoot(request.stagingRoot);
-        const disk = await statfs(base);
-        if (disk.bavail * disk.bsize < limits.stagingBytes + limits.freeReserveBytes)
-            return failure(requestId, 'limit-exceeded');
+        const url = new URL(sourceUrl);
+        if (!['http:', 'https:', 'file:'].includes(url.protocol) || url.username || url.password)
+            return;
+        if (sourceUrl.length > 16384 || (url.protocol === 'file:' && (url.host || url.search || url.hash)))
+            return;
+        return url;
     }
     catch {
-        return failure(requestId, 'invalid-input');
+        return;
     }
-    const deadline = request.deadlineMs ?? limits.captureDeadlineMs;
-    if (!Number.isSafeInteger(deadline) || deadline < 100 || deadline > limits.captureDeadlineMs)
-        return failure(requestId, 'invalid-input');
-    if (url.protocol === 'file:' && (url.host || url.search || url.hash || request.connection))
-        return failure(requestId, 'invalid-input');
-    if (request.connection || request.allowPrivateNetwork)
-        return failure(requestId, 'unsupported');
-    const captureHelper = fileURLToPath(new URL(import.meta.url.endsWith('.ts') ? './capture-helper.ts' : './capture-helper.js', import.meta.url));
-    const args = [
-        captureHelper,
-        runtime.root,
-        request.sourceUrl,
-        '--slug',
-        'document',
-        '--timeout-ms',
-        String(deadline),
-        '--max-pdf-bytes',
-        String(limits.pdfBytes),
-        '--max-pages',
-        '500',
-        '--max-images',
-        String(limits.imagesPerItem),
-        '--max-asset-bytes',
-        String(limits.imageBytes),
-        '--max-total-asset-bytes',
-        String(500 * 1024 ** 2),
-        '--quiet',
-        '--json',
-    ];
-    const root = await realpath(await mkdtemp(join(base, 'capture-'))), home = join(root, 'home'), output = join(root, 'bundles');
-    args.push('--output', output);
-    const env = isolatedEnvironment(home, home);
-    const controller = new AbortController();
-    const abort = () => controller.abort();
-    signal?.addEventListener('abort', abort, { once: true });
-    if (signal?.aborted)
-        abort();
-    let cleanup = 'settled', keep = false, quota = false, checking = false;
-    const monitor = setInterval(async () => {
-        if (checking)
-            return;
-        checking = true;
-        try {
-            if ((await physicalBytes(root)) > limits.stagingBytes) {
-                quota = true;
-                controller.abort();
+}
+export async function capturePdf(request) {
+    const url = sourceOf(request.sourceUrl);
+    // A PDF is fetched directly; a signed-in browser adds nothing.
+    if (url && request.connection)
+        return failure(request.requestId, 'unsupported');
+    let sourceHash;
+    return superviseChild({
+        ...request,
+        valid: url !== undefined,
+        prepare: async ({ root, output, deadline, signal }) => {
+            let source = { url: request.sourceUrl };
+            if (url.protocol === 'file:') {
+                const path = join(root, 'input.pdf');
+                sourceHash = await copyPdfInput(fileURLToPath(url), path, signal);
+                source = { path };
             }
-        }
-        catch {
-            quota = true;
-            controller.abort();
-        }
-        finally {
-            checking = false;
-        }
-    }, 100);
-    const started = Date.now();
-    try {
-        await mkdir(home, { mode: 0o700 });
-        await mkdir(output, { mode: 0o700 });
-        await writeFile(join(root, 'ownership.json'), JSON.stringify({ schemaVersion: 1, requestId, state: 'acquiring' }), {
-            mode: 0o600,
-        });
-        let sourceHash;
-        if (url.protocol === 'file:') {
-            sourceHash = await copyPdfInput(fileURLToPath(url), join(root, 'input.pdf'), controller.signal);
-            args[2] = join(root, 'input.pdf');
-        }
-        cleanup = 'pending';
-        const result = await (request.runProcess ?? runOwned)(runtime.bun, args, {
-            cwd: root,
-            env,
-            signal: controller.signal,
-            deadlineMs: Math.max(1, deadline - (Date.now() - started)),
-            maxOutputBytes: limits.metadataBytes,
-            graceMs: 5000,
-        });
-        cleanup = result.cleanup;
-        let summary;
-        try {
-            summary = JSON.parse(result.stdout);
-        }
-        catch {
-            summary = null;
-        }
-        const upstream = pdfFailure(summary, requestId);
-        if (upstream.code === 'cleanup-required')
-            cleanup = 'pending';
-        if (cleanup !== 'settled')
-            return failure(requestId, 'cleanup-required', 'cleanup');
-        if (quota)
-            return failure(requestId, 'limit-exceeded');
-        if (result.stopped)
-            return failure(requestId, result.stopped);
-        if (result.code !== 0 || object(summary).ok !== true)
-            return upstream;
-        const outputDirectory = object(summary).outputDirectory;
-        if (typeof outputDirectory !== 'string' ||
-            !outputDirectory.startsWith(output + '/') ||
-            (await realpath(outputDirectory)) !== outputDirectory)
-            return failure(requestId, 'integrity', 'verification');
-        const helper = fileURLToPath(new URL(import.meta.url.endsWith('.ts') ? './verify-helper.ts' : './verify-helper.js', import.meta.url));
-        cleanup = 'pending';
-        const checked = await (request.runProcess ?? runOwned)(runtime.bun, [helper, runtime.root, outputDirectory, request.sourceUrl], {
-            cwd: root,
-            env,
-            signal: controller.signal,
-            deadlineMs: Math.max(1, deadline - (Date.now() - started)),
-            maxOutputBytes: limits.helperFrameBytes,
-        });
-        cleanup = checked.cleanup;
-        if (cleanup !== 'settled')
-            return failure(requestId, 'cleanup-required', 'cleanup');
-        if (checked.stopped)
-            return failure(requestId, checked.stopped);
-        if (checked.code !== 0)
-            return failure(requestId, 'integrity', 'verification');
-        const capture = await normalizePdfBundle({
-            bundleRoot: outputDirectory,
-            inspection: JSON.parse(checked.stdout),
-            summary,
-            requestId,
+            return {
+                schemaVersion: 1,
+                kind: 'pdf',
+                source,
+                outputDirectory: output,
+                timeoutMs: deadline,
+                maxPdfBytes: limits.pdfBytes,
+                maxPages: 500,
+                maxImages: limits.imagesPerItem,
+                maxAssetBytes: limits.imageBytes,
+                maxTotalAssetBytes: 500 * 1024 ** 2,
+                allowPrivateNetwork: request.allowPrivateNetwork === true,
+            };
+        },
+        verify: (bundleRoot) => normalizePdfBundle({
+            bundleRoot,
+            requestId: request.requestId,
             sourceUrl: request.sourceUrl,
-            wordcellVersion: runtime.pins.wordcell.version,
             ...(sourceHash ? { sourceHash } : {}),
-        });
-        if (controller.signal.aborted)
-            return failure(requestId, quota ? 'limit-exceeded' : 'cancelled');
-        await writeFile(join(root, 'ownership.json'), JSON.stringify({ schemaVersion: 1, requestId, state: 'verified' }), {
-            mode: 0o600,
-        });
-        keep = true;
-        return {
-            ok: true,
-            capture,
-            bundleRoot: outputDirectory,
-            dispose: async () => {
-                if ((await settleCaptureRoot(root, requestId)) === 'pending')
-                    throw new SpongeError('cleanup-required');
-            },
-        };
-    }
-    catch (error) {
-        if (cleanup !== 'settled')
-            return failure(requestId, 'cleanup-required', 'cleanup');
-        const raw = codeOf(error), code = raw === 'integrity' ||
-            raw === 'limit-exceeded' ||
-            raw === 'schema-newer' ||
-            raw === 'invalid-input' ||
-            raw === 'cancelled'
-            ? raw
-            : 'unclassified';
-        return failure(requestId, code, 'verification');
-    }
-    finally {
-        clearInterval(monitor);
-        while (checking)
-            await new Promise((r) => setTimeout(r, 5));
-        signal?.removeEventListener('abort', abort);
-        if (!keep && cleanup === 'settled' && (await settleCaptureRoot(root, requestId)) === 'pending')
-            return failure(requestId, 'cleanup-required', 'cleanup');
-    }
+        }),
+    });
 }

package/dist/src/pdf/handler.d.ts

@@ -1,5 +1,5 @@
 import type { ErrorEnvelope, JobCheckpoint } from '../../contracts/index.js';
-import { type CaptureRuntime, type CaptureConnection } from '../acquisition/index.js';
+import { type CaptureConnection } from '../acquisition/index.js';
 import { type PdfCaptureRequest as CaptureRequest, type CaptureOutcome } from './capture.js';
 import type { LibraryStore, AttemptToken } from '../store/index.js';
 interface PdfResult {
@@ -14,7 +14,6 @@ export type PdfOutcome = (PdfResult | ErrorEnvelope) & {
 export interface PdfOptions {
     checkpoint?: (stage: JobCheckpoint['stage']) => Promise<void>;
     runProcess?: CaptureRequest['runProcess'];
-    runtime: CaptureRuntime;
     connection?: CaptureConnection;
     signal?: AbortSignal;
     allowPrivateNetwork?: boolean;

package/dist/src/pdf/handler.js

@@ -24,7 +24,6 @@ export function ingestPdf(store, token, options) {
             throw new SpongeError('invalid-input');
         const result = await (options.capture ?? capturePdf)({
             runProcess: options.runProcess,
-            runtime: options.runtime,
             stagingRoot: join(store.root, 'staging'),
             sourceUrl: source.locator,
             requestId: token.attemptId,

package/dist/src/pdf/normalize.d.ts

@@ -1,10 +1,9 @@
 import type { CaptureResult } from '../../contracts/index.js';
+/** Re-verify a PDF bundle written by the owned child: every declared file is re-read and rehashed here. */
 export declare function normalizePdfBundle(input: {
     bundleRoot: string;
-    inspection: unknown;
-    summary: unknown;
     requestId: string;
     sourceUrl: string;
-    wordcellVersion: string;
+    /** The hash of the parent's own copy of a local input. */
     sourceHash?: string;
 }): Promise<CaptureResult>;

package/dist/src/pdf/normalize.js

@@ -4,16 +4,16 @@ import { readConfined } from '../acquisition/files.js';
 import { sha256 as hash } from '../util/hash.js';
 import { object } from '../acquisition/normalize.js';
 import { SpongeError } from '../errors.js';
+/** Identifies Sponge's own PDF representation; bump when the bundle meaning changes. */
+const pdfExtractor = { name: 'sponge', version: 'pdf-v1' }, rule = 'sponge-pdf-v1';
+/** Re-verify a PDF bundle written by the owned child: every declared file is re-read and rehashed here. */
 export async function normalizePdfBundle(input) {
-    const verified = object(input.inspection), inspection = object(verified.inspection), summary = object(input.summary);
-    if (verified.ok !== true || inspection.schemaVersion !== 1 || !Array.isArray(inspection.assets))
-        throw new SpongeError('integrity');
     const manifestBytes = await readConfined(input.bundleRoot, 'capture.json', limits.metadataBytes), manifest = object(JSON.parse(manifestBytes.toString('utf8')));
     if (manifest.schemaVersion !== 1 ||
         manifest.kind !== 'pdf' ||
         !['complete', 'partial'].includes(String(manifest.status)))
         throw new SpongeError('integrity');
-    function asset(raw, mediaType) {
+    async function verified(raw, mediaType, maxBytes) {
         const value = object(raw);
         const result = {
             relativePath: String(value.path),
@@ -22,55 +22,51 @@ export async function normalizePdfBundle(input) {
             mediaType,
         };
         validateAsset(result);
+        if (result.bytes > maxBytes)
+            throw new SpongeError('limit-exceeded');
+        const content = await readConfined(input.bundleRoot, result.relativePath, maxBytes);
+        if (content.length !== result.bytes || hash(content) !== result.sha256)
+            throw new SpongeError('integrity');
         return result;
     }
-    function assertVerified(value, checked) {
-        const record = object(checked);
-        if (record.integrity !== 'verified' ||
-            record.path !== value.relativePath ||
-            record.sha256 !== value.sha256 ||
-            record.bytes !== value.bytes)
+    const markdown = await verified(manifest.markdown, 'text/markdown', limits.markdownBytes), sourcePdf = await verified(manifest.source, 'application/pdf', limits.pdfBytes);
+    if (input.sourceHash !== undefined) {
+        if (sourcePdf.sha256 !== input.sourceHash)
+            throw new SpongeError('integrity');
+    }
+    else {
+        // The child records the requested URL with secrets redacted; its origin and path must still be ours.
+        const requested = new URL(String(object(manifest.source).requestedUrl)), expected = new URL(input.sourceUrl);
+        if (requested.origin !== expected.origin || requested.pathname !== expected.pathname)
             throw new SpongeError('integrity');
     }
-    const markdown = asset(manifest.markdown, 'text/markdown'), sourcePdf = asset(manifest.source, 'application/pdf');
-    assertVerified(markdown, inspection.markdown);
-    assertVerified(sourcePdf, inspection.source);
-    if (input.sourceHash && sourcePdf.sha256 !== input.sourceHash)
-        throw new SpongeError('integrity');
-    const source = object(manifest.source);
-    if (!input.sourceHash &&
-        (typeof verified.expectedSourceUrl !== 'string' || source.requestedUrl !== verified.expectedSourceUrl))
-        throw new SpongeError('integrity');
-    const markdownBytes = await readConfined(input.bundleRoot, markdown.relativePath, limits.markdownBytes);
-    if (hash(markdownBytes) !== markdown.sha256 || markdownBytes.length !== markdown.bytes)
-        throw new SpongeError('integrity');
     if (!Array.isArray(manifest.images) || manifest.images.length > limits.imagesPerItem)
         throw new SpongeError('limit-exceeded');
     const byPath = new Map();
     for (const raw of manifest.images) {
-        const image = object(raw), record = object(image.asset), value = asset(record, String(record.mimeType));
-        if (!value.mediaType.startsWith('image/'))
-            throw new SpongeError('integrity');
-        assertVerified(value, inspection.assets.find((raw) => object(raw).path === value.relativePath));
-        const prior = byPath.get(value.relativePath);
-        if (prior && JSON.stringify(prior) !== JSON.stringify(value))
+        const record = object(object(raw).asset), mediaType = String(record.mimeType), path = String(record.path);
+        if (!mediaType.startsWith('image/') || !path.startsWith('assets/'))
             throw new SpongeError('integrity');
-        byPath.set(value.relativePath, value);
+        const prior = byPath.get(path);
+        if (prior) {
+            if (prior.sha256 !== record.sha256 || prior.bytes !== record.bytes || prior.mediaType !== mediaType)
+                throw new SpongeError('integrity');
+            continue;
+        }
+        byPath.set(path, await verified(record, mediaType, limits.imageBytes));
     }
-    if (byPath.size !== inspection.assets.length)
-        throw new SpongeError('integrity');
     const metadata = [], document = object(manifest.document), observedAt = String(manifest.capturedAt);
     for (const field of ['title', 'author', 'subject', 'creator', 'producer', 'createdAt', 'modifiedAt']) {
         const value = document[field];
         if (typeof value === 'string' && value.length > 0 && value.length <= 16384)
-            metadata.push({ field, value, source: 'pdf', observedAt, rule: 'wordcell-pdf-v1', selected: true });
+            metadata.push({ field, value, source: 'pdf', observedAt, rule, selected: true });
     }
     metadata.push({
         field: 'pageCount',
         value: String(document.pageCount),
         source: 'pdf',
         observedAt,
-        rule: 'wordcell-pdf-v1',
+        rule,
     });
     if (manifest.quality) {
         const quality = object(manifest.quality);
@@ -80,7 +76,7 @@ export async function normalizePdfBundle(input) {
                 value: String(quality.issues.length),
                 source: 'pdf',
                 observedAt,
-                rule: 'wordcell-pdf-v1',
+                rule,
             });
             if (quality.issues.length > 400)
                 metadata.push({
@@ -88,7 +84,7 @@ export async function normalizePdfBundle(input) {
                     value: 'true',
                     source: 'pdf',
                     observedAt,
-                    rule: 'wordcell-pdf-v1',
+                    rule,
                 });
             for (const issue of quality.issues.slice(0, 400)) {
                 const row = object(issue);
@@ -97,12 +93,12 @@ export async function normalizePdfBundle(input) {
                     value: JSON.stringify({ code: row.code, page: row.page }),
                     source: 'pdf',
                     observedAt,
-                    rule: 'wordcell-pdf-v1',
+                    rule,
                 });
             }
         }
     }
-    const partial = manifest.status !== 'complete' || inspection.status !== 'complete' || summary.status !== 'complete';
+    const partial = manifest.status !== 'complete';
     const result = {
         schemaVersion: 1,
         requestId: input.requestId,
@@ -114,7 +110,7 @@ export async function normalizePdfBundle(input) {
         assets: [...byPath.values()],
         metadata,
         warnings: partial ? ['partial'] : [],
-        extractor: { name: 'wordcell', version: input.wordcellVersion },
+        extractor: { ...pdfExtractor },
         manifestSha256: hash(manifestBytes),
         completeness: { text: partial ? 'partial' : 'complete', images: partial ? 'partial' : 'complete' },
     };

package/dist/src/pdf/tools.d.ts

@@ -0,0 +1,2 @@
+/** Whether the Poppler converters PDF capture runs are installed. */
+export declare function pdfToolsAvailable(): boolean;

package/dist/src/pdf/tools.js

@@ -0,0 +1,21 @@
+import { accessSync, constants } from 'node:fs';
+import { join } from 'node:path';
+// The child finds the same tools: on PATH, then in the usual package-manager directories.
+const directories = () => [
+    ...(process.env.PATH ?? '').split(':').filter((d) => d.startsWith('/')),
+    '/opt/homebrew/bin',
+    '/usr/local/bin',
+    '/usr/bin',
+];
+/** Whether the Poppler converters PDF capture runs are installed. */
+export function pdfToolsAvailable() {
+    return ['pdfinfo', 'pdftohtml'].every((tool) => directories().some((directory) => {
+        try {
+            accessSync(join(directory, tool), constants.X_OK);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }));
+}

package/native/runtime-manifest.json

@@ -33,16 +33,16 @@
     }
   ],
   "build": {
-    "asrSourceSha256": "7520570edfaae2e511814a9c463b977b87523ea8bb6494b244a59a5ca5e73d42",
+    "asrSourceSha256": "17934fedff0c472a1a11f93849b20ec142334a25e035d5dfad40c9151b72c9eb",
     "clangVersion": "Apple clang version 21.0.0 (clang-2100.1.1.101)",
-    "embeddingSourceSha256": "0938c0368b954bc609ee2438b579121850919cefb51144270dfb0fb62ec86c34",
-    "mediaSourceSha256": "db88df9dcfd4077b26bf1ef3595f1ce295fb3cdb085eccf4e1eea894698902bd",
+    "embeddingSourceSha256": "ca8d8bc22b03a3ca65a8ff4fb43f88d3d0cb3bd93591fe5842fd0106601aa6f1",
+    "mediaSourceSha256": "dc775291679fd69e554e28ee4cbab457f4e79080fb58542d92a83bc0004cac82",
     "rustcVersion": "rustc 1.97.1 (8bab26f4f 2026-07-14)",
     "swiftVersion": "Apple Swift version 6.3.3 (swiftlang-6.3.3.1.3 clang-2100.1.1.101)"
   },
   "minimumMacOS": "26.0",
-  "packageVersion": "0.5.3",
+  "packageVersion": "0.6.1",
   "platform": "darwin",
   "schemaVersion": 1,
-  "sourceCommit": "2191f9e2296d89d5ef5b26750e262469ccf6c0d3"
+  "sourceCommit": "e5785b1c0df60222bf5be92d24ecefeab921712d"
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hraness/sponge",
-  "version": "0.5.3",
+  "version": "0.6.1",
   "description": "A local library for captured articles and verified offline assets.",
   "repository": {
     "type": "git",
@@ -13,6 +13,7 @@
   },
   "files": [
     "bin/",
+    "dist/capture/",
     "dist/contracts/",
     "dist/migrations/",
     "dist/src/",
@@ -28,7 +29,7 @@
   },
   "scripts": {
     "check": "node --check bin/sponge.js",
-    "build": "tsc -p tsconfig.build.json",
+    "build": "node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\" && tsc -p tsconfig.build.json && node scripts/bundle-capture.mjs",
     "typecheck": "tsc --noEmit",
     "cycles": "madge --circular --extensions ts src",
     "deadcode": "knip --no-progress",
@@ -48,10 +49,18 @@
   "devDependencies": {
     "@types/mdast": "4.0.4",
     "@types/node": "24.10.1",
+    "defuddle": "0.19.1",
+    "entities": "7.0.1",
+    "esbuild": "0.25.11",
+    "htmlparser2": "10.1.0",
     "knip": "6.39.0",
+    "linkedom": "0.18.14",
     "madge": "8.0.0",
+    "mathml-to-latex": "1.8.0",
     "prettier": "3.9.10",
+    "temml": "0.13.4",
     "tsx": "4.20.6",
+    "turndown": "7.2.4",
     "typescript": "5.9.3"
   },
   "dependencies": {

package/dist/src/acquisition/clip-helper.js

@@ -1,14 +0,0 @@
-// Executed only by the pinned Bun runtime. The import is Wordcell's public clip entry point.
-const runtime = process.argv[2];
-try {
-    const { main } = await import(import.meta.resolve('@hraness/wordcell/clip/cli', `${runtime}/package.json`));
-    process.exitCode = await main(process.argv.slice(3), process.env, {
-        stdout: (value) => process.stdout.write(value),
-        stderr: (value) => process.stderr.write(value),
-    });
-}
-catch {
-    process.stdout.write(JSON.stringify({ ok: false }));
-    process.exitCode = 1;
-}
-export {};