@hraness/sponge 0.1.0 → 0.2.0

package/dist/migrations/002.js

@@ -0,0 +1,6 @@
+/** Additive retry evidence; schema 1 and its checksum remain immutable. */
+export const version = 2;
+export const sql = `
+ALTER TABLE jobs ADD COLUMN retryable INTEGER NOT NULL DEFAULT 0 CHECK(retryable IN (0,1));
+ALTER TABLE jobs ADD COLUMN retry_after_ms INTEGER CHECK(retry_after_ms>=0 AND retry_after_ms<=86400000);
+`;

package/dist/migrations/003.d.ts

@@ -0,0 +1,3 @@
+/** Optional provenance preserves legacy rows and earlier migration checksums. */
+export declare const version = 3;
+export declare const sql = "\nALTER TABLE metadata_claims ADD COLUMN observed_json TEXT;\nALTER TABLE metadata_claims ADD COLUMN locator TEXT;\nALTER TABLE metadata_claims ADD COLUMN selected INTEGER CHECK(selected IN (0,1));\nALTER TABLE metadata_claims ADD COLUMN extraction_version TEXT;\nALTER TABLE metadata_claims ADD COLUMN date_precision TEXT CHECK(date_precision IN ('date','datetime','invalid'));\nALTER TABLE revisions ADD COLUMN metadata_warnings_json TEXT NOT NULL DEFAULT '[]';\n";

package/dist/migrations/003.js

@@ -0,0 +1,10 @@
+/** Optional provenance preserves legacy rows and earlier migration checksums. */
+export const version = 3;
+export const sql = `
+ALTER TABLE metadata_claims ADD COLUMN observed_json TEXT;
+ALTER TABLE metadata_claims ADD COLUMN locator TEXT;
+ALTER TABLE metadata_claims ADD COLUMN selected INTEGER CHECK(selected IN (0,1));
+ALTER TABLE metadata_claims ADD COLUMN extraction_version TEXT;
+ALTER TABLE metadata_claims ADD COLUMN date_precision TEXT CHECK(date_precision IN ('date','datetime','invalid'));
+ALTER TABLE revisions ADD COLUMN metadata_warnings_json TEXT NOT NULL DEFAULT '[]';
+`;

package/dist/migrations/004.d.ts

@@ -0,0 +1,3 @@
+/** Additive library-level work; capture jobs retain their existing item foreign key. */
+export declare const version = 4;
+export declare const sql = "\nALTER TABLE vectors ADD COLUMN input_hash TEXT CHECK(input_hash IS NULL OR length(input_hash)=64);\nCREATE TABLE index_jobs(id TEXT PRIMARY KEY,generation_id TEXT NOT NULL UNIQUE REFERENCES index_generations(id),state TEXT NOT NULL CHECK(state IN ('queued','running','failed','cancelled','succeeded','cleanup-pending')),cancel_requested INTEGER NOT NULL DEFAULT 0 CHECK(cancel_requested IN (0,1)),attempt_count INTEGER NOT NULL DEFAULT 0,reason TEXT,created_at TEXT NOT NULL) STRICT;\nCREATE INDEX index_jobs_ready ON index_jobs(state,created_at);\nCREATE TABLE index_attempts(id TEXT PRIMARY KEY,job_id TEXT NOT NULL REFERENCES index_jobs(id),owner_incarnation TEXT NOT NULL,fence INTEGER NOT NULL,stage TEXT NOT NULL,cleanup TEXT NOT NULL CHECK(cleanup IN ('pending','settled')),created_at TEXT NOT NULL,finished_at TEXT) STRICT;\nCREATE INDEX index_attempts_job ON index_attempts(job_id);\n";

package/dist/migrations/004.js

@@ -0,0 +1,9 @@
+/** Additive library-level work; capture jobs retain their existing item foreign key. */
+export const version = 4;
+export const sql = `
+ALTER TABLE vectors ADD COLUMN input_hash TEXT CHECK(input_hash IS NULL OR length(input_hash)=64);
+CREATE TABLE index_jobs(id TEXT PRIMARY KEY,generation_id TEXT NOT NULL UNIQUE REFERENCES index_generations(id),state TEXT NOT NULL CHECK(state IN ('queued','running','failed','cancelled','succeeded','cleanup-pending')),cancel_requested INTEGER NOT NULL DEFAULT 0 CHECK(cancel_requested IN (0,1)),attempt_count INTEGER NOT NULL DEFAULT 0,reason TEXT,created_at TEXT NOT NULL) STRICT;
+CREATE INDEX index_jobs_ready ON index_jobs(state,created_at);
+CREATE TABLE index_attempts(id TEXT PRIMARY KEY,job_id TEXT NOT NULL REFERENCES index_jobs(id),owner_incarnation TEXT NOT NULL,fence INTEGER NOT NULL,stage TEXT NOT NULL,cleanup TEXT NOT NULL CHECK(cleanup IN ('pending','settled')),created_at TEXT NOT NULL,finished_at TEXT) STRICT;
+CREATE INDEX index_attempts_job ON index_attempts(job_id);
+`;

package/dist/src/acquisition/capture.d.ts

@@ -0,0 +1,29 @@
+import type { CaptureResult, ErrorEnvelope } from '../../contracts/index.js';
+import { runOwned } from './process.js';
+import { type CaptureRuntime } from './runtime.js';
+export type CaptureConnection = {
+    kind: 'account';
+    stateHome: string;
+    accountId: string;
+} | {
+    kind: 'cdp';
+    endpoint: string;
+};
+export interface CaptureRequest {
+    runtime: CaptureRuntime;
+    stagingRoot: string;
+    sourceUrl: string;
+    requestId: string;
+    connection?: CaptureConnection;
+    signal?: AbortSignal;
+    deadlineMs?: number;
+    allowPrivateNetwork?: boolean;
+    runProcess?: typeof runOwned;
+}
+export type CaptureOutcome = {
+    ok: true;
+    capture: CaptureResult;
+    bundleRoot: string;
+    dispose: () => Promise<void>;
+} | ErrorEnvelope;
+export declare function captureArticle(request: CaptureRequest): Promise<CaptureOutcome>;

package/dist/src/acquisition/capture.js

@@ -0,0 +1,160 @@
+import { mkdir, mkdtemp, writeFile, statfs, realpath } from 'node:fs/promises';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { limits } from '../../contracts/index.js';
+import { privateRoot, physicalBytes } from './files.js';
+import { runOwned, isolatedEnvironment } from './process.js';
+import { inspectCaptureRuntime, CaptureSetupError } from './runtime.js';
+import { settleCaptureRoot, CaptureCleanupError } from './cleanup.js';
+import { failure, upstreamFailure } from './errors.js';
+import { normalizeBundle, object } from './normalize.js';
+export async function captureArticle(request) {
+    const { requestId, signal } = request;
+    if (!/^[a-zA-Z0-9_-]{1,128}$/.test(requestId))
+        return failure('invalid-request', 'invalid-input');
+    let url;
+    try {
+        url = new URL(request.sourceUrl);
+        if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password || request.sourceUrl.length > 16384)
+            throw Error();
+    }
+    catch {
+        return failure(requestId, 'invalid-input');
+    }
+    if (signal?.aborted)
+        return failure(requestId, 'cancelled');
+    let runtime;
+    try {
+        runtime = await inspectCaptureRuntime(request.runtime.root, request.runtime.pins);
+    }
+    catch (error) {
+        return failure(requestId, error instanceof CaptureSetupError ? error.code : 'runtime-missing', 'setup');
+    }
+    let base;
+    try {
+        base = await privateRoot(request.stagingRoot);
+        const disk = await statfs(base);
+        if (disk.bavail * disk.bsize < limits.stagingBytes + limits.freeReserveBytes)
+            return failure(requestId, 'limit-exceeded');
+    }
+    catch {
+        return failure(requestId, 'invalid-input');
+    }
+    const deadline = request.deadlineMs ?? limits.captureDeadlineMs;
+    if (!Number.isSafeInteger(deadline) || deadline < 100 || deadline > limits.captureDeadlineMs)
+        return failure(requestId, 'invalid-input');
+    const args = [runtime.ghostget, 'clip', request.sourceUrl, 'article', '--media', 'images', '--timeout-ms', String(deadline), '--max-html-bytes', String(limits.markdownBytes), '--max-asset-bytes', String(limits.imageBytes), '--max-total-asset-bytes', String(500 * 1024 ** 2), '--quiet', '--json'];
+    if (request.allowPrivateNetwork)
+        args.push('--allow-private-network');
+    let selectedState;
+    if (request.connection?.kind === 'account') {
+        if (!/^[a-zA-Z0-9_.-]{1,128}$/.test(request.connection.accountId))
+            return failure(requestId, 'invalid-input');
+        try {
+            selectedState = await privateRoot(request.connection.stateHome);
+        }
+        catch {
+            return failure(requestId, 'repair-auth');
+        }
+        args.push('--auth', request.connection.accountId, '--mode', 'http');
+    }
+    else if (request.connection?.kind === 'cdp') {
+        if (!/^(?:[1-9][0-9]{0,4}|(?:http|ws):\/\/127\.0\.0\.1:[1-9][0-9]{0,4}(?:\/[^\s]*)?)$/.test(request.connection.endpoint))
+            return failure(requestId, 'invalid-input');
+        args.push('--cdp', request.connection.endpoint);
+    }
+    else
+        args.push('--mode', 'http');
+    const root = await realpath(await mkdtemp(join(base, 'capture-'))), home = join(root, 'home'), output = join(root, 'bundles');
+    args.push('--output', output);
+    const env = isolatedEnvironment(home, home, selectedState ?? join(home, 'state'));
+    const controller = new AbortController();
+    const abort = () => controller.abort();
+    signal?.addEventListener('abort', abort, { once: true });
+    if (signal?.aborted)
+        abort();
+    let cleanup = 'settled', keep = false, quota = false, checking = false;
+    const monitor = setInterval(async () => { if (checking)
+        return; checking = true; try {
+        if (await physicalBytes(root) > limits.stagingBytes) {
+            quota = true;
+            controller.abort();
+        }
+    }
+    catch {
+        quota = true;
+        controller.abort();
+    }
+    finally {
+        checking = false;
+    } }, 100);
+    const started = Date.now();
+    try {
+        await mkdir(home, { mode: 0o700 });
+        await mkdir(output, { mode: 0o700 });
+        await writeFile(join(root, 'ownership.json'), JSON.stringify({ schemaVersion: 1, requestId, state: 'acquiring' }), { mode: 0o600 });
+        cleanup = 'pending';
+        const result = await (request.runProcess ?? runOwned)(runtime.bun, args, { cwd: root, env, signal: controller.signal, deadlineMs: deadline, maxOutputBytes: limits.metadataBytes, graceMs: 5000 });
+        cleanup = result.cleanup;
+        // Ghostget reports escaped browser custody separately from generic descendants.
+        let summary;
+        try {
+            summary = JSON.parse(result.stdout);
+        }
+        catch {
+            summary = null;
+        }
+        const upstream = upstreamFailure(summary, requestId);
+        if (upstream.code === 'cleanup-required')
+            cleanup = 'pending';
+        if (cleanup === 'settled' && request.connection?.kind === 'cdp') {
+            cleanup = 'pending';
+            const census = await (request.runProcess ?? runOwned)('/bin/ps', ['-axo', 'command'], { env, deadlineMs: 3000, maxOutputBytes: 4 * 1024 ** 2 });
+            cleanup = census.cleanup;
+            if (census.code !== 0 || census.stopped || census.stdout.split('\n').some(line => line.includes(root)))
+                cleanup = 'pending';
+        }
+        if (cleanup !== 'settled')
+            return failure(requestId, 'cleanup-required', 'cleanup');
+        if (quota)
+            return failure(requestId, 'limit-exceeded');
+        if (result.stopped)
+            return failure(requestId, result.stopped);
+        if (result.code !== 0 || object(summary).ok !== true)
+            return upstream;
+        const outputDirectory = object(summary).outputDirectory;
+        if (typeof outputDirectory !== 'string' || !outputDirectory.startsWith(output + '/') || await realpath(outputDirectory) !== outputDirectory)
+            return failure(requestId, 'integrity', 'verification');
+        const helper = fileURLToPath(new URL(import.meta.url.endsWith('.ts') ? './verify-helper.ts' : './verify-helper.js', import.meta.url));
+        cleanup = 'pending';
+        const checked = await (request.runProcess ?? runOwned)(runtime.bun, [helper, runtime.root, outputDirectory], { cwd: root, env, signal: controller.signal, deadlineMs: Math.max(1, deadline - (Date.now() - started)), maxOutputBytes: limits.helperFrameBytes });
+        cleanup = checked.cleanup;
+        if (cleanup !== 'settled')
+            return failure(requestId, 'cleanup-required', 'cleanup');
+        if (checked.stopped)
+            return failure(requestId, checked.stopped);
+        if (checked.code !== 0)
+            return failure(requestId, 'integrity', 'verification');
+        const capture = await normalizeBundle({ bundleRoot: outputDirectory, inspection: JSON.parse(checked.stdout), summary, requestId, sourceUrl: request.sourceUrl, wordcellVersion: runtime.pins.wordcell.version });
+        if (controller.signal.aborted)
+            return failure(requestId, quota ? 'limit-exceeded' : 'cancelled');
+        await writeFile(join(root, 'ownership.json'), JSON.stringify({ schemaVersion: 1, requestId, state: 'verified' }), { mode: 0o600 });
+        keep = true;
+        return { ok: true, capture, bundleRoot: outputDirectory, dispose: async () => { if (await settleCaptureRoot(root, requestId) === 'pending')
+                throw new CaptureCleanupError(); } };
+    }
+    catch (error) {
+        if (cleanup !== 'settled')
+            return failure(requestId, 'cleanup-required', 'cleanup');
+        const code = (error instanceof Error && ['integrity', 'limit-exceeded', 'schema-newer'].includes(error.message) ? error.message : 'unclassified');
+        return failure(requestId, code, 'verification');
+    }
+    finally {
+        clearInterval(monitor);
+        while (checking)
+            await new Promise(r => setTimeout(r, 5));
+        signal?.removeEventListener('abort', abort);
+        if (!keep && cleanup === 'settled' && await settleCaptureRoot(root, requestId) === 'pending')
+            return failure(requestId, 'cleanup-required', 'cleanup');
+    }
+}

package/dist/src/acquisition/cleanup.d.ts

@@ -0,0 +1,6 @@
+export declare class CaptureCleanupError extends Error {
+    readonly code = "cleanup-required";
+    constructor();
+}
+/** Cleanup failure retains ownership evidence and must not settle the queue attempt. */
+export declare function settleCaptureRoot(root: string, requestId: string): Promise<'settled' | 'pending'>;

package/dist/src/acquisition/cleanup.js

@@ -0,0 +1,20 @@
+import { rm, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+export class CaptureCleanupError extends Error {
+    code = 'cleanup-required';
+    constructor() { super('cleanup-required'); }
+}
+/** Cleanup failure retains ownership evidence and must not settle the queue attempt. */
+export async function settleCaptureRoot(root, requestId) {
+    try {
+        await rm(root, { recursive: true, force: true });
+        return 'settled';
+    }
+    catch {
+        try {
+            await writeFile(join(root, 'ownership.json'), JSON.stringify({ schemaVersion: 1, requestId, state: 'cleanup-required' }), { mode: 0o600 });
+        }
+        catch { /* Existing ownership record remains when directory access itself failed. */ }
+        return 'pending';
+    }
+}

package/dist/src/acquisition/errors.d.ts

@@ -0,0 +1,3 @@
+import type { ErrorCode, ErrorEnvelope } from '../../contracts/index.js';
+export declare const failure: (requestId: string, code: ErrorCode, phase?: ErrorEnvelope["phase"], retryable?: boolean) => ErrorEnvelope;
+export declare function upstreamFailure(value: unknown, requestId: string): ErrorEnvelope;

package/dist/src/acquisition/errors.js

@@ -0,0 +1,20 @@
+const mappings = { 'unknown': 'unclassified', 'permission-required': 'grant-permission', 'auth-required': 'repair-auth', 'invalid-input': 'invalid-input', 'cancelled': 'cancelled', 'timeout': 'unavailable', 'transient': 'unavailable', 'unsupported': 'unsupported', 'cleanup-pending': 'cleanup-required', 'rate-limited': 'rate-limited' };
+export const failure = (requestId, code, phase = 'acquisition', retryable = false) => ({ schemaVersion: 1, requestId, ok: false, code, phase, retryable });
+export function upstreamFailure(value, requestId) {
+    const fallback = failure(requestId, 'unclassified');
+    if (!value || typeof value !== 'object' || Array.isArray(value))
+        return fallback;
+    const outer = value;
+    if (outer.ok !== false)
+        return fallback;
+    const raw = outer.captureError;
+    if (!raw || typeof raw !== 'object' || Array.isArray(raw))
+        return fallback;
+    const typed = raw;
+    if (Object.keys(typed).some(k => !['schemaVersion', 'code', 'retryable'].includes(k)) || typed.schemaVersion !== 1 || typeof typed.code !== 'string' || !Object.hasOwn(mappings, typed.code) || typeof typed.retryable !== 'boolean')
+        return fallback;
+    const code = mappings[typed.code];
+    if (typed.retryable && !['timeout', 'transient', 'rate-limited'].includes(typed.code))
+        return fallback;
+    return failure(requestId, code, code === 'cleanup-required' ? 'cleanup' : 'acquisition', typed.retryable);
+}

package/dist/src/acquisition/files.d.ts

@@ -0,0 +1,4 @@
+export declare const hash: (bytes: Uint8Array | string) => string;
+export declare function privateRoot(path: string): Promise<string>;
+export declare function readConfined(root: string, name: string, maxBytes: number): Promise<Buffer>;
+export declare function physicalBytes(root: string): Promise<number>;

package/dist/src/acquisition/files.js

@@ -0,0 +1,49 @@
+import { constants } from 'node:fs';
+import { open, lstat, realpath, readdir } from 'node:fs/promises';
+import { resolve, relative, isAbsolute, join } from 'node:path';
+import { createHash } from 'node:crypto';
+export const hash = (bytes) => createHash('sha256').update(bytes).digest('hex');
+export async function privateRoot(path) {
+    if (!isAbsolute(path))
+        throw Error('invalid-input');
+    const canonical = await realpath(path), stat = await lstat(path);
+    if (canonical !== path || !stat.isDirectory() || stat.isSymbolicLink() || stat.uid !== process.getuid?.() || (stat.mode & 0o077) !== 0)
+        throw Error('invalid-input');
+    return canonical;
+}
+export async function readConfined(root, name, maxBytes) {
+    if (!name || isAbsolute(name) || name.includes('\\') || name.split('/').some(p => !p || p === '.' || p === '..'))
+        throw Error('integrity');
+    const path = resolve(root, name);
+    if (relative(root, path).startsWith('..') || await realpath(path) !== path)
+        throw Error('integrity');
+    const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+    try {
+        const stat = await file.stat();
+        if (!stat.isFile() || stat.size > maxBytes)
+            throw Error('limit-exceeded');
+        const data = Buffer.alloc(stat.size + 1);
+        const { bytesRead } = await file.read(data, 0, data.length, 0);
+        if (bytesRead !== stat.size)
+            throw Error('integrity');
+        return data.subarray(0, bytesRead);
+    }
+    finally {
+        await file.close();
+    }
+}
+export async function physicalBytes(root) {
+    let total = 0;
+    for (const entry of await readdir(root)) {
+        const path = join(root, entry);
+        try {
+            const stat = await lstat(path);
+            total += stat.isDirectory() ? await physicalBytes(path) : stat.size;
+        }
+        catch (e) {
+            if (e.code !== 'ENOENT')
+                throw e;
+        }
+    }
+    return total;
+}

package/dist/src/acquisition/index.d.ts

@@ -0,0 +1,6 @@
+export { captureArticle } from './capture.js';
+export type { CaptureConnection, CaptureRequest, CaptureOutcome } from './capture.js';
+export { provisionCapture, inspectCaptureRuntime, captureSetupPlan, CaptureSetupError } from './runtime.js';
+export type { CaptureRuntime, CapturePins, SetupPlan } from './runtime.js';
+export { defaultCapturePins } from './pins.js';
+export { CaptureCleanupError } from './cleanup.js';

package/dist/src/acquisition/index.js

@@ -0,0 +1,4 @@
+export { captureArticle } from './capture.js';
+export { provisionCapture, inspectCaptureRuntime, captureSetupPlan, CaptureSetupError } from './runtime.js';
+export { defaultCapturePins } from './pins.js';
+export { CaptureCleanupError } from './cleanup.js';