@hraness/sponge 0.1.2 → 0.2.0

package/dist/src/indexing/index.js

@@ -0,0 +1,123 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { spaceFingerprint, encodeEmbeddingRequest } from '../inference/embedding/index.js';
+export class IndexError extends Error {
+    code;
+    constructor(code) {
+        super(code);
+        this.code = code;
+    }
+}
+const fail = (code = 'invalid-index') => { throw new IndexError(code); };
+const hash = (value) => createHash('sha256').update(value).digest('hex');
+function atomic(db, fn) { if (db.isTransaction)
+    return fn(); db.exec('BEGIN IMMEDIATE'); try {
+    const value = fn();
+    db.exec('COMMIT');
+    return value;
+}
+catch (e) {
+    db.exec('ROLLBACK');
+    throw e;
+} }
+function epoch(db) { return Number(db.prepare('SELECT revision_epoch FROM library_info WHERE id=1').get()?.revision_epoch); }
+function revisions(db) { return db.prepare('SELECT id AS itemId,current_revision AS revisionId FROM items WHERE deleted_at IS NULL AND current_revision IS NOT NULL ORDER BY id').all(); }
+/** Unicode-safe chunks leave room for the fixed document prompt within 8192 UTF-8 bytes. */
+export function chunkDocument(text) { const chunks = []; let part = '', size = 0; for (const char of text) {
+    const bytes = Buffer.byteLength(char);
+    if (size + bytes > 7600) {
+        if (part.trim())
+            chunks.push(part);
+        part = '';
+        size = 0;
+    }
+    part += char;
+    size += bytes;
+} if (part.trim())
+    chunks.push(part); return chunks; }
+function snapshot(db) {
+    const current = revisions(db), inputs = [];
+    for (const r of current) {
+        const docs = db.prepare('SELECT id,markdown FROM representations WHERE revision_id=? AND markdown IS NOT NULL ORDER BY id').all(r.revisionId);
+        for (const doc of docs) {
+            const chunks = chunkDocument(String(doc.markdown));
+            for (let ordinal = 0; ordinal < chunks.length; ordinal++) {
+                const text = chunks[ordinal], derivationHash = hash(JSON.stringify(['sponge-chunk-v1', String(doc.id), ordinal, text]));
+                inputs.push({ ...r, sourceId: JSON.stringify([r.revisionId, 'document', String(doc.id), ordinal]), kind: 'document', representationId: String(doc.id), ordinal, derivationHash });
+            }
+        }
+        const images = db.prepare("SELECT DISTINCT a.sha256 FROM revision_assets ra JOIN assets a ON a.sha256=ra.sha256 WHERE ra.revision_id=? AND a.media_type LIKE 'image/%' ORDER BY a.sha256").all(r.revisionId);
+        for (const image of images) {
+            const assetSha256 = String(image.sha256);
+            inputs.push({ ...r, sourceId: JSON.stringify([r.revisionId, 'image', assetSha256]), kind: 'image', assetSha256, derivationHash: hash(JSON.stringify(['sponge-image-source-v1', assetSha256])) });
+        }
+    }
+    return { version: 1, revisions: current, inputs };
+}
+function generation(db, id) { const row = db.prepare('SELECT g.*,s.dimensions FROM index_generations g JOIN embedding_spaces s ON s.fingerprint=g.fingerprint WHERE g.id=?').get(id); if (!row)
+    return fail(); const coverage = JSON.parse(String(row.coverage_json)); const profile = JSON.parse(String(row.profile_json)); if (coverage.version !== 1 || !Array.isArray(coverage.inputs) || !Array.isArray(coverage.revisions) || typeof profile.profileId !== 'string')
+    fail(); return { row, coverage, profile }; }
+/** Identity must already be the verified embedder fingerprint. Existing identities are immutable. */
+export function createGeneration(db, options) {
+    if (!options.profileId || options.profileId.length > 128 || !/^[a-f0-9]{64}$/.test(options.fingerprint) || !Number.isSafeInteger(options.dimensions) || options.dimensions < 1 || options.dimensions > 4096)
+        fail();
+    if (spaceFingerprint(options.identity) !== options.fingerprint || options.identity.dimensions !== options.dimensions)
+        fail();
+    return atomic(db, () => { const identity = JSON.stringify(Object.fromEntries(Object.keys(options.identity).sort().map(k => [k, options.identity[k]]))); if (!identity || Buffer.byteLength(identity) > 16384)
+        fail(); const existing = db.prepare('SELECT * FROM embedding_spaces WHERE fingerprint=?').get(options.fingerprint); if (existing && (existing.identity_json !== identity || Number(existing.dimensions) !== options.dimensions || existing.normalization !== 'unit-l2' || existing.metric !== 'cosine'))
+        fail(); if (!existing)
+        db.prepare('INSERT INTO embedding_spaces VALUES(?,?,?,?,?)').run(options.fingerprint, identity, options.dimensions, 'unit-l2', 'cosine'); const id = randomUUID(); db.prepare('INSERT INTO index_generations VALUES(?,?,?,?,?,?)').run(id, options.fingerprint, JSON.stringify({ profileId: options.profileId }), 'building', epoch(db), JSON.stringify(snapshot(db))); return generationStatus(db, id); });
+}
+export function generationStatus(db, id) { const { row, coverage, profile } = generation(db, id); return { id, fingerprint: String(row.fingerprint), state: String(row.state), revisionEpoch: Number(row.revision_epoch), expected: coverage.inputs.length, completed: Number(db.prepare('SELECT count(*) AS n FROM vectors WHERE generation_id=?').get(id)?.n), profileId: profile.profileId }; }
+export function pendingInputs(db, id, limit = 16) { if (!Number.isSafeInteger(limit) || limit < 1 || limit > 256)
+    fail(); const { row, coverage } = generation(db, id); if (row.state === 'cancelled')
+    fail('cancelled-index'); if (row.state !== 'building')
+    return []; const done = new Set(db.prepare('SELECT source_id FROM vectors WHERE generation_id=? AND input_hash IS NOT NULL').all(id).map(r => String(r.source_id))); return coverage.inputs.filter(i => !done.has(i.sourceId)).slice(0, limit).map(input => { if (input.kind === 'image')
+    return input; const doc = db.prepare('SELECT markdown FROM representations WHERE id=? AND revision_id=?').get(input.representationId, input.revisionId); if (!doc || typeof doc.markdown !== 'string')
+    return fail(); const text = chunkDocument(doc.markdown)[input.ordinal]; if (text === undefined || hash(JSON.stringify(['sponge-chunk-v1', input.representationId, input.ordinal, text])) !== input.derivationHash)
+    return fail(); return { ...input, text }; }); }
+export function writeBatch(db, id, fingerprint, batch) {
+    if (!batch.length || batch.length > 256)
+        fail();
+    atomic(db, () => { const { row, coverage } = generation(db, id); if (row.state === 'cancelled')
+        fail('cancelled-index'); if (row.state !== 'building' || row.fingerprint !== fingerprint)
+        fail(); const expected = new Map(coverage.inputs.map(i => [i.sourceId, i])); const seen = new Set(); for (const result of batch) {
+        const input = expected.get(result.sourceId);
+        if (!input)
+            return fail();
+        if (seen.has(result.sourceId) || input.revisionId !== result.revisionId || input.derivationHash !== result.derivationHash || !/^[a-f0-9]{64}$/.test(result.inputHash) || result.vector.length !== Number(row.dimensions) || result.vector.some(n => !Number.isFinite(n)) || Math.abs(Math.hypot(...result.vector) - 1) > .001)
+            fail();
+        if (input.kind === 'document') {
+            const doc = db.prepare('SELECT markdown FROM representations WHERE id=? AND revision_id=?').get(input.representationId, input.revisionId);
+            if (!doc || typeof doc.markdown !== 'string')
+                return fail();
+            const text = chunkDocument(String(doc.markdown))[input.ordinal];
+            if (text === undefined || encodeEmbeddingRequest([{ kind: 'document', text }]).inputHashes[0] !== result.inputHash)
+                fail();
+        }
+        seen.add(result.sourceId);
+        const bytes = Buffer.alloc(result.vector.length * 4);
+        result.vector.forEach((n, i) => bytes.writeFloatLE(n, i * 4));
+        const prior = db.prepare('SELECT vector,input_hash FROM vectors WHERE generation_id=? AND source_id=?').get(id, input.sourceId);
+        if (prior && prior.input_hash !== null) {
+            if (prior.input_hash !== result.inputHash || !bytes.equals(Buffer.from(prior.vector)))
+                fail();
+            continue;
+        }
+        db.prepare('INSERT INTO vectors(generation_id,source_id,revision_id,asset_sha256,derivation_hash,vector,input_hash) VALUES(?,?,?,?,?,?,?) ON CONFLICT(generation_id,source_id) DO UPDATE SET vector=excluded.vector,input_hash=excluded.input_hash').run(id, input.sourceId, input.revisionId, input.assetSha256 ?? null, input.derivationHash, bytes, result.inputHash);
+    } });
+}
+export function activateGeneration(db, id) {
+    return atomic(db, () => { const { row, coverage, profile } = generation(db, id); if (!['building', 'active', 'retired'].includes(String(row.state)))
+        fail('cancelled-index'); if (Number(row.revision_epoch) !== epoch(db) || JSON.stringify(coverage.revisions) !== JSON.stringify(revisions(db)))
+        fail('stale-index'); const actual = db.prepare('SELECT source_id,revision_id,derivation_hash,input_hash FROM vectors WHERE generation_id=?').all(id); const expected = new Map(coverage.inputs.map(i => [i.sourceId, i])); if (actual.length !== expected.size || actual.some(v => { const input = expected.get(String(v.source_id)); return !input || input.revisionId !== v.revision_id || input.derivationHash !== v.derivation_hash || typeof v.input_hash !== 'string' || !/^[a-f0-9]{64}$/.test(v.input_hash); }))
+        fail('incomplete-index'); const prior = db.prepare('SELECT generation_id FROM active_indexes WHERE profile_id=?').get(profile.profileId); if (prior && prior.generation_id !== id)
+        db.prepare("UPDATE index_generations SET state='retired' WHERE id=?").run(prior.generation_id); db.prepare('INSERT INTO active_indexes VALUES(?,?) ON CONFLICT(profile_id) DO UPDATE SET generation_id=excluded.generation_id').run(profile.profileId, id); db.prepare("UPDATE index_generations SET state='active' WHERE id=?").run(id); return generationStatus(db, id); });
+}
+export function cancelGeneration(db, id) { atomic(db, () => { const { row } = generation(db, id); if (row.state === 'active' || row.state === 'retired')
+    fail(); db.prepare("UPDATE index_generations SET state='cancelled' WHERE id=?").run(id); }); }
+/** Read-side join removes deleted/refreshed revisions even when an old generation stays active. */
+export function activeVectors(db, profileId) { const active = db.prepare('SELECT generation_id FROM active_indexes WHERE profile_id=?').get(profileId); if (!active)
+    return []; const id = String(active.generation_id), { row, coverage } = generation(db, id), inputs = new Map(coverage.inputs.map(i => [i.sourceId, i])); return db.prepare('SELECT v.* FROM vectors v JOIN revisions r ON r.id=v.revision_id JOIN items i ON i.id=r.item_id WHERE v.generation_id=? AND i.deleted_at IS NULL AND i.current_revision=v.revision_id ORDER BY v.source_id').all(id).map(v => { const bytes = Buffer.from(v.vector), source = inputs.get(String(v.source_id)); if (!source || bytes.length !== Number(row.dimensions) * 4)
+    fail(); const vector = Array.from({ length: Number(row.dimensions) }, (_, i) => bytes.readFloatLE(i * 4)); if (vector.some(n => !Number.isFinite(n)))
+    fail(); return { fingerprint: String(row.fingerprint), generationId: id, source: source, vector }; }); }
+export const rollbackGeneration = activateGeneration;

package/dist/src/indexing/jobs.d.ts

@@ -0,0 +1,14 @@
+import type { DatabaseSync } from 'node:sqlite';
+import { type OwnerToken, type AttemptToken, type OwnedAttempt } from '../store/types.js';
+export declare function submitIndexJob(db: DatabaseSync, generationId: string): string;
+export declare function listIndexJobs(db: DatabaseSync): Record<string, import("node:sqlite").SQLOutputValue>[];
+export declare function indexJob(db: DatabaseSync, id: string): Record<string, import("node:sqlite").SQLOutputValue> | undefined;
+export declare function claimIndexJob(db: DatabaseSync, token: OwnerToken, id?: string): AttemptToken | null;
+export declare function withIndexAttempt<T>(db: DatabaseSync, token: AttemptToken, fn: (generationId: string) => T): T;
+export declare function checkpointIndexAttempt(db: DatabaseSync, token: AttemptToken, stage: string): void;
+export declare function finishIndexAttempt(db: DatabaseSync, token: AttemptToken, reason?: string): void;
+export declare function settleIndexAttempt(db: DatabaseSync, token: AttemptToken): void;
+export declare function cancelIndexJob(db: DatabaseSync, id: string): void;
+export declare function retryIndexJob(db: DatabaseSync, id: string): void;
+export declare function indexOwnerAttempts(db: DatabaseSync, token: OwnerToken): OwnedAttempt[];
+export declare function recoverIndexAttempt(db: DatabaseSync, token: AttemptToken): void;

package/dist/src/indexing/jobs.js

@@ -0,0 +1,49 @@
+import { randomUUID } from 'node:crypto';
+import { StoreError } from '../store/types.js';
+const now = () => new Date().toISOString();
+function atomic(db, fn) { if (db.isTransaction)
+    return fn(); db.exec('BEGIN IMMEDIATE'); try {
+    const result = fn();
+    db.exec('COMMIT');
+    return result;
+}
+catch (error) {
+    db.exec('ROLLBACK');
+    throw error;
+} }
+function owner(db, token) { const row = db.prepare('SELECT * FROM worker_owner WHERE id=1').get(); if (row?.incarnation !== token.ownerIncarnation || row.fence !== token.fence)
+    throw new StoreError('owner-busy'); }
+function attempt(db, token, active = true) { owner(db, token); const row = db.prepare('SELECT a.*,j.state,j.cancel_requested,j.generation_id FROM index_attempts a JOIN index_jobs j ON j.id=a.job_id WHERE a.id=? AND a.job_id=?').get(token.attemptId, token.jobId); if (!row || row.owner_incarnation !== token.ownerIncarnation || row.fence !== token.fence || (active && (row.finished_at !== null || row.state !== 'running')))
+    throw new StoreError('owner-busy'); if (active && row.cancel_requested === 1)
+    throw new StoreError('cancelled'); return row; }
+export function submitIndexJob(db, generationId) { return atomic(db, () => { const generation = db.prepare('SELECT state FROM index_generations WHERE id=?').get(generationId); if (!generation || generation.state !== 'building')
+    throw new StoreError('unavailable'); const prior = db.prepare('SELECT id FROM index_jobs WHERE generation_id=?').get(generationId); if (prior)
+    return String(prior.id); const id = randomUUID(); db.prepare("INSERT INTO index_jobs(id,generation_id,state,created_at) VALUES(?,?,'queued',?)").run(id, generationId, now()); return id; }); }
+export function listIndexJobs(db) { return db.prepare("SELECT *, 'index' AS operation, NULL AS item_id FROM index_jobs ORDER BY created_at,id").all(); }
+export function indexJob(db, id) { return db.prepare('SELECT * FROM index_jobs WHERE id=?').get(id); }
+export function claimIndexJob(db, token, id) { return atomic(db, () => { owner(db, token); const job = id ? indexJob(db, id) : db.prepare("SELECT * FROM index_jobs WHERE state='queued' AND cancel_requested=0 ORDER BY created_at,id LIMIT 1").get(); if (!job) {
+    if (id)
+        throw new StoreError('unavailable');
+    return null;
+} if (job.state !== 'queued' || job.cancel_requested)
+    throw new StoreError('unavailable'); if (db.prepare("SELECT (SELECT count(*) FROM attempts WHERE cleanup='pending')+(SELECT count(*) FROM index_attempts WHERE cleanup='pending') AS n").get()?.n)
+    throw new StoreError('owner-busy'); const attemptId = randomUUID(); db.prepare("UPDATE index_jobs SET state='running',attempt_count=attempt_count+1 WHERE id=?").run(job.id); db.prepare("INSERT INTO index_attempts VALUES(?,?,?,?,'acquiring','pending',?,NULL)").run(attemptId, job.id, token.ownerIncarnation, token.fence, now()); return { ...token, jobId: String(job.id), attemptId }; }); }
+export function withIndexAttempt(db, token, fn) { return atomic(db, () => { const row = attempt(db, token); return fn(String(row.generation_id)); }); }
+export function checkpointIndexAttempt(db, token, stage) { const stages = ['acquiring', 'extracting', 'indexing', 'promoting', 'cleanup']; atomic(db, () => { const row = attempt(db, token); if (!stages.includes(stage) || stages.indexOf(stage) < stages.indexOf(String(row.stage)))
+    throw new StoreError('invalid-input'); db.prepare('UPDATE index_attempts SET stage=? WHERE id=?').run(stage, token.attemptId); }); }
+export function finishIndexAttempt(db, token, reason) { atomic(db, () => { const row = attempt(db, token, false); if (row.finished_at !== null || row.state !== 'running')
+    throw new StoreError('owner-busy'); if (reason !== undefined && !/^[a-z][a-z-]{0,63}$/.test(reason))
+    throw new StoreError('invalid-input'); if (!reason && row.cancel_requested)
+    throw new StoreError('cancelled'); db.prepare('UPDATE index_jobs SET state=?,reason=? WHERE id=?').run(row.cancel_requested ? 'cancelled' : reason ? 'failed' : 'succeeded', reason ?? null, token.jobId); db.prepare("UPDATE index_attempts SET stage='cleanup',finished_at=? WHERE id=?").run(now(), token.attemptId); }); }
+export function settleIndexAttempt(db, token) { atomic(db, () => { const row = attempt(db, token, false); if (row.finished_at === null)
+    throw new StoreError('owner-busy'); db.prepare("UPDATE index_attempts SET cleanup='settled',stage='settled' WHERE id=?").run(token.attemptId); }); }
+export function cancelIndexJob(db, id) { atomic(db, () => { const row = indexJob(db, id); if (!row)
+    throw new StoreError('unavailable'); if (row.state === 'succeeded')
+    return; db.prepare("UPDATE index_jobs SET cancel_requested=1,state=CASE WHEN state IN ('queued','failed','cancelled') THEN 'cancelled' ELSE state END WHERE id=?").run(id); }); }
+export function retryIndexJob(db, id) { atomic(db, () => { const row = indexJob(db, id); if (!row || !['failed', 'cancelled'].includes(String(row.state)))
+    throw new StoreError('unavailable'); if (db.prepare("SELECT count(*) AS n FROM index_attempts WHERE job_id=? AND cleanup='pending'").get(id)?.n)
+    throw new StoreError('owner-busy'); const generation = db.prepare('SELECT state,revision_epoch FROM index_generations WHERE id=?').get(row.generation_id); if (!generation || !['building', 'active'].includes(String(generation.state)) || generation.revision_epoch !== db.prepare('SELECT revision_epoch FROM library_info WHERE id=1').get()?.revision_epoch)
+    throw new StoreError('unavailable'); db.prepare("UPDATE index_jobs SET state='queued',cancel_requested=0,reason=NULL WHERE id=?").run(id); }); }
+export function indexOwnerAttempts(db, token) { owner(db, token); return db.prepare('SELECT a.*,j.state,j.cancel_requested FROM index_attempts a JOIN index_jobs j ON j.id=a.job_id WHERE a.owner_incarnation=? AND a.fence=? ORDER BY a.created_at,a.id').all(token.ownerIncarnation, token.fence).map(a => ({ ...token, kind: 'index', attemptId: String(a.id), jobId: String(a.job_id), stage: String(a.stage), cleanup: a.cleanup, finished: a.finished_at !== null, jobState: String(a.state), cancelRequested: a.cancel_requested === 1 })); }
+export function recoverIndexAttempt(db, token) { const row = attempt(db, token, false); db.prepare("UPDATE index_attempts SET cleanup='settled',stage='settled',finished_at=COALESCE(finished_at,?) WHERE id=?").run(now(), token.attemptId); if (row.finished_at === null)
+    db.prepare("UPDATE index_jobs SET state=CASE WHEN cancel_requested=1 THEN 'cancelled' ELSE 'failed' END,reason='unavailable' WHERE id=?").run(token.jobId); }

package/dist/src/indexing/runner.d.ts

@@ -0,0 +1,23 @@
+import type { LibraryStore, AttemptToken } from '../store/index.js';
+import { embedBatch, type EmbeddingRuntime } from '../inference/embedding/index.js';
+import { deriveEmbeddingImage } from './image.js';
+import type { ErrorEnvelope } from '../../contracts/index.js';
+export type IndexRunResult = ({
+    ok: true;
+    jobId: string;
+    generationId: string;
+} | ErrorEnvelope) & {
+    cleanup: 'settled' | 'pending';
+};
+export interface IndexRunOptions {
+    runtime: EmbeddingRuntime;
+    jobId?: string;
+    signal?: AbortSignal;
+    embed?: typeof embedBatch;
+    deriveImage?: typeof deriveEmbeddingImage;
+    onResult?: (result: IndexRunResult) => void;
+    onCheckpoint?: (attempt: AttemptToken, stage: string) => Promise<void>;
+    onBatch?: (attempt: AttemptToken, completed: number) => Promise<void>;
+}
+/** Same durable worker lock/owner/supervisor as acquisition, with sequential bounded batches. */
+export declare function drainIndexQueue(store: LibraryStore, options: IndexRunOptions): Promise<IndexRunResult[]>;

package/dist/src/indexing/runner.js

@@ -0,0 +1,114 @@
+import { join } from 'node:path';
+import { StoreError } from '../store/index.js';
+import { acquireWorkerCustody, proveAttemptCleanup } from '../worker/custody.js';
+import { supervisedRunner } from '../worker/supervisor.js';
+import { directory, writeJournal } from '../worker/journal.js';
+import { embedBatch, embeddingFingerprint, encodeEmbeddingRequest } from '../inference/embedding/index.js';
+import { deriveEmbeddingImage } from './image.js';
+import { failure } from '../acquisition/errors.js';
+/** Same durable worker lock/owner/supervisor as acquisition, with sequential bounded batches. */
+export async function drainIndexQueue(store, options) {
+    const custody = await acquireWorkerCustody(store.root);
+    let owner, settled = true;
+    const results = [];
+    try {
+        owner = store.acquireOwner(custody.incarnation);
+        while (!options.signal?.aborted) {
+            const attempt = store.claimIndexJob(owner, options.jobId);
+            if (!attempt)
+                break;
+            settled = false;
+            const attemptDirectory = await directory(join(custody.directory, 'attempts', attempt.attemptId)), scratchRoot = await directory(join(store.root, 'staging', attempt.attemptId));
+            await writeJournal(join(scratchRoot, 'ownership.json'), { schemaVersion: 1, requestId: attempt.attemptId, ownerIncarnation: attempt.ownerIncarnation });
+            const checkpoint = async (stage) => { store.checkpointIndexAttempt(attempt, stage); await writeJournal(join(attemptDirectory, 'checkpoint.json'), { schemaVersion: 1, ...attempt, stage }); await options.onCheckpoint?.(attempt, stage); };
+            const controller = new AbortController(), cancel = () => { store.requestCancellation(attempt.jobId); controller.abort(); };
+            options.signal?.addEventListener('abort', cancel, { once: true });
+            const poll = setInterval(() => { try {
+                if (store.jobCancellationRequested(attempt.jobId))
+                    controller.abort();
+            }
+            catch {
+                controller.abort();
+            } }, 100);
+            const runProcess = supervisedRunner(custody.directory, custody.incarnation, attempt.attemptId);
+            let result;
+            try {
+                if (options.signal?.aborted)
+                    cancel();
+                await checkpoint('acquiring');
+                const job = store.indexJob(attempt.jobId);
+                if (!job)
+                    throw new StoreError('unavailable');
+                const generationId = String(job.generation_id), status = store.indexStatus(generationId);
+                if (embeddingFingerprint(options.runtime.helperSha256) !== status.fingerprint)
+                    throw new StoreError('integrity');
+                await checkpoint('indexing');
+                for (;;) {
+                    if (controller.signal.aborted || store.jobCancellationRequested(attempt.jobId))
+                        throw new StoreError('cancelled');
+                    if (store.revisionEpoch !== status.revisionEpoch)
+                        throw Object.assign(Error('stale-index'), { code: 'stale-index' });
+                    const inputs = store.indexInputs(generationId, 16);
+                    if (!inputs.length)
+                        break;
+                    const batch = [];
+                    for (const input of inputs) {
+                        if (input.kind === 'document') {
+                            batch.push({ kind: 'document', text: input.text });
+                            continue;
+                        }
+                        const item = store.read(input.itemId), asset = item?.assets.find(a => a.sha256 === input.assetSha256);
+                        if (!item || item.revisionId !== input.revisionId || !asset)
+                            throw new StoreError('unavailable');
+                        const derived = await (options.deriveImage ?? deriveEmbeddingImage)(store.readBlob(asset), { scratchRoot, runProcess, signal: controller.signal });
+                        batch.push({ kind: 'image', png: derived.png });
+                    }
+                    const expected = encodeEmbeddingRequest(batch).inputHashes, embedded = await (options.embed ?? embedBatch)(batch, { runtime: options.runtime, scratchRoot, runProcess, signal: controller.signal });
+                    if (embedded.fingerprint !== status.fingerprint || embedded.inputHashes.length !== expected.length || embedded.inputHashes.some((hash, i) => hash !== expected[i]))
+                        throw new StoreError('integrity');
+                    store.writeIndexAttemptBatch(attempt, embedded.fingerprint, inputs.map((input, i) => ({ ...input, inputHash: expected[i], vector: embedded.vectors[i] })));
+                    await options.onBatch?.(attempt, store.indexStatus(generationId).completed);
+                }
+                await checkpoint('promoting');
+                store.completeIndexAttempt(attempt);
+                result = { ok: true, jobId: attempt.jobId, generationId, cleanup: 'settled' };
+            }
+            catch (error) {
+                const raw = error.code, reason = raw && /^[a-z][a-z-]{0,63}$/.test(raw) ? raw : 'unclassified';
+                store.failIndexAttempt(attempt, reason);
+                const known = ['integrity', 'unavailable', 'unsupported', 'cancelled', 'limit-exceeded', 'cleanup-required', 'runtime-missing', 'owner-busy'];
+                result = { ...failure(attempt.attemptId, known.includes(reason) ? reason : 'unavailable', 'inference'), cleanup: 'pending' };
+            }
+            finally {
+                clearInterval(poll);
+                options.signal?.removeEventListener('abort', cancel);
+            }
+            settled = !(result.ok === false && result.code === 'cleanup-required') && await proveAttemptCleanup(custody.directory, custody.incarnation, attempt.attemptId);
+            if (settled) {
+                store.settleIndexAttempt(attempt);
+                result.cleanup = 'settled';
+            }
+            else
+                result.cleanup = 'pending';
+            results.push(result);
+            options.onResult?.(result);
+            if (!settled || options.jobId)
+                break;
+        }
+        if (!settled)
+            throw Object.assign(Error('cleanup-required'), { code: 'cleanup-required' });
+        return results;
+    }
+    finally {
+        if (owner && settled) {
+            try {
+                store.releaseOwner(owner);
+            }
+            catch {
+                settled = false;
+            }
+        }
+        if (settled)
+            await custody.release();
+    }
+}

package/dist/src/inference/asr/bridge.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/src/inference/asr/bridge.js

@@ -0,0 +1,16 @@
+/** Private one-request transport. Outer ProcessRunner owns the whole inherited group. */
+import { readFile } from 'node:fs/promises';
+import { spawn } from 'node:child_process';
+try {
+    const config = JSON.parse(await readFile(process.argv[2], 'utf8'));
+    const child = spawn('/usr/bin/sandbox-exec', ['-p', '(version 1) (allow default) (deny network*)', config.helper, '--model', config.model, '--once'], { stdio: ['pipe', 'pipe', 'pipe'], detached: false });
+    child.stdout.pipe(process.stdout);
+    child.stderr.pipe(process.stderr);
+    child.stdin.on('error', () => { });
+    child.stdin.end(JSON.stringify(config.request) + '\n');
+    child.once('error', () => { process.exitCode = 1; });
+    child.once('close', code => { process.exitCode = code ?? 1; });
+}
+catch {
+    process.exitCode = 1;
+}

package/dist/src/inference/asr/protocol.d.ts

@@ -0,0 +1,23 @@
+import type { Transcript } from '../../../contracts/index.js';
+export declare const asrIdentity: {
+    readonly engine: "Cactus-Compute/needle3@93e88ede867e4311c39151285e74df513e4b835b";
+    readonly model: "Cactus-Compute/whistle@ca5287601bef25af26dcf2e1b2bdc0843a7c19e5";
+    readonly modelSha256: "b6e02f048568ac5d01a2042556c658061e699acbc0aa2a1439f52f3d461dffeb";
+    readonly modelBytes: 16919407;
+};
+export declare const asrLimits: {
+    readonly outputBytes: number;
+    readonly frameBytes: number;
+    readonly words: 30000;
+    readonly maximumDurationMs: 3600000;
+    readonly windowDurationMs: 60000;
+    readonly overlapMs: 4000;
+    readonly deadlineMs: 180000;
+};
+export interface SourceTranscript extends Transcript {
+    sourceSha256: string;
+    timing: 'source-captions' | 'decoded-audio-map';
+    durationMs: number;
+}
+/** Decode only correlated native events; diagnostics never become error classifications. */
+export declare function parseAsrOutput(stdout: string, requestId: string, sourceSha256: string, helperSha256: string): SourceTranscript;

package/dist/src/inference/asr/protocol.js

@@ -0,0 +1,60 @@
+import { createHash } from 'node:crypto';
+export const asrIdentity = { engine: 'Cactus-Compute/needle3@93e88ede867e4311c39151285e74df513e4b835b', model: 'Cactus-Compute/whistle@ca5287601bef25af26dcf2e1b2bdc0843a7c19e5', modelSha256: 'b6e02f048568ac5d01a2042556c658061e699acbc0aa2a1439f52f3d461dffeb', modelBytes: 16919407 };
+export const asrLimits = { outputBytes: 4 * 1024 ** 2, frameBytes: 4 * 1024 ** 2, words: 30000, maximumDurationMs: 3600000, windowDurationMs: 60000, overlapMs: 4000, deadlineMs: 180000 };
+const fail = () => { throw Error('integrity'); };
+const obj = (v) => v && typeof v === 'object' && !Array.isArray(v) ? v : fail();
+const finite = (v, min = 0, max = Number.MAX_SAFE_INTEGER) => typeof v === 'number' && Number.isFinite(v) && v >= min && v <= max ? v : fail();
+function time(v) { const t = obj(v); if (!Number.isSafeInteger(t.value) || !Number.isSafeInteger(t.timescale) || t.timescale <= 0)
+    fail(); return finite(t.value / t.timescale, 0, asrLimits.maximumDurationMs / 1000); }
+/** Decode only correlated native events; diagnostics never become error classifications. */
+export function parseAsrOutput(stdout, requestId, sourceSha256, helperSha256) {
+    if (Buffer.byteLength(stdout) > asrLimits.outputBytes || !/^[a-f0-9]{64}$/.test(sourceSha256) || !/^[a-f0-9]{64}$/.test(helperSha256))
+        fail();
+    let ready = false, result;
+    for (const line of stdout.split('\n')) {
+        if (!line.trim())
+            continue;
+        if (Buffer.byteLength(line) > asrLimits.frameBytes)
+            fail();
+        const event = obj(JSON.parse(line));
+        if (event.type === 'ready') {
+            if (ready || event.protocolVersion !== 2 || event.engine !== asrIdentity.engine || event.model !== asrIdentity.model)
+                fail();
+            ready = true;
+        }
+        else if (event.type === 'progress') {
+            if (!ready || result || event.id !== requestId || !['verifying', 'extracting', 'recognizing'].includes(event.stage))
+                fail();
+        }
+        else if (event.type === 'result') {
+            if (!ready || result || event.id !== requestId || event.sourceSha256 !== sourceSha256 || event.engine !== asrIdentity.engine || event.model !== asrIdentity.model)
+                fail();
+            result = event;
+        }
+        else if (event.type === 'error' || event.type === 'fatal')
+            throw Error('unclassified');
+        else
+            fail();
+    }
+    if (!ready || !result || result.hasAudio !== true)
+        return fail();
+    const recognition = obj(result.recognition), map = obj(result.audioMap);
+    if (recognition.language !== 'en' || typeof recognition.text !== 'string' || !Array.isArray(recognition.words) || recognition.words.length > asrLimits.words || map.sampleRate !== 16000 || !Number.isSafeInteger(map.sampleCount) || map.sampleCount < 0 || map.sampleCount > asrLimits.windowDurationMs * 16 || !Array.isArray(map.spans) || map.spans.length > 30000)
+        fail();
+    const durationMs = time(map.sourceDuration) * 1000;
+    if (durationMs <= 0)
+        fail();
+    let sampleEnd = 0, sourceEnd = 0;
+    const spans = map.spans.map((raw) => { const s = obj(raw), range = obj(s.source), start = time(range.start), end = time(range.end); if (!Number.isSafeInteger(s.sampleStart) || !Number.isSafeInteger(s.sampleEnd) || s.sampleStart !== sampleEnd || s.sampleEnd <= s.sampleStart || s.sampleEnd > map.sampleCount || start < sourceEnd || end <= start || end * 1000 > durationMs + .1 || !['audio', 'silence'].includes(s.kind))
+        fail(); sampleEnd = s.sampleEnd; sourceEnd = end; return { sampleStart: s.sampleStart, sampleEnd: s.sampleEnd, start, end }; });
+    if (sampleEnd !== map.sampleCount)
+        fail();
+    const sourceTime = (seconds, end) => { const sample = seconds * 16000; const span = spans.find(s => sample >= s.sampleStart && (end ? sample <= s.sampleEnd : sample < s.sampleEnd)) ?? (sample === map.sampleCount ? spans.at(-1) : undefined); if (!span)
+        return fail(); return (span.start + (sample - span.sampleStart) / (span.sampleEnd - span.sampleStart) * (span.end - span.start)) * 1000; };
+    let prior = 0;
+    const segments = recognition.words.map((raw) => { const w = obj(raw); if (typeof w.word !== 'string' || !w.word.trim() || Buffer.byteLength(w.word) > 16384)
+        fail(); const start = finite(w.start, prior, map.sampleCount / 16000), end = finite(w.end, start, map.sampleCount / 16000); prior = start; return { startMs: sourceTime(start, false), endMs: sourceTime(end, true), text: w.word.trim() }; });
+    if (!segments.length && recognition.text.trim())
+        fail();
+    return { schemaVersion: 1, requestId, source: 'local-asr', language: 'en', modelFingerprint: createHash('sha256').update(JSON.stringify(['sponge-asr-v1', asrIdentity, helperSha256, 'decoded-audio-map-window-v2', { windowDurationMs: asrLimits.windowDurationMs, overlapMs: asrLimits.overlapMs, join: 'source-midpoint-overlap-dedupe-v1' }])).digest('hex'), segments, cleanup: 'settled', sourceSha256, timing: 'decoded-audio-map', durationMs };
+}

package/dist/src/inference/asr/runtime.d.ts

@@ -0,0 +1,29 @@
+import type { ErrorEnvelope } from '../../../contracts/index.js';
+import type { ProcessRunner } from '../../worker/supervisor.js';
+import { type SourceTranscript } from './protocol.js';
+export interface AsrRuntime {
+    helper: string;
+    helperSha256: string;
+    model: string;
+}
+export interface AudioInput {
+    path: string;
+    sha256: string;
+    bytes: number;
+    durationMs: number;
+}
+export interface AsrOptions {
+    scratchRoot: string;
+    runtime?: AsrRuntime;
+    runProcess: ProcessRunner;
+    signal?: AbortSignal;
+}
+export type AsrOutcome = ({
+    ok: true;
+    transcript: SourceTranscript;
+} | ErrorEnvelope) & {
+    cleanup: 'settled' | 'pending';
+};
+/** Portable callers can use captions; this pinned native artifact requires macOS26 arm64. */
+export declare function supportsNativeAsr(platform?: NodeJS.Platform, arch?: NodeJS.Architecture, kernel?: string): boolean;
+export declare function transcribeAudio(requestId: string, audio: AudioInput, language: string | undefined, options: AsrOptions): Promise<AsrOutcome>;

package/dist/src/inference/asr/runtime.js

@@ -0,0 +1,103 @@
+import { constants } from 'node:fs';
+import { open, realpath, mkdtemp, writeFile, rm } from 'node:fs/promises';
+import { join, isAbsolute } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { release } from 'node:os';
+import { createHash } from 'node:crypto';
+import { failure } from '../../acquisition/errors.js';
+import { privateRoot } from '../../acquisition/files.js';
+import { asrIdentity, asrLimits, parseAsrOutput } from './protocol.js';
+import { audioWindows, joinWindows } from './windows.js';
+async function verify(path, sha256, maximum, expected) {
+    if (!isAbsolute(path) || !/^[a-f0-9]{64}$/.test(sha256) || await realpath(path) !== path)
+        throw Error('integrity');
+    const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+    try {
+        const stat = await file.stat();
+        if (!stat.isFile() || stat.size > maximum || expected !== undefined && stat.size !== expected)
+            throw Error('integrity');
+        const hash = createHash('sha256'), buffer = Buffer.alloc(65536);
+        let bytes = 0;
+        for (;;) {
+            const { bytesRead } = await file.read(buffer, 0, buffer.length, null);
+            if (!bytesRead)
+                break;
+            bytes += bytesRead;
+            if (bytes > maximum)
+                throw Error('limit-exceeded');
+            hash.update(buffer.subarray(0, bytesRead));
+        }
+        if (bytes !== stat.size || hash.digest('hex') !== sha256)
+            throw Error('integrity');
+    }
+    finally {
+        await file.close();
+    }
+}
+/** Portable callers can use captions; this pinned native artifact requires macOS26 arm64. */
+export function supportsNativeAsr(platform = process.platform, arch = process.arch, kernel = release()) { return platform === 'darwin' && arch === 'arm64' && /^\d+\./.test(kernel) && Number(kernel.split('.')[0]) >= 25; }
+export async function transcribeAudio(requestId, audio, language, options) {
+    const error = (code, cleanup = 'settled') => ({ ...failure(requestId, code, cleanup === 'pending' ? 'cleanup' : 'inference'), cleanup });
+    if (options.signal?.aborted)
+        return error('cancelled');
+    if (!supportsNativeAsr() || language !== 'en')
+        return error('unsupported');
+    if (!options.runtime)
+        return error('runtime-missing');
+    if (!/^[A-Za-z0-9_-]{1,128}$/.test(requestId) || !Number.isSafeInteger(audio.bytes) || audio.bytes < 1 || audio.bytes > 512 * 1024 ** 2 || !Number.isFinite(audio.durationMs) || audio.durationMs <= 0 || audio.durationMs > asrLimits.maximumDurationMs)
+        return error('limit-exceeded');
+    let scratch, cleanup = 'settled', outcome;
+    try {
+        const root = await privateRoot(options.scratchRoot), runtime = options.runtime;
+        await verify(audio.path, audio.sha256, 512 * 1024 ** 2, audio.bytes);
+        await verify(runtime.helper, runtime.helperSha256, 32 * 1024 ** 2);
+        await verify(runtime.model, asrIdentity.modelSha256, asrIdentity.modelBytes, asrIdentity.modelBytes);
+        if (options.signal?.aborted)
+            return error('cancelled');
+        scratch = await mkdtemp(join(root, 'sponge-asr-'));
+        const requestFile = join(scratch, 'request.json');
+        const source = import.meta.url.endsWith('.ts'), entry = fileURLToPath(new URL(source ? './bridge.ts' : './bridge.js', import.meta.url));
+        const windows = audioWindows(audio.durationMs), transcripts = [];
+        let totalWords = 0;
+        for (const window of windows) {
+            await writeFile(requestFile, JSON.stringify({ helper: runtime.helper, model: runtime.model, request: { id: requestId, sourcePath: audio.path, expectedSourceSha256: audio.sha256, windowStartMs: window.startMs, windowEndMs: window.endMs } }), { mode: 0o600, flag: transcripts.length ? 'w' : 'wx' });
+            cleanup = 'pending';
+            const result = await options.runProcess(process.execPath, source ? ['--import', import.meta.resolve('tsx'), entry, requestFile] : [entry, requestFile], { env: { PATH: '/usr/bin:/bin', HOME: scratch, TMPDIR: scratch, LANG: 'en_US.UTF-8' }, deadlineMs: asrLimits.deadlineMs, maxOutputBytes: asrLimits.outputBytes, signal: options.signal });
+            cleanup = result.cleanup;
+            if (cleanup === 'pending')
+                throw Error('cleanup-required');
+            if (options.signal?.aborted || result.stopped === 'cancelled')
+                throw Error('cancelled');
+            if (result.stopped)
+                throw Error(result.stopped);
+            if (result.code !== 0)
+                throw Error('unclassified');
+            const transcript = parseAsrOutput(result.stdout, requestId, audio.sha256, runtime.helperSha256);
+            if (Math.abs(transcript.durationMs - audio.durationMs) > 1000)
+                throw Error('integrity');
+            if (transcript.segments.some(s => s.startMs < window.startMs - .1 || s.endMs > window.endMs + .1))
+                throw Error('integrity');
+            totalWords += transcript.segments.length;
+            if (totalWords > asrLimits.words)
+                throw Error('limit-exceeded');
+            transcripts.push(transcript);
+        }
+        await verify(audio.path, audio.sha256, 512 * 1024 ** 2, audio.bytes);
+        outcome = { ok: true, transcript: joinWindows(windows, transcripts), cleanup };
+    }
+    catch (e) {
+        const code = e instanceof Error ? e.message : '';
+        outcome = error(cleanup === 'pending' ? 'cleanup-required' : code === 'integrity' ? 'integrity' : code === 'limit-exceeded' ? 'limit-exceeded' : code === 'cancelled' ? 'cancelled' : code === 'unavailable' ? 'unavailable' : 'unclassified', cleanup);
+    }
+    finally {
+        if (scratch && cleanup === 'settled') {
+            try {
+                await rm(scratch, { recursive: true, force: true });
+            }
+            catch {
+                cleanup = 'pending';
+            }
+        }
+    }
+    return cleanup === 'pending' ? error('cleanup-required', 'pending') : outcome;
+}

package/dist/src/inference/asr/windows.d.ts

@@ -0,0 +1,9 @@
+import { type SourceTranscript } from './protocol.js';
+export interface AudioWindow {
+    startMs: number;
+    endMs: number;
+}
+export declare function audioWindows(durationMs: number): AudioWindow[];
+/** Choose one owner on each overlap midpoint using source timestamps. Repeated
+ * words are removed only when their time ranges overlap, never globally. */
+export declare function joinWindows(windows: AudioWindow[], transcripts: SourceTranscript[]): SourceTranscript;

package/dist/src/inference/asr/windows.js

@@ -0,0 +1,41 @@
+import { asrLimits } from './protocol.js';
+export function audioWindows(durationMs) {
+    if (!Number.isFinite(durationMs) || durationMs <= 0 || durationMs > asrLimits.maximumDurationMs)
+        throw Error('limit-exceeded');
+    durationMs = Math.ceil(durationMs);
+    const result = [];
+    for (let startMs = 0; startMs < durationMs; startMs += asrLimits.windowDurationMs - asrLimits.overlapMs) {
+        const endMs = Math.min(durationMs, startMs + asrLimits.windowDurationMs);
+        result.push({ startMs, endMs });
+        if (endMs === durationMs)
+            break;
+    }
+    return result;
+}
+/** Choose one owner on each overlap midpoint using source timestamps. Repeated
+ * words are removed only when their time ranges overlap, never globally. */
+export function joinWindows(windows, transcripts) {
+    if (!transcripts.length || windows.length !== transcripts.length)
+        throw Error('integrity');
+    const first = transcripts[0], segments = [];
+    for (let i = 0; i < windows.length; i++) {
+        const window = windows[i], transcript = transcripts[i];
+        if (transcript.sourceSha256 !== first.sourceSha256 || transcript.durationMs !== first.durationMs || transcript.modelFingerprint !== first.modelFingerprint || transcript.requestId !== first.requestId)
+            throw Error('integrity');
+        const lower = i ? (windows[i - 1].endMs + window.startMs) / 2 : 0, upper = i + 1 < windows.length ? (window.endMs + windows[i + 1].startMs) / 2 : Infinity;
+        for (const segment of transcript.segments) {
+            const center = (segment.startMs + segment.endMs) / 2;
+            if (center < lower || center >= upper)
+                continue;
+            const prior = segments.at(-1);
+            if (prior && segment.startMs < prior.endMs && segment.text.toLowerCase() === prior.text.toLowerCase()) {
+                prior.endMs = Math.max(prior.endMs, segment.endMs);
+                continue;
+            }
+            if (prior && segment.startMs < prior.startMs)
+                throw Error('integrity');
+            segments.push({ ...segment });
+        }
+    }
+    return { ...first, segments };
+}