hippo-memory 1.69.0 → 1.70.0

package/dist/store/{entry-writes-group.js → sqlite/entry-writes-group.js}

@@ -1,54 +1,41 @@
 // hippo.db's half of the EntryWrites store group: the queries remember, outcome, supersede, archive and forget run today.
-import { ConflictError, NotFoundError } from '../api-errors.js';
+import { ConflictError, NotFoundError } from '../../core/api-errors.js';
 import { appendAuditEvent } from '../audit.js';
-import { isSqliteBusy, withWriteScope } from '../db.js';
-import { log } from '../log.js';
-import { entryAfterOutcome } from '../memory.js';
+import { isSqliteBusy, withWriteScope } from '../../db/index.js';
+import { errorMessage, log } from '../../util/log.js';
+import { entryAfterOutcome } from '../../core/memory.js';
 import { archiveRawMemory } from '../raw-archive.js';
 import { ownScopeTouches } from '../recall-scope.js';
+import { recordTraceOutcome } from '../recall-trace.js';
 import { RejectedValueError } from '../rejection.js';
 import { markSummaryDirtyInTx } from '../summary-dirty.js';
-import { auditRejectionRefusal } from './audit-event.js';
-import { deleteEntryCore } from './delete-and-batch.js';
-import { selectEntriesByIds } from './entry-reads.js';
-import { stampOriginProject } from './entry-row.js';
-import { writeEntryDbOnly, writeEntryMirrors } from './entry-writes.js';
-import { updateStatsUnlessBusy } from './index-and-stats.js';
-import { purgeMirrorBestEffort, removeEntryMirrors } from './mirrors.js';
-import { onHandle, openStore } from './open.js';
-import { selectMemoryReach } from './tenant-lookup.js';
+import { auditRejectionRefusal } from '../audit-event.js';
+import { deleteEntryCore } from '../delete-and-batch.js';
+import { selectEntriesByIds } from '../entry-reads.js';
+import { stampOriginProject } from '../entry-row.js';
+import { writeEntryDbOnly, writeEntryMirrors } from '../entry-writes.js';
+import { updateStatsUnlessBusy } from '../index-and-stats.js';
+import { purgeMirrorBestEffort, removeEntryMirrors } from '../mirrors.js';
+import { onHandle, openStore } from '../open.js';
+import { selectMemoryReach } from '../tenant-lookup.js';
 /** Each call on its own handle; mirrors and the forgotten counter follow the commit, so a rolled-back write leaves neither. */
 export function sqliteEntryWrites(hippoRoot) {
     return {
-        async writeEntry({ entry, actor }) {
-            const stamped = stampOriginProject(hippoRoot, entry);
-            onHandle(hippoRoot, (db) => writeInOwnTenant(db, stamped, actor), openStore);
-            writeEntryMirrors(hippoRoot, stamped);
+        writeEntry(write) {
+            writeEntryAt(hippoRoot, write);
         },
-        async applyOutcome(outcome) {
-            const applied = onHandle(hippoRoot, (db) => applyOutcomeOn(db, outcome), openStore);
-            for (const entry of applied)
-                writeEntryMirrors(hippoRoot, entry);
-            return applied.map((entry) => entry.id);
+        applyOutcome(outcome) {
+            return applyOutcomeAt(hippoRoot, outcome);
         },
-        async supersede(write) {
+        supersede(write) {
             const successor = stampOriginProject(hippoRoot, write.successor);
             onHandle(hippoRoot, (db) => commitSupersede(db, { ...write, successor }));
             writeEntryMirrors(hippoRoot, successor);
         },
-        async archiveRaw(archive) {
-            const archivedAt = onHandle(hippoRoot, (db) => {
-                const at = withWriteScope(db, 'archive_raw_in_reach', () => {
-                    assertInReach(db, archive, archive.id);
-                    return archiveRawMemory(db, archive.id, { reason: archive.reason, who: archive.actor });
-                });
-                cleanArchivedMirrors(db, hippoRoot, archive.id);
-                return at;
-            });
-            updateStatsUnlessBusy(hippoRoot, { forgotten: 1 }, `removed ${archive.id}`);
-            return archivedAt;
+        archiveRaw(archive) {
+            return archiveRawAt(hippoRoot, archive);
         },
-        async forget(removal) {
+        forget(removal) {
             onHandle(hippoRoot, (db) => withWriteScope(db, 'forget_in_reach', () => {
                 assertInReach(db, removal, removal.id);
                 if (!deleteEntryCore(db, removal.id, { actor: removal.actor }))
@@ -59,6 +46,39 @@ export function sqliteEntryWrites(hippoRoot) {
         },
     };
 }
+/** A store-side hook writes on the row's handle inside its write scope, so its throw undoes the row. */
+export function writeEntryAt(hippoRoot, { entry, actor }, afterWrite) {
+    const stamped = stampOriginProject(hippoRoot, entry);
+    onHandle(hippoRoot, (db) => writeInOwnTenant(db, stamped, actor, afterWrite), openStore);
+    writeEntryMirrors(hippoRoot, stamped);
+}
+/** A trace link follows the commit on the same handle and names only the ids applied, so a rolled-back outcome leaves none. */
+export function applyOutcomeAt(hippoRoot, outcome, traceId) {
+    const applied = onHandle(hippoRoot, (db) => {
+        const rows = applyOutcomeOn(db, outcome);
+        const memoryIds = rows.map((entry) => entry.id);
+        if (traceId !== undefined && rows.length > 0) {
+            recordTraceOutcome(db, { traceId, tenantId: outcome.tenantId, outcome: outcome.good ? 'positive' : 'negative', memoryIds });
+        }
+        return rows;
+    }, openStore);
+    for (const entry of applied)
+        writeEntryMirrors(hippoRoot, entry);
+    return applied.map((entry) => entry.id);
+}
+/** Reach is checked inside the archive's write scope. A store-side hook writes on the same handle inside that scope, so its throw undoes the archive. */
+export function archiveRawAt(hippoRoot, archive, afterArchive) {
+    const archivedAt = onHandle(hippoRoot, (db) => {
+        const at = withWriteScope(db, 'archive_raw_in_reach', () => {
+            assertInReach(db, archive, archive.id);
+            return archiveRawMemory(db, archive.id, { reason: archive.reason, who: archive.actor, afterArchive });
+        });
+        cleanArchivedMirrors(db, hippoRoot, archive.id);
+        return at;
+    });
+    updateStatsUnlessBusy(hippoRoot, { forgotten: 1 }, `removed ${archive.id}`);
+    return archivedAt;
+}
 function notFound(id) {
     return new NotFoundError(`memory not found: ${id}`);
 }
@@ -74,13 +94,10 @@ function assertIdInTenant(db, entry) {
     if (holder !== undefined && holder !== entry.tenantId)
         throw new ConflictError(`Memory ${entry.id} belongs to another tenant`);
 }
-/** The refusal row lands after the rollback, in a fresh implicit transaction the aborted one cannot undo. */
-function writeInOwnTenant(db, entry, actor) {
+/** The refusal row lands after `write` has rolled back, in a fresh implicit transaction the aborted one cannot undo. */
+export function auditingRefusal(db, actor, write) {
     try {
-        withWriteScope(db, 'write_entry_in_tenant', () => {
-            assertIdInTenant(db, entry);
-            writeEntryDbOnly(db, entry, { actor });
-        });
+        return write();
     }
     catch (err) {
         if (err instanceof RejectedValueError)
@@ -88,9 +105,16 @@ function writeInOwnTenant(db, entry, actor) {
         throw err;
     }
 }
+/** writeEntryAt on the caller's handle, so a sibling group can read on it after the scope has unwound. */
+export function writeInOwnTenant(db, entry, actor, afterWrite) {
+    auditingRefusal(db, actor, () => withWriteScope(db, 'write_entry_in_tenant', () => {
+        assertIdInTenant(db, entry);
+        writeEntryDbOnly(db, entry, { actor, afterWrite });
+    }));
+}
 /** The reads, rewrites and outcome rows share one BEGIN IMMEDIATE transaction, hippo.db's form of the port's row lock; the caller writes the mirrors after. */
 function applyOutcomeOn(db, outcome) {
-    return withWriteScope(db, 'apply_outcome', () => {
+    return auditingRefusal(db, outcome.actor, () => withWriteScope(db, 'apply_outcome', () => {
         const live = selectEntriesByIds(db, outcome.ids, outcome.tenantId);
         const applied = [];
         for (const id of outcome.ids) {
@@ -104,13 +128,13 @@ function applyOutcomeOn(db, outcome) {
             applied.push(updated);
         }
         return applied;
-    });
+    }));
 }
 /** Reach, the CAS on the old row, the successor's insert and the supersede row in one BEGIN IMMEDIATE transaction. Two racing supersedes: one CAS wins. */
-export function commitSupersede(db, write) {
+function commitSupersede(db, write) {
     const { tenantId, actor, oldId, successor } = write;
-    db.exec('BEGIN IMMEDIATE');
-    try {
+    // The refusal row lands after the scope has rolled back, in a fresh implicit transaction the aborted one cannot undo.
+    auditingRefusal(db, actor, () => withWriteScope(db, 'supersede', () => {
         assertInReach(db, write, oldId);
         // SAFETY: RETURNING names one column, dag_parent_id; no row means another writer got there first.
         const won = db.prepare('UPDATE memories SET superseded_by = ? WHERE id = ? AND tenant_id = ? AND superseded_by IS NULL RETURNING dag_parent_id')
@@ -123,26 +147,15 @@ export function commitSupersede(db, write) {
         assertIdInTenant(db, successor);
         writeEntryDbOnly(db, successor, { actor });
         appendAuditEvent(db, { tenantId, actor, op: 'supersede', targetId: oldId, metadata: { newId: successor.id } });
-        db.exec('COMMIT');
-    }
-    catch (err) {
-        try {
-            db.exec('ROLLBACK');
-        }
-        catch { /* already rolled back */ }
-        // The refusal row lands after ROLLBACK, in a fresh implicit transaction the aborted one cannot undo.
-        if (err instanceof RejectedValueError)
-            auditRejectionRefusal(db, err, actor);
-        throw err;
-    }
+    }));
 }
 /** A mirror left on disk would bring the archived row back on the next import; on failure the reaper retries, as mirror_cleaned_at stays NULL. */
-export function cleanArchivedMirrors(db, hippoRoot, id) {
+function cleanArchivedMirrors(db, hippoRoot, id) {
     try {
         removeEntryMirrors(hippoRoot, id);
     }
     catch (mirrorErr) {
-        log.error(`archiveRaw: mirror cleanup failed for ${id} (will retry via reaper on next openHippoDb): ${mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)}`);
+        log.error(`archiveRaw: mirror cleanup failed for ${id} (will retry via reaper on next openHippoDb): ${errorMessage(mirrorErr)}`);
         return;
     }
     try {

package/dist/store/sqlite/error-codec.d.ts

@@ -0,0 +1,17 @@
+type FieldValue = string | number | boolean | null;
+/** One error as a thread message can carry it. */
+export interface WireError {
+    /** Constructor names from the thrown class up to Error: several classes set no `name`, so a name alone cannot pick the class. */
+    readonly classes: readonly string[];
+    readonly message: string;
+    readonly stack: string | undefined;
+    /** Own enumerable fields with a primitive value: `status`, `errcode`, `storeKind`, a `name` its constructor set. */
+    readonly fields: Readonly<Record<string, FieldValue>>;
+    readonly cause: WireError | undefined;
+}
+/** Worker side. A thrown value that is not an Error carries its text and no class. */
+export declare function encodeError<E>(err: E, depth?: number): WireError;
+/** Server side: an instance of the thrown class, or of its nearest ancestor in the table, with the worker's stack, cause and fields. */
+export declare function decodeError(wire: WireError): Error;
+export {};
+//# sourceMappingURL=error-codec.d.ts.map

package/dist/store/sqlite/error-codec.js

@@ -0,0 +1,59 @@
+// A thrown error crosses the worker boundary as plain data and arrives as an instance of its class: the server's mappers test `instanceof`, `errcode` and fields.
+import { BadRequestError, ConflictError, ForbiddenError, NotFoundError } from '../../core/api-errors.js';
+import { IncompatibleBinaryError, StoreBusyError } from '../../db/index.js';
+import { BodyTimeoutError, BodyTooLargeError, HttpError } from '../../util/http-util.js';
+import { errorMessage, log } from '../../util/log.js';
+import { ScopeForbiddenError } from '../recall-scope.js';
+import { RejectedValueError } from '../rejection.js';
+import { OtherStoreFolderError, SqliteBlockedError, StoreNotPortedError } from '../../util/sqlite-blocked.js';
+// Every class the server maps by, at or below the store layer. Abstract ApiError is absent: only its four subclasses are thrown.
+const CLASSES = [
+    Error, TypeError, RangeError, SyntaxError, ReferenceError, EvalError, URIError,
+    HttpError, BodyTooLargeError, BodyTimeoutError,
+    BadRequestError, ForbiddenError, NotFoundError, ConflictError, RejectedValueError, ScopeForbiddenError,
+    SqliteBlockedError, StoreNotPortedError, OtherStoreFolderError,
+    StoreBusyError, IncompatibleBinaryError,
+];
+const PROTOTYPES = new Map(CLASSES.map((known) => [known.name, known.prototype]));
+// A chain of causes deeper than this is cut, so a cycle cannot recurse for ever.
+const MAX_CAUSES = 4;
+function isFieldValue(value) {
+    return value === null || typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean';
+}
+function classNames(err) {
+    const names = [];
+    let proto = Object.getPrototypeOf(err);
+    while (proto !== null && proto !== Object.prototype) {
+        names.push(proto.constructor.name);
+        proto = Object.getPrototypeOf(proto);
+    }
+    return names;
+}
+function ownFields(err) {
+    return Object.fromEntries(Object.entries(err).filter((field) => isFieldValue(field[1])));
+}
+/** Worker side. A thrown value that is not an Error carries its text and no class. */
+export function encodeError(err, depth = 0) {
+    if (!(err instanceof Error))
+        return { classes: [], message: errorMessage(err), stack: undefined, fields: {}, cause: undefined };
+    const cause = err.cause === undefined || depth >= MAX_CAUSES ? undefined : encodeError(err.cause, depth + 1);
+    return { classes: classNames(err), message: err.message, stack: err.stack, fields: ownFields(err), cause };
+}
+/** Server side: an instance of the thrown class, or of its nearest ancestor in the table, with the worker's stack, cause and fields. */
+export function decodeError(wire) {
+    const thrown = wire.classes[0];
+    const known = wire.classes.find((name) => PROTOTYPES.has(name)) ?? 'Error';
+    const unlisted = thrown !== undefined && thrown !== known;
+    if (unlisted)
+        log.once(`store-worker-error:${thrown}`, 'warn', `a store worker threw ${thrown}, which the error codec does not list; it arrives as ${known}`);
+    // A listed ancestor still maps to its status, so its text stays as thrown; under plain Error the class survives only in the message.
+    const message = unlisted && known === 'Error' ? `${thrown}: ${wire.message}` : wire.message;
+    const rebuilt = wire.cause === undefined ? new Error(message) : new Error(message, { cause: decodeError(wire.cause) });
+    // The constructor is not run again: its arguments did not cross, and the fields below are what it set.
+    Object.setPrototypeOf(rebuilt, PROTOTYPES.get(known) ?? Error.prototype);
+    Object.assign(rebuilt, wire.fields);
+    if (wire.stack !== undefined)
+        rebuilt.stack = wire.stack;
+    return rebuilt;
+}
+//# sourceMappingURL=error-codec.js.map

package/dist/store/sqlite/executor-counts.d.ts

@@ -0,0 +1,10 @@
+/** Counted on the server thread, across every executor of the process. */
+export declare const workerCounts: {
+    /** Calls refused because a thread's queue was full. */
+    queueRefusals: number;
+    /** Calls whose request deadline passed, waiting or running. */
+    jobsExpired: number;
+    /** Threads stopped at a deadline and replaced. */
+    workersReplaced: number;
+};
+//# sourceMappingURL=executor-counts.d.ts.map

package/dist/store/sqlite/executor-counts.js

@@ -0,0 +1,11 @@
+// What the store workers refused, timed out and replaced since the process started; /health reports them beside the audit failure count.
+/** Counted on the server thread, across every executor of the process. */
+export const workerCounts = {
+    /** Calls refused because a thread's queue was full. */
+    queueRefusals: 0,
+    /** Calls whose request deadline passed, waiting or running. */
+    jobsExpired: 0,
+    /** Threads stopped at a deadline and replaced. */
+    workersReplaced: 0,
+};
+//# sourceMappingURL=executor-counts.js.map

package/dist/store/sqlite/executor.d.ts

@@ -0,0 +1,28 @@
+import { type OpMode } from './worker-ops.js';
+export interface CallOptions {
+    readonly mode: OpMode;
+    readonly requestId: string | undefined;
+    /** Epoch milliseconds by which the call's request must be answered; absent means the call waits as long as it takes. */
+    readonly deadlineAt?: number | undefined;
+}
+export interface ExecutorOptions {
+    /** Lock wait of each thread's connection; defaults to the server's. */
+    readonly busyWaitMs?: number;
+    /** Calls that may wait for one thread; defaults to HIPPO_STORE_QUEUE_MAX, else 256. */
+    readonly queueMax?: number;
+    /** How long past its deadline a write may still report its own outcome; defaults to 2000 ms. */
+    readonly writeGraceMs?: number;
+}
+export interface SqliteExecutor {
+    /** Runs `op` of the synchronous store on a worker thread. A throw there rejects here as an instance of its own class. */
+    call<T>(op: string, args: readonly unknown[], opts: CallOptions): Promise<T>;
+    /** Resolves once every thread has exited: Windows cannot delete a store folder while a thread holds its files. */
+    close(): Promise<void>;
+    /** Threads started and not yet exited. */
+    liveThreads(): number;
+    /** Stops the writer thread as a crash would: its running and queued calls reject, and the next write starts a fresh thread. */
+    terminateWriter(): void;
+}
+/** Threads for the hippo.db under `hippoRoot`: one writer and two readers. The first call starts the writer for the store's setup; a reader starts at its own first call. */
+export declare function createSqliteExecutor(hippoRoot: string, opts?: ExecutorOptions): SqliteExecutor;
+//# sourceMappingURL=executor.d.ts.map

package/dist/store/sqlite/executor.js

@@ -0,0 +1,286 @@
+// The server thread's side of the store workers: SQLite's synchronous statements and lock waits run on their threads, so the event loop keeps answering.
+import { fileURLToPath } from 'node:url';
+import { SHARE_ENV, Worker } from 'node:worker_threads';
+import { addAuditWriteFailures } from '../audit.js';
+import { getHippoDbPath, SERVER_DB_WAIT_MS, StoreBusyError } from '../../db/index.js';
+import { autoCheckpointPages } from '../../db/wal-checkpointer.js';
+import { envStoreQueueMax } from '../../util/env.js';
+import { DeadlineExceededError } from '../../util/http-util.js';
+import { errorMessage, log } from '../../util/log.js';
+import { requestScopes } from '../../util/request-scope.js';
+import { decodeError } from './error-codec.js';
+import { workerCounts } from './executor-counts.js';
+import { STORE_SETUP_OP } from './worker-ops.js';
+// Resolved from the package root, so a server run from TypeScript source starts the same built file.
+const WORKER_ENTRY = fileURLToPath(new URL('../../../dist/store/sqlite/worker-entry.js', import.meta.url));
+const WORKER_GONE = 'the store worker stopped before it answered; a write it was running may or may not be saved';
+const EXECUTOR_CLOSED = 'the store is closed';
+const QUEUE_FULL = 'the store worker has a full queue, so the call was refused before it ran';
+const EXPIRED_WAITING = "the call was still waiting for a store worker at its request's deadline, so it never ran";
+const READ_LATE = "the store did not answer the read by the request's deadline";
+const WRITE_NOT_SAVED = "the store did not finish the write by the request's deadline; the write was stopped and nothing was saved";
+const WRITE_UNKNOWN = "the store did not confirm the write by the request's deadline; it may or may not be saved";
+// Far above any burst a healthy thread clears, small enough that a stuck thread refuses new work in bounded memory.
+const DEFAULT_QUEUE_MAX = 256;
+// How long past its deadline a write may still report its own outcome before the caller is told it is unknown.
+const DEFAULT_WRITE_GRACE_MS = 2000;
+function laneOf(mode) {
+    return { mode, queue: [], running: undefined, worker: undefined, stopping: undefined, commitFlag: undefined };
+}
+/** Gives `pending`'s caller its one answer; a second call does nothing. */
+function finish(pending, answer) {
+    if (pending.answered)
+        return;
+    pending.answered = true;
+    clearTimeout(pending.timer);
+    answer();
+}
+function mayHaveCommitted(lane, pending) {
+    return lane.commitFlag !== undefined && Atomics.load(lane.commitFlag, 0) === pending.job.id;
+}
+function load(lane) {
+    return lane.queue.length + (lane.running === undefined ? 0 : 1);
+}
+class WorkerPool {
+    #hippoRoot;
+    #busyWaitMs;
+    // One writer, so writes run in call order and never wait on each other's lock.
+    #writer = laneOf('write');
+    #readers = [laneOf('read'), laneOf('read')];
+    // Keyed by thread and not by lane: close() must also wait for a thread its lane already replaced.
+    #exits = new Map();
+    #queueMax;
+    #writeGraceMs;
+    #nextId = 1;
+    #closed = false;
+    // 'done' once the writer has run the store's open-time setup, which a reader's connection could not write.
+    #setup = 'none';
+    constructor(hippoRoot, opts) {
+        this.#hippoRoot = hippoRoot;
+        this.#busyWaitMs = opts.busyWaitMs ?? SERVER_DB_WAIT_MS;
+        this.#queueMax = opts.queueMax ?? envStoreQueueMax() ?? DEFAULT_QUEUE_MAX;
+        this.#writeGraceMs = opts.writeGraceMs ?? DEFAULT_WRITE_GRACE_MS;
+    }
+    call(op, args, { mode, requestId, deadlineAt }) {
+        return new Promise((resolve, reject) => {
+            if (this.#closed) {
+                reject(new StoreBusyError(EXECUTOR_CLOSED));
+                return;
+            }
+            const settle = (reply) => {
+                if (!reply.ok) {
+                    reject(decodeError(reply.error));
+                    return;
+                }
+                // SAFETY: the worker ran `op` of the synchronous store, whose result is the T the caller's port method declares.
+                resolve(reply.value);
+            };
+            const lane = mode === 'write' ? this.#writer : this.#readers.reduce((least, next) => (load(next) < load(least) ? next : least));
+            const refusal = this.#refusal(lane, deadlineAt);
+            if (refusal !== undefined) {
+                reject(refusal);
+                return;
+            }
+            const pending = { job: { id: this.#nextId++, op, args, requestId }, settle, fail: reject, deadlineAt, timer: undefined, answered: false };
+            this.#watchDeadline(lane, pending);
+            lane.queue.push(pending);
+            this.#ensureSetup();
+            this.#pump(lane);
+        });
+    }
+    // Ahead of the call that started it, and with no deadline: the writer's own lock wait bounds it.
+    #ensureSetup() {
+        if (this.#setup !== 'none')
+            return;
+        this.#setup = 'running';
+        const pending = {
+            job: { id: this.#nextId++, op: STORE_SETUP_OP, args: [], requestId: undefined },
+            settle: (reply) => this.#setupEnded(reply.ok ? undefined : decodeError(reply.error)),
+            fail: (err) => this.#setupEnded(err),
+            deadlineAt: undefined,
+            timer: undefined,
+            answered: false,
+        };
+        this.#writer.queue.unshift(pending);
+        this.#pump(this.#writer);
+    }
+    // A failed setup fails the reads that waited for it with its own error, as each read's own open would have, and the next call tries it again.
+    #setupEnded(failure) {
+        this.#setup = failure === undefined ? 'done' : 'none';
+        for (const lane of this.#readers) {
+            if (failure !== undefined)
+                for (const waiting of lane.queue.splice(0))
+                    finish(waiting, () => waiting.fail(failure));
+            this.#pump(lane);
+        }
+    }
+    // Nothing is queued and nothing ran, so both refusals are the retryable 503 of a busy store.
+    #refusal(lane, deadlineAt) {
+        if (lane.queue.length >= this.#queueMax) {
+            workerCounts.queueRefusals += 1;
+            return new StoreBusyError(QUEUE_FULL);
+        }
+        if (deadlineAt !== undefined && deadlineAt <= Date.now()) {
+            workerCounts.jobsExpired += 1;
+            return new StoreBusyError(EXPIRED_WAITING);
+        }
+        return undefined;
+    }
+    #watchDeadline(lane, pending) {
+        if (pending.deadlineAt === undefined)
+            return;
+        pending.timer = setTimeout(() => this.#expire(lane, pending), pending.deadlineAt - Date.now());
+        // The request that waits on the call keeps the process alive; this timer alone must not.
+        pending.timer.unref();
+    }
+    #expire(lane, pending) {
+        workerCounts.jobsExpired += 1;
+        const place = lane.queue.indexOf(pending);
+        if (place >= 0) {
+            lane.queue.splice(place, 1);
+            finish(pending, () => pending.fail(new StoreBusyError(EXPIRED_WAITING)));
+            return;
+        }
+        if (lane.running !== pending || lane.worker === undefined)
+            return;
+        log.warn(`the store's ${lane.mode} worker ran '${pending.job.op}' past its request's deadline`, { requestId: pending.job.requestId });
+        if (lane.mode === 'read')
+            this.#stopReader(lane, lane.worker, pending);
+        else
+            this.#stopWriter(lane, lane.worker, pending);
+    }
+    // A read changes nothing, so its caller is answered at once; the thread is replaced when it has exited.
+    #stopReader(lane, worker, pending) {
+        finish(pending, () => pending.fail(new DeadlineExceededError(READ_LATE)));
+        lane.stopping = worker;
+        void worker.terminate();
+    }
+    // Before its commit flag a stopped write is a known rollback; after it the thread is left to finish, since stopping it could not undo the write.
+    #stopWriter(lane, worker, pending) {
+        if (!mayHaveCommitted(lane, pending)) {
+            lane.stopping = worker;
+            this.terminateWriter();
+        }
+        // A statement in flight runs to its end even after the stop, so the caller waits for the thread's own word only this long.
+        pending.timer = setTimeout(() => finish(pending, () => pending.fail(new DeadlineExceededError(WRITE_UNKNOWN))), this.#writeGraceMs);
+        pending.timer.unref();
+    }
+    // The thread has exited, so the write lock is free and what the stopped job did is final.
+    #replaceStopped(lane) {
+        const stopped = lane.running;
+        lane.stopping = undefined;
+        lane.running = undefined;
+        workerCounts.workersReplaced += 1;
+        if (stopped !== undefined) {
+            // Read only now: the thread could still set its flag between the deadline and its exit.
+            const message = mayHaveCommitted(lane, stopped) ? WRITE_UNKNOWN : WRITE_NOT_SAVED;
+            finish(stopped, () => stopped.fail(new DeadlineExceededError(message)));
+        }
+        if (!this.#closed)
+            this.#pump(lane);
+    }
+    async close() {
+        this.#closed = true;
+        for (const lane of [this.#writer, ...this.#readers]) {
+            for (const pending of lane.queue.splice(0))
+                finish(pending, () => pending.fail(new StoreBusyError(EXECUTOR_CLOSED)));
+            // An idle thread is unreferenced, and the process must not end before the thread has closed its connection.
+            lane.worker?.ref();
+            // Read after the reply of a job still running; the thread then closes its connection and ends.
+            lane.worker?.postMessage('stop');
+        }
+        await Promise.all(this.#exits.values());
+    }
+    liveThreads() {
+        return this.#exits.size;
+    }
+    terminateWriter() {
+        void this.#writer.worker?.terminate();
+    }
+    #pump(lane) {
+        if (lane.running !== undefined)
+            return;
+        // Setup comes first: a reader sent a job before it would open a store whose setup it cannot write.
+        if (lane.mode === 'read' && this.#setup !== 'done')
+            return;
+        const next = lane.queue.shift();
+        if (next === undefined)
+            return;
+        // Started by the first call and never at boot, so a server that only answers /health opens no thread.
+        // Outside that call's request scope, or the thread's listeners would log every later event under its request id.
+        lane.worker ??= requestScopes.exit(() => this.#start(lane));
+        lane.running = next;
+        // An idle thread never keeps the process alive; one with a call in flight does.
+        lane.worker.ref();
+        try {
+            lane.worker.postMessage({ ...next.job, walPages: autoCheckpointPages(getHippoDbPath(this.#hippoRoot)) });
+        }
+        catch (err) {
+            // Arguments that cannot be copied to the thread fail this call alone.
+            this.#idle(lane, lane.worker);
+            finish(next, () => next.fail(err instanceof Error ? err : new Error(errorMessage(err))));
+            this.#pump(lane);
+        }
+    }
+    #idle(lane, worker) {
+        lane.running = undefined;
+        worker.unref();
+    }
+    #start(lane) {
+        // A fresh flag per thread: a job id left by the thread before it could match no later job, but a shared buffer would outlive its writer.
+        const shared = lane.mode === 'write' ? new SharedArrayBuffer(4) : undefined;
+        lane.commitFlag = shared && new Int32Array(shared);
+        const init = { hippoRoot: this.#hippoRoot, mode: lane.mode, busyWaitMs: this.#busyWaitMs, ...(shared && { commitFlag: shared }) };
+        // An explicit execArgv keeps the parent's loaders out of the thread and silences its repeat of the node:sqlite warning.
+        const worker = new Worker(WORKER_ENTRY, { env: SHARE_ENV, execArgv: ['--no-warnings'], workerData: init });
+        worker.unref();
+        worker.on('message', (reply) => this.#settle(lane, worker, reply));
+        worker.on('error', (err) => this.#retire(lane, worker, err));
+        const exited = new Promise((resolve) => {
+            worker.once('exit', () => {
+                this.#exits.delete(worker);
+                this.#retire(lane, worker);
+                resolve();
+            });
+        });
+        this.#exits.set(worker, exited);
+        return worker;
+    }
+    #settle(lane, worker, reply) {
+        const done = lane.running;
+        // A reply for a call that was already rejected with its thread has no caller left.
+        if (lane.worker !== worker || done === undefined || done.job.id !== reply.id)
+            return;
+        addAuditWriteFailures(reply.auditFailures);
+        finish(done, () => done.settle(reply));
+        // A thread being stopped answered first; its lane stays taken until it has exited.
+        if (lane.stopping === worker)
+            return;
+        this.#idle(lane, worker);
+        this.#pump(lane);
+    }
+    // The lane keeps a stopped thread until it has exited, so a new thread never starts while the old one may still hold the write lock.
+    #retire(lane, worker, cause) {
+        if (lane.worker !== worker)
+            return;
+        lane.worker = undefined;
+        if (lane.stopping === worker) {
+            // Stopped on purpose: the calls still waiting run on the replacement.
+            this.#replaceStopped(lane);
+            return;
+        }
+        const lost = lane.running === undefined ? lane.queue.splice(0) : [lane.running, ...lane.queue.splice(0)];
+        lane.running = undefined;
+        if (lost.length > 0 || cause !== undefined) {
+            log.warn(`the store's ${lane.mode} worker stopped with ${lost.length} call(s) unanswered${cause === undefined ? '' : `: ${cause.message}`}`);
+        }
+        // The server answers this class as its retryable 503.
+        for (const pending of lost)
+            finish(pending, () => pending.fail(new StoreBusyError(WORKER_GONE, { cause })));
+    }
+}
+/** Threads for the hippo.db under `hippoRoot`: one writer and two readers. The first call starts the writer for the store's setup; a reader starts at its own first call. */
+export function createSqliteExecutor(hippoRoot, opts = {}) {
+    return new WorkerPool(hippoRoot, opts);
+}
+//# sourceMappingURL=executor.js.map

package/dist/store/sqlite/graph-reads-group.d.ts

@@ -0,0 +1,5 @@
+import type { GraphReads, Sync } from '../port.js';
+export declare function sqliteGraphReads(hippoRoot: string): Sync<GraphReads>;
+/** The group as a served store answers it: the call runs at once and resolves through a Promise, so a throw rejects as another store's would. */
+export declare function servedGraphReads(sync: Sync<GraphReads>): GraphReads;
+//# sourceMappingURL=graph-reads-group.d.ts.map

package/dist/store/sqlite/graph-reads-group.js

@@ -0,0 +1,18 @@
+// hippo.db's half of the GraphReads store group: one read snapshot around the walk the CLI graph commands run.
+import { canReadScope } from '../recall-scope.js';
+import { withGraphReadSnapshot } from '../graph-reads.js';
+import { graphViewRows } from '../graph-view-rows.js';
+export function sqliteGraphReads(hippoRoot) {
+    return {
+        graphRows: (tenantId, { entity, limit, reader }) => withGraphReadSnapshot(hippoRoot, (db) => graphViewRows(hippoRoot, db, tenantId, {
+            entity,
+            limit,
+            canRead: reader && ((scope) => scope === null || canReadScope(reader, scope)),
+        })),
+    };
+}
+/** The group as a served store answers it: the call runs at once and resolves through a Promise, so a throw rejects as another store's would. */
+export function servedGraphReads(sync) {
+    return { graphRows: async (tenantId, query) => sync.graphRows(tenantId, query) };
+}
+//# sourceMappingURL=graph-reads-group.js.map