@hraness/sponge 0.2.5 → 0.3.0

package/dist/src/service/loop.js

@@ -1,12 +1,14 @@
 import { constants, openSync, closeSync, fstatSync, writeSync, renameSync, lstatSync, existsSync } from 'node:fs';
 import { join } from 'node:path';
-import { safeDirectory, regularFile, syncDirectory } from '../store/files.js';
+import { safeDirectory, regularFile } from '../store/files.js';
+import { syncDirectory } from '../util/fsync.js';
 import { writePrivateExclusive } from '../portability/files.js';
 import { processIdentity } from '../worker/identity.js';
 import { randomUUID } from 'node:crypto';
 import { acquireServiceLifetime } from './lifetime.js';
-/** Fixed event vocabulary; source URLs, credentials, content and raw errors never enter logs. */
-export function serviceLog(root, event) {
+import { SpongeError, codeOf } from '../errors.js';
+/** Fixed event vocabulary plus a stable error code token; source URLs, credentials, content and raw errors never enter logs. */
+export function serviceLog(root, event, code) {
     const directory = safeDirectory(join(root, 'runtime', 'service'), true), path = join(directory, 'worker.log');
     if (existsSync(path)) {
         regularFile(path);
@@ -26,30 +28,56 @@ export function serviceLog(root, event) {
     try {
         const stat = fstatSync(fd);
         if (!stat.isFile() || stat.nlink !== 1)
-            throw Error('integrity');
-        writeSync(fd, JSON.stringify({ at: new Date().toISOString(), event }) + '\n');
+            throw new SpongeError('integrity');
+        writeSync(fd, JSON.stringify({
+            at: new Date().toISOString(),
+            event,
+            ...(code === undefined
+                ? {}
+                : { code: typeof code === 'string' && /^[a-z][a-z0-9-]{0,63}$/u.test(code) ? code : 'unclassified' }),
+        }) + '\n');
     }
     finally {
         closeSync(fd);
     }
 }
-export function idleWait(ms, signal) { return new Promise(resolve => { if (signal.aborted) {
-    resolve();
-    return;
-} const done = () => { clearTimeout(timer); signal.removeEventListener('abort', done); resolve(); }; const timer = setTimeout(done, ms); signal.addEventListener('abort', done, { once: true }); }); }
+export function idleWait(ms, signal) {
+    return new Promise((resolve) => {
+        if (signal.aborted) {
+            resolve();
+            return;
+        }
+        const done = () => {
+            clearTimeout(timer);
+            signal.removeEventListener('abort', done);
+            resolve();
+        };
+        const timer = setTimeout(done, ms);
+        signal.addEventListener('abort', done, { once: true });
+    });
+}
 /** Uses the existing drain engine; it owns locking, cancellation and descendant collection. */
 export async function runWorkerService(store, options) {
     const idleMs = options.idleMs ?? 15000;
-    if (!Number.isSafeInteger(idleMs) || idleMs < 1000 || idleMs > 60000 || !/^[a-f0-9-]{36}$/u.test(options.installationId))
-        throw Error('invalid-input');
+    if (!Number.isSafeInteger(idleMs) ||
+        idleMs < 1000 ||
+        idleMs > 60000 ||
+        !/^[a-f0-9-]{36}$/u.test(options.installationId))
+        throw new SpongeError('invalid-input');
     const root = safeDirectory(join(store.root, 'runtime', 'service'), true), identity = await (options.identity ?? (() => processIdentity(process.pid)))();
     if (!identity)
-        throw Error('unavailable');
+        throw new SpongeError('unavailable');
     const lifetime = await acquireServiceLifetime(root, options.installationId, identity);
     const runId = randomUUID(), runPath = join(root, 'run.json');
-    const record = (state) => { const temp = join(root, `run-${runId}.tmp`); writePrivateExclusive(temp, JSON.stringify({ schemaVersion: 1, installationId: options.installationId, runId, identity, state }) + '\n'); if (existsSync(runPath))
-        regularFile(runPath); renameSync(temp, runPath); syncDirectory(root); };
-    const log = options.log ?? (event => serviceLog(store.root, event));
+    const record = (state) => {
+        const temp = join(root, `run-${runId}.tmp`);
+        writePrivateExclusive(temp, JSON.stringify({ schemaVersion: 1, installationId: options.installationId, runId, identity, state }) + '\n');
+        if (existsSync(runPath))
+            regularFile(runPath);
+        renameSync(temp, runPath);
+        syncDirectory(root);
+    };
+    const log = options.log ?? ((event, code) => serviceLog(store.root, event, code));
     let cycles = 0, cleanup = 'settled';
     try {
         record('running');
@@ -61,11 +89,11 @@ export async function runWorkerService(store, options) {
                 log(results.length ? 'drained' : 'idle');
             }
             catch (error) {
-                const code = error.code;
+                const code = codeOf(error);
                 if (code === 'owner-busy')
                     log('busy');
                 else {
-                    log('repair-required');
+                    log('repair-required', code ?? 'unclassified');
                     if (code === 'cleanup-required')
                         cleanup = 'pending';
                     break;
@@ -76,13 +104,13 @@ export async function runWorkerService(store, options) {
                 break;
             await (options.wait ?? idleWait)(idleMs, options.signal);
         }
-        if (store.pendingCleanupCount() > 0)
+        if (store.queue.pendingCleanup() > 0)
             cleanup = 'pending';
         return { cycles, cleanup };
     }
     finally {
         try {
-            if (store.pendingCleanupCount() > 0)
+            if (store.queue.pendingCleanup() > 0)
                 cleanup = 'pending';
             record(cleanup === 'settled' ? 'stopped' : 'cleanup-required');
             log('stopped');

package/dist/src/store/annotations.d.ts

@@ -0,0 +1,32 @@
+import * as annotations from '../annotations/store.js';
+import { type Context } from './context.js';
+/** Highlights, comments and imported notes on items; every write goes through the change log. */
+export declare class Annotations {
+    private readonly c;
+    constructor(c: Context);
+    addHighlight(input: Parameters<typeof annotations.addHighlight>[1]): annotations.Annotation;
+    addComment(input: Parameters<typeof annotations.addComment>[1]): annotations.Annotation;
+    update(id: string, body: string): annotations.Annotation;
+    delete(id: string): void;
+    list(itemId: string): (annotations.Annotation & {
+        anchored?: boolean;
+    })[];
+    /** Export's view: the one annotation reader, without the redundant item id, plus reading state. */
+    forExport(itemId: string): {
+        annotations: {
+            id: string;
+            kind: "note" | "highlight" | "comment";
+            revisionId: string | null;
+            parentId: string | null;
+            anchor: unknown;
+            body: string;
+            createdAt: string;
+            updatedAt: string | null;
+            anchored?: boolean;
+        }[];
+        readingState: {
+            state: unknown;
+            updatedAt: string;
+        } | null;
+    };
+}

package/dist/src/store/annotations.js

@@ -0,0 +1,44 @@
+import * as annotations from '../annotations/store.js';
+import { text } from './files.js';
+import { transaction } from './context.js';
+/** Highlights, comments and imported notes on items; every write goes through the change log. */
+export class Annotations {
+    c;
+    constructor(c) {
+        this.c = c;
+    }
+    addHighlight(input) {
+        text(input.itemId, 256);
+        return transaction(this.c.db, () => annotations.addHighlight(this.c.db, input));
+    }
+    addComment(input) {
+        text(input.itemId, 256);
+        if (input.parentId !== undefined)
+            text(input.parentId, 256);
+        return transaction(this.c.db, () => annotations.addComment(this.c.db, input));
+    }
+    update(id, body) {
+        text(id, 256);
+        return transaction(this.c.db, () => annotations.updateAnnotation(this.c.db, id, body));
+    }
+    delete(id) {
+        text(id, 256);
+        transaction(this.c.db, () => annotations.deleteAnnotation(this.c.db, id));
+    }
+    list(itemId) {
+        text(itemId, 256);
+        return annotations.listAnnotations(this.c.db, itemId);
+    }
+    /** Export's view: the one annotation reader, without the redundant item id, plus reading state. */
+    forExport(itemId) {
+        text(itemId, 256);
+        const list = annotations.listAnnotations(this.c.db, itemId).map(({ itemId: _, ...a }) => a);
+        const reading = this.c.db.prepare('SELECT state_json,updated_at FROM reading_state WHERE item_id=?').get(itemId);
+        return {
+            annotations: list,
+            readingState: reading
+                ? { state: JSON.parse(String(reading.state_json)), updatedAt: String(reading.updated_at) }
+                : null,
+        };
+    }
+}

package/dist/src/store/backups.d.ts

@@ -0,0 +1,18 @@
+import { type Context } from './context.js';
+/** Online database snapshots, each held open by an active persistent backup fence that keeps GC away. */
+export declare class Backups {
+    private readonly c;
+    constructor(c: Context);
+    /** Online DB snapshot only; P12 pairs this with blobs while the returned fence remains active. */
+    snapshot(): Promise<{
+        id: string;
+        path: string;
+        revisionEpoch: number;
+    }>;
+    /** The fence must still be this backup's active fence: anything else means GC was not held off for the whole backup. */
+    finish(id: string): void;
+    private removeSnapshot;
+    /** Expire active backup fences whose recorded holder is provably dead (exact identity, never age), removing their
+     * snapshot files; then remove snapshot files that no active fence protects. Live, unknown and legacy (NULL owner) fences stay. */
+    expireFences(): Promise<number>;
+}

package/dist/src/store/backups.js

@@ -0,0 +1,110 @@
+import { backup } from 'node:sqlite';
+import { chmodSync, closeSync, constants, fsyncSync, openSync, readdirSync, rmSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { text } from './files.js';
+import { syncDirectory } from '../util/fsync.js';
+import { now, revisionEpoch, transaction } from './context.js';
+import { processIdentity, identityAlive } from '../worker/identity.js';
+import { SpongeError } from '../errors.js';
+/** Online database snapshots, each held open by an active persistent backup fence that keeps GC away. */
+export class Backups {
+    c;
+    constructor(c) {
+        this.c = c;
+    }
+    /** Online DB snapshot only; P12 pairs this with blobs while the returned fence remains active. */
+    async snapshot() {
+        const { root, db, options } = this.c, id = randomUUID(), identity = await processIdentity(process.pid);
+        if (!identity)
+            throw new SpongeError('unavailable');
+        // The fence names its holder so a crashed backup's fence can be expired on exact death proof.
+        const epoch = transaction(db, () => {
+            const e = revisionEpoch(db);
+            db.prepare("INSERT INTO backup_fences(id,revision_epoch,state,created_at,owner_json) VALUES(?,?,'active',?,?)").run(id, e, now(), JSON.stringify(identity));
+            return e;
+        });
+        const path = join(root, 'backups', id + '.sqlite');
+        try {
+            await backup(db, path);
+            options.fault?.('backup');
+            chmodSync(path, 0o600);
+            const fd = openSync(path, constants.O_RDONLY);
+            try {
+                fsyncSync(fd);
+            }
+            finally {
+                closeSync(fd);
+            }
+            syncDirectory(dirname(path));
+            return { id, path, revisionEpoch: epoch };
+        }
+        catch (error) {
+            // Partial snapshot bytes go first; the fence is dropped only once nothing it protects remains.
+            this.removeSnapshot(id);
+            transaction(db, () => {
+                db.prepare("DELETE FROM backup_fences WHERE id=? AND state='active'").run(id);
+            });
+            throw error;
+        }
+    }
+    /** The fence must still be this backup's active fence: anything else means GC was not held off for the whole backup. */
+    finish(id) {
+        text(id, 256);
+        transaction(this.c.db, () => {
+            if (this.c.db.prepare("UPDATE backup_fences SET state='complete' WHERE id=? AND state='active'").run(id).changes !==
+                1)
+                throw new SpongeError('integrity');
+        });
+    }
+    removeSnapshot(id) {
+        const path = join(this.c.root, 'backups', id + '.sqlite');
+        for (const suffix of ['', '-wal', '-shm'])
+            rmSync(path + suffix, { force: true });
+        syncDirectory(dirname(path));
+    }
+    /** Expire active backup fences whose recorded holder is provably dead (exact identity, never age), removing their
+     * snapshot files; then remove snapshot files that no active fence protects. Live, unknown and legacy (NULL owner) fences stay. */
+    async expireFences() {
+        const db = this.c.db;
+        let expired = 0;
+        for (const row of db
+            .prepare("SELECT id,owner_json FROM backup_fences WHERE state='active' AND owner_json IS NOT NULL ORDER BY id")
+            .all()) {
+            let identity;
+            try {
+                identity = JSON.parse(String(row.owner_json));
+                if (!Number.isSafeInteger(identity?.pid) ||
+                    identity.pid < 1 ||
+                    identity.uid !== process.getuid?.() ||
+                    typeof identity.started !== 'string' ||
+                    (await identityAlive(identity)))
+                    continue;
+            }
+            catch {
+                continue;
+            }
+            this.removeSnapshot(String(row.id));
+            expired += transaction(db, () => Number(db
+                .prepare("DELETE FROM backup_fences WHERE id=? AND state='active' AND owner_json=?")
+                .run(String(row.id), String(row.owner_json)).changes));
+        }
+        // A fence row is committed before its snapshot file exists, so list files first and fences second.
+        const dir = join(this.c.root, 'backups'), names = readdirSync(dir);
+        const active = new Set(db
+            .prepare("SELECT id FROM backup_fences WHERE state='active'")
+            .all()
+            .map((r) => String(r.id)));
+        let removed = false;
+        for (const name of names) {
+            const match = /^([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.sqlite(?:-wal|-shm)?$/.exec(name);
+            if (match && !active.has(match[1])) {
+                rmSync(join(dir, name), { force: true });
+                removed = true;
+            }
+        }
+        if (removed)
+            syncDirectory(dir);
+        return expired;
+    }
+}

package/dist/src/store/blobs.d.ts

@@ -0,0 +1,24 @@
+import { type Context } from './context.js';
+import type { AttemptToken, BlobReference } from './types.js';
+/** Immutable content-addressed bytes under `blobs/sha256`, and the fence GC holds while it inspects them. */
+export declare class Blobs {
+    private readonly c;
+    constructor(c: Context);
+    /** Stream bytes under a durable staging pin, verify hash and size, then publish create-if-absent. */
+    put(token: AttemptToken, expectedHash: string, expectedBytes: number, source: AsyncIterable<Uint8Array>): Promise<BlobReference>;
+    /** Bytes checked and returned from the same bounded file descriptor. */
+    read(reference: BlobReference): Uint8Array;
+    /** Hold the SQLite writer fence across synchronous GC inspection and removal. */
+    withGcFence<T>(expectedEpoch: number, callback: (references: readonly BlobReference[]) => T): T;
+    /** GC must honor both durable staging pins and active backup fences. */
+    hasGcFence(): boolean;
+    /** Runtime GC pins: recorded model artifacts and embedding helpers of active or building generations. */
+    runtimeReferences(): {
+        modelArtifacts: {
+            id: string;
+            manifest: string;
+        }[];
+        embeddingHelpers: string[];
+        unmappedSpaces: number;
+    };
+}

package/dist/src/store/blobs.js

@@ -0,0 +1,139 @@
+import { closeSync, constants, fsyncSync, linkSync, openSync, statfsSync, unlinkSync, writeSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { createHash, randomUUID } from 'node:crypto';
+import { limits } from '../../contracts/index.js';
+import { confinedFile, hash, integer, readVerifiedFile, safeDirectory, verifyFile } from './files.js';
+import { syncDirectory } from '../util/fsync.js';
+import { checkAttempt, hasGcFence, ownerSnapshot, pendingCleanup } from './lease.js';
+import { revisionEpoch, transaction } from './context.js';
+import { SpongeError } from '../errors.js';
+/** Immutable content-addressed bytes under `blobs/sha256`, and the fence GC holds while it inspects them. */
+export class Blobs {
+    c;
+    constructor(c) {
+        this.c = c;
+    }
+    /** Stream bytes under a durable staging pin, verify hash and size, then publish create-if-absent. */
+    async put(token, expectedHash, expectedBytes, source) {
+        hash(expectedHash);
+        integer(expectedBytes, limits.pdfBytes);
+        const { root, db, options } = this.c, relativePath = `blobs/sha256/${expectedHash.slice(0, 2)}/${expectedHash}`;
+        transaction(db, () => {
+            checkAttempt(db, token);
+            const pins = db.prepare('SELECT COALESCE(sum(bytes),0) n FROM staging_pins').get();
+            if (Number(pins.n) + expectedBytes > limits.stagingBytes)
+                throw new SpongeError('limit-exceeded');
+            const free = statfsSync(root);
+            if (free.bavail * free.bsize < limits.freeReserveBytes + expectedBytes)
+                throw new SpongeError('limit-exceeded');
+            const existing = db
+                .prepare('SELECT bytes FROM staging_pins WHERE attempt_id=? AND sha256=?')
+                .get(token.attemptId, expectedHash);
+            if (existing && existing.bytes !== expectedBytes)
+                throw new SpongeError('integrity');
+            db.prepare("INSERT OR IGNORE INTO staging_pins VALUES(?,?,?,'staging')").run(token.attemptId, expectedHash, expectedBytes);
+        });
+        const stage = safeDirectory(join(root, 'staging', token.attemptId), true), temp = join(stage, randomUUID() + '.tmp');
+        const fd = openSync(temp, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
+        let count = 0;
+        const h = createHash('sha256');
+        try {
+            for await (const chunk of source) {
+                if (!(chunk instanceof Uint8Array))
+                    throw new SpongeError('invalid-input');
+                count += chunk.byteLength;
+                if (count > expectedBytes)
+                    throw new SpongeError('limit-exceeded');
+                h.update(chunk);
+                let offset = 0;
+                while (offset < chunk.byteLength)
+                    offset += writeSync(fd, chunk, offset, chunk.byteLength - offset);
+            }
+            if (count !== expectedBytes || h.digest('hex') !== expectedHash)
+                throw new SpongeError('integrity');
+            fsyncSync(fd);
+        }
+        catch (error) {
+            closeSync(fd);
+            unlinkSync(temp);
+            throw error;
+        }
+        closeSync(fd);
+        syncDirectory(stage);
+        options.fault?.('staged');
+        const target = join(root, relativePath);
+        safeDirectory(dirname(target), true);
+        // Atomic create-if-absent publication. A hard-link plus unlink avoids rename overwriting immutable bytes.
+        transaction(db, () => {
+            checkAttempt(db, token);
+            try {
+                linkSync(temp, target);
+            }
+            catch (error) {
+                if (error.code !== 'EEXIST')
+                    throw error;
+                verifyFile(target, expectedHash, expectedBytes);
+            }
+            unlinkSync(temp);
+            syncDirectory(dirname(target));
+            syncDirectory(stage);
+            verifyFile(target, expectedHash, expectedBytes);
+            db.prepare("UPDATE staging_pins SET state='published' WHERE attempt_id=? AND sha256=?").run(token.attemptId, expectedHash);
+        });
+        options.fault?.('published');
+        return { sha256: expectedHash, bytes: expectedBytes, relativePath };
+    }
+    /** Bytes checked and returned from the same bounded file descriptor. */
+    read(reference) {
+        hash(reference.sha256);
+        integer(reference.bytes, limits.pdfBytes);
+        if (reference.relativePath !== `blobs/sha256/${reference.sha256.slice(0, 2)}/${reference.sha256}`)
+            throw new SpongeError('integrity');
+        const path = confinedFile(this.c.root, join(this.c.root, reference.relativePath));
+        return readVerifiedFile(path, reference.sha256, reference.bytes);
+    }
+    /** Hold the SQLite writer fence across synchronous GC inspection and removal. */
+    withGcFence(expectedEpoch, callback) {
+        integer(expectedEpoch);
+        const db = this.c.db;
+        return transaction(db, () => {
+            if (revisionEpoch(db) !== expectedEpoch || ownerSnapshot(db) !== null || pendingCleanup(db) > 0 || hasGcFence(db))
+                throw new SpongeError('owner-busy');
+            const references = db
+                .prepare('SELECT sha256,bytes,relative_path relativePath FROM assets ORDER BY sha256')
+                .all();
+            return callback(references);
+        });
+    }
+    /** GC must honor both durable staging pins and active backup fences. */
+    hasGcFence() {
+        return hasGcFence(this.c.db);
+    }
+    /** Runtime GC pins: recorded model artifacts and embedding helpers of active or building generations. */
+    runtimeReferences() {
+        const db = this.c.db;
+        const modelArtifacts = db
+            .prepare('SELECT id,manifest_json FROM model_artifacts ORDER BY id')
+            .all()
+            .map((r) => ({ id: String(r.id), manifest: String(r.manifest_json) }));
+        const helpers = new Set();
+        let unmapped = 0;
+        for (const row of db
+            .prepare("SELECT DISTINCT s.identity_json FROM embedding_spaces s JOIN index_generations g ON g.fingerprint=s.fingerprint WHERE g.state='building' OR g.id IN (SELECT generation_id FROM active_indexes)")
+            .all()) {
+            let identity;
+            try {
+                identity = JSON.parse(String(row.identity_json));
+            }
+            catch {
+                throw new SpongeError('integrity');
+            }
+            const sha = identity?.helperSha256;
+            if (typeof sha === 'string' && /^[a-f0-9]{64}$/.test(sha))
+                helpers.add(sha);
+            else
+                unmapped++;
+        }
+        return { modelArtifacts, embeddingHelpers: [...helpers].sort(), unmappedSpaces: unmapped };
+    }
+}

package/dist/src/store/changes.d.ts

@@ -1,5 +1,4 @@
 import type { DatabaseSync } from 'node:sqlite';
-type Row = Record<string, any>;
 export type Entity = 'item' | 'annotation' | 'reading' | 'derivation';
 /** One field-level write to user state. The sync contract: replaying every change in order rebuilds user state. */
 export interface Change {
@@ -14,29 +13,23 @@ export interface LoggedChange extends Change {
     deviceId: string;
     hlc: string;
 }
-export type ReviewStatus = 'accepted' | 'rejected';
-/** Bound on one change's JSON; imported note bodies (64 KiB) plus anchors fit with room for escaping. */
-export declare const maxChangeBytes = 262144;
-/** Append without applying: for state that already exists (seeding). Callers hold the write transaction. */
-export declare function append(db: DatabaseSync, change: Change): void;
 /** The only write path for user state: tick the clock, apply the row update, append the change. */
 export declare function mutate(db: DatabaseSync, entity: Entity, entityId: string, field: string, value: unknown): void;
 export declare function listChanges(db: DatabaseSync, after?: number, limit?: number): LoggedChange[];
 /** Deterministic source-ref id, so replayed item creation yields the same row on every device. */
 export declare const sourceRefId: (itemId: string) => string;
-/**
- * Review JSON in one canonical shape, whatever order its paths were written in. A digest is reviewed per field;
- * an editorial derivation (a reviewed source note kept whole) has one path, `review.note`.
- */
-export declare function setReviewPath(review: Row, output: Row, path: string, value: ReviewStatus | null, kind?: string): Row;
 /**
  * Map one change onto its row. Table-driven: every user-state field Sponge writes is listed here,
  * and nothing else writes those columns. Objects (revisions, derivation outputs) must already exist.
  */
 export declare function applyChange(db: DatabaseSync, change: Change): void;
+/**
+ * `items.title` is derived: the user's override, else an importer's owner title, else the captured title. A changed title
+ * is re-indexed for lexical search at once; replay on a device without the FTS row leaves indexing to capture.
+ */
+export declare function refreshTitle(db: DatabaseSync, id: string): void;
 /**
  * Seed the log with user state written before schema 5, so another device can rebuild it.
  * Idempotent: a field that already has a change is never seeded again. Rows already exist, so this appends only.
  */
 export declare function seedChanges(db: DatabaseSync): number;
-export {};