@hraness/sponge 0.2.6 → 0.3.0

package/dist/src/store/search-index.js

@@ -0,0 +1,190 @@
+import { closeSync, constants, fstatSync, openSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { safeDirectory, text } from './files.js';
+import * as indexing from '../indexing/index.js';
+import * as jobs from '../indexing/jobs.js';
+import * as prune from '../indexing/prune.js';
+import { hybridSearch } from '../search/hybrid.js';
+import { checkOwner, hasGcFence, pendingCleanup, removeStaging } from './lease.js';
+import { transaction } from './context.js';
+import { SpongeError } from '../errors.js';
+/**
+ * Embedding index generations, the index jobs that build them under the shared worker lease, retired-generation pruning
+ * and hybrid search. The generation and job SQL lives in `src/indexing/`; this is its store surface.
+ */
+export class SearchIndex {
+    c;
+    /** Index jobs as one `JobKind` of the queue. */
+    jobs;
+    constructor(c) {
+        this.c = c;
+        const db = c.db;
+        this.jobs = {
+            owns: (id) => !!jobs.indexJob(db, id),
+            list: () => jobs.listIndexJobs(db),
+            attempts: (token) => jobs.indexOwnerAttempts(db, token),
+            cancel: (id) => jobs.cancelIndexJob(db, id),
+            retry: (id, options) => {
+                if (options.automatic || options.notBefore !== undefined)
+                    throw new SpongeError('invalid-input');
+                jobs.retryIndexJob(db, id);
+            },
+            cancelRequested: (id) => jobs.indexJob(db, id).cancel_requested === 1,
+            recover: (attempt) => {
+                removeStaging(c.root, attempt.attemptId, (path) => this.checkStaging(attempt, path));
+                jobs.recoverIndexAttempt(db, attempt);
+            },
+        };
+    }
+    create(options) {
+        return indexing.createGeneration(this.c.db, options);
+    }
+    status(id) {
+        return indexing.generationStatus(this.c.db, id);
+    }
+    inputs(id, limit = 16) {
+        return indexing.pendingInputs(this.c.db, id, limit);
+    }
+    writeBatch(id, fingerprint, batch) {
+        indexing.writeBatch(this.c.db, id, fingerprint, batch);
+    }
+    activate(id) {
+        return indexing.activateGeneration(this.c.db, id);
+    }
+    cancel(id) {
+        indexing.cancelGeneration(this.c.db, id);
+    }
+    submitJob(generationId) {
+        return jobs.submitIndexJob(this.c.db, generationId);
+    }
+    job(id) {
+        return jobs.indexJob(this.c.db, id);
+    }
+    claim(owner, id) {
+        return jobs.claimIndexJob(this.c.db, owner, id);
+    }
+    checkpoint(token, stage) {
+        jobs.checkpointIndexAttempt(this.c.db, token, stage);
+    }
+    writeAttemptBatch(token, fingerprint, batch) {
+        jobs.withIndexAttempt(this.c.db, token, (id) => indexing.writeBatch(this.c.db, id, fingerprint, batch));
+    }
+    /** Coverage catches up with the library before a run reads its pending inputs. */
+    refresh(token) {
+        return jobs.withIndexAttempt(this.c.db, token, (id) => indexing.syncGeneration(this.c.db, id));
+    }
+    /** A complete building generation activates; an active one stays active. Content still pending (a capture after the last
+     * batch) queues the job again, so the same drain picks it up. */
+    complete(token) {
+        const db = this.c.db;
+        return jobs.withIndexAttempt(db, token, (id) => {
+            indexing.syncGeneration(db, id);
+            let status = indexing.generationStatus(db, id);
+            if (status.state !== 'building' && status.state !== 'active')
+                throw new SpongeError('cancelled-index');
+            if (status.state === 'building' && !status.pending)
+                status = indexing.activateGeneration(db, id);
+            jobs.finishIndexAttempt(db, token);
+            if (status.pending)
+                indexing.armJob(db, id);
+            return status;
+        });
+    }
+    /** The profile's active generation with its coverage and pending counts and its index job, or nulls when none is active. */
+    overview(profileId = 'default') {
+        const db = this.c.db;
+        return transaction(db, () => {
+            const id = indexing.activeGeneration(db, profileId);
+            if (!id)
+                return { generation: null, job: null };
+            const job = db.prepare('SELECT id,state,reason FROM index_jobs WHERE generation_id=?').get(id);
+            return {
+                generation: indexing.generationStatus(db, id),
+                job: job
+                    ? { id: String(job.id), state: String(job.state), reason: job.reason === null ? null : String(job.reason) }
+                    : null,
+            };
+        });
+    }
+    fail(token, reason) {
+        jobs.finishIndexAttempt(this.c.db, token, reason);
+    }
+    settle(token) {
+        const db = this.c.db;
+        transaction(db, () => {
+            checkOwner(db, token);
+            const a = db.prepare('SELECT * FROM index_attempts WHERE id=? AND job_id=?').get(token.attemptId, token.jobId);
+            if (!a || a.owner_incarnation !== token.ownerIncarnation || a.fence !== token.fence || a.finished_at === null)
+                throw new SpongeError('owner-busy');
+            removeStaging(this.c.root, token.attemptId, (path) => this.checkStaging(token, path));
+            jobs.settleIndexAttempt(db, token);
+        });
+    }
+    /** An index attempt's staging directory is removed only when its private ownership marker names this attempt and owner. */
+    checkStaging(token, path) {
+        safeDirectory(path);
+        const fd = openSync(join(path, 'ownership.json'), constants.O_RDONLY | constants.O_NOFOLLOW);
+        try {
+            const stat = fstatSync(fd);
+            if (!stat.isFile() || stat.size > 65536 || (stat.mode & 0o077) !== 0)
+                throw new SpongeError('integrity');
+            const marker = JSON.parse(readFileSync(fd, 'utf8'));
+            if (marker.schemaVersion !== 1 ||
+                marker.requestId !== token.attemptId ||
+                marker.ownerIncarnation !== token.ownerIncarnation)
+                throw new SpongeError('integrity');
+        }
+        finally {
+            closeSync(fd);
+        }
+    }
+    /** Retired-generation pruning; see src/indexing/prune.ts. The plan probes rollback inside a rolled-back savepoint. */
+    prunePlan(keep) {
+        const db = this.c.db;
+        return transaction(db, () => {
+            const generations = prune.planPrune(db, keep);
+            return {
+                generations,
+                orphans: prune.orphanVectors(db, generations.filter((g) => g.action === 'prune').map((g) => g.id)),
+                stats: prune.pageStats(db),
+            };
+        });
+    }
+    pruneStep(owner, id, keep) {
+        text(id, 256);
+        const db = this.c.db;
+        return transaction(db, () => {
+            checkOwner(db, owner);
+            if (pendingCleanup(db) || hasGcFence(db))
+                throw new SpongeError('owner-busy');
+            return prune.pruneStep(db, id, keep);
+        });
+    }
+    /** One bounded batch of embeddings no retained generation covers. Returns rows deleted; 0 means done. */
+    gcStep(owner) {
+        const db = this.c.db;
+        return transaction(db, () => {
+            checkOwner(db, owner);
+            if (pendingCleanup(db) || hasGcFence(db))
+                throw new SpongeError('owner-busy');
+            return prune.gcStep(db);
+        });
+    }
+    /** VACUUM cannot run in a transaction: fences are checked under the owner token immediately before it. */
+    vacuum(owner) {
+        const db = this.c.db;
+        const stats = transaction(db, () => {
+            checkOwner(db, owner);
+            if (pendingCleanup(db) || hasGcFence(db))
+                throw new SpongeError('owner-busy');
+            return prune.pageStats(db);
+        });
+        prune.vacuumSpace(this.c.root, stats.bytes);
+        db.exec('VACUUM');
+        db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
+        return prune.pageStats(db);
+    }
+    search(query, options = {}) {
+        return hybridSearch(this.c.db, query, options);
+    }
+}

package/dist/src/store/sync-ready.d.ts

@@ -0,0 +1,6 @@
+import type { DatabaseSync } from 'node:sqlite';
+/**
+ * Migration 9's code step: no absolute paths in synced data, split titles, content-addressed objects. Runs inside the
+ * migration transaction after the pre-migration backup; every phase is total per row. See docs/decisions/content-ids.md.
+ */
+export declare function syncReady(db: DatabaseSync): void;

package/dist/src/store/sync-ready.js

@@ -0,0 +1,121 @@
+import { sha256 } from '../util/hash.js';
+import { chunkDocument, chunkHash } from '../indexing/index.js';
+import { refreshTitle } from './changes.js';
+import { blockId, claimId, isFileLocator, portableLocator, representationId } from './ids.js';
+/**
+ * Migration 9's code step: no absolute paths in synced data, split titles, content-addressed objects. Runs inside the
+ * migration transaction after the pre-migration backup; every phase is total per row. See docs/decisions/content-ids.md.
+ */
+export function syncReady(db) {
+    portableLocators(db);
+    splitTitles(db);
+    rekeyObjects(db);
+}
+const parse = (json) => {
+    try {
+        return JSON.parse(String(json));
+    }
+    catch {
+        return undefined;
+    }
+};
+/** `file:` locators move to `local_sources`; source refs and logged item creations keep only the portable form. */
+function portableLocators(db) {
+    for (const r of db
+        .prepare("SELECT item_id,locator FROM source_refs WHERE locator LIKE 'file:%' ORDER BY rowid")
+        .all()) {
+        db.prepare('INSERT INTO local_sources(item_id,locator) VALUES(?,?) ON CONFLICT(item_id) DO NOTHING').run(r.item_id, r.locator);
+        db.prepare('UPDATE source_refs SET locator=? WHERE item_id=? AND locator=?').run(portableLocator(String(r.locator)), r.item_id, r.locator);
+    }
+    for (const c of db.prepare("SELECT seq,value_json FROM changes WHERE entity='item' AND field='created'").all()) {
+        const v = parse(c.value_json);
+        if (v && typeof v === 'object' && typeof v.locator === 'string' && isFileLocator(v.locator))
+            db.prepare('UPDATE changes SET value_json=? WHERE seq=?').run(JSON.stringify({ ...v, locator: portableLocator(v.locator) }), c.seq);
+    }
+}
+/** Logged `title` writes become `noteTitle` when the Jungle importer left a receipt for them, else `capturedTitle`. */
+function splitTitles(db) {
+    const receipted = db.prepare("SELECT 1 FROM import_receipts WHERE namespace='jungle-title' AND source_key=? AND source_hash=?");
+    for (const c of db
+        .prepare("SELECT seq,id,entity_id FROM changes WHERE entity='item' AND field='title' ORDER BY seq")
+        .all())
+        db.prepare('UPDATE changes SET field=? WHERE seq=?').run(receipted.get(c.entity_id, sha256(String(c.id))) ? 'noteTitle' : 'capturedTitle', c.seq);
+    const last = db.prepare("SELECT value_json FROM changes WHERE entity='item' AND entity_id=? AND field=? ORDER BY seq DESC LIMIT 1");
+    for (const i of db.prepare('SELECT id,title FROM items ORDER BY rowid').all()) {
+        const id = String(i.id), captured = last.get(id, 'capturedTitle'), note = last.get(id, 'noteTitle'), text = (r) => {
+            const v = r && parse(r.value_json);
+            return typeof v === 'string' && Buffer.byteLength(v) <= 4096 ? v : '';
+        };
+        db.prepare('UPDATE items SET title_captured=?,title_note=? WHERE id=?').run(captured ? text(captured) : note ? '' : String(i.title), text(note) || null, id);
+        refreshTitle(db, id);
+    }
+}
+/** Representations, blocks and claims take content ids; index generations follow the representation ids they name. */
+function rekeyObjects(db) {
+    const taken = (table, id) => !!db.prepare(`SELECT 1 FROM ${table} WHERE id=?`).get(id);
+    const reps = new Map();
+    for (const r of db
+        .prepare('SELECT rowid,id,revision_id,kind,mime_type,sha256 FROM representations ORDER BY rowid')
+        .all()) {
+        const next = representationId(String(r.revision_id), String(r.kind), String(r.mime_type), String(r.sha256));
+        // An identical second row keeps its old id rather than failing the migration.
+        if (next === r.id || taken('representations', next))
+            continue;
+        db.prepare('UPDATE representations SET id=? WHERE rowid=?').run(next, r.rowid);
+        reps.set(String(r.id), next);
+    }
+    for (const b of db.prepare('SELECT rowid,id,revision_id,ordinal,kind,text FROM blocks ORDER BY rowid').all()) {
+        const next = blockId(String(b.revision_id), Number(b.ordinal), String(b.kind), String(b.text));
+        if (next !== b.id && !taken('blocks', next))
+            db.prepare('UPDATE blocks SET id=? WHERE rowid=?').run(next, b.rowid);
+    }
+    let revision, ordinal = 0;
+    for (const c of db.prepare('SELECT rowid,* FROM metadata_claims ORDER BY revision_id,rowid').all()) {
+        ordinal = c.revision_id === revision ? ordinal + 1 : 0;
+        revision = c.revision_id;
+        const columns = [
+            c.field,
+            c.value_json,
+            c.source,
+            c.observed_at,
+            c.rule,
+            c.timezone_known,
+            c.observed_json,
+            c.locator,
+            c.selected,
+            c.extraction_version,
+            c.date_precision,
+        ];
+        const next = claimId(String(c.revision_id), ordinal, columns);
+        if (next !== c.id && !taken('metadata_claims', next))
+            db.prepare('UPDATE metadata_claims SET id=? WHERE rowid=?').run(next, c.rowid);
+    }
+    if (!reps.size)
+        return;
+    for (const g of db.prepare('SELECT id,coverage_json FROM index_generations ORDER BY rowid').all()) {
+        const coverage = parse(g.coverage_json);
+        if (!coverage || !Array.isArray(coverage.inputs))
+            continue;
+        const moved = [];
+        for (const input of coverage.inputs) {
+            const next = input && input.kind === 'document' && reps.get(input.representationId);
+            if (!next)
+                continue;
+            const markdown = db.prepare('SELECT markdown FROM representations WHERE id=?').get(next)?.markdown, text = typeof markdown === 'string' && Number.isSafeInteger(input.ordinal)
+                ? chunkDocument(markdown)[input.ordinal]
+                : undefined;
+            const prior = input.sourceId;
+            // A chunk hash that did not verify before stays unverifiable: re-keying never repairs a broken generation.
+            if (text !== undefined && input.derivationHash === chunkHash(input.representationId, input.ordinal, text))
+                input.derivationHash = chunkHash(next, input.ordinal, text);
+            input.representationId = next;
+            input.sourceId = JSON.stringify([input.revisionId, 'document', next, input.ordinal]);
+            moved.push([String(prior), input]);
+        }
+        if (!moved.length)
+            continue;
+        db.prepare('UPDATE index_generations SET coverage_json=? WHERE id=?').run(JSON.stringify(coverage), g.id);
+        for (const [prior, input] of moved)
+            db.prepare('UPDATE vectors SET source_id=?,derivation_hash=? WHERE generation_id=? AND source_id=?').run(input.sourceId, input.derivationHash, g.id, prior);
+    }
+}

package/dist/src/store/types.d.ts

@@ -1,6 +1,6 @@
 import type { MetadataClaim, MetadataWarning } from '../../contracts/index.js';
 export type ItemKind = 'article' | 'pdf' | 'media';
-export type FaultPoint = 'staged' | 'published' | 'transaction' | 'committed';
+export type FaultPoint = 'staged' | 'published' | 'transaction' | 'committed' | 'migration' | 'backup';
 export interface StoreOptions {
     fault?: (point: FaultPoint) => void;
     appVersion?: string;
@@ -84,7 +84,3 @@ export interface ReadableItem {
     metadata: MetadataClaim[];
     metadataWarnings: MetadataWarning[];
 }
-export declare class StoreError extends Error {
-    readonly code: 'invalid-input' | 'integrity' | 'limit-exceeded' | 'schema-newer' | 'owner-busy' | 'cancelled' | 'unavailable';
-    constructor(code: 'invalid-input' | 'integrity' | 'limit-exceeded' | 'schema-newer' | 'owner-busy' | 'cancelled' | 'unavailable', message?: "unavailable" | "cancelled" | "invalid-input" | "integrity" | "limit-exceeded" | "schema-newer" | "owner-busy");
-}

package/dist/src/store/types.js

@@ -1,8 +1 @@
-export class StoreError extends Error {
-    code;
-    constructor(code, message = code) {
-        super(message);
-        this.code = code;
-        this.name = 'StoreError';
-    }
-}
+export {};

package/dist/src/util/canonical.d.ts

@@ -0,0 +1,2 @@
+/** Sorted keys, no whitespace, undefined fields dropped: equal values always give equal bytes. Publish signatures depend on these exact bytes. */
+export declare function canonicalJson(value: unknown, invalid: () => never): string;

package/dist/src/util/canonical.js

@@ -0,0 +1,17 @@
+/** Sorted keys, no whitespace, undefined fields dropped: equal values always give equal bytes. Publish signatures depend on these exact bytes. */
+export function canonicalJson(value, invalid) {
+    if (Array.isArray(value))
+        return '[' + value.map((v) => canonicalJson(v, invalid)).join(',') + ']';
+    if (value && typeof value === 'object')
+        return ('{' +
+            Object.keys(value)
+                .sort()
+                .filter((k) => value[k] !== undefined)
+                .map((k) => JSON.stringify(k) + ':' + canonicalJson(value[k], invalid))
+                .join(',') +
+            '}');
+    if ((typeof value === 'number' && !Number.isSafeInteger(value)) ||
+        (!['string', 'number', 'boolean'].includes(typeof value) && value !== null))
+        invalid();
+    return JSON.stringify(value);
+}

package/dist/src/util/exact.d.ts

@@ -0,0 +1,2 @@
+/** A plain object with exactly these keys, or `invalid()`. */
+export declare function exactKeys(value: unknown, keys: readonly string[], invalid: () => never): Record<string, unknown>;

package/dist/src/util/exact.js

@@ -0,0 +1,9 @@
+/** A plain object with exactly these keys, or `invalid()`. */
+export function exactKeys(value, keys, invalid) {
+    if (!value ||
+        typeof value !== 'object' ||
+        Array.isArray(value) ||
+        Object.keys(value).sort().join(',') !== [...keys].sort().join(','))
+        invalid();
+    return value;
+}

package/dist/src/util/fsync.d.ts

@@ -0,0 +1,4 @@
+/** Make a directory's entry changes (create, rename, unlink) durable. */
+export declare function syncDirectory(path: string): void;
+/** `syncDirectory` without blocking the event loop. */
+export declare function syncDirectoryAsync(path: string): Promise<void>;

package/dist/src/util/fsync.js

@@ -0,0 +1,22 @@
+import { constants, openSync, fsyncSync, closeSync } from 'node:fs';
+import { open } from 'node:fs/promises';
+/** Make a directory's entry changes (create, rename, unlink) durable. */
+export function syncDirectory(path) {
+    const fd = openSync(path, constants.O_RDONLY);
+    try {
+        fsyncSync(fd);
+    }
+    finally {
+        closeSync(fd);
+    }
+}
+/** `syncDirectory` without blocking the event loop. */
+export async function syncDirectoryAsync(path) {
+    const file = await open(path, constants.O_RDONLY);
+    try {
+        await file.sync();
+    }
+    finally {
+        await file.close();
+    }
+}

package/dist/src/util/hash.d.ts

@@ -0,0 +1,4 @@
+/** Lowercase hex sha256 of a string (UTF-8) or bytes. */
+export declare const sha256: (value: string | Uint8Array) => string;
+/** Domain-tagged content id: sha256 of `sponge:<domain>\n` and the canonical JSON of `value`. Equal values give equal ids; distinct domains never collide. */
+export declare const contentId: (domain: string, value: unknown) => string;

package/dist/src/util/hash.js

@@ -0,0 +1,8 @@
+import { createHash } from 'node:crypto';
+import { canonicalJson } from './canonical.js';
+/** Lowercase hex sha256 of a string (UTF-8) or bytes. */
+export const sha256 = (value) => createHash('sha256').update(value).digest('hex');
+/** Domain-tagged content id: sha256 of `sponge:<domain>\n` and the canonical JSON of `value`. Equal values give equal ids; distinct domains never collide. */
+export const contentId = (domain, value) => sha256(`sponge:${domain}\n${canonicalJson(value, () => {
+    throw new TypeError('contentId: not canonical JSON');
+})}`);

package/dist/src/worker/basic.d.ts

@@ -7,4 +7,3 @@ export declare function drainLibrary(store: LibraryStore, options: ArticleOption
     resolveConnection?: (realm: string) => ArticleOptions['connection'];
     onResult?: (result: ArticleOutcome) => void;
 }): Promise<ArticleOutcome[]>;
-export declare const drainArticles: typeof drainLibrary;

package/dist/src/worker/basic.js

@@ -6,22 +6,38 @@ import { drainQueue } from './engine.js';
 /** Content handlers over the durable sequential worker and its process supervisor. */
 export async function drainLibrary(store, options) {
     const handler = async (context) => {
-        const source = store.sourceForJob(context.attempt.jobId);
+        const source = store.queue.source(context.attempt.jobId);
         let connection;
         try {
             connection = options.resolveConnection?.(source?.realm ?? 'public') ?? options.connection;
         }
         catch {
-            store.failAttempt(context.attempt, 'repair-auth');
-            store.settleAttempt(context.attempt);
+            store.queue.fail(context.attempt, 'repair-auth');
+            store.queue.settle(context.attempt);
             return { ...failure(context.attempt.attemptId, 'repair-auth', 'acquisition'), cleanup: 'settled' };
         }
         if (source?.kind === 'media') {
             const { capture: _articleCapture, ...mediaOptions } = options;
-            return ingestMedia(store, context.attempt, { ...mediaOptions, connection, signal: context.signal, runProcess: context.runProcess, checkpoint: context.checkpoint });
+            return ingestMedia(store, context.attempt, {
+                ...mediaOptions,
+                connection,
+                signal: context.signal,
+                runProcess: context.runProcess,
+                checkpoint: context.checkpoint,
+            });
         }
-        return (source?.kind === 'pdf' ? ingestPdf : ingestArticle)(store, context.attempt, { ...options, connection, signal: context.signal, runProcess: context.runProcess, checkpoint: context.checkpoint });
+        return (source?.kind === 'pdf' ? ingestPdf : ingestArticle)(store, context.attempt, {
+            ...options,
+            connection,
+            signal: context.signal,
+            runProcess: context.runProcess,
+            checkpoint: context.checkpoint,
+        });
     };
-    return drainQueue(store, { signal: options.signal, jobId: options.jobId, onResult: options.onResult, handlers: { article: handler, pdf: handler, media: handler } });
+    return drainQueue(store, {
+        signal: options.signal,
+        jobId: options.jobId,
+        onResult: options.onResult,
+        handlers: { article: handler, pdf: handler, media: handler },
+    });
 }
-export const drainArticles = drainLibrary;

package/dist/src/worker/custody.d.ts

@@ -7,7 +7,10 @@ export interface WorkerCustody {
 export declare function acquireWorkerCustody(root: string): Promise<WorkerCustody>;
 /** Receipts are specific to every pre-launch ticket. Missing receipts never mean absent children. */
 export declare function proveAttemptCleanup(ownerDirectory: string, incarnation: string, attemptId: string): Promise<boolean>;
-export declare function recoverWorker(store: LibraryStore): Promise<{
+export type RecoveryFault = 'guarded' | 'recovered';
+export declare function recoverWorker(store: LibraryStore, options?: {
+    fault?: (point: RecoveryFault) => void;
+}): Promise<{
     recovered: boolean;
     attempts: number;
 }>;