@hraness/sponge 0.2.5 → 0.3.0

package/dist/src/search/hybrid.d.ts

@@ -1,6 +1,6 @@
 import type { DatabaseSync } from 'node:sqlite';
-import { type SearchGeneration, type VectorHit } from './vector.js';
-export interface SearchHit {
+import { type SearchGeneration } from './vector.js';
+interface SearchHit {
     itemId: string;
     revisionId: string;
     title: string;
@@ -24,12 +24,6 @@ export type QueryEmbedder = (query: string, pin: SearchGeneration, signal: Abort
     fingerprint: string;
     vector: readonly number[];
 }>;
-/** Reciprocal rank fusion after per-channel item collapsing; chunks cannot amplify rank. */
-export declare function fuse(lexicalHits: {
-    itemId: string;
-    revisionId: string;
-    title: string;
-}[], semanticHits: VectorHit[], limit: number): SearchHit[];
 export declare function hybridSearch(db: DatabaseSync, query: string, options?: {
     profileId?: string;
     limit?: number;
@@ -37,3 +31,4 @@ export declare function hybridSearch(db: DatabaseSync, query: string, options?:
     embed?: QueryEmbedder;
     signal?: AbortSignal;
 }): Promise<SearchResult>;
+export {};

package/dist/src/search/hybrid.js

@@ -1,10 +1,14 @@
 import { chunkDocument } from '../indexing/index.js';
 import { lexicalSearch } from './lexical.js';
 import { pinSearchGeneration, vectorSearch } from './vector.js';
+import { SpongeError } from '../errors.js';
 /** Reciprocal rank fusion after per-channel item collapsing; chunks cannot amplify rank. */
-export function fuse(lexicalHits, semanticHits, limit) {
+function fuse(lexicalHits, semanticHits, limit) {
     const hits = new Map();
-    for (const [method, rows] of [['lexical', lexicalHits], ['semantic', semanticHits]]) {
+    for (const [method, rows] of [
+        ['lexical', lexicalHits],
+        ['semantic', semanticHits],
+    ]) {
         const seen = new Set();
         let rank = 0;
         for (const row of rows) {
@@ -13,7 +17,14 @@ export function fuse(lexicalHits, semanticHits, limit) {
             seen.add(row.itemId);
             rank++;
             const previous = hits.get(row.itemId), vector = method === 'semantic' ? row : undefined;
-            const anchor = vector ? { kind: vector.source.kind === 'image' ? 'image' : 'document', representationId: vector.source.representationId, chunk: vector.source.ordinal, assetSha256: vector.source.assetSha256 } : { kind: 'document' };
+            const anchor = vector
+                ? {
+                    kind: vector.source.kind === 'image' ? 'image' : 'document',
+                    representationId: vector.source.representationId,
+                    chunk: vector.source.ordinal,
+                    assetSha256: vector.source.assetSha256,
+                }
+                : { kind: 'document' };
             if (previous) {
                 previous.score += 1 / (60 + rank);
                 previous.methods.push(method);
@@ -21,7 +32,14 @@ export function fuse(lexicalHits, semanticHits, limit) {
                     previous.anchor = anchor;
             }
             else
-                hits.set(row.itemId, { itemId: row.itemId, revisionId: row.revisionId, title: row.title, score: 1 / (60 + rank), methods: [method], anchor });
+                hits.set(row.itemId, {
+                    itemId: row.itemId,
+                    revisionId: row.revisionId,
+                    title: row.title,
+                    score: 1 / (60 + rank),
+                    methods: [method],
+                    anchor,
+                });
         }
     }
     return [...hits.values()].sort((a, b) => b.score - a.score || a.itemId.localeCompare(b.itemId)).slice(0, limit);
@@ -31,7 +49,9 @@ function projectAnchors(db, hits) {
     for (const hit of hits) {
         if (hit.anchor.kind !== 'document' || !hit.anchor.representationId || hit.anchor.chunk === undefined)
             continue;
-        const row = db.prepare('SELECT p.markdown,i.kind,r.extractor_name AS extractor FROM representations p JOIN revisions r ON r.id=p.revision_id JOIN items i ON i.id=r.item_id WHERE p.id=? AND p.revision_id=?').get(hit.anchor.representationId, hit.revisionId);
+        const row = db
+            .prepare('SELECT p.markdown,i.kind,r.extractor_name AS extractor FROM representations p JOIN revisions r ON r.id=p.revision_id JOIN items i ON i.id=r.item_id WHERE p.id=? AND p.revision_id=?')
+            .get(hit.anchor.representationId, hit.revisionId);
         if (!row || typeof row.markdown !== 'string')
             continue;
         const chunks = chunkDocument(row.markdown), chunk = chunks[hit.anchor.chunk];
@@ -51,13 +71,13 @@ function projectAnchors(db, hits) {
         const end = start + chunk.length;
         if (row.kind === 'pdf') {
             const markers = [...row.markdown.matchAll(/^## Page ([1-9][0-9]*)$/gm)];
-            const marker = markers.filter(m => m.index <= start).at(-1) ?? markers.find(m => m.index < end);
+            const marker = markers.filter((m) => m.index <= start).at(-1) ?? markers.find((m) => m.index < end);
             if (marker)
                 hit.anchor.page = Number(marker[1]);
         }
         else if (row.extractor === 'sponge-media') {
             const markers = [...row.markdown.matchAll(/^\[(\d+):(\d{2}):(\d{2})\] /gm)];
-            const marker = markers.filter(m => m.index <= start).at(-1) ?? markers.find(m => m.index < end);
+            const marker = markers.filter((m) => m.index <= start).at(-1) ?? markers.find((m) => m.index < end);
             if (marker)
                 hit.anchor.startMs = (Number(marker[1]) * 3600 + Number(marker[2]) * 60 + Number(marker[3])) * 1000;
         }
@@ -65,8 +85,15 @@ function projectAnchors(db, hits) {
 }
 export async function hybridSearch(db, query, options = {}) {
     const limit = options.limit ?? 20, timeout = options.semanticTimeoutMs ?? 10000;
-    if (!query.trim() || Buffer.byteLength(query) > 4096 || !Number.isSafeInteger(limit) || limit < 1 || limit > 100 || !Number.isSafeInteger(timeout) || timeout < 1 || timeout > 120000)
-        throw Error('invalid-search-query');
+    if (!query.trim() ||
+        Buffer.byteLength(query) > 4096 ||
+        !Number.isSafeInteger(limit) ||
+        limit < 1 ||
+        limit > 100 ||
+        !Number.isSafeInteger(timeout) ||
+        timeout < 1 ||
+        timeout > 120000)
+        throw new SpongeError('invalid-search-query');
     const pin = pinSearchGeneration(db, options.profileId ?? 'default'), warnings = [];
     let encoded;
     if (pin && options.embed) {
@@ -75,7 +102,10 @@ export async function hybridSearch(db, query, options = {}) {
         if (options.signal?.aborted)
             abort();
         let timedOut = false;
-        const timer = setTimeout(() => { timedOut = true; controller.abort(); }, timeout);
+        const timer = setTimeout(() => {
+            timedOut = true;
+            controller.abort();
+        }, timeout);
         try {
             const result = await options.embed(query, pin, controller.signal);
             if (timedOut)
@@ -106,7 +136,11 @@ export async function hybridSearch(db, query, options = {}) {
                 warnings.push('semantic-unavailable');
             }
         }
-        const result = { hits: fuse(lexicalSearch(db, query, Math.min(500, limit * 5)), semantic, limit), warnings, generation: pin };
+        const result = {
+            hits: fuse(lexicalSearch(db, query, Math.min(500, limit * 5)), semantic, limit),
+            warnings,
+            generation: pin,
+        };
         projectAnchors(db, result.hits);
         db.exec('COMMIT');
         return result;

package/dist/src/search/lexical.js

@@ -1,6 +1,20 @@
 /** Contentless FTS rows are keyed by readable_fts_rows.id; items.rowid is not VACUUM-stable. Callers own the transaction. */
-export function indexReadable(db, itemId, revisionId, title, markdown) { unindexReadable(db, itemId); const row = db.prepare('INSERT INTO readable_fts_rows(item_id,revision_id) VALUES(?,?) RETURNING id').get(itemId, revisionId); db.prepare('INSERT INTO readable_fts(rowid,title,markdown) VALUES(?,?,?)').run(row.id, title, markdown); }
-export function unindexReadable(db, itemId) { const row = db.prepare('DELETE FROM readable_fts_rows WHERE item_id=? RETURNING id').get(itemId); if (row)
-    db.prepare('DELETE FROM readable_fts WHERE rowid=?').run(row.id); }
+export function indexReadable(db, itemId, revisionId, title, markdown) {
+    unindexReadable(db, itemId);
+    const row = db
+        .prepare('INSERT INTO readable_fts_rows(item_id,revision_id) VALUES(?,?) RETURNING id')
+        .get(itemId, revisionId);
+    db.prepare('INSERT INTO readable_fts(rowid,title,markdown) VALUES(?,?,?)').run(row.id, title, markdown);
+}
+export function unindexReadable(db, itemId) {
+    const row = db.prepare('DELETE FROM readable_fts_rows WHERE item_id=? RETURNING id').get(itemId);
+    if (row)
+        db.prepare('DELETE FROM readable_fts WHERE rowid=?').run(row.id);
+}
 /** Literal phrase, bm25 rank then item id; excludes tombstones and superseded revisions. */
-export function lexicalSearch(db, query, limit) { const literal = '"' + query.replaceAll('"', '""') + '"'; return db.prepare('SELECT i.id AS itemId,i.title,i.current_revision AS revisionId FROM readable_fts f JOIN readable_fts_rows m ON m.id=f.rowid JOIN items i ON i.id=m.item_id WHERE readable_fts MATCH ? AND i.deleted_at IS NULL AND i.current_revision=m.revision_id ORDER BY rank,i.id LIMIT ?').all(literal, limit); }
+export function lexicalSearch(db, query, limit) {
+    const literal = '"' + query.replaceAll('"', '""') + '"';
+    return db
+        .prepare('SELECT i.id AS itemId,i.title,i.current_revision AS revisionId FROM readable_fts f JOIN readable_fts_rows m ON m.id=f.rowid JOIN items i ON i.id=m.item_id WHERE readable_fts MATCH ? AND i.deleted_at IS NULL AND i.current_revision=m.revision_id ORDER BY rank,i.id LIMIT ?')
+        .all(literal, limit);
+}

package/dist/src/search/query.js

@@ -2,39 +2,62 @@ import { randomUUID } from 'node:crypto';
 import { join } from 'node:path';
 import { rm } from 'node:fs/promises';
 import { acquireWorkerCustody, proveAttemptCleanup } from '../worker/custody.js';
-import { directory, writeJournal, readJournal, syncDir } from '../worker/journal.js';
+import { directory, writeJournal, readJournal } from '../worker/journal.js';
+import { syncDirectoryAsync } from '../util/fsync.js';
 import { supervisedRunner } from '../worker/supervisor.js';
-import { embedBatch, embeddingFingerprint, embeddingTextBytes, resolveIndexSpace, projectVector, encodeEmbeddingRequest, EmbeddingError } from '../inference/embedding/index.js';
+import { embedBatch, embeddingFingerprint, embeddingTextBytes, resolveIndexSpace, projectVector, encodeEmbeddingRequest, } from '../inference/embedding/index.js';
+import { SpongeError, codeOf } from '../errors.js';
 /** One custody-held helper run. Mutates coordination journals only, never library content or job tables. */
 async function custodyEmbed(store, runtime, inputs, signal, options) {
     const custody = await acquireWorkerCustody(store.root), attemptId = randomUUID();
     let scratchRoot, cleanupUnknown = false;
     try {
         const attempts = await directory(join(custody.directory, 'attempts', attemptId));
-        await writeJournal(join(attempts, 'query.json'), { schemaVersion: 1, kind: 'query', attemptId, ownerIncarnation: custody.incarnation });
+        await writeJournal(join(attempts, 'query.json'), {
+            schemaVersion: 1,
+            kind: 'query',
+            attemptId,
+            ownerIncarnation: custody.incarnation,
+        });
         scratchRoot = await directory(join(store.root, 'staging', 'query-' + attemptId));
-        await writeJournal(join(scratchRoot, 'ownership.json'), { schemaVersion: 1, kind: 'query', requestId: attemptId, ownerIncarnation: custody.incarnation });
+        await writeJournal(join(scratchRoot, 'ownership.json'), {
+            schemaVersion: 1,
+            kind: 'query',
+            requestId: attemptId,
+            ownerIncarnation: custody.incarnation,
+        });
         await options.onReady?.({ attemptId, ownerDirectory: custody.directory, scratchRoot });
         const expected = encodeEmbeddingRequest(inputs).inputHashes;
-        const result = await (options.embed ?? embedBatch)(inputs, { runtime, scratchRoot, signal, runProcess: supervisedRunner(custody.directory, custody.incarnation, attemptId) });
-        if (result.fingerprint !== embeddingFingerprint(runtime.helperSha256) || result.inputHashes.length !== expected.length || result.inputHashes.some((h, i) => h !== expected[i]) || result.vectors.length !== expected.length)
-            throw new EmbeddingError('integrity');
+        const result = await (options.embed ?? embedBatch)(inputs, {
+            runtime,
+            scratchRoot,
+            signal,
+            runProcess: supervisedRunner(custody.directory, custody.incarnation, attemptId),
+        });
+        if (result.fingerprint !== embeddingFingerprint(runtime.helperSha256) ||
+            result.inputHashes.length !== expected.length ||
+            result.inputHashes.some((h, i) => h !== expected[i]) ||
+            result.vectors.length !== expected.length)
+            throw new SpongeError('integrity');
         return result.vectors;
     }
     catch (error) {
-        cleanupUnknown = error.code === 'cleanup-required';
+        cleanupUnknown = codeOf(error) === 'cleanup-required';
         throw error;
     }
     finally {
-        const proved = !cleanupUnknown && await proveAttemptCleanup(custody.directory, custody.incarnation, attemptId);
+        const proved = !cleanupUnknown && (await proveAttemptCleanup(custody.directory, custody.incarnation, attemptId));
         if (!proved)
-            throw new EmbeddingError('cleanup-required');
+            throw new SpongeError('cleanup-required');
         if (scratchRoot) {
             const marker = await readJournal(join(scratchRoot, 'ownership.json'));
-            if (marker.schemaVersion !== 1 || marker.kind !== 'query' || marker.requestId !== attemptId || marker.ownerIncarnation !== custody.incarnation)
-                throw new EmbeddingError('cleanup-required');
+            if (marker.schemaVersion !== 1 ||
+                marker.kind !== 'query' ||
+                marker.requestId !== attemptId ||
+                marker.ownerIncarnation !== custody.incarnation)
+                throw new SpongeError('cleanup-required');
             await rm(scratchRoot, { recursive: true });
-            await syncDir(join(store.root, 'staging'));
+            await syncDirectoryAsync(join(store.root, 'staging'));
         }
         await custody.release();
     }
@@ -43,24 +66,32 @@ async function custodyEmbed(store, runtime, inputs, signal, options) {
 export function createQueryEmbedder(store, runtime, options = {}) {
     return async (query, pin, signal) => {
         if (signal.aborted)
-            throw new EmbeddingError('cancelled');
+            throw new SpongeError('cancelled');
         const space = resolveIndexSpace(runtime.helperSha256, pin.fingerprint);
         if (!space || space.dimensions !== pin.dimensions)
-            throw new EmbeddingError('integrity');
+            throw new SpongeError('integrity');
         const [vector] = await custodyEmbed(store, runtime, [{ kind: 'query', text: query }], signal, options);
         return { fingerprint: pin.fingerprint, vector: projectVector(vector, space.dimensions) };
     };
 }
 /** Unit 768-dimension document vectors for short texts, 16 per helper run; text is clipped to the helper's byte budget on a character boundary. */
 export function createTextEmbedder(store, runtime, options = {}) {
-    const room = embeddingTextBytes - Buffer.byteLength('title: none | text: '), clip = (t) => { let out = ''; for (const ch of t) {
-        if (Buffer.byteLength(out + ch) > room)
-            break;
-        out += ch;
-    } return out; };
-    return async (texts, signal) => { const out = []; for (let i = 0; i < texts.length; i += 16) {
-        if (signal.aborted)
-            throw new EmbeddingError('cancelled');
-        out.push(...await custodyEmbed(store, runtime, texts.slice(i, i + 16).map(t => ({ kind: 'document', text: clip(t) })), signal, options));
-    } return out; };
+    const room = embeddingTextBytes - Buffer.byteLength('title: none | text: '), clip = (t) => {
+        let out = '';
+        for (const ch of t) {
+            if (Buffer.byteLength(out + ch) > room)
+                break;
+            out += ch;
+        }
+        return out;
+    };
+    return async (texts, signal) => {
+        const out = [];
+        for (let i = 0; i < texts.length; i += 16) {
+            if (signal.aborted)
+                throw new SpongeError('cancelled');
+            out.push(...(await custodyEmbed(store, runtime, texts.slice(i, i + 16).map((t) => ({ kind: 'document', text: clip(t) })), signal, options)));
+        }
+        return out;
+    };
 }

package/dist/src/search/vector.js

@@ -1,22 +1,49 @@
-import { encodeVector, spaceEncoding } from '../indexing/index.js';
+import { encodeVector, sourceInput, spaceEncoding } from '../indexing/index.js';
+import { SpongeError } from '../errors.js';
 export function pinSearchGeneration(db, profileId) {
-    const row = db.prepare('SELECT g.id,g.fingerprint,s.dimensions FROM active_indexes a JOIN index_generations g ON g.id=a.generation_id JOIN embedding_spaces s ON s.fingerprint=g.fingerprint WHERE a.profile_id=?').get(profileId);
+    const row = db
+        .prepare('SELECT g.id,g.fingerprint,s.dimensions FROM active_indexes a JOIN index_generations g ON g.id=a.generation_id JOIN embedding_spaces s ON s.fingerprint=g.fingerprint WHERE a.profile_id=?')
+        .get(profileId);
     if (!row)
         return undefined;
     return { id: String(row.id), fingerprint: String(row.fingerprint), dimensions: Number(row.dimensions), profileId };
 }
 /** Native scalar scan keeps vectors inside SQLite; SQL bounds materialized result rows. */
 export function vectorSearch(db, pin, query, limit = 50) {
-    if (!Number.isSafeInteger(limit) || limit < 1 || limit > 500 || query.fingerprint !== pin.fingerprint || query.vector.length !== pin.dimensions || query.vector.some(n => !Number.isFinite(n)) || Math.abs(Math.hypot(...query.vector) - 1) > .001)
-        throw Error('invalid-search-vector');
+    if (!Number.isSafeInteger(limit) ||
+        limit < 1 ||
+        limit > 500 ||
+        query.fingerprint !== pin.fingerprint ||
+        query.vector.length !== pin.dimensions ||
+        query.vector.some((n) => !Number.isFinite(n)) ||
+        Math.abs(Math.hypot(...query.vector) - 1) > 0.001)
+        throw new SpongeError('invalid-search-vector');
     if (db.prepare('SELECT vec_version() AS version').get()?.version !== 'v0.1.9')
-        throw Error('unsupported-vector-runtime');
-    const generation = db.prepare('SELECT g.fingerprint,s.dimensions,s.identity_json FROM index_generations g JOIN embedding_spaces s ON s.fingerprint=g.fingerprint WHERE g.id=?').get(pin.id);
+        throw new SpongeError('unsupported-vector-runtime');
+    const generation = db
+        .prepare('SELECT g.fingerprint,s.dimensions,s.identity_json FROM index_generations g JOIN embedding_spaces s ON s.fingerprint=g.fingerprint WHERE g.id=?')
+        .get(pin.id);
     if (!generation || generation.fingerprint !== pin.fingerprint || Number(generation.dimensions) !== pin.dimensions)
-        throw Error('search-generation-missing');
+        throw new SpongeError('search-generation-missing');
     // int8 spaces quantize the query the same way; cosine over codes is the cosine of the dequantized vectors because per-vector scales cancel.
     const encoding = spaceEncoding(JSON.parse(String(generation.identity_json))), bytes = encodeVector(query.vector, encoding), distance = encoding === 'int8' ? 'vec_distance_cosine(vec_int8(v.vector),vec_int8(?))' : 'vec_distance_cosine(v.vector,?)';
-    const rows = db.prepare('SELECT v.source_id,v.revision_id,v.derivation_hash,v.asset_sha256,i.id AS item_id,i.title,' + distance + ' AS distance FROM vectors v JOIN revisions r ON r.id=v.revision_id JOIN items i ON i.id=r.item_id WHERE v.generation_id=? AND i.deleted_at IS NULL AND i.current_revision=v.revision_id ORDER BY distance,v.source_id LIMIT ?').all(bytes, pin.id, limit);
-    return rows.map(row => { const descriptor = JSON.parse(String(row.source_id)), distance = Number(row.distance); if (!Array.isArray(descriptor) || descriptor[0] !== row.revision_id || !['document', 'image'].includes(String(descriptor[1])) || typeof descriptor[2] !== 'string' || !Number.isFinite(distance))
-        throw Error('invalid-search-index'); const source = { sourceId: String(row.source_id), itemId: String(row.item_id), revisionId: String(row.revision_id), derivationHash: String(row.derivation_hash), kind: descriptor[1], ...(descriptor[1] === 'image' ? { assetSha256: String(row.asset_sha256) } : { representationId: descriptor[2], ordinal: Number(descriptor[3]) }) }; return { itemId: String(row.item_id), revisionId: String(row.revision_id), title: String(row.title), source, distance }; });
+    // Coverage rows name content; the space stores one vector per content hash, shared by every item and generation that has it.
+    const rows = db
+        .prepare('SELECT c.source_id,c.revision_id,c.content_hash,i.id AS item_id,i.title,' +
+        distance +
+        ' AS distance FROM index_inputs c JOIN embeddings v ON v.fingerprint=? AND v.content_hash=c.content_hash JOIN revisions r ON r.id=c.revision_id JOIN items i ON i.id=r.item_id WHERE c.generation_id=? AND i.deleted_at IS NULL AND i.current_revision=c.revision_id ORDER BY distance,c.source_id LIMIT ?')
+        .all(bytes, pin.fingerprint, pin.id, limit);
+    return rows.map((row) => {
+        const distance = Number(row.distance);
+        let source;
+        try {
+            source = sourceInput(row);
+        }
+        catch {
+            throw new SpongeError('invalid-search-index');
+        }
+        if (!Number.isFinite(distance))
+            throw new SpongeError('invalid-search-index');
+        return { itemId: source.itemId, revisionId: source.revisionId, title: String(row.title), source, distance };
+    });
 }

package/dist/src/service/launch-agent.d.ts

@@ -1,6 +1,6 @@
 import type { LibraryStore } from '../store/index.js';
 import { type ProcessIdentity } from '../worker/identity.js';
-export interface LaunchResult {
+interface LaunchResult {
     code: number;
     stdout: string;
 }
@@ -22,7 +22,7 @@ interface Pin {
     sha256: string;
 }
 interface Installation {
-    language: "en" | null;
+    language: 'en' | null;
     schemaVersion: 1;
     id: string;
     label: string;
@@ -37,7 +37,9 @@ interface Installation {
     launcherHash: string;
     pins: Pin[];
 }
-export declare function agentPlist(record: Pick<Installation, 'label' | 'node' | 'launcher'>): string;
+export declare function agentPlist(record: Pick<Installation, 'label' | 'node' | 'launcher'> & {
+    log: string;
+}): string;
 export declare function serviceStatus(store: LibraryStore, env: ServiceEnvironment): Promise<Record<string, unknown>>;
 export declare function installService(store: LibraryStore, env: ServiceEnvironment, options?: {
     language?: 'en';