@hraness/sponge 0.2.3 → 0.2.5

package/dist/src/search/query.js

@@ -4,44 +4,63 @@ import { rm } from 'node:fs/promises';
 import { acquireWorkerCustody, proveAttemptCleanup } from '../worker/custody.js';
 import { directory, writeJournal, readJournal, syncDir } from '../worker/journal.js';
 import { supervisedRunner } from '../worker/supervisor.js';
-import { embedBatch, embeddingFingerprint, encodeEmbeddingRequest, EmbeddingError } from '../inference/embedding/index.js';
+import { embedBatch, embeddingFingerprint, embeddingTextBytes, resolveIndexSpace, projectVector, encodeEmbeddingRequest, EmbeddingError } from '../inference/embedding/index.js';
+/** One custody-held helper run. Mutates coordination journals only, never library content or job tables. */
+async function custodyEmbed(store, runtime, inputs, signal, options) {
+    const custody = await acquireWorkerCustody(store.root), attemptId = randomUUID();
+    let scratchRoot, cleanupUnknown = false;
+    try {
+        const attempts = await directory(join(custody.directory, 'attempts', attemptId));
+        await writeJournal(join(attempts, 'query.json'), { schemaVersion: 1, kind: 'query', attemptId, ownerIncarnation: custody.incarnation });
+        scratchRoot = await directory(join(store.root, 'staging', 'query-' + attemptId));
+        await writeJournal(join(scratchRoot, 'ownership.json'), { schemaVersion: 1, kind: 'query', requestId: attemptId, ownerIncarnation: custody.incarnation });
+        await options.onReady?.({ attemptId, ownerDirectory: custody.directory, scratchRoot });
+        const expected = encodeEmbeddingRequest(inputs).inputHashes;
+        const result = await (options.embed ?? embedBatch)(inputs, { runtime, scratchRoot, signal, runProcess: supervisedRunner(custody.directory, custody.incarnation, attemptId) });
+        if (result.fingerprint !== embeddingFingerprint(runtime.helperSha256) || result.inputHashes.length !== expected.length || result.inputHashes.some((h, i) => h !== expected[i]) || result.vectors.length !== expected.length)
+            throw new EmbeddingError('integrity');
+        return result.vectors;
+    }
+    catch (error) {
+        cleanupUnknown = error.code === 'cleanup-required';
+        throw error;
+    }
+    finally {
+        const proved = !cleanupUnknown && await proveAttemptCleanup(custody.directory, custody.incarnation, attemptId);
+        if (!proved)
+            throw new EmbeddingError('cleanup-required');
+        if (scratchRoot) {
+            const marker = await readJournal(join(scratchRoot, 'ownership.json'));
+            if (marker.schemaVersion !== 1 || marker.kind !== 'query' || marker.requestId !== attemptId || marker.ownerIncarnation !== custody.incarnation)
+                throw new EmbeddingError('cleanup-required');
+            await rm(scratchRoot, { recursive: true });
+            await syncDir(join(store.root, 'staging'));
+        }
+        await custody.release();
+    }
+}
 /** Queries mutate coordination journals only, never library content or job tables. */
 export function createQueryEmbedder(store, runtime, options = {}) {
     return async (query, pin, signal) => {
         if (signal.aborted)
             throw new EmbeddingError('cancelled');
-        if (pin.fingerprint !== embeddingFingerprint(runtime.helperSha256))
+        const space = resolveIndexSpace(runtime.helperSha256, pin.fingerprint);
+        if (!space || space.dimensions !== pin.dimensions)
             throw new EmbeddingError('integrity');
-        const custody = await acquireWorkerCustody(store.root), attemptId = randomUUID();
-        let scratchRoot, cleanupUnknown = false;
-        try {
-            const attempts = await directory(join(custody.directory, 'attempts', attemptId));
-            await writeJournal(join(attempts, 'query.json'), { schemaVersion: 1, kind: 'query', attemptId, ownerIncarnation: custody.incarnation });
-            scratchRoot = await directory(join(store.root, 'staging', 'query-' + attemptId));
-            await writeJournal(join(scratchRoot, 'ownership.json'), { schemaVersion: 1, kind: 'query', requestId: attemptId, ownerIncarnation: custody.incarnation });
-            await options.onReady?.({ attemptId, ownerDirectory: custody.directory, scratchRoot });
-            const inputs = [{ kind: 'query', text: query }], expected = encodeEmbeddingRequest(inputs).inputHashes[0];
-            const result = await (options.embed ?? embedBatch)(inputs, { runtime, scratchRoot, signal, runProcess: supervisedRunner(custody.directory, custody.incarnation, attemptId) });
-            if (result.fingerprint !== pin.fingerprint || result.inputHashes.length !== 1 || result.inputHashes[0] !== expected || result.vectors.length !== 1)
-                throw new EmbeddingError('integrity');
-            return { fingerprint: result.fingerprint, vector: result.vectors[0] };
-        }
-        catch (error) {
-            cleanupUnknown = error.code === 'cleanup-required';
-            throw error;
-        }
-        finally {
-            const proved = !cleanupUnknown && await proveAttemptCleanup(custody.directory, custody.incarnation, attemptId);
-            if (!proved)
-                throw new EmbeddingError('cleanup-required');
-            if (scratchRoot) {
-                const marker = await readJournal(join(scratchRoot, 'ownership.json'));
-                if (marker.schemaVersion !== 1 || marker.kind !== 'query' || marker.requestId !== attemptId || marker.ownerIncarnation !== custody.incarnation)
-                    throw new EmbeddingError('cleanup-required');
-                await rm(scratchRoot, { recursive: true });
-                await syncDir(join(store.root, 'staging'));
-            }
-            await custody.release();
-        }
+        const [vector] = await custodyEmbed(store, runtime, [{ kind: 'query', text: query }], signal, options);
+        return { fingerprint: pin.fingerprint, vector: projectVector(vector, space.dimensions) };
     };
 }
+/** Unit 768-dimension document vectors for short texts, 16 per helper run; text is clipped to the helper's byte budget on a character boundary. */
+export function createTextEmbedder(store, runtime, options = {}) {
+    const room = embeddingTextBytes - Buffer.byteLength('title: none | text: '), clip = (t) => { let out = ''; for (const ch of t) {
+        if (Buffer.byteLength(out + ch) > room)
+            break;
+        out += ch;
+    } return out; };
+    return async (texts, signal) => { const out = []; for (let i = 0; i < texts.length; i += 16) {
+        if (signal.aborted)
+            throw new EmbeddingError('cancelled');
+        out.push(...await custodyEmbed(store, runtime, texts.slice(i, i + 16).map(t => ({ kind: 'document', text: clip(t) })), signal, options));
+    } return out; };
+}

package/dist/src/search/vector.d.ts

@@ -1,5 +1,5 @@
 import type { DatabaseSync } from 'node:sqlite';
-import type { IndexInput } from '../indexing/index.js';
+import { type IndexInput } from '../indexing/index.js';
 export interface SearchGeneration {
     id: string;
     fingerprint: string;

package/dist/src/search/vector.js

@@ -1,3 +1,4 @@
+import { encodeVector, spaceEncoding } from '../indexing/index.js';
 export function pinSearchGeneration(db, profileId) {
     const row = db.prepare('SELECT g.id,g.fingerprint,s.dimensions FROM active_indexes a JOIN index_generations g ON g.id=a.generation_id JOIN embedding_spaces s ON s.fingerprint=g.fingerprint WHERE a.profile_id=?').get(profileId);
     if (!row)
@@ -10,12 +11,12 @@ export function vectorSearch(db, pin, query, limit = 50) {
         throw Error('invalid-search-vector');
     if (db.prepare('SELECT vec_version() AS version').get()?.version !== 'v0.1.9')
         throw Error('unsupported-vector-runtime');
-    const generation = db.prepare('SELECT fingerprint FROM index_generations WHERE id=?').get(pin.id);
-    if (!generation || generation.fingerprint !== pin.fingerprint)
+    const generation = db.prepare('SELECT g.fingerprint,s.dimensions,s.identity_json FROM index_generations g JOIN embedding_spaces s ON s.fingerprint=g.fingerprint WHERE g.id=?').get(pin.id);
+    if (!generation || generation.fingerprint !== pin.fingerprint || Number(generation.dimensions) !== pin.dimensions)
         throw Error('search-generation-missing');
-    const bytes = Buffer.alloc(query.vector.length * 4);
-    query.vector.forEach((v, i) => bytes.writeFloatLE(v, i * 4));
-    const rows = db.prepare('SELECT v.source_id,v.revision_id,v.derivation_hash,v.asset_sha256,i.id AS item_id,i.title,vec_distance_cosine(v.vector,?) AS distance FROM vectors v JOIN revisions r ON r.id=v.revision_id JOIN items i ON i.id=r.item_id WHERE v.generation_id=? AND i.deleted_at IS NULL AND i.current_revision=v.revision_id ORDER BY distance,v.source_id LIMIT ?').all(bytes, pin.id, limit);
+    // int8 spaces quantize the query the same way; cosine over codes is the cosine of the dequantized vectors because per-vector scales cancel.
+    const encoding = spaceEncoding(JSON.parse(String(generation.identity_json))), bytes = encodeVector(query.vector, encoding), distance = encoding === 'int8' ? 'vec_distance_cosine(vec_int8(v.vector),vec_int8(?))' : 'vec_distance_cosine(v.vector,?)';
+    const rows = db.prepare('SELECT v.source_id,v.revision_id,v.derivation_hash,v.asset_sha256,i.id AS item_id,i.title,' + distance + ' AS distance FROM vectors v JOIN revisions r ON r.id=v.revision_id JOIN items i ON i.id=r.item_id WHERE v.generation_id=? AND i.deleted_at IS NULL AND i.current_revision=v.revision_id ORDER BY distance,v.source_id LIMIT ?').all(bytes, pin.id, limit);
     return rows.map(row => { const descriptor = JSON.parse(String(row.source_id)), distance = Number(row.distance); if (!Array.isArray(descriptor) || descriptor[0] !== row.revision_id || !['document', 'image'].includes(String(descriptor[1])) || typeof descriptor[2] !== 'string' || !Number.isFinite(distance))
         throw Error('invalid-search-index'); const source = { sourceId: String(row.source_id), itemId: String(row.item_id), revisionId: String(row.revision_id), derivationHash: String(row.derivation_hash), kind: descriptor[1], ...(descriptor[1] === 'image' ? { assetSha256: String(row.asset_sha256) } : { representationId: descriptor[2], ordinal: Number(descriptor[3]) }) }; return { itemId: String(row.item_id), revisionId: String(row.revision_id), title: String(row.title), source, distance }; });
 }

package/dist/src/store/changes.d.ts

@@ -0,0 +1,42 @@
+import type { DatabaseSync } from 'node:sqlite';
+type Row = Record<string, any>;
+export type Entity = 'item' | 'annotation' | 'reading' | 'derivation';
+/** One field-level write to user state. The sync contract: replaying every change in order rebuilds user state. */
+export interface Change {
+    entity: Entity;
+    entityId: string;
+    field: string;
+    value: unknown;
+}
+export interface LoggedChange extends Change {
+    seq: number;
+    id: string;
+    deviceId: string;
+    hlc: string;
+}
+export type ReviewStatus = 'accepted' | 'rejected';
+/** Bound on one change's JSON; imported note bodies (64 KiB) plus anchors fit with room for escaping. */
+export declare const maxChangeBytes = 262144;
+/** Append without applying: for state that already exists (seeding). Callers hold the write transaction. */
+export declare function append(db: DatabaseSync, change: Change): void;
+/** The only write path for user state: tick the clock, apply the row update, append the change. */
+export declare function mutate(db: DatabaseSync, entity: Entity, entityId: string, field: string, value: unknown): void;
+export declare function listChanges(db: DatabaseSync, after?: number, limit?: number): LoggedChange[];
+/** Deterministic source-ref id, so replayed item creation yields the same row on every device. */
+export declare const sourceRefId: (itemId: string) => string;
+/**
+ * Review JSON in one canonical shape, whatever order its paths were written in. A digest is reviewed per field;
+ * an editorial derivation (a reviewed source note kept whole) has one path, `review.note`.
+ */
+export declare function setReviewPath(review: Row, output: Row, path: string, value: ReviewStatus | null, kind?: string): Row;
+/**
+ * Map one change onto its row. Table-driven: every user-state field Sponge writes is listed here,
+ * and nothing else writes those columns. Objects (revisions, derivation outputs) must already exist.
+ */
+export declare function applyChange(db: DatabaseSync, change: Change): void;
+/**
+ * Seed the log with user state written before schema 5, so another device can rebuild it.
+ * Idempotent: a field that already has a change is never seeded again. Rows already exist, so this appends only.
+ */
+export declare function seedChanges(db: DatabaseSync): number;
+export {};

package/dist/src/store/changes.js

@@ -0,0 +1,178 @@
+import { randomUUID } from 'node:crypto';
+import { StoreError } from './types.js';
+import { digest } from './files.js';
+const now = () => new Date().toISOString();
+const fail = (code = 'invalid-input') => { throw new StoreError(code); };
+/** Bound on one change's JSON; imported note bodies (64 KiB) plus anchors fit with room for escaping. */
+export const maxChangeBytes = 262144;
+/** Hybrid logical clock `ms.counter.device`: zero-padded so every device sorts the strings alike. */
+function tick(db) {
+    const row = db.prepare('SELECT device_id,clock_ms,clock_counter FROM device WHERE id=1').get() ?? fail('integrity');
+    const wall = Date.now(), prior = Number(row.clock_ms), ms = Math.max(wall, prior), counter = ms === prior ? Number(row.clock_counter) + 1 : 0;
+    db.prepare('UPDATE device SET clock_ms=?,clock_counter=? WHERE id=1').run(ms, counter);
+    return { deviceId: String(row.device_id), hlc: `${String(ms).padStart(15, '0')}.${String(counter).padStart(9, '0')}.${row.device_id}` };
+}
+/** Append without applying: for state that already exists (seeding). Callers hold the write transaction. */
+export function append(db, change) {
+    const { deviceId, hlc } = tick(db);
+    db.prepare('INSERT INTO changes(id,device_id,hlc,entity,entity_id,field,value_json,created_at) VALUES(?,?,?,?,?,?,?,?)').run(randomUUID(), deviceId, hlc, change.entity, change.entityId, change.field, JSON.stringify(change.value ?? null), now());
+}
+/** The only write path for user state: tick the clock, apply the row update, append the change. */
+export function mutate(db, entity, entityId, field, value) {
+    const change = { entity, entityId, field, value: value ?? null };
+    if (Buffer.byteLength(JSON.stringify(change.value)) > maxChangeBytes)
+        fail();
+    applyChange(db, change);
+    append(db, change);
+}
+export function listChanges(db, after = 0, limit = 500) {
+    if (!Number.isSafeInteger(after) || after < 0 || !Number.isSafeInteger(limit) || limit < 1 || limit > 5000)
+        fail();
+    return db.prepare('SELECT * FROM changes WHERE seq>? ORDER BY seq LIMIT ?').all(after, limit).map(r => ({ seq: Number(r.seq), id: String(r.id), deviceId: String(r.device_id), hlc: String(r.hlc), entity: r.entity, entityId: String(r.entity_id), field: String(r.field), value: JSON.parse(String(r.value_json)) }));
+}
+const object = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
+const str = (v, max = 65536) => typeof v === 'string' && v.length > 0 && Buffer.byteLength(v) <= max;
+const optional = (v, max = 256) => v === null || str(v, max);
+const status = (v) => v === null || v === 'accepted' || v === 'rejected';
+/** Deterministic source-ref id, so replayed item creation yields the same row on every device. */
+export const sourceRefId = (itemId) => digest(JSON.stringify(['source-ref-v1', itemId]));
+const parsed = (json) => { try {
+    return JSON.parse(String(json));
+}
+catch {
+    return String(json);
+} };
+const changed = (r) => { if (!Number(r.changes))
+    fail('unavailable'); };
+/**
+ * Review JSON in one canonical shape, whatever order its paths were written in. A digest is reviewed per field;
+ * an editorial derivation (a reviewed source note kept whole) has one path, `review.note`.
+ */
+export function setReviewPath(review, output, path, value, kind = 'digest') {
+    if (kind === 'editorial') {
+        if (path !== 'review.note')
+            fail();
+        return value ? { note: value } : {};
+    }
+    if (kind !== 'digest')
+        fail();
+    const ideas = output.ideas.map((_, i) => review.ideas?.[i] ?? null);
+    const quotes = output.ideas.map((idea, i) => idea.quotes.map((_, j) => review.quotes?.[i]?.[j] ?? null));
+    let gist = review.gist ?? null;
+    const m = /^review\.(?:(gist)|ideas\.(\d{1,2})|quotes\.(\d{1,2})\.(\d{1,2}))$/.exec(path) ?? fail();
+    if (m[1])
+        gist = value;
+    else if (m[2] !== undefined) {
+        const i = Number(m[2]);
+        if (i >= ideas.length)
+            fail();
+        ideas[i] = value;
+    }
+    else {
+        const i = Number(m[3]), j = Number(m[4]);
+        if (i >= quotes.length || j >= quotes[i].length)
+            fail();
+        quotes[i][j] = value;
+    }
+    return { ...(gist ? { gist } : {}), ...(ideas.some(v => v) ? { ideas } : {}), ...(quotes.some(q => q.some(v => v)) ? { quotes } : {}) };
+}
+/**
+ * Map one change onto its row. Table-driven: every user-state field Sponge writes is listed here,
+ * and nothing else writes those columns. Objects (revisions, derivation outputs) must already exist.
+ */
+export function applyChange(db, change) {
+    // Untrusted once a transport delivers it: every case validates before writing.
+    const { entity, entityId: id, field } = change, v = change.value;
+    if (!str(id, 256))
+        fail();
+    switch (`${entity}.${field.startsWith('review.') ? 'review' : field}`) {
+        case 'item.created': {
+            if (!object(v) || !['article', 'pdf', 'media'].includes(v.kind) || !str(v.createdAt, 64) || !optional(v.sourceKey, 8192) || !(v.realm === null || typeof v.realm === 'string' && Buffer.byteLength(v.realm) <= 256) || !optional(v.locator, 16384))
+                fail();
+            db.prepare('INSERT INTO items(id,kind,created_at) VALUES(?,?,?) ON CONFLICT(id) DO NOTHING').run(id, v.kind, v.createdAt);
+            if (v.sourceKey !== null && v.locator !== null)
+                db.prepare('INSERT INTO source_refs(id,item_id,source_key,realm_key,locator) VALUES(?,?,?,?,?) ON CONFLICT DO NOTHING').run(sourceRefId(id), id, v.sourceKey, v.realm ?? '', v.locator);
+            return;
+        }
+        case 'item.currentRevision':
+            if (!optional(v))
+                fail();
+            if (v !== null && !db.prepare('SELECT 1 FROM revisions WHERE id=? AND item_id=?').get(v, id))
+                fail('unavailable');
+            return changed(db.prepare('UPDATE items SET current_revision=? WHERE id=?').run(v, id));
+        case 'item.title':
+            if (typeof v !== 'string' || Buffer.byteLength(v) > 4096)
+                fail();
+            return changed(db.prepare('UPDATE items SET title=? WHERE id=?').run(v, id));
+        case 'item.visibility':
+            if (v !== null && v !== 'public' && v !== 'private')
+                fail();
+            return changed(db.prepare('UPDATE items SET visibility=? WHERE id=?').run(v, id));
+        case 'item.deleted':
+            if (!str(v, 64))
+                fail();
+            return changed(db.prepare('UPDATE items SET deleted_at=COALESCE(deleted_at,?) WHERE id=?').run(v, id));
+        case 'annotation.created':
+            if (!object(v) || !str(v.itemId, 256) || !['note', 'highlight', 'comment'].includes(v.kind) || !optional(v.revisionId) || !optional(v.parentId) || typeof v.body !== 'string' || !str(v.createdAt, 64) || !('anchor' in v))
+                fail();
+            db.prepare('INSERT INTO annotations(id,item_id,revision_id,anchor_json,body,created_at,kind,parent_id) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(id) DO NOTHING').run(id, v.itemId, v.revisionId, JSON.stringify(v.anchor), v.body, v.createdAt, v.kind, v.parentId);
+            return;
+        case 'annotation.body':
+            if (!object(v) || typeof v.body !== 'string' || !str(v.updatedAt, 64))
+                fail();
+            return changed(db.prepare('UPDATE annotations SET body=?,updated_at=? WHERE id=?').run(v.body, v.updatedAt, id));
+        case 'annotation.deleted':
+            if (!str(v, 64))
+                fail();
+            return changed(db.prepare('UPDATE annotations SET deleted_at=COALESCE(deleted_at,?) WHERE id=?').run(v, id));
+        case 'reading.state':
+            if (!object(v) || v.state === undefined || !str(v.updatedAt, 64))
+                fail();
+            db.prepare('INSERT INTO reading_state(item_id,state_json,updated_at) VALUES(?,?,?) ON CONFLICT(item_id) DO UPDATE SET state_json=excluded.state_json,updated_at=excluded.updated_at').run(id, JSON.stringify(v.state), v.updatedAt);
+            return;
+        case 'derivation.review': {
+            if (!status(v))
+                fail();
+            const row = db.prepare('SELECT kind,output_json,review_json FROM derivations WHERE id=?').get(id) ?? fail('unavailable');
+            db.prepare('UPDATE derivations SET review_json=? WHERE id=?').run(JSON.stringify(setReviewPath(JSON.parse(String(row.review_json)), JSON.parse(String(row.output_json)), field, v, String(row.kind))), id);
+            return;
+        }
+        default: fail();
+    }
+}
+/**
+ * Seed the log with user state written before schema 5, so another device can rebuild it.
+ * Idempotent: a field that already has a change is never seeded again. Rows already exist, so this appends only.
+ */
+export function seedChanges(db) {
+    let count = 0;
+    const seed = (entity, entityId, field, value) => {
+        if (db.prepare('SELECT 1 FROM changes WHERE entity=? AND entity_id=? AND field=? LIMIT 1').get(entity, entityId, field))
+            return;
+        append(db, { entity, entityId, field, value });
+        count++;
+    };
+    for (const i of db.prepare('SELECT * FROM items ORDER BY created_at,rowid').all()) {
+        const id = String(i.id), source = db.prepare('SELECT source_key,realm_key,locator FROM source_refs WHERE item_id=? ORDER BY rowid LIMIT 1').get(id);
+        seed('item', id, 'created', { kind: i.kind, sourceKey: source ? String(source.source_key) : null, realm: source ? String(source.realm_key) : null, locator: source ? String(source.locator) : null, createdAt: String(i.created_at) });
+        if (i.current_revision !== null)
+            seed('item', id, 'currentRevision', String(i.current_revision));
+        if (i.title !== '')
+            seed('item', id, 'title', String(i.title));
+        if (i.visibility !== null && i.visibility !== undefined)
+            seed('item', id, 'visibility', i.visibility);
+        if (i.deleted_at !== null)
+            seed('item', id, 'deleted', String(i.deleted_at));
+    }
+    for (const a of db.prepare('SELECT * FROM annotations ORDER BY parent_id IS NOT NULL,created_at,rowid').all()) {
+        const id = String(a.id);
+        seed('annotation', id, 'created', { itemId: String(a.item_id), kind: a.kind, revisionId: a.revision_id ?? null, parentId: a.parent_id ?? null, anchor: parsed(a.anchor_json), body: String(a.body), createdAt: String(a.created_at) });
+        if (a.updated_at !== null)
+            seed('annotation', id, 'body', { body: String(a.body), updatedAt: String(a.updated_at) });
+        if (a.deleted_at !== null)
+            seed('annotation', id, 'deleted', String(a.deleted_at));
+    }
+    for (const r of db.prepare('SELECT * FROM reading_state ORDER BY rowid').all())
+        seed('reading', String(r.item_id), 'state', { state: parsed(r.state_json), updatedAt: String(r.updated_at) });
+    return count;
+}

package/dist/src/store/foundation.d.ts

@@ -0,0 +1,7 @@
+/** Sync and publishing foundation, split by concern; this barrel keeps existing imports working. See docs/decisions/sync-and-publishing.md. */
+export { applyChange, append, listChanges, mutate, seedChanges, setReviewPath, sourceRefId, maxChangeBytes, type Change, type Entity, type LoggedChange } from './changes.js';
+export { backfillUris, itemUris, itemsForUri, setVisibility, syncUris, visibility, type Visibility, type VisibilityState } from '../identity/visibility.js';
+export { addComment, addHighlight, deleteAnnotation, importNote, listAnnotations, markdownFor, readAnnotation, setReadingState, updateAnnotation, type Annotation } from '../annotations/store.js';
+export { acceptedDigest, canonicalDigest, currentDigest, derivationId, findDigest, putDigest, readDigest, reviewDigest, type Derivation, type DigestReview, type ReviewStatus } from '../digest/store.js';
+export { currentEditorial, listEditorials, publicEditorial, putEditorial, readEditorial, reviewEditorial, type Editorial } from '../digest/editorial.js';
+export { publicProjection, type PublicEntry } from '../publish/projection.js';

package/dist/src/store/foundation.js

@@ -0,0 +1,7 @@
+/** Sync and publishing foundation, split by concern; this barrel keeps existing imports working. See docs/decisions/sync-and-publishing.md. */
+export { applyChange, append, listChanges, mutate, seedChanges, setReviewPath, sourceRefId, maxChangeBytes } from './changes.js';
+export { backfillUris, itemUris, itemsForUri, setVisibility, syncUris, visibility } from '../identity/visibility.js';
+export { addComment, addHighlight, deleteAnnotation, importNote, listAnnotations, markdownFor, readAnnotation, setReadingState, updateAnnotation } from '../annotations/store.js';
+export { acceptedDigest, canonicalDigest, currentDigest, derivationId, findDigest, putDigest, readDigest, reviewDigest } from '../digest/store.js';
+export { currentEditorial, listEditorials, publicEditorial, putEditorial, readEditorial, reviewEditorial } from '../digest/editorial.js';
+export { publicProjection } from '../publish/projection.js';

package/dist/src/store/index.d.ts

@@ -1,9 +1,13 @@
 import { DatabaseSync } from 'node:sqlite';
+import * as foundation from './foundation.js';
+import * as jungle from '../importers/jungle-store.js';
 import type { JobCheckpoint } from '../../contracts/index.js';
 import { type AttemptToken, type OwnedAttempt, type BlobReference, type CommitRevision, type OwnerToken, type ReadableItem, type SourceSubmission, type StoreOptions, type Submission } from './types.js';
 import * as indexing from '../indexing/index.js';
+import * as prune from '../indexing/prune.js';
 import { hybridSearch } from '../search/hybrid.js';
 export * from './types.js';
+export declare const libraryVersion = 7;
 export declare const libraryMigrations: readonly string[];
 type Row = Record<string, any>;
 export declare function openLibrary(path: string, options?: StoreOptions): Promise<LibraryStore>;
@@ -29,20 +33,76 @@ export declare class LibraryStore {
     writeIndexBatch(id: string, fingerprint: string, batch: Parameters<typeof indexing.writeBatch>[3]): void;
     activateIndex(id: string): indexing.GenerationStatus;
     cancelIndex(id: string): void;
+    /** Retired-generation pruning; see src/indexing/prune.ts. The plan probes rollback inside a rolled-back savepoint. */
+    pruneIndexPlan(keep: number): {
+        generations: prune.PruneEntry[];
+        stats: prune.PageStats;
+    };
+    pruneIndexStep(owner: OwnerToken, id: string, keep: number): number;
+    /** VACUUM cannot run in a transaction: fences are checked under the owner token immediately before it. */
+    vacuumLibrary(owner: OwnerToken): prune.PageStats;
     searchHybrid(query: string, options?: Parameters<typeof hybridSearch>[2]): Promise<import("../search/hybrid.js").SearchResult>;
+    /** Export's view: the one annotation reader, without the redundant item id, plus reading state. */
     readAnnotations(itemId: string): {
         annotations: {
             id: string;
+            kind: "note" | "highlight" | "comment";
             revisionId: string | null;
+            parentId: string | null;
             anchor: unknown;
             body: string;
             createdAt: string;
+            updatedAt: string | null;
+            anchored?: boolean;
         }[];
         readingState: {
             state: unknown;
             updatedAt: string;
         } | null;
     };
+    /** Sync and publishing foundation; see docs/decisions/sync-and-publishing.md. */
+    identity(itemId: string): {
+        uris: import("../identity/uri.js").ItemUri[];
+        visibility: foundation.VisibilityState;
+    };
+    itemsForUri(uri: string): string[];
+    setVisibility(itemId: string, value: foundation.Visibility | null): foundation.VisibilityState;
+    addHighlight(input: Parameters<typeof foundation.addHighlight>[1]): foundation.Annotation;
+    addComment(input: Parameters<typeof foundation.addComment>[1]): foundation.Annotation;
+    updateAnnotation(id: string, body: string): foundation.Annotation;
+    deleteAnnotation(id: string): void;
+    listAnnotations(itemId: string): (foundation.Annotation & {
+        anchored?: boolean;
+    })[];
+    putDigest(input: Parameters<typeof foundation.putDigest>[1]): {
+        id: string;
+        duplicate: boolean;
+    };
+    currentDigest(itemId: string): foundation.Derivation | undefined;
+    readDigest(id: string): foundation.Derivation | undefined;
+    findDigest(revisionId: string, inputHash: string): foundation.Derivation | undefined;
+    reviewDigest(id: string, review: foundation.DigestReview): foundation.Derivation;
+    currentEditorial(itemId: string): foundation.Editorial | undefined;
+    readEditorial(id: string): foundation.Editorial | undefined;
+    reviewEditorial(id: string, status: foundation.ReviewStatus | null): foundation.Editorial;
+    /** The item an importer realm holds for one source key, with its current revision; deleted items are reported, not hidden. */
+    importedItem(realm: string, sourceKey: string): {
+        itemId: string;
+        deleted: boolean;
+        revisionId: string | null;
+    } | null;
+    importedRevision(itemId: string, bundleHash: string): string;
+    revisionMarkdown(revisionId: string): string;
+    planJungleNote(input: jungle.JungleNoteInput): jungle.JunglePlan;
+    applyJungleNote(input: jungle.JungleNoteInput): jungle.JunglePlan;
+    listChanges(after?: number, limit?: number): foundation.LoggedChange[];
+    publicProjection(): {
+        entries: foundation.PublicEntry[];
+        skipped: {
+            itemId: string;
+            reason: string;
+        }[];
+    };
     lookupImportReceipt(namespace: string, sourceKey: string, sourceHash: string): {
         itemId: string;
         outcome: string;
@@ -112,6 +172,15 @@ export declare class LibraryStore {
     finishBackup(id: string): void;
     /** Hold the SQLite writer fence across synchronous GC inspection and removal. */
     withGcFence<T>(expectedEpoch: number, callback: (references: readonly BlobReference[]) => T): T;
+    /** Runtime GC pins: recorded model artifacts and embedding helpers of active or building generations. */
+    runtimeReferences(): {
+        modelArtifacts: {
+            id: string;
+            manifest: string;
+        }[];
+        embeddingHelpers: string[];
+        unmappedSpaces: number;
+    };
     /** GC must honor both durable staging pins and active backup fences. */
     pendingCleanupCount(): number;
     hasGcFence(): boolean;