@hraness/sponge 0.2.4 → 0.2.5

package/dist/src/publish/index.js

@@ -0,0 +1,137 @@
+import { closeSync, constants, fsyncSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, rmSync } from 'node:fs';
+import { randomUUID } from 'node:crypto';
+import { basename, dirname, isAbsolute, join, parse, relative, resolve, sep } from 'node:path';
+import { digest, pathExists, regularFile, safeDirectory, syncDirectory, verifyFile } from '../store/files.js';
+import { StoreError } from '../store/types.js';
+import { writeAll } from '../portability/files.js';
+import { baseUrl, buildSite, manifestPath, signaturePath } from './feed.js';
+import { publishIdentity } from './identity.js';
+export { buildSite, canonical, xml, feedLimit } from './feed.js';
+export const publishMarker = '.sponge-publish', markerBody = 'sponge-publish-v1\n';
+const fail = (code = 'invalid-input') => { throw new StoreError(code); };
+/** Drop entries listed fewer than `days` days ago, so listing dates do not reveal what is read right now. */
+export function minAge(entries, days = 0, now = new Date()) {
+    if (!Number.isSafeInteger(days) || days < 0 || days > 3650)
+        fail();
+    if (!days)
+        return entries;
+    const cutoff = new Date(now.getTime() - days * 86_400_000).toISOString().slice(0, 10);
+    return entries.filter(e => e.listedOn <= cutoff);
+}
+/** Files a previous publish wrote, or a refusal: an unmarked directory or unlisted files are not ours to replace. */
+function previousOutput(target, stat) {
+    if (!stat.isDirectory() || stat.isSymbolicLink())
+        fail('integrity');
+    const marker = join(target, publishMarker);
+    if (!pathExists(marker))
+        fail();
+    regularFile(marker);
+    verifyFile(marker, digest(markerBody), Buffer.byteLength(markerBody));
+    let listed;
+    try {
+        const manifest = join(target, manifestPath);
+        regularFile(manifest);
+        listed = new Set(JSON.parse(readFileSync(manifest, 'utf8')).files.map(f => f.path));
+    }
+    catch {
+        return fail();
+    }
+    for (const path of [manifestPath, signaturePath, publishMarker])
+        listed.add(path);
+    const walk = (dir) => { for (const name of readdirSync(dir)) {
+        const path = join(dir, name), s = lstatSync(path), rel = relative(target, path).split(sep).join('/');
+        if (s.isSymbolicLink())
+            fail('integrity');
+        if (s.isDirectory())
+            walk(path);
+        else if (!s.isFile() || !listed.has(rel))
+            fail();
+    } };
+    walk(target);
+}
+function writeTree(root, files) {
+    mkdirSync(root, { mode: 0o755 });
+    const dirs = new Set([root]);
+    for (const [path, bytes] of [...files, [publishMarker, Buffer.from(markerBody)]]) {
+        const target = join(root, ...path.split('/')), dir = dirname(target);
+        if (!dirs.has(dir)) {
+            mkdirSync(dir, { mode: 0o755 });
+            dirs.add(dir);
+        }
+        const fd = openSync(target, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o644);
+        try {
+            writeAll(fd, bytes);
+            fsyncSync(fd);
+        }
+        finally {
+            closeSync(fd);
+        }
+    }
+    for (const dir of [...dirs].reverse())
+        syncDirectory(dir);
+}
+/**
+ * Write the signed static site for the public projection. The site is built in a temporary sibling and renamed
+ * into place; an existing directory is replaced only when it is a previous publish output with no foreign files.
+ */
+export function publishSite(source, options) {
+    const output = options.output;
+    if (!isAbsolute(output) || resolve(output) !== output || output === parse(output).root)
+        fail();
+    let parent;
+    try {
+        parent = safeDirectory(dirname(output));
+    }
+    catch (error) {
+        if (error.code === 'ENOENT')
+            fail();
+        throw error;
+    }
+    const target = join(parent, basename(output)), library = safeDirectory(source.root), inside = relative(library, target);
+    if (!inside || !inside.startsWith('..') && !isAbsolute(inside) || !relative(target, library).startsWith('..'))
+        fail();
+    const title = options.title ?? 'Reading list';
+    if (!title.trim() || title.length > 200 || /[\u0000-\u001f\u007f]/.test(title))
+        fail();
+    const base = options.baseUrl === undefined ? undefined : baseUrl(options.baseUrl);
+    let previous;
+    try {
+        previous = lstatSync(target);
+    }
+    catch (error) {
+        if (error.code !== 'ENOENT')
+            throw error;
+    }
+    if (previous)
+        previousOutput(target, previous);
+    const projection = source.publicProjection(), entries = minAge(projection.entries, options.minAgeDays, options.now), identity = publishIdentity(library);
+    const files = buildSite(entries, identity, { title, ...(base ? { baseUrl: base } : {}) }, bytes => identity.sign(bytes));
+    const id = randomUUID(), temp = join(parent, `.${basename(target)}.${id}.tmp`);
+    try {
+        writeTree(temp, files);
+    }
+    catch (error) {
+        rmSync(temp, { recursive: true, force: true });
+        throw error;
+    }
+    if (previous) {
+        const old = join(parent, `.${basename(target)}.${id}.old`);
+        renameSync(target, old);
+        try {
+            renameSync(temp, target);
+        }
+        catch (error) {
+            renameSync(old, target);
+            rmSync(temp, { recursive: true, force: true });
+            throw error;
+        }
+        const moved = lstatSync(old);
+        if (moved.dev !== previous.dev || moved.ino !== previous.ino)
+            fail('integrity');
+        rmSync(old, { recursive: true });
+    }
+    else
+        renameSync(temp, target);
+    syncDirectory(parent);
+    return { output: target, entries: entries.length, held: projection.entries.length - entries.length, skipped: projection.skipped.length, files: files.size, publicKey: identity.publicKey };
+}

package/dist/src/publish/projection.d.ts

@@ -0,0 +1,29 @@
+import type { DatabaseSync } from 'node:sqlite';
+/** One public list entry. Metadata only: no captured text, paths, account realms or local fingerprints. */
+export interface PublicEntry {
+    uri: string;
+    aliases: string[];
+    kind: string;
+    title: string;
+    authors?: string;
+    published?: string;
+    site?: string;
+    listedOn: string;
+    digest?: {
+        lead?: string;
+        ideas: {
+            claim: string;
+            detail: string;
+            kind?: string;
+            quotes: string[];
+        }[];
+    };
+}
+/** What `sponge publish` would publish now, newest first. Items without a resolvable URI are listed as skipped. */
+export declare function publicProjection(db: DatabaseSync): {
+    entries: PublicEntry[];
+    skipped: {
+        itemId: string;
+        reason: string;
+    }[];
+};

package/dist/src/publish/projection.js

@@ -0,0 +1,43 @@
+import { itemUris, visibility } from '../identity/visibility.js';
+import { acceptedDigest, currentDigest } from '../digest/store.js';
+import { currentEditorial, publicEditorial } from '../digest/editorial.js';
+/** The first selected (or unmarked) claim among `fields`; string lists are joined. */
+function selectedClaim(db, revisionId, fields) {
+    for (const field of fields) {
+        const r = db.prepare('SELECT value_json FROM metadata_claims WHERE revision_id=? AND field=? AND selected IS NOT 0 ORDER BY selected IS NOT 1,rowid LIMIT 1').get(revisionId, field);
+        const raw = r ? JSON.parse(String(r.value_json)) : undefined, v = Array.isArray(raw) && raw.every(x => typeof x === 'string') ? raw.join(', ') : raw;
+        if (typeof v === 'string' && v.trim() && v.length <= 1024)
+            return v.trim();
+    }
+    return undefined;
+}
+/**
+ * The public digest: the accepted fields of the current Digest v1, else the public subset of an accepted editorial
+ * note (lead, claims, verbatim quotes within the v1 budget). Never both, and never the note's other content.
+ */
+function publicDigest(db, id) { const d = currentDigest(db, id); if (d)
+    return acceptedDigest(d); const e = currentEditorial(db, id); return e && publicEditorial(db, e); }
+/** What `sponge publish` would publish now, newest first. Items without a resolvable URI are listed as skipped. */
+export function publicProjection(db) {
+    const entries = [], skipped = [];
+    for (const item of db.prepare('SELECT id,kind,title,current_revision,created_at FROM items WHERE deleted_at IS NULL ORDER BY created_at DESC,id').all()) {
+        const id = String(item.id);
+        if (visibility(db, id).effective !== 'public')
+            continue;
+        if (!item.current_revision) {
+            skipped.push({ itemId: id, reason: 'not-captured' });
+            continue;
+        }
+        const uris = itemUris(db, id).filter(u => u.resolvable), primary = uris.find(u => u.role === 'primary') ?? uris[0];
+        if (!primary) {
+            skipped.push({ itemId: id, reason: 'no-public-uri' });
+            continue;
+        }
+        const revision = String(item.current_revision), authors = selectedClaim(db, revision, ['authors', 'author']), published = selectedClaim(db, revision, ['publishedAt', 'published']), accepted = publicDigest(db, id);
+        let site;
+        if (/^https?:/.test(primary.uri))
+            site = new URL(primary.uri).hostname.replace(/^www\./, '');
+        entries.push({ uri: primary.uri, aliases: uris.filter(u => u !== primary).map(u => u.uri), kind: String(item.kind), title: String(item.title), ...(authors ? { authors } : {}), ...(published ? { published } : {}), ...(site ? { site } : {}), listedOn: String(item.created_at).slice(0, 10), ...(accepted ? { digest: accepted } : {}) });
+    }
+    return { entries, skipped };
+}

package/dist/src/search/hybrid.js

@@ -1,6 +1,6 @@
 import { chunkDocument } from '../indexing/index.js';
+import { lexicalSearch } from './lexical.js';
 import { pinSearchGeneration, vectorSearch } from './vector.js';
-function lexical(db, query, limit) { const literal = '"' + query.replaceAll('"', '""') + '"'; return db.prepare('SELECT i.id AS itemId,i.title,i.current_revision AS revisionId FROM readable_fts f JOIN items i ON i.id=f.item_id WHERE readable_fts MATCH ? AND i.deleted_at IS NULL AND i.current_revision=f.revision_id ORDER BY rank,i.id LIMIT ?').all(literal, limit); }
 /** Reciprocal rank fusion after per-channel item collapsing; chunks cannot amplify rank. */
 export function fuse(lexicalHits, semanticHits, limit) {
     const hits = new Map();
@@ -106,7 +106,7 @@ export async function hybridSearch(db, query, options = {}) {
                 warnings.push('semantic-unavailable');
             }
         }
-        const result = { hits: fuse(lexical(db, query, Math.min(500, limit * 5)), semantic, limit), warnings, generation: pin };
+        const result = { hits: fuse(lexicalSearch(db, query, Math.min(500, limit * 5)), semantic, limit), warnings, generation: pin };
         projectAnchors(db, result.hits);
         db.exec('COMMIT');
         return result;

package/dist/src/search/lexical.d.ts

@@ -0,0 +1,11 @@
+import type { DatabaseSync } from 'node:sqlite';
+export interface LexicalHit {
+    itemId: string;
+    revisionId: string;
+    title: string;
+}
+/** Contentless FTS rows are keyed by readable_fts_rows.id; items.rowid is not VACUUM-stable. Callers own the transaction. */
+export declare function indexReadable(db: DatabaseSync, itemId: string, revisionId: string, title: string, markdown: string): void;
+export declare function unindexReadable(db: DatabaseSync, itemId: string): void;
+/** Literal phrase, bm25 rank then item id; excludes tombstones and superseded revisions. */
+export declare function lexicalSearch(db: DatabaseSync, query: string, limit: number): LexicalHit[];

package/dist/src/search/lexical.js

@@ -0,0 +1,6 @@
+/** Contentless FTS rows are keyed by readable_fts_rows.id; items.rowid is not VACUUM-stable. Callers own the transaction. */
+export function indexReadable(db, itemId, revisionId, title, markdown) { unindexReadable(db, itemId); const row = db.prepare('INSERT INTO readable_fts_rows(item_id,revision_id) VALUES(?,?) RETURNING id').get(itemId, revisionId); db.prepare('INSERT INTO readable_fts(rowid,title,markdown) VALUES(?,?,?)').run(row.id, title, markdown); }
+export function unindexReadable(db, itemId) { const row = db.prepare('DELETE FROM readable_fts_rows WHERE item_id=? RETURNING id').get(itemId); if (row)
+    db.prepare('DELETE FROM readable_fts WHERE rowid=?').run(row.id); }
+/** Literal phrase, bm25 rank then item id; excludes tombstones and superseded revisions. */
+export function lexicalSearch(db, query, limit) { const literal = '"' + query.replaceAll('"', '""') + '"'; return db.prepare('SELECT i.id AS itemId,i.title,i.current_revision AS revisionId FROM readable_fts f JOIN readable_fts_rows m ON m.id=f.rowid JOIN items i ON i.id=m.item_id WHERE readable_fts MATCH ? AND i.deleted_at IS NULL AND i.current_revision=m.revision_id ORDER BY rank,i.id LIMIT ?').all(literal, limit); }

package/dist/src/search/query.d.ts

@@ -1,12 +1,16 @@
 import type { LibraryStore } from '../store/index.js';
 import { embedBatch, type EmbeddingRuntime } from '../inference/embedding/index.js';
 import type { QueryEmbedder } from './hybrid.js';
-/** Queries mutate coordination journals only, never library content or job tables. */
-export declare function createQueryEmbedder(store: LibraryStore, runtime: EmbeddingRuntime, options?: {
+type EmbedOptions = {
     embed?: typeof embedBatch;
     onReady?: (context: {
         attemptId: string;
         ownerDirectory: string;
         scratchRoot: string;
     }) => Promise<void>;
-}): QueryEmbedder;
+};
+/** Queries mutate coordination journals only, never library content or job tables. */
+export declare function createQueryEmbedder(store: LibraryStore, runtime: EmbeddingRuntime, options?: EmbedOptions): QueryEmbedder;
+/** Unit 768-dimension document vectors for short texts, 16 per helper run; text is clipped to the helper's byte budget on a character boundary. */
+export declare function createTextEmbedder(store: LibraryStore, runtime: EmbeddingRuntime, options?: EmbedOptions): (texts: string[], signal: AbortSignal) => Promise<number[][]>;
+export {};

package/dist/src/search/query.js

@@ -4,44 +4,63 @@ import { rm } from 'node:fs/promises';
 import { acquireWorkerCustody, proveAttemptCleanup } from '../worker/custody.js';
 import { directory, writeJournal, readJournal, syncDir } from '../worker/journal.js';
 import { supervisedRunner } from '../worker/supervisor.js';
-import { embedBatch, embeddingFingerprint, encodeEmbeddingRequest, EmbeddingError } from '../inference/embedding/index.js';
+import { embedBatch, embeddingFingerprint, embeddingTextBytes, resolveIndexSpace, projectVector, encodeEmbeddingRequest, EmbeddingError } from '../inference/embedding/index.js';
+/** One custody-held helper run. Mutates coordination journals only, never library content or job tables. */
+async function custodyEmbed(store, runtime, inputs, signal, options) {
+    const custody = await acquireWorkerCustody(store.root), attemptId = randomUUID();
+    let scratchRoot, cleanupUnknown = false;
+    try {
+        const attempts = await directory(join(custody.directory, 'attempts', attemptId));
+        await writeJournal(join(attempts, 'query.json'), { schemaVersion: 1, kind: 'query', attemptId, ownerIncarnation: custody.incarnation });
+        scratchRoot = await directory(join(store.root, 'staging', 'query-' + attemptId));
+        await writeJournal(join(scratchRoot, 'ownership.json'), { schemaVersion: 1, kind: 'query', requestId: attemptId, ownerIncarnation: custody.incarnation });
+        await options.onReady?.({ attemptId, ownerDirectory: custody.directory, scratchRoot });
+        const expected = encodeEmbeddingRequest(inputs).inputHashes;
+        const result = await (options.embed ?? embedBatch)(inputs, { runtime, scratchRoot, signal, runProcess: supervisedRunner(custody.directory, custody.incarnation, attemptId) });
+        if (result.fingerprint !== embeddingFingerprint(runtime.helperSha256) || result.inputHashes.length !== expected.length || result.inputHashes.some((h, i) => h !== expected[i]) || result.vectors.length !== expected.length)
+            throw new EmbeddingError('integrity');
+        return result.vectors;
+    }
+    catch (error) {
+        cleanupUnknown = error.code === 'cleanup-required';
+        throw error;
+    }
+    finally {
+        const proved = !cleanupUnknown && await proveAttemptCleanup(custody.directory, custody.incarnation, attemptId);
+        if (!proved)
+            throw new EmbeddingError('cleanup-required');
+        if (scratchRoot) {
+            const marker = await readJournal(join(scratchRoot, 'ownership.json'));
+            if (marker.schemaVersion !== 1 || marker.kind !== 'query' || marker.requestId !== attemptId || marker.ownerIncarnation !== custody.incarnation)
+                throw new EmbeddingError('cleanup-required');
+            await rm(scratchRoot, { recursive: true });
+            await syncDir(join(store.root, 'staging'));
+        }
+        await custody.release();
+    }
+}
 /** Queries mutate coordination journals only, never library content or job tables. */
 export function createQueryEmbedder(store, runtime, options = {}) {
     return async (query, pin, signal) => {
         if (signal.aborted)
             throw new EmbeddingError('cancelled');
-        if (pin.fingerprint !== embeddingFingerprint(runtime.helperSha256))
+        const space = resolveIndexSpace(runtime.helperSha256, pin.fingerprint);
+        if (!space || space.dimensions !== pin.dimensions)
             throw new EmbeddingError('integrity');
-        const custody = await acquireWorkerCustody(store.root), attemptId = randomUUID();
-        let scratchRoot, cleanupUnknown = false;
-        try {
-            const attempts = await directory(join(custody.directory, 'attempts', attemptId));
-            await writeJournal(join(attempts, 'query.json'), { schemaVersion: 1, kind: 'query', attemptId, ownerIncarnation: custody.incarnation });
-            scratchRoot = await directory(join(store.root, 'staging', 'query-' + attemptId));
-            await writeJournal(join(scratchRoot, 'ownership.json'), { schemaVersion: 1, kind: 'query', requestId: attemptId, ownerIncarnation: custody.incarnation });
-            await options.onReady?.({ attemptId, ownerDirectory: custody.directory, scratchRoot });
-            const inputs = [{ kind: 'query', text: query }], expected = encodeEmbeddingRequest(inputs).inputHashes[0];
-            const result = await (options.embed ?? embedBatch)(inputs, { runtime, scratchRoot, signal, runProcess: supervisedRunner(custody.directory, custody.incarnation, attemptId) });
-            if (result.fingerprint !== pin.fingerprint || result.inputHashes.length !== 1 || result.inputHashes[0] !== expected || result.vectors.length !== 1)
-                throw new EmbeddingError('integrity');
-            return { fingerprint: result.fingerprint, vector: result.vectors[0] };
-        }
-        catch (error) {
-            cleanupUnknown = error.code === 'cleanup-required';
-            throw error;
-        }
-        finally {
-            const proved = !cleanupUnknown && await proveAttemptCleanup(custody.directory, custody.incarnation, attemptId);
-            if (!proved)
-                throw new EmbeddingError('cleanup-required');
-            if (scratchRoot) {
-                const marker = await readJournal(join(scratchRoot, 'ownership.json'));
-                if (marker.schemaVersion !== 1 || marker.kind !== 'query' || marker.requestId !== attemptId || marker.ownerIncarnation !== custody.incarnation)
-                    throw new EmbeddingError('cleanup-required');
-                await rm(scratchRoot, { recursive: true });
-                await syncDir(join(store.root, 'staging'));
-            }
-            await custody.release();
-        }
+        const [vector] = await custodyEmbed(store, runtime, [{ kind: 'query', text: query }], signal, options);
+        return { fingerprint: pin.fingerprint, vector: projectVector(vector, space.dimensions) };
     };
 }
+/** Unit 768-dimension document vectors for short texts, 16 per helper run; text is clipped to the helper's byte budget on a character boundary. */
+export function createTextEmbedder(store, runtime, options = {}) {
+    const room = embeddingTextBytes - Buffer.byteLength('title: none | text: '), clip = (t) => { let out = ''; for (const ch of t) {
+        if (Buffer.byteLength(out + ch) > room)
+            break;
+        out += ch;
+    } return out; };
+    return async (texts, signal) => { const out = []; for (let i = 0; i < texts.length; i += 16) {
+        if (signal.aborted)
+            throw new EmbeddingError('cancelled');
+        out.push(...await custodyEmbed(store, runtime, texts.slice(i, i + 16).map(t => ({ kind: 'document', text: clip(t) })), signal, options));
+    } return out; };
+}

package/dist/src/search/vector.d.ts

@@ -1,5 +1,5 @@
 import type { DatabaseSync } from 'node:sqlite';
-import type { IndexInput } from '../indexing/index.js';
+import { type IndexInput } from '../indexing/index.js';
 export interface SearchGeneration {
     id: string;
     fingerprint: string;

package/dist/src/search/vector.js

@@ -1,3 +1,4 @@
+import { encodeVector, spaceEncoding } from '../indexing/index.js';
 export function pinSearchGeneration(db, profileId) {
     const row = db.prepare('SELECT g.id,g.fingerprint,s.dimensions FROM active_indexes a JOIN index_generations g ON g.id=a.generation_id JOIN embedding_spaces s ON s.fingerprint=g.fingerprint WHERE a.profile_id=?').get(profileId);
     if (!row)
@@ -10,12 +11,12 @@ export function vectorSearch(db, pin, query, limit = 50) {
         throw Error('invalid-search-vector');
     if (db.prepare('SELECT vec_version() AS version').get()?.version !== 'v0.1.9')
         throw Error('unsupported-vector-runtime');
-    const generation = db.prepare('SELECT fingerprint FROM index_generations WHERE id=?').get(pin.id);
-    if (!generation || generation.fingerprint !== pin.fingerprint)
+    const generation = db.prepare('SELECT g.fingerprint,s.dimensions,s.identity_json FROM index_generations g JOIN embedding_spaces s ON s.fingerprint=g.fingerprint WHERE g.id=?').get(pin.id);
+    if (!generation || generation.fingerprint !== pin.fingerprint || Number(generation.dimensions) !== pin.dimensions)
         throw Error('search-generation-missing');
-    const bytes = Buffer.alloc(query.vector.length * 4);
-    query.vector.forEach((v, i) => bytes.writeFloatLE(v, i * 4));
-    const rows = db.prepare('SELECT v.source_id,v.revision_id,v.derivation_hash,v.asset_sha256,i.id AS item_id,i.title,vec_distance_cosine(v.vector,?) AS distance FROM vectors v JOIN revisions r ON r.id=v.revision_id JOIN items i ON i.id=r.item_id WHERE v.generation_id=? AND i.deleted_at IS NULL AND i.current_revision=v.revision_id ORDER BY distance,v.source_id LIMIT ?').all(bytes, pin.id, limit);
+    // int8 spaces quantize the query the same way; cosine over codes is the cosine of the dequantized vectors because per-vector scales cancel.
+    const encoding = spaceEncoding(JSON.parse(String(generation.identity_json))), bytes = encodeVector(query.vector, encoding), distance = encoding === 'int8' ? 'vec_distance_cosine(vec_int8(v.vector),vec_int8(?))' : 'vec_distance_cosine(v.vector,?)';
+    const rows = db.prepare('SELECT v.source_id,v.revision_id,v.derivation_hash,v.asset_sha256,i.id AS item_id,i.title,' + distance + ' AS distance FROM vectors v JOIN revisions r ON r.id=v.revision_id JOIN items i ON i.id=r.item_id WHERE v.generation_id=? AND i.deleted_at IS NULL AND i.current_revision=v.revision_id ORDER BY distance,v.source_id LIMIT ?').all(bytes, pin.id, limit);
     return rows.map(row => { const descriptor = JSON.parse(String(row.source_id)), distance = Number(row.distance); if (!Array.isArray(descriptor) || descriptor[0] !== row.revision_id || !['document', 'image'].includes(String(descriptor[1])) || typeof descriptor[2] !== 'string' || !Number.isFinite(distance))
         throw Error('invalid-search-index'); const source = { sourceId: String(row.source_id), itemId: String(row.item_id), revisionId: String(row.revision_id), derivationHash: String(row.derivation_hash), kind: descriptor[1], ...(descriptor[1] === 'image' ? { assetSha256: String(row.asset_sha256) } : { representationId: descriptor[2], ordinal: Number(descriptor[3]) }) }; return { itemId: String(row.item_id), revisionId: String(row.revision_id), title: String(row.title), source, distance }; });
 }

package/dist/src/store/changes.d.ts

@@ -0,0 +1,42 @@
+import type { DatabaseSync } from 'node:sqlite';
+type Row = Record<string, any>;
+export type Entity = 'item' | 'annotation' | 'reading' | 'derivation';
+/** One field-level write to user state. The sync contract: replaying every change in order rebuilds user state. */
+export interface Change {
+    entity: Entity;
+    entityId: string;
+    field: string;
+    value: unknown;
+}
+export interface LoggedChange extends Change {
+    seq: number;
+    id: string;
+    deviceId: string;
+    hlc: string;
+}
+export type ReviewStatus = 'accepted' | 'rejected';
+/** Bound on one change's JSON; imported note bodies (64 KiB) plus anchors fit with room for escaping. */
+export declare const maxChangeBytes = 262144;
+/** Append without applying: for state that already exists (seeding). Callers hold the write transaction. */
+export declare function append(db: DatabaseSync, change: Change): void;
+/** The only write path for user state: tick the clock, apply the row update, append the change. */
+export declare function mutate(db: DatabaseSync, entity: Entity, entityId: string, field: string, value: unknown): void;
+export declare function listChanges(db: DatabaseSync, after?: number, limit?: number): LoggedChange[];
+/** Deterministic source-ref id, so replayed item creation yields the same row on every device. */
+export declare const sourceRefId: (itemId: string) => string;
+/**
+ * Review JSON in one canonical shape, whatever order its paths were written in. A digest is reviewed per field;
+ * an editorial derivation (a reviewed source note kept whole) has one path, `review.note`.
+ */
+export declare function setReviewPath(review: Row, output: Row, path: string, value: ReviewStatus | null, kind?: string): Row;
+/**
+ * Map one change onto its row. Table-driven: every user-state field Sponge writes is listed here,
+ * and nothing else writes those columns. Objects (revisions, derivation outputs) must already exist.
+ */
+export declare function applyChange(db: DatabaseSync, change: Change): void;
+/**
+ * Seed the log with user state written before schema 5, so another device can rebuild it.
+ * Idempotent: a field that already has a change is never seeded again. Rows already exist, so this appends only.
+ */
+export declare function seedChanges(db: DatabaseSync): number;
+export {};

package/dist/src/store/changes.js

@@ -0,0 +1,178 @@
+import { randomUUID } from 'node:crypto';
+import { StoreError } from './types.js';
+import { digest } from './files.js';
+const now = () => new Date().toISOString();
+const fail = (code = 'invalid-input') => { throw new StoreError(code); };
+/** Bound on one change's JSON; imported note bodies (64 KiB) plus anchors fit with room for escaping. */
+export const maxChangeBytes = 262144;
+/** Hybrid logical clock `ms.counter.device`: zero-padded so every device sorts the strings alike. */
+function tick(db) {
+    const row = db.prepare('SELECT device_id,clock_ms,clock_counter FROM device WHERE id=1').get() ?? fail('integrity');
+    const wall = Date.now(), prior = Number(row.clock_ms), ms = Math.max(wall, prior), counter = ms === prior ? Number(row.clock_counter) + 1 : 0;
+    db.prepare('UPDATE device SET clock_ms=?,clock_counter=? WHERE id=1').run(ms, counter);
+    return { deviceId: String(row.device_id), hlc: `${String(ms).padStart(15, '0')}.${String(counter).padStart(9, '0')}.${row.device_id}` };
+}
+/** Append without applying: for state that already exists (seeding). Callers hold the write transaction. */
+export function append(db, change) {
+    const { deviceId, hlc } = tick(db);
+    db.prepare('INSERT INTO changes(id,device_id,hlc,entity,entity_id,field,value_json,created_at) VALUES(?,?,?,?,?,?,?,?)').run(randomUUID(), deviceId, hlc, change.entity, change.entityId, change.field, JSON.stringify(change.value ?? null), now());
+}
+/** The only write path for user state: tick the clock, apply the row update, append the change. */
+export function mutate(db, entity, entityId, field, value) {
+    const change = { entity, entityId, field, value: value ?? null };
+    if (Buffer.byteLength(JSON.stringify(change.value)) > maxChangeBytes)
+        fail();
+    applyChange(db, change);
+    append(db, change);
+}
+export function listChanges(db, after = 0, limit = 500) {
+    if (!Number.isSafeInteger(after) || after < 0 || !Number.isSafeInteger(limit) || limit < 1 || limit > 5000)
+        fail();
+    return db.prepare('SELECT * FROM changes WHERE seq>? ORDER BY seq LIMIT ?').all(after, limit).map(r => ({ seq: Number(r.seq), id: String(r.id), deviceId: String(r.device_id), hlc: String(r.hlc), entity: r.entity, entityId: String(r.entity_id), field: String(r.field), value: JSON.parse(String(r.value_json)) }));
+}
+const object = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
+const str = (v, max = 65536) => typeof v === 'string' && v.length > 0 && Buffer.byteLength(v) <= max;
+const optional = (v, max = 256) => v === null || str(v, max);
+const status = (v) => v === null || v === 'accepted' || v === 'rejected';
+/** Deterministic source-ref id, so replayed item creation yields the same row on every device. */
+export const sourceRefId = (itemId) => digest(JSON.stringify(['source-ref-v1', itemId]));
+const parsed = (json) => { try {
+    return JSON.parse(String(json));
+}
+catch {
+    return String(json);
+} };
+const changed = (r) => { if (!Number(r.changes))
+    fail('unavailable'); };
+/**
+ * Review JSON in one canonical shape, whatever order its paths were written in. A digest is reviewed per field;
+ * an editorial derivation (a reviewed source note kept whole) has one path, `review.note`.
+ */
+export function setReviewPath(review, output, path, value, kind = 'digest') {
+    if (kind === 'editorial') {
+        if (path !== 'review.note')
+            fail();
+        return value ? { note: value } : {};
+    }
+    if (kind !== 'digest')
+        fail();
+    const ideas = output.ideas.map((_, i) => review.ideas?.[i] ?? null);
+    const quotes = output.ideas.map((idea, i) => idea.quotes.map((_, j) => review.quotes?.[i]?.[j] ?? null));
+    let gist = review.gist ?? null;
+    const m = /^review\.(?:(gist)|ideas\.(\d{1,2})|quotes\.(\d{1,2})\.(\d{1,2}))$/.exec(path) ?? fail();
+    if (m[1])
+        gist = value;
+    else if (m[2] !== undefined) {
+        const i = Number(m[2]);
+        if (i >= ideas.length)
+            fail();
+        ideas[i] = value;
+    }
+    else {
+        const i = Number(m[3]), j = Number(m[4]);
+        if (i >= quotes.length || j >= quotes[i].length)
+            fail();
+        quotes[i][j] = value;
+    }
+    return { ...(gist ? { gist } : {}), ...(ideas.some(v => v) ? { ideas } : {}), ...(quotes.some(q => q.some(v => v)) ? { quotes } : {}) };
+}
+/**
+ * Map one change onto its row. Table-driven: every user-state field Sponge writes is listed here,
+ * and nothing else writes those columns. Objects (revisions, derivation outputs) must already exist.
+ */
+export function applyChange(db, change) {
+    // Untrusted once a transport delivers it: every case validates before writing.
+    const { entity, entityId: id, field } = change, v = change.value;
+    if (!str(id, 256))
+        fail();
+    switch (`${entity}.${field.startsWith('review.') ? 'review' : field}`) {
+        case 'item.created': {
+            if (!object(v) || !['article', 'pdf', 'media'].includes(v.kind) || !str(v.createdAt, 64) || !optional(v.sourceKey, 8192) || !(v.realm === null || typeof v.realm === 'string' && Buffer.byteLength(v.realm) <= 256) || !optional(v.locator, 16384))
+                fail();
+            db.prepare('INSERT INTO items(id,kind,created_at) VALUES(?,?,?) ON CONFLICT(id) DO NOTHING').run(id, v.kind, v.createdAt);
+            if (v.sourceKey !== null && v.locator !== null)
+                db.prepare('INSERT INTO source_refs(id,item_id,source_key,realm_key,locator) VALUES(?,?,?,?,?) ON CONFLICT DO NOTHING').run(sourceRefId(id), id, v.sourceKey, v.realm ?? '', v.locator);
+            return;
+        }
+        case 'item.currentRevision':
+            if (!optional(v))
+                fail();
+            if (v !== null && !db.prepare('SELECT 1 FROM revisions WHERE id=? AND item_id=?').get(v, id))
+                fail('unavailable');
+            return changed(db.prepare('UPDATE items SET current_revision=? WHERE id=?').run(v, id));
+        case 'item.title':
+            if (typeof v !== 'string' || Buffer.byteLength(v) > 4096)
+                fail();
+            return changed(db.prepare('UPDATE items SET title=? WHERE id=?').run(v, id));
+        case 'item.visibility':
+            if (v !== null && v !== 'public' && v !== 'private')
+                fail();
+            return changed(db.prepare('UPDATE items SET visibility=? WHERE id=?').run(v, id));
+        case 'item.deleted':
+            if (!str(v, 64))
+                fail();
+            return changed(db.prepare('UPDATE items SET deleted_at=COALESCE(deleted_at,?) WHERE id=?').run(v, id));
+        case 'annotation.created':
+            if (!object(v) || !str(v.itemId, 256) || !['note', 'highlight', 'comment'].includes(v.kind) || !optional(v.revisionId) || !optional(v.parentId) || typeof v.body !== 'string' || !str(v.createdAt, 64) || !('anchor' in v))
+                fail();
+            db.prepare('INSERT INTO annotations(id,item_id,revision_id,anchor_json,body,created_at,kind,parent_id) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(id) DO NOTHING').run(id, v.itemId, v.revisionId, JSON.stringify(v.anchor), v.body, v.createdAt, v.kind, v.parentId);
+            return;
+        case 'annotation.body':
+            if (!object(v) || typeof v.body !== 'string' || !str(v.updatedAt, 64))
+                fail();
+            return changed(db.prepare('UPDATE annotations SET body=?,updated_at=? WHERE id=?').run(v.body, v.updatedAt, id));
+        case 'annotation.deleted':
+            if (!str(v, 64))
+                fail();
+            return changed(db.prepare('UPDATE annotations SET deleted_at=COALESCE(deleted_at,?) WHERE id=?').run(v, id));
+        case 'reading.state':
+            if (!object(v) || v.state === undefined || !str(v.updatedAt, 64))
+                fail();
+            db.prepare('INSERT INTO reading_state(item_id,state_json,updated_at) VALUES(?,?,?) ON CONFLICT(item_id) DO UPDATE SET state_json=excluded.state_json,updated_at=excluded.updated_at').run(id, JSON.stringify(v.state), v.updatedAt);
+            return;
+        case 'derivation.review': {
+            if (!status(v))
+                fail();
+            const row = db.prepare('SELECT kind,output_json,review_json FROM derivations WHERE id=?').get(id) ?? fail('unavailable');
+            db.prepare('UPDATE derivations SET review_json=? WHERE id=?').run(JSON.stringify(setReviewPath(JSON.parse(String(row.review_json)), JSON.parse(String(row.output_json)), field, v, String(row.kind))), id);
+            return;
+        }
+        default: fail();
+    }
+}
+/**
+ * Seed the log with user state written before schema 5, so another device can rebuild it.
+ * Idempotent: a field that already has a change is never seeded again. Rows already exist, so this appends only.
+ */
+export function seedChanges(db) {
+    let count = 0;
+    const seed = (entity, entityId, field, value) => {
+        if (db.prepare('SELECT 1 FROM changes WHERE entity=? AND entity_id=? AND field=? LIMIT 1').get(entity, entityId, field))
+            return;
+        append(db, { entity, entityId, field, value });
+        count++;
+    };
+    for (const i of db.prepare('SELECT * FROM items ORDER BY created_at,rowid').all()) {
+        const id = String(i.id), source = db.prepare('SELECT source_key,realm_key,locator FROM source_refs WHERE item_id=? ORDER BY rowid LIMIT 1').get(id);
+        seed('item', id, 'created', { kind: i.kind, sourceKey: source ? String(source.source_key) : null, realm: source ? String(source.realm_key) : null, locator: source ? String(source.locator) : null, createdAt: String(i.created_at) });
+        if (i.current_revision !== null)
+            seed('item', id, 'currentRevision', String(i.current_revision));
+        if (i.title !== '')
+            seed('item', id, 'title', String(i.title));
+        if (i.visibility !== null && i.visibility !== undefined)
+            seed('item', id, 'visibility', i.visibility);
+        if (i.deleted_at !== null)
+            seed('item', id, 'deleted', String(i.deleted_at));
+    }
+    for (const a of db.prepare('SELECT * FROM annotations ORDER BY parent_id IS NOT NULL,created_at,rowid').all()) {
+        const id = String(a.id);
+        seed('annotation', id, 'created', { itemId: String(a.item_id), kind: a.kind, revisionId: a.revision_id ?? null, parentId: a.parent_id ?? null, anchor: parsed(a.anchor_json), body: String(a.body), createdAt: String(a.created_at) });
+        if (a.updated_at !== null)
+            seed('annotation', id, 'body', { body: String(a.body), updatedAt: String(a.updated_at) });
+        if (a.deleted_at !== null)
+            seed('annotation', id, 'deleted', String(a.deleted_at));
+    }
+    for (const r of db.prepare('SELECT * FROM reading_state ORDER BY rowid').all())
+        seed('reading', String(r.item_id), 'state', { state: parsed(r.state_json), updatedAt: String(r.updated_at) });
+    return count;
+}

package/dist/src/store/foundation.d.ts

@@ -0,0 +1,7 @@
+/** Sync and publishing foundation, split by concern; this barrel keeps existing imports working. See docs/decisions/sync-and-publishing.md. */
+export { applyChange, append, listChanges, mutate, seedChanges, setReviewPath, sourceRefId, maxChangeBytes, type Change, type Entity, type LoggedChange } from './changes.js';
+export { backfillUris, itemUris, itemsForUri, setVisibility, syncUris, visibility, type Visibility, type VisibilityState } from '../identity/visibility.js';
+export { addComment, addHighlight, deleteAnnotation, importNote, listAnnotations, markdownFor, readAnnotation, setReadingState, updateAnnotation, type Annotation } from '../annotations/store.js';
+export { acceptedDigest, canonicalDigest, currentDigest, derivationId, findDigest, putDigest, readDigest, reviewDigest, type Derivation, type DigestReview, type ReviewStatus } from '../digest/store.js';
+export { currentEditorial, listEditorials, publicEditorial, putEditorial, readEditorial, reviewEditorial, type Editorial } from '../digest/editorial.js';
+export { publicProjection, type PublicEntry } from '../publish/projection.js';