@hraness/sponge 0.2.4 → 0.2.6

package/README.md

@@ -101,6 +101,36 @@ full-text search. Use `index create --profile NAME` and `search QUERY --profile
 to keep separate search profiles. Model changes create separate index generations; a new one
 becomes active only after it covers the current library. `index rollback
 GENERATION` requires a complete generation compatible with the current revisions.
+`index prune --dry-run` lists what pruning would delete; `index prune` deletes the
+vectors of cancelled, failed and retired generations, keeping the active and any
+building generation plus the most recent rollback-ready one per profile (`--keep N`,
+0-16). It refuses while index work, cleanup or a backup is running. Freed pages are
+reused; `--vacuum` also shrinks the file after checking free space. Rollback to a
+pruned generation needs a rebuild.
+
+### Digests
+
+A digest (Digest v2) is a short gist led by one sentence and 3–5 ideas, each a
+claim with detail and 0–2 verbatim quotes, with 1–6 quotes in all.
+A local model proposes it; Sponge checks every rule against the captured text
+and stores only a passing draft. Drafting is opt-in and needs a model server you
+start yourself, such as llama.cpp `llama-server`, LM Studio or Ollama:
+
+```sh
+llama-server -m gemma-4-12b-it-Q4_K_M.gguf --port 8080
+sponge digest ITEM --endpoint http://127.0.0.1:8080 --model NAME
+sponge digest ITEM
+sponge review DIGEST --accept all --reject quote:1.0
+```
+
+`--model` must match an id the server lists at `/v1/models`. Endpoints must be
+literal loopback HTTP (`127.0.0.1` or `[::1]`); Sponge refuses anything else so
+captured text never leaves the machine, and never starts or downloads a model.
+Suggested models are Gemma 4 12B on 16 GB Macs and Gemma 4 E4B on 8 GB, with
+Qwen3.5 9B or 4B as alternatives. Rerunning with the same inputs returns the stored
+draft. Review targets are `all`, `gist`, `idea:N` and `quote:N.M` (0-based); only
+accepted fields can be published. `--check-support` also requires each quote to
+support its claim by local embedding similarity. See [digest drafting](src/digest/README.md).
 
 ### Export, recovery and background work
 
@@ -110,6 +140,7 @@ sponge backup --output /absolute/new/library.sponge
 sponge restore /absolute/library.sponge --into /absolute/new/library
 sponge gc --dry-run
 sponge gc --apply /absolute/path/to/report.json
+sponge gc --runtimes --dry-run
 sponge worker install --language en
 sponge worker status
 sponge worker uninstall
@@ -119,6 +150,10 @@ Exports contain Markdown, images and saved notes. Backups preserve revisions,
 assets, reading state and notes. Restore requires a new directory, verifies every
 file and leaves connections unresolved; runtimes and credentials are excluded.
 Garbage collection removes only unreferenced blobs after rechecking the report.
+`gc --runtimes` removes capture runtimes and model packages this Sponge no longer
+pins, plus settled worker journals; `--dry-run` only lists them. It holds worker
+custody, keeps models an active or building index generation needs, and refuses
+symlinks and anything in use. Successful `setup` runs the same collection.
 
 The background worker is opt-in and processes work sequentially. Its LaunchAgent
 pins the installed command; after an upgrade, reinstall the worker to update that
@@ -126,6 +161,25 @@ binding. Uninstall stops the worker and preserves the library. In-session startu
 and cleanup are checked; fresh-login startup remains unqualified. See
 [importing existing captures](docs/importing.md) for a preview-first import flow.
 
+### Public reading list
+
+```sh
+sponge publish --dry-run
+sponge publish --output /absolute/site --title 'My reads' --base-url https://reads.example.com/ --min-age-days 7
+```
+
+Items captured from public sources are public by default; `visibility ITEM private`
+hides one. `publish --output` writes a static site: a JSON Feed (`feed.json`,
+newest 100), monthly shards under `items/`, an Atom `feed.xml`, and an
+`index.html` that uses the bundled `<sponge-list>` element. An Ed25519 key created
+in the library on first publish signs a manifest of every file's hash; the
+private key stays in the library. The same library state gives byte-identical
+files. `--min-age-days N` holds back items listed in the last N days. Nothing is
+uploaded. Embed the list elsewhere with
+`<sponge-list src="https://reads.example.com/feed.json" verify></sponge-list>` and
+`sponge-list.js`; the host must allow cross-origin reads, and `verify` needs
+HTTPS. See [the decision record](docs/decisions/publish-feed.md).
+
 ### Selected authenticated connections
 
 The initial development interface accepts private `connections.json` in the

package/dist/migrations/005.d.ts

@@ -0,0 +1,3 @@
+/** Additive sync and publishing foundation: shareable URIs, visibility, change log, annotations, content-addressed derivations. */
+export declare const version = 5;
+export declare const sql = "\nCREATE TABLE item_uris(item_id TEXT NOT NULL REFERENCES items(id),uri TEXT NOT NULL CHECK(length(uri) BETWEEN 3 AND 8192),role TEXT NOT NULL CHECK(role IN ('primary','alias')),resolvable INTEGER NOT NULL CHECK(resolvable IN (0,1)),origin TEXT NOT NULL CHECK(origin IN ('locator','revision','metadata','user')),PRIMARY KEY(item_id,uri)) STRICT;\nCREATE UNIQUE INDEX item_uris_primary ON item_uris(item_id) WHERE role='primary';\nCREATE INDEX item_uris_uri ON item_uris(uri);\nALTER TABLE items ADD COLUMN visibility TEXT CHECK(visibility IN ('public','private'));\nALTER TABLE annotations ADD COLUMN kind TEXT NOT NULL DEFAULT 'note' CHECK(kind IN ('note','highlight','comment'));\nALTER TABLE annotations ADD COLUMN parent_id TEXT REFERENCES annotations(id);\nALTER TABLE annotations ADD COLUMN updated_at TEXT;\nALTER TABLE annotations ADD COLUMN deleted_at TEXT;\nCREATE TABLE device(id INTEGER PRIMARY KEY CHECK(id=1),device_id TEXT NOT NULL CHECK(length(device_id)=32),clock_ms INTEGER NOT NULL DEFAULT 0 CHECK(clock_ms>=0),clock_counter INTEGER NOT NULL DEFAULT 0 CHECK(clock_counter>=0)) STRICT;\nINSERT INTO device(id,device_id) VALUES(1,lower(hex(randomblob(16))));\nCREATE TABLE changes(seq INTEGER PRIMARY KEY AUTOINCREMENT,id TEXT NOT NULL UNIQUE,device_id TEXT NOT NULL,hlc TEXT NOT NULL,entity TEXT NOT NULL CHECK(entity IN ('item','annotation','reading','derivation')),entity_id TEXT NOT NULL,field TEXT NOT NULL,value_json TEXT NOT NULL,created_at TEXT NOT NULL) STRICT;\nCREATE INDEX changes_entity ON changes(entity,entity_id,field,hlc);\nCREATE TABLE derivations(id TEXT PRIMARY KEY,item_id TEXT NOT NULL REFERENCES items(id),revision_id TEXT NOT NULL REFERENCES revisions(id),kind TEXT NOT NULL CHECK(kind IN ('digest')),schema_version INTEGER NOT NULL CHECK(schema_version>0),producer_json TEXT NOT NULL,input_hash TEXT NOT NULL CHECK(length(input_hash)=64),output_json TEXT NOT NULL,review_json TEXT NOT NULL DEFAULT '{}',created_at TEXT NOT NULL) STRICT;\nCREATE INDEX derivations_revision ON derivations(revision_id,kind,input_hash);\nCREATE INDEX derivations_item ON derivations(item_id,kind,created_at);\n";

package/dist/migrations/005.js

@@ -0,0 +1,19 @@
+/** Additive sync and publishing foundation: shareable URIs, visibility, change log, annotations, content-addressed derivations. */
+export const version = 5;
+export const sql = `
+CREATE TABLE item_uris(item_id TEXT NOT NULL REFERENCES items(id),uri TEXT NOT NULL CHECK(length(uri) BETWEEN 3 AND 8192),role TEXT NOT NULL CHECK(role IN ('primary','alias')),resolvable INTEGER NOT NULL CHECK(resolvable IN (0,1)),origin TEXT NOT NULL CHECK(origin IN ('locator','revision','metadata','user')),PRIMARY KEY(item_id,uri)) STRICT;
+CREATE UNIQUE INDEX item_uris_primary ON item_uris(item_id) WHERE role='primary';
+CREATE INDEX item_uris_uri ON item_uris(uri);
+ALTER TABLE items ADD COLUMN visibility TEXT CHECK(visibility IN ('public','private'));
+ALTER TABLE annotations ADD COLUMN kind TEXT NOT NULL DEFAULT 'note' CHECK(kind IN ('note','highlight','comment'));
+ALTER TABLE annotations ADD COLUMN parent_id TEXT REFERENCES annotations(id);
+ALTER TABLE annotations ADD COLUMN updated_at TEXT;
+ALTER TABLE annotations ADD COLUMN deleted_at TEXT;
+CREATE TABLE device(id INTEGER PRIMARY KEY CHECK(id=1),device_id TEXT NOT NULL CHECK(length(device_id)=32),clock_ms INTEGER NOT NULL DEFAULT 0 CHECK(clock_ms>=0),clock_counter INTEGER NOT NULL DEFAULT 0 CHECK(clock_counter>=0)) STRICT;
+INSERT INTO device(id,device_id) VALUES(1,lower(hex(randomblob(16))));
+CREATE TABLE changes(seq INTEGER PRIMARY KEY AUTOINCREMENT,id TEXT NOT NULL UNIQUE,device_id TEXT NOT NULL,hlc TEXT NOT NULL,entity TEXT NOT NULL CHECK(entity IN ('item','annotation','reading','derivation')),entity_id TEXT NOT NULL,field TEXT NOT NULL,value_json TEXT NOT NULL,created_at TEXT NOT NULL) STRICT;
+CREATE INDEX changes_entity ON changes(entity,entity_id,field,hlc);
+CREATE TABLE derivations(id TEXT PRIMARY KEY,item_id TEXT NOT NULL REFERENCES items(id),revision_id TEXT NOT NULL REFERENCES revisions(id),kind TEXT NOT NULL CHECK(kind IN ('digest')),schema_version INTEGER NOT NULL CHECK(schema_version>0),producer_json TEXT NOT NULL,input_hash TEXT NOT NULL CHECK(length(input_hash)=64),output_json TEXT NOT NULL,review_json TEXT NOT NULL DEFAULT '{}',created_at TEXT NOT NULL) STRICT;
+CREATE INDEX derivations_revision ON derivations(revision_id,kind,input_hash);
+CREATE INDEX derivations_item ON derivations(item_id,kind,created_at);
+`;

package/dist/migrations/006.d.ts

@@ -0,0 +1,3 @@
+/** Storage diet: readable FTS keeps no text copy; representations.markdown and items.title stay authoritative. */
+export declare const version = 6;
+export declare const sql = "\nCREATE TABLE readable_fts_rows(id INTEGER PRIMARY KEY,item_id TEXT NOT NULL UNIQUE REFERENCES items(id),revision_id TEXT NOT NULL REFERENCES revisions(id)) STRICT;\nDROP TABLE readable_fts;\nCREATE VIRTUAL TABLE readable_fts USING fts5(title,markdown,content='',contentless_delete=1,tokenize='unicode61');\nINSERT INTO readable_fts_rows(item_id,revision_id) SELECT i.id,i.current_revision FROM items i WHERE i.deleted_at IS NULL AND i.current_revision IS NOT NULL AND EXISTS(SELECT 1 FROM representations p WHERE p.revision_id=i.current_revision AND p.kind='markdown' AND p.markdown IS NOT NULL) ORDER BY i.created_at,i.id;\nINSERT INTO readable_fts(rowid,title,markdown) SELECT m.id,i.title,(SELECT p.markdown FROM representations p WHERE p.revision_id=m.revision_id AND p.kind='markdown' AND p.markdown IS NOT NULL ORDER BY p.rowid LIMIT 1) FROM readable_fts_rows m JOIN items i ON i.id=m.item_id ORDER BY m.id;\n";

package/dist/migrations/006.js

@@ -0,0 +1,9 @@
+/** Storage diet: readable FTS keeps no text copy; representations.markdown and items.title stay authoritative. */
+export const version = 6;
+export const sql = `
+CREATE TABLE readable_fts_rows(id INTEGER PRIMARY KEY,item_id TEXT NOT NULL UNIQUE REFERENCES items(id),revision_id TEXT NOT NULL REFERENCES revisions(id)) STRICT;
+DROP TABLE readable_fts;
+CREATE VIRTUAL TABLE readable_fts USING fts5(title,markdown,content='',contentless_delete=1,tokenize='unicode61');
+INSERT INTO readable_fts_rows(item_id,revision_id) SELECT i.id,i.current_revision FROM items i WHERE i.deleted_at IS NULL AND i.current_revision IS NOT NULL AND EXISTS(SELECT 1 FROM representations p WHERE p.revision_id=i.current_revision AND p.kind='markdown' AND p.markdown IS NOT NULL) ORDER BY i.created_at,i.id;
+INSERT INTO readable_fts(rowid,title,markdown) SELECT m.id,i.title,(SELECT p.markdown FROM representations p WHERE p.revision_id=m.revision_id AND p.kind='markdown' AND p.markdown IS NOT NULL ORDER BY p.rowid LIMIT 1) FROM readable_fts_rows m JOIN items i ON i.id=m.item_id ORDER BY m.id;
+`;

package/dist/migrations/007.d.ts

@@ -0,0 +1,3 @@
+/** Editorial derivations: a reviewed source note (Jungle Reading) kept losslessly beside Digest v1. SQLite cannot widen a CHECK in place, so the table is rebuilt; rows and ids are unchanged. */
+export declare const version = 7;
+export declare const sql = "\nCREATE TABLE derivations_v7(id TEXT PRIMARY KEY,item_id TEXT NOT NULL REFERENCES items(id),revision_id TEXT NOT NULL REFERENCES revisions(id),kind TEXT NOT NULL CHECK(kind IN ('digest','editorial')),schema_version INTEGER NOT NULL CHECK(schema_version>0),producer_json TEXT NOT NULL,input_hash TEXT NOT NULL CHECK(length(input_hash)=64),output_json TEXT NOT NULL,review_json TEXT NOT NULL DEFAULT '{}',created_at TEXT NOT NULL) STRICT;\nINSERT INTO derivations_v7(rowid,id,item_id,revision_id,kind,schema_version,producer_json,input_hash,output_json,review_json,created_at) SELECT rowid,id,item_id,revision_id,kind,schema_version,producer_json,input_hash,output_json,review_json,created_at FROM derivations ORDER BY rowid;\nDROP TABLE derivations;\nALTER TABLE derivations_v7 RENAME TO derivations;\nCREATE INDEX derivations_revision ON derivations(revision_id,kind,input_hash);\nCREATE INDEX derivations_item ON derivations(item_id,kind,created_at);\n";

package/dist/migrations/007.js

@@ -0,0 +1,10 @@
+/** Editorial derivations: a reviewed source note (Jungle Reading) kept losslessly beside Digest v1. SQLite cannot widen a CHECK in place, so the table is rebuilt; rows and ids are unchanged. */
+export const version = 7;
+export const sql = `
+CREATE TABLE derivations_v7(id TEXT PRIMARY KEY,item_id TEXT NOT NULL REFERENCES items(id),revision_id TEXT NOT NULL REFERENCES revisions(id),kind TEXT NOT NULL CHECK(kind IN ('digest','editorial')),schema_version INTEGER NOT NULL CHECK(schema_version>0),producer_json TEXT NOT NULL,input_hash TEXT NOT NULL CHECK(length(input_hash)=64),output_json TEXT NOT NULL,review_json TEXT NOT NULL DEFAULT '{}',created_at TEXT NOT NULL) STRICT;
+INSERT INTO derivations_v7(rowid,id,item_id,revision_id,kind,schema_version,producer_json,input_hash,output_json,review_json,created_at) SELECT rowid,id,item_id,revision_id,kind,schema_version,producer_json,input_hash,output_json,review_json,created_at FROM derivations ORDER BY rowid;
+DROP TABLE derivations;
+ALTER TABLE derivations_v7 RENAME TO derivations;
+CREATE INDEX derivations_revision ON derivations(revision_id,kind,input_hash);
+CREATE INDEX derivations_item ON derivations(item_id,kind,created_at);
+`;

package/dist/src/annotations/quote.d.ts

@@ -0,0 +1,32 @@
+/** A located quotation: raw offsets into the revision's Markdown plus context that survives re-capture. */
+export interface TextQuote {
+    type: 'text-quote';
+    exact: string;
+    prefix: string;
+    suffix: string;
+    start: number;
+    end: number;
+}
+export declare function foldText(value: string): string;
+/** Every occurrence of `quote` in `source`, compared after folding. */
+export declare function locateAll(source: string, quote: string): {
+    start: number;
+    end: number;
+}[];
+export declare function textQuote(source: string, start: number, end: number): TextQuote;
+/** The single match for `quote`, or undefined when absent or ambiguous. */
+export declare function locateQuote(source: string, quote: string): TextQuote | undefined;
+/** A stored selector still names its exact text in `source`. */
+export declare function anchorHolds(source: string, anchor: TextQuote): boolean;
+export declare function words(value: string): number;
+/**
+ * Every occurrence of `quote` in `source` ignoring inline Markdown and case, within optional raw bounds.
+ * Use only to snap a model's near-verbatim quote to the exact raw span; checking still uses `locateAll`.
+ */
+export declare function locateLoose(source: string, quote: string, bounds?: {
+    start: number;
+    end: number;
+}): {
+    start: number;
+    end: number;
+}[];

package/dist/src/annotations/quote.js

@@ -0,0 +1,135 @@
+const context = 32;
+const graphemes = new Intl.Segmenter(undefined, { granularity: 'grapheme' });
+const escapable = /^[\\`*_{}[\]()#+\-.!|>~<"']$/;
+/**
+ * Fold the differences a reader cannot see: Markdown escapes, curly quotes, dashes, runs of
+ * whitespace and Unicode normalization. Folding works on grapheme clusters, each NFC-normalized;
+ * every folded code unit of a cluster maps to the cluster's raw bounds, so `map[i]` and `ends[i]`
+ * bound the raw text of folded unit i and matches map back exactly.
+ */
+function fold(raw) {
+    let text = '', space = false;
+    const map = [], ends = [], clusters = [...graphemes.segment(raw)];
+    const emit = (out, start, end) => { text += out; for (let k = 0; k < out.length; k++) {
+        map.push(start);
+        ends.push(end);
+    } };
+    for (let c = 0; c < clusters.length; c++) {
+        const { segment, index } = clusters[c];
+        // An escaped character maps to its backslash, so a match starting there keeps the escape.
+        let ch = segment, end = index + segment.length;
+        if (segment === '\\' && c + 1 < clusters.length && escapable.test(clusters[c + 1].segment)) {
+            ch = clusters[++c].segment;
+            end = clusters[c].index + ch.length;
+        }
+        if (/^\s+$/.test(ch)) {
+            if (!space && text.length)
+                emit(' ', index, end);
+            space = true;
+            continue;
+        }
+        space = false;
+        emit(ch.replace(/[‘’‚′]/g, "'").replace(/[“”„″]/g, '"').replace(/[\u2010-\u2015\u2212]/g, '-').normalize('NFC'), index, end);
+    }
+    if (text.endsWith(' ')) {
+        text = text.slice(0, -1);
+        map.pop();
+        ends.pop();
+    }
+    return { text, map, ends };
+}
+export function foldText(value) { return fold(value).text; }
+/** Every occurrence of `quote` in `source`, compared after folding. */
+export function locateAll(source, quote) {
+    const s = fold(source), q = fold(quote).text, out = [];
+    if (!q)
+        return out;
+    for (let at = s.text.indexOf(q); at >= 0; at = s.text.indexOf(q, at + 1)) {
+        // A match must start and end on cluster boundaries; half a cluster has no raw offsets.
+        const last = at + q.length - 1;
+        if (at > 0 && s.map[at - 1] === s.map[at] || last + 1 < s.text.length && s.map[last + 1] === s.map[last])
+            continue;
+        out.push({ start: s.map[at], end: s.ends[last] });
+    }
+    return out;
+}
+export function textQuote(source, start, end) {
+    return { type: 'text-quote', exact: source.slice(start, end), prefix: source.slice(Math.max(0, start - context), start), suffix: source.slice(end, end + context), start, end };
+}
+/** The single match for `quote`, or undefined when absent or ambiguous. */
+export function locateQuote(source, quote) {
+    const all = locateAll(source, quote);
+    return all.length === 1 ? textQuote(source, all[0].start, all[0].end) : undefined;
+}
+/** A stored selector still names its exact text in `source`. */
+export function anchorHolds(source, anchor) {
+    return Number.isSafeInteger(anchor.start) && Number.isSafeInteger(anchor.end) && anchor.start >= 0 && anchor.end > anchor.start && source.slice(anchor.start, anchor.end) === anchor.exact;
+}
+export function words(value) { const t = foldText(value); return t ? t.split(' ').length : 0; }
+/**
+ * Rendered-text projection for snapping a near-verbatim quote: drops inline Markdown syntax (link targets, brackets,
+ * emphasis, code ticks, line-start block markers), folds like `fold`, and lowercases. Offsets map back to raw text.
+ */
+function rendered(raw) {
+    let text = '', map = [], ends = [], space = false, lineStart = true;
+    for (let i = 0; i < raw.length; i++) {
+        let ch = raw[i], at = i;
+        if (lineStart) {
+            const m = /^[ \t]*(?:>[ \t]?|#{1,6}[ \t]+|[-*+][ \t]+|\d{1,3}[.)][ \t]+)*/.exec(raw.slice(i, i + 64));
+            if (m[0].length) {
+                i += m[0].length - 1;
+                lineStart = false;
+                continue;
+            }
+        }
+        lineStart = ch === '\n';
+        if (ch === ']' && raw[i + 1] === '(') {
+            const close = raw.indexOf(')', i + 2);
+            if (close > 0 && close - i < 2048 && !raw.slice(i + 2, close).includes('\n')) {
+                i = close;
+                continue;
+            }
+        }
+        if (ch === '!' && raw[i + 1] === '[' || /[[\]*_`]/.test(ch))
+            continue;
+        if (ch === '\\' && i + 1 < raw.length && /[\\`*_{}[\]()#+\-.!|>~<"']/.test(raw[i + 1])) {
+            ch = raw[++i];
+        }
+        if (/\s/.test(ch)) {
+            if (!space && text.length) {
+                text += ' ';
+                map.push(i);
+                ends.push(i + 1);
+            }
+            space = true;
+            continue;
+        }
+        space = false;
+        ch = ch.replace(/[‘’‚′]/g, "'").replace(/[“”„″]/g, '"').replace(/[‐-―−]/g, '-').normalize('NFC').toLowerCase();
+        // One entry per output code unit: normalization and case mapping may change the length.
+        text += ch;
+        for (let k = 0; k < ch.length; k++) {
+            map.push(at);
+            ends.push(i + 1);
+        }
+    }
+    if (text.endsWith(' ')) {
+        text = text.slice(0, -1);
+        map.pop();
+        ends.pop();
+    }
+    return { text, map, ends };
+}
+/**
+ * Every occurrence of `quote` in `source` ignoring inline Markdown and case, within optional raw bounds.
+ * Use only to snap a model's near-verbatim quote to the exact raw span; checking still uses `locateAll`.
+ */
+export function locateLoose(source, quote, bounds = { start: 0, end: source.length }) {
+    const q = rendered(quote).text.replace(/^["'\s]+|["'\s]+$/g, ''), out = [];
+    if (q.length < 8)
+        return out;
+    const s = rendered(source.slice(bounds.start, bounds.end));
+    for (let at = s.text.indexOf(q); at >= 0; at = s.text.indexOf(q, at + 1))
+        out.push({ start: bounds.start + s.map[at], end: bounds.start + s.ends[at + q.length - 1] });
+    return out;
+}

package/dist/src/annotations/store.d.ts

@@ -0,0 +1,50 @@
+import type { DatabaseSync } from 'node:sqlite';
+export interface Annotation {
+    id: string;
+    itemId: string;
+    kind: 'note' | 'highlight' | 'comment';
+    revisionId: string | null;
+    parentId: string | null;
+    anchor: unknown;
+    body: string;
+    createdAt: string;
+    updatedAt: string | null;
+}
+export declare function markdownFor(db: DatabaseSync, revisionId: string): string;
+/**
+ * Highlight `quote` in the item's current revision. An ambiguous quote needs `occurrence` (0-based),
+ * so a highlight never silently lands on the wrong passage.
+ */
+export declare function addHighlight(db: DatabaseSync, input: {
+    itemId: string;
+    quote: string;
+    occurrence?: number;
+    note?: string;
+}): Annotation;
+export declare function addComment(db: DatabaseSync, input: {
+    itemId: string;
+    body: string;
+    parentId?: string;
+}): Annotation;
+/**
+ * An imported note keeps a stable id per source note. Re-importing an unchanged note is a no-op;
+ * once the user edits or deletes it, the user's choice wins and the import skips it.
+ */
+export declare function importNote(db: DatabaseSync, input: {
+    itemId: string;
+    revisionId: string;
+    namespace: string;
+    sourceKey: string;
+    noteId: string;
+    body: string;
+}): void;
+export declare function updateAnnotation(db: DatabaseSync, id: string, body: string): Annotation;
+/** Tombstone, never a row delete, so another device learns the deletion. */
+export declare function deleteAnnotation(db: DatabaseSync, id: string): void;
+/** Reading state is one field per item; `updatedAt` travels with it. */
+export declare function setReadingState(db: DatabaseSync, itemId: string, state: unknown): void;
+export declare function readAnnotation(db: DatabaseSync, id: string): Annotation | undefined;
+/** The one annotation reader: live annotations in creation order, highlights checked against their revision. */
+export declare function listAnnotations(db: DatabaseSync, itemId: string): (Annotation & {
+    anchored?: boolean;
+})[];

package/dist/src/annotations/store.js

@@ -0,0 +1,89 @@
+import { randomUUID } from 'node:crypto';
+import { anchorHolds, locateAll, textQuote } from './quote.js';
+import { mutate } from '../store/changes.js';
+import { StoreError } from '../store/types.js';
+import { digest, text } from '../store/files.js';
+const now = () => new Date().toISOString();
+const fail = (code = 'invalid-input') => { throw new StoreError(code); };
+export function markdownFor(db, revisionId) {
+    const row = db.prepare("SELECT markdown FROM representations WHERE revision_id=? AND kind='markdown'").get(revisionId);
+    return typeof row?.markdown === 'string' ? row.markdown : fail('unavailable');
+}
+/** Create through the log with the full row image, so a replay rebuilds the same row. */
+function create(db, id, row) { mutate(db, 'annotation', id, 'created', row); }
+/**
+ * Highlight `quote` in the item's current revision. An ambiguous quote needs `occurrence` (0-based),
+ * so a highlight never silently lands on the wrong passage.
+ */
+export function addHighlight(db, input) {
+    text(input.quote, 4000);
+    if (input.note !== undefined)
+        text(input.note, 16384);
+    const item = db.prepare('SELECT current_revision FROM items WHERE id=? AND deleted_at IS NULL').get(input.itemId);
+    if (!item?.current_revision)
+        fail('unavailable');
+    const revisionId = String(item.current_revision), source = markdownFor(db, revisionId), matches = locateAll(source, input.quote);
+    const index = input.occurrence ?? (matches.length === 1 ? 0 : -1);
+    if (!Number.isSafeInteger(index) || index < 0 || index >= matches.length)
+        fail();
+    const anchor = textQuote(source, matches[index].start, matches[index].end), id = randomUUID();
+    create(db, id, { itemId: input.itemId, kind: 'highlight', revisionId, parentId: null, anchor, body: input.note ?? '', createdAt: now() });
+    return readAnnotation(db, id);
+}
+export function addComment(db, input) {
+    text(input.body, 16384);
+    if (!db.prepare('SELECT 1 FROM items WHERE id=? AND deleted_at IS NULL').get(input.itemId))
+        fail('unavailable');
+    if (input.parentId !== undefined) {
+        const parent = db.prepare('SELECT item_id,deleted_at FROM annotations WHERE id=?').get(input.parentId);
+        if (!parent || parent.item_id !== input.itemId || parent.deleted_at)
+            fail('unavailable');
+    }
+    const id = randomUUID();
+    create(db, id, { itemId: input.itemId, kind: 'comment', revisionId: null, parentId: input.parentId ?? null, anchor: null, body: input.body, createdAt: now() });
+    return readAnnotation(db, id);
+}
+/**
+ * An imported note keeps a stable id per source note. Re-importing an unchanged note is a no-op;
+ * once the user edits or deletes it, the user's choice wins and the import skips it.
+ */
+export function importNote(db, input) {
+    const id = digest(JSON.stringify(['import-note-v1', input.namespace, input.sourceKey, input.noteId]));
+    const prior = db.prepare('SELECT item_id,body,updated_at,deleted_at FROM annotations WHERE id=?').get(id);
+    if (prior) {
+        if (prior.item_id !== input.itemId || prior.body !== input.body && prior.updated_at === null && prior.deleted_at === null)
+            fail('integrity');
+        return;
+    }
+    create(db, id, { itemId: input.itemId, kind: 'note', revisionId: input.revisionId, parentId: null, anchor: { kind: 'imported-note', sourceId: input.noteId }, body: input.body, createdAt: now() });
+}
+export function updateAnnotation(db, id, body) {
+    text(body, 16384);
+    const row = db.prepare('SELECT body,deleted_at FROM annotations WHERE id=?').get(id);
+    if (!row || row.deleted_at)
+        fail('unavailable');
+    if (row.body !== body)
+        mutate(db, 'annotation', id, 'body', { body, updatedAt: now() });
+    return readAnnotation(db, id);
+}
+/** Tombstone, never a row delete, so another device learns the deletion. */
+export function deleteAnnotation(db, id) {
+    const row = db.prepare('SELECT deleted_at FROM annotations WHERE id=?').get(id);
+    if (!row)
+        fail('unavailable');
+    if (!row.deleted_at)
+        mutate(db, 'annotation', id, 'deleted', now());
+}
+/** Reading state is one field per item; `updatedAt` travels with it. */
+export function setReadingState(db, itemId, state) { mutate(db, 'reading', itemId, 'state', { state, updatedAt: now() }); }
+function annotationRow(r) { return { id: String(r.id), itemId: String(r.item_id), kind: r.kind, revisionId: r.revision_id === null ? null : String(r.revision_id), parentId: r.parent_id === null ? null : String(r.parent_id), anchor: JSON.parse(String(r.anchor_json)), body: String(r.body), createdAt: String(r.created_at), updatedAt: r.updated_at === null ? null : String(r.updated_at) }; }
+export function readAnnotation(db, id) { const r = db.prepare('SELECT * FROM annotations WHERE id=? AND deleted_at IS NULL').get(id); return r ? annotationRow(r) : undefined; }
+/** The one annotation reader: live annotations in creation order, highlights checked against their revision. */
+export function listAnnotations(db, itemId) {
+    return db.prepare('SELECT * FROM annotations WHERE item_id=? AND deleted_at IS NULL ORDER BY created_at,rowid').all(itemId).map(r => {
+        const a = annotationRow(r);
+        if (a.kind !== 'highlight' || !a.revisionId)
+            return a;
+        return { ...a, anchored: anchorHolds(markdownFor(db, a.revisionId), a.anchor) };
+    });
+}

package/dist/src/app/source.d.ts

@@ -2,6 +2,8 @@ export declare class InputError extends Error {
     readonly code = "invalid-input";
     constructor();
 }
+/** The identity form of an http(s) locator; source keys and shareable URIs both use it. Its output pins every existing source key. */
+export declare function normalizeWebLocator(locator: string): string;
 export declare function articleSource(locator: string, realm?: string): {
     sourceKey: string;
     locator: string;

package/dist/src/app/source.js

@@ -1,4 +1,5 @@
 import { createHash } from 'node:crypto';
+import { campaignFields, filterQuery } from '../identity/query-policy.js';
 export class InputError extends Error {
     code = 'invalid-input';
     constructor() { super('invalid-input'); }
@@ -7,6 +8,14 @@ export class InputError extends Error {
 function webSource(locator, realm, kind) {
     if (Buffer.byteLength(locator) > 8192 || !realm || Buffer.byteLength(realm) > 256 || /[\u0000-\u001f\u007f]/u.test(locator + realm))
         throw new InputError();
+    const normalized = normalizeWebLocator(locator);
+    const sourceKey = createHash('sha256').update(JSON.stringify([kind, realm, normalized])).digest('hex');
+    return { sourceKey, locator, realm };
+}
+/** The identity form of an http(s) locator; source keys and shareable URIs both use it. Its output pins every existing source key. */
+export function normalizeWebLocator(locator) {
+    if (Buffer.byteLength(locator) > 8192 || /[\u0000-\u001f\u007f]/u.test(locator))
+        throw new InputError();
     let url;
     try {
         url = new URL(locator);
@@ -16,24 +25,11 @@ function webSource(locator, realm, kind) {
     }
     if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password || !url.hostname)
         throw new InputError();
-    // Filter raw fields: URLSearchParams mutation rewrites unknown signed query bytes.
-    if (url.search) {
-        const fields = url.search.slice(1).split('&').filter(field => {
-            let key;
-            try {
-                key = decodeURIComponent(field.split('=', 1)[0].replaceAll('+', ' '));
-            }
-            catch {
-                return true;
-            }
-            return !['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'utm_id'].includes(key.toLowerCase());
-        });
-        url.search = fields.length ? '?' + fields.join('&') : '';
-    }
+    // Undecodable keys are kept: they cannot be campaign fields.
+    if (url.search)
+        url.search = filterQuery(url.search, key => campaignFields.has(key));
     // URL normalizes host casing/default ports. Preserve query order, unknown fields and fragments.
-    const normalized = url.href;
-    const sourceKey = createHash('sha256').update(JSON.stringify([kind, realm, normalized])).digest('hex');
-    return { sourceKey, locator, realm };
+    return url.href;
 }
 export function articleSource(locator, realm = 'public') { return webSource(locator, realm, 'article'); }
 /** A page and its media can be saved independently without changing existing article identities. */

package/dist/src/cli/arguments.js

@@ -1,5 +1,6 @@
 import { isAbsolute } from 'node:path';
 import { InputError } from '../app/source.js';
+import { reviewSpec } from './digest.js';
 const specifications = {
     init: { count: 0, flags: {} },
     add: { count: 1, flags: { connection: 'value', kind: 'value', wait: 'boolean', language: 'value', 'force-local': 'boolean', 'allow-private-network': 'boolean' } },
@@ -16,10 +17,16 @@ const specifications = {
     export: { count: 1, flags: { output: 'value' } },
     backup: { count: 0, flags: { output: 'value' } },
     restore: { count: 1, flags: { into: 'value' } },
-    gc: { count: 0, flags: { 'dry-run': 'boolean', apply: 'value' } },
+    gc: { count: 0, flags: { 'dry-run': 'boolean', apply: 'value', runtimes: 'boolean' } },
     worker: { count: 1, flags: { language: 'value', 'force-local': 'boolean' } },
-    index: { count: -1, flags: { profile: 'value' } },
-    import: { count: 1, flags: { apply: 'boolean', 'dry-run': 'boolean', limit: 'value', offset: 'value' } },
+    index: { count: -1, flags: { profile: 'value', keep: 'value', 'dry-run': 'boolean', vacuum: 'boolean' } },
+    visibility: { count: 2, flags: {} },
+    highlight: { count: 2, flags: { note: 'value', occurrence: 'value' } },
+    comment: { count: 2, flags: { on: 'value' } },
+    publish: { count: 0, flags: { 'dry-run': 'boolean', output: 'value', 'min-age-days': 'value', title: 'value', 'base-url': 'value' } },
+    digest: { count: 1, flags: { endpoint: 'value', model: 'value', 'check-support': 'boolean' } },
+    review: { count: 1, flags: { accept: 'value', reject: 'value' } },
+    import: { count: -1, flags: { apply: 'boolean', 'dry-run': 'boolean', limit: 'value', offset: 'value' } },
 };
 /** No shell grammar, implicit option coercion, duplicate flags, or unbounded argv. */
 export function parseArguments(argv) {
@@ -63,7 +70,7 @@ export function parseArguments(argv) {
             positional.push(arg);
     }
     const spec = command && specifications[command];
-    if (!command || !spec || (command === 'queue' ? !(['status', 'recover'].includes(positional[0]) && positional.length === 1 || ['cancel', 'retry'].includes(positional[0]) && positional.length === 2) : command === 'index' ? !(['create', 'run'].includes(positional[0]) && positional.length === 1 || ['status', 'rollback'].includes(positional[0]) && positional.length === 2) : positional.length !== spec.count))
+    if (!command || !spec || (command === 'queue' ? !(['status', 'recover'].includes(positional[0]) && positional.length === 1 || ['cancel', 'retry'].includes(positional[0]) && positional.length === 2) : command === 'import' ? !(positional.length === 1 || positional[0] === 'jungle' && positional.length === 2) : command === 'index' ? !(['create', 'run', 'prune'].includes(positional[0]) && positional.length === 1 || ['status', 'rollback'].includes(positional[0]) && positional.length === 2) : positional.length !== spec.count))
         throw new InputError();
     if (Object.keys(flags).some(key => !['library', 'json'].includes(key) && !Object.hasOwn(spec.flags, key)))
         throw new InputError();
@@ -76,7 +83,7 @@ export function parseArguments(argv) {
         throw new InputError();
     if (flags.mode && !['lexical', 'hybrid'].includes(String(flags.mode)))
         throw new InputError();
-    if (flags.limit && (!/^[1-9]\d{0,2}$/u.test(String(flags.limit)) || Number(flags.limit) > 100))
+    if (flags.limit && (!/^[1-9]\d{0,3}$/u.test(String(flags.limit)) || Number(flags.limit) > (command === 'import' && positional[0] === 'jungle' ? 1000 : 100)))
         throw new InputError();
     if (command === 'run' && !flags.drain || command === 'setup' && [flags.capture, flags.asr, flags.embedding].filter(Boolean).length !== 1)
         throw new InputError();
@@ -86,11 +93,19 @@ export function parseArguments(argv) {
         throw new InputError();
     if (command === 'restore' && (typeof flags.into !== 'string' || !isAbsolute(flags.into) || flags.library !== undefined || !isAbsolute(positional[0])))
         throw new InputError();
-    if (command === 'gc' && (Boolean(flags['dry-run']) === Boolean(flags.apply) || flags.apply !== undefined && !isAbsolute(String(flags.apply))))
+    if (command === 'gc' && (flags.runtimes ? flags.apply !== undefined : Boolean(flags['dry-run']) === Boolean(flags.apply) || flags.apply !== undefined && !isAbsolute(String(flags.apply))))
         throw new InputError();
     if (command === 'worker' && !['install', 'status', 'uninstall', 'serve'].includes(positional[0]))
         throw new InputError();
-    if (command === 'import' && (!isAbsolute(positional[0]) || flags.apply && flags['dry-run'] || flags.offset !== undefined && !/^(0|[1-9][0-9]{0,5})$/.test(String(flags.offset))))
+    if (command === 'import' && (!isAbsolute(positional.at(-1)) || flags.apply && flags['dry-run'] || flags.offset !== undefined && !/^(0|[1-9][0-9]{0,5})$/.test(String(flags.offset))))
+        throw new InputError();
+    if (command === 'visibility' && !['public', 'private', 'default'].includes(positional[1]) || command === 'publish' && (Boolean(flags['dry-run']) === Boolean(flags.output) || flags.output !== undefined && (typeof flags.output !== 'string' || !isAbsolute(flags.output)) || (flags.title !== undefined || flags['base-url'] !== undefined) && !flags.output || flags['min-age-days'] !== undefined && !/^(0|[1-9][0-9]{0,3})$/.test(String(flags['min-age-days'])) || Number(flags['min-age-days'] ?? 0) > 3650 || flags.title !== undefined && String(flags.title).length > 200 || flags['base-url'] !== undefined && !/^https:\/\//.test(String(flags['base-url']))))
+        throw new InputError();
+    if (command === 'digest' && (Boolean(flags.endpoint) !== Boolean(flags.model) || flags['check-support'] && !flags.endpoint || flags.model !== undefined && !/^[\w.:/@+-]{1,256}$/.test(String(flags.model))))
+        throw new InputError();
+    if (command === 'review' && (flags.accept === undefined && flags.reject === undefined || [flags.accept, flags.reject].some(v => v !== undefined && !reviewSpec.test(String(v)))))
+        throw new InputError();
+    if (flags.occurrence !== undefined && !/^(0|[1-9][0-9]{0,4})$/.test(String(flags.occurrence)))
         throw new InputError();
     if (flags.language !== undefined && flags.language !== 'en')
         throw new InputError();
@@ -100,6 +115,8 @@ export function parseArguments(argv) {
         throw new InputError();
     if ((flags.language || flags['force-local']) && (['add', 'refresh'].includes(command) && !flags.wait || command === 'worker' && !['serve', 'install'].includes(positional[0])))
         throw new InputError();
+    if (command === 'index' && positional[0] !== 'prune' && (flags.keep !== undefined || flags['dry-run'] || flags.vacuum) || flags.keep !== undefined && !/^(0|[1-9]|1[0-6])$/.test(String(flags.keep)))
+        throw new InputError();
     if (flags.profile && (!(command === 'search' || command === 'index' && positional[0] === 'create') || !/^[a-zA-Z0-9_-]{1,128}$/.test(String(flags.profile))))
         throw new InputError();
     return { command, positional, flags, library: library, json: flags.json === true || flags.format === 'json' };

package/dist/src/cli/digest.d.ts

@@ -0,0 +1,10 @@
+import type { Derivation, DigestReview } from '../store/foundation.js';
+/** Review targets: `all`, `gist`, `idea:N` or `quote:N.M` (0-based), comma-separated. */
+export declare const reviewSpec: RegExp;
+/**
+ * Merge `--accept` and `--reject` targets into the derivation's existing review. `all` applies first and specific
+ * targets override it; a target named under both flags is ambiguous and rejected.
+ */
+export declare function reviewFromSpec(d: Derivation, accept?: string, reject?: string): DigestReview;
+/** Plain-text digest with the review targets `sponge review` accepts. */
+export declare function digestText(d: Derivation): string;