@hraness/sponge 0.2.3 → 0.2.5

package/README.md

@@ -101,6 +101,35 @@ full-text search. Use `index create --profile NAME` and `search QUERY --profile
 to keep separate search profiles. Model changes create separate index generations; a new one
 becomes active only after it covers the current library. `index rollback
 GENERATION` requires a complete generation compatible with the current revisions.
+`index prune --dry-run` lists what pruning would delete; `index prune` deletes the
+vectors of cancelled, failed and retired generations, keeping the active and any
+building generation plus the most recent rollback-ready one per profile (`--keep N`,
+0-16). It refuses while index work, cleanup or a backup is running. Freed pages are
+reused; `--vacuum` also shrinks the file after checking free space. Rollback to a
+pruned generation needs a rebuild.
+
+### Digests
+
+A digest is a one-sentence gist and 3–5 ideas, each with 1–2 verbatim quotes.
+A local model proposes it; Sponge checks every rule against the captured text
+and stores only a passing draft. Drafting is opt-in and needs a model server you
+start yourself, such as llama.cpp `llama-server`, LM Studio or Ollama:
+
+```sh
+llama-server -m gemma-4-12b-it-Q4_K_M.gguf --port 8080
+sponge digest ITEM --endpoint http://127.0.0.1:8080 --model NAME
+sponge digest ITEM
+sponge review DIGEST --accept all --reject quote:1.0
+```
+
+`--model` must match an id the server lists at `/v1/models`. Endpoints must be
+literal loopback HTTP (`127.0.0.1` or `[::1]`); Sponge refuses anything else so
+captured text never leaves the machine, and never starts or downloads a model.
+Suggested models are Gemma 4 12B on 16 GB Macs and Gemma 4 E4B on 8 GB, with
+Qwen3.5 9B or 4B as alternatives. Rerunning with the same inputs returns the stored
+draft. Review targets are `all`, `gist`, `idea:N` and `quote:N.M` (0-based); only
+accepted fields can be published. `--check-support` also requires each quote to
+support its claim by local embedding similarity. See [digest drafting](src/digest/README.md).
 
 ### Export, recovery and background work
 
@@ -110,6 +139,7 @@ sponge backup --output /absolute/new/library.sponge
 sponge restore /absolute/library.sponge --into /absolute/new/library
 sponge gc --dry-run
 sponge gc --apply /absolute/path/to/report.json
+sponge gc --runtimes --dry-run
 sponge worker install --language en
 sponge worker status
 sponge worker uninstall
@@ -119,6 +149,10 @@ Exports contain Markdown, images and saved notes. Backups preserve revisions,
 assets, reading state and notes. Restore requires a new directory, verifies every
 file and leaves connections unresolved; runtimes and credentials are excluded.
 Garbage collection removes only unreferenced blobs after rechecking the report.
+`gc --runtimes` removes capture runtimes and model packages this Sponge no longer
+pins, plus settled worker journals; `--dry-run` only lists them. It holds worker
+custody, keeps models an active or building index generation needs, and refuses
+symlinks and anything in use. Successful `setup` runs the same collection.
 
 The background worker is opt-in and processes work sequentially. Its LaunchAgent
 pins the installed command; after an upgrade, reinstall the worker to update that
@@ -126,6 +160,25 @@ binding. Uninstall stops the worker and preserves the library. In-session startu
 and cleanup are checked; fresh-login startup remains unqualified. See
 [importing existing captures](docs/importing.md) for a preview-first import flow.
 
+### Public reading list
+
+```sh
+sponge publish --dry-run
+sponge publish --output /absolute/site --title 'My reads' --base-url https://reads.example.com/ --min-age-days 7
+```
+
+Items captured from public sources are public by default; `visibility ITEM private`
+hides one. `publish --output` writes a static site: a JSON Feed (`feed.json`,
+newest 100), monthly shards under `items/`, an Atom `feed.xml`, and an
+`index.html` that uses the bundled `<sponge-list>` element. An Ed25519 key created
+in the library on first publish signs a manifest of every file's hash; the
+private key stays in the library. The same library state gives byte-identical
+files. `--min-age-days N` holds back items listed in the last N days. Nothing is
+uploaded. Embed the list elsewhere with
+`<sponge-list src="https://reads.example.com/feed.json" verify></sponge-list>` and
+`sponge-list.js`; the host must allow cross-origin reads, and `verify` needs
+HTTPS. See [the decision record](docs/decisions/publish-feed.md).
+
 ### Selected authenticated connections
 
 The initial development interface accepts private `connections.json` in the

package/dist/migrations/005.d.ts

@@ -0,0 +1,3 @@
+/** Additive sync and publishing foundation: shareable URIs, visibility, change log, annotations, content-addressed derivations. */
+export declare const version = 5;
+export declare const sql = "\nCREATE TABLE item_uris(item_id TEXT NOT NULL REFERENCES items(id),uri TEXT NOT NULL CHECK(length(uri) BETWEEN 3 AND 8192),role TEXT NOT NULL CHECK(role IN ('primary','alias')),resolvable INTEGER NOT NULL CHECK(resolvable IN (0,1)),origin TEXT NOT NULL CHECK(origin IN ('locator','revision','metadata','user')),PRIMARY KEY(item_id,uri)) STRICT;\nCREATE UNIQUE INDEX item_uris_primary ON item_uris(item_id) WHERE role='primary';\nCREATE INDEX item_uris_uri ON item_uris(uri);\nALTER TABLE items ADD COLUMN visibility TEXT CHECK(visibility IN ('public','private'));\nALTER TABLE annotations ADD COLUMN kind TEXT NOT NULL DEFAULT 'note' CHECK(kind IN ('note','highlight','comment'));\nALTER TABLE annotations ADD COLUMN parent_id TEXT REFERENCES annotations(id);\nALTER TABLE annotations ADD COLUMN updated_at TEXT;\nALTER TABLE annotations ADD COLUMN deleted_at TEXT;\nCREATE TABLE device(id INTEGER PRIMARY KEY CHECK(id=1),device_id TEXT NOT NULL CHECK(length(device_id)=32),clock_ms INTEGER NOT NULL DEFAULT 0 CHECK(clock_ms>=0),clock_counter INTEGER NOT NULL DEFAULT 0 CHECK(clock_counter>=0)) STRICT;\nINSERT INTO device(id,device_id) VALUES(1,lower(hex(randomblob(16))));\nCREATE TABLE changes(seq INTEGER PRIMARY KEY AUTOINCREMENT,id TEXT NOT NULL UNIQUE,device_id TEXT NOT NULL,hlc TEXT NOT NULL,entity TEXT NOT NULL CHECK(entity IN ('item','annotation','reading','derivation')),entity_id TEXT NOT NULL,field TEXT NOT NULL,value_json TEXT NOT NULL,created_at TEXT NOT NULL) STRICT;\nCREATE INDEX changes_entity ON changes(entity,entity_id,field,hlc);\nCREATE TABLE derivations(id TEXT PRIMARY KEY,item_id TEXT NOT NULL REFERENCES items(id),revision_id TEXT NOT NULL REFERENCES revisions(id),kind TEXT NOT NULL CHECK(kind IN ('digest')),schema_version INTEGER NOT NULL CHECK(schema_version>0),producer_json TEXT NOT NULL,input_hash TEXT NOT NULL CHECK(length(input_hash)=64),output_json TEXT NOT NULL,review_json TEXT NOT NULL DEFAULT '{}',created_at TEXT NOT NULL) STRICT;\nCREATE INDEX derivations_revision ON derivations(revision_id,kind,input_hash);\nCREATE INDEX derivations_item ON derivations(item_id,kind,created_at);\n";

package/dist/migrations/005.js

@@ -0,0 +1,19 @@
+/** Additive sync and publishing foundation: shareable URIs, visibility, change log, annotations, content-addressed derivations. */
+export const version = 5;
+export const sql = `
+CREATE TABLE item_uris(item_id TEXT NOT NULL REFERENCES items(id),uri TEXT NOT NULL CHECK(length(uri) BETWEEN 3 AND 8192),role TEXT NOT NULL CHECK(role IN ('primary','alias')),resolvable INTEGER NOT NULL CHECK(resolvable IN (0,1)),origin TEXT NOT NULL CHECK(origin IN ('locator','revision','metadata','user')),PRIMARY KEY(item_id,uri)) STRICT;
+CREATE UNIQUE INDEX item_uris_primary ON item_uris(item_id) WHERE role='primary';
+CREATE INDEX item_uris_uri ON item_uris(uri);
+ALTER TABLE items ADD COLUMN visibility TEXT CHECK(visibility IN ('public','private'));
+ALTER TABLE annotations ADD COLUMN kind TEXT NOT NULL DEFAULT 'note' CHECK(kind IN ('note','highlight','comment'));
+ALTER TABLE annotations ADD COLUMN parent_id TEXT REFERENCES annotations(id);
+ALTER TABLE annotations ADD COLUMN updated_at TEXT;
+ALTER TABLE annotations ADD COLUMN deleted_at TEXT;
+CREATE TABLE device(id INTEGER PRIMARY KEY CHECK(id=1),device_id TEXT NOT NULL CHECK(length(device_id)=32),clock_ms INTEGER NOT NULL DEFAULT 0 CHECK(clock_ms>=0),clock_counter INTEGER NOT NULL DEFAULT 0 CHECK(clock_counter>=0)) STRICT;
+INSERT INTO device(id,device_id) VALUES(1,lower(hex(randomblob(16))));
+CREATE TABLE changes(seq INTEGER PRIMARY KEY AUTOINCREMENT,id TEXT NOT NULL UNIQUE,device_id TEXT NOT NULL,hlc TEXT NOT NULL,entity TEXT NOT NULL CHECK(entity IN ('item','annotation','reading','derivation')),entity_id TEXT NOT NULL,field TEXT NOT NULL,value_json TEXT NOT NULL,created_at TEXT NOT NULL) STRICT;
+CREATE INDEX changes_entity ON changes(entity,entity_id,field,hlc);
+CREATE TABLE derivations(id TEXT PRIMARY KEY,item_id TEXT NOT NULL REFERENCES items(id),revision_id TEXT NOT NULL REFERENCES revisions(id),kind TEXT NOT NULL CHECK(kind IN ('digest')),schema_version INTEGER NOT NULL CHECK(schema_version>0),producer_json TEXT NOT NULL,input_hash TEXT NOT NULL CHECK(length(input_hash)=64),output_json TEXT NOT NULL,review_json TEXT NOT NULL DEFAULT '{}',created_at TEXT NOT NULL) STRICT;
+CREATE INDEX derivations_revision ON derivations(revision_id,kind,input_hash);
+CREATE INDEX derivations_item ON derivations(item_id,kind,created_at);
+`;

package/dist/migrations/006.d.ts

@@ -0,0 +1,3 @@
+/** Storage diet: readable FTS keeps no text copy; representations.markdown and items.title stay authoritative. */
+export declare const version = 6;
+export declare const sql = "\nCREATE TABLE readable_fts_rows(id INTEGER PRIMARY KEY,item_id TEXT NOT NULL UNIQUE REFERENCES items(id),revision_id TEXT NOT NULL REFERENCES revisions(id)) STRICT;\nDROP TABLE readable_fts;\nCREATE VIRTUAL TABLE readable_fts USING fts5(title,markdown,content='',contentless_delete=1,tokenize='unicode61');\nINSERT INTO readable_fts_rows(item_id,revision_id) SELECT i.id,i.current_revision FROM items i WHERE i.deleted_at IS NULL AND i.current_revision IS NOT NULL AND EXISTS(SELECT 1 FROM representations p WHERE p.revision_id=i.current_revision AND p.kind='markdown' AND p.markdown IS NOT NULL) ORDER BY i.created_at,i.id;\nINSERT INTO readable_fts(rowid,title,markdown) SELECT m.id,i.title,(SELECT p.markdown FROM representations p WHERE p.revision_id=m.revision_id AND p.kind='markdown' AND p.markdown IS NOT NULL ORDER BY p.rowid LIMIT 1) FROM readable_fts_rows m JOIN items i ON i.id=m.item_id ORDER BY m.id;\n";

package/dist/migrations/006.js

@@ -0,0 +1,9 @@
+/** Storage diet: readable FTS keeps no text copy; representations.markdown and items.title stay authoritative. */
+export const version = 6;
+export const sql = `
+CREATE TABLE readable_fts_rows(id INTEGER PRIMARY KEY,item_id TEXT NOT NULL UNIQUE REFERENCES items(id),revision_id TEXT NOT NULL REFERENCES revisions(id)) STRICT;
+DROP TABLE readable_fts;
+CREATE VIRTUAL TABLE readable_fts USING fts5(title,markdown,content='',contentless_delete=1,tokenize='unicode61');
+INSERT INTO readable_fts_rows(item_id,revision_id) SELECT i.id,i.current_revision FROM items i WHERE i.deleted_at IS NULL AND i.current_revision IS NOT NULL AND EXISTS(SELECT 1 FROM representations p WHERE p.revision_id=i.current_revision AND p.kind='markdown' AND p.markdown IS NOT NULL) ORDER BY i.created_at,i.id;
+INSERT INTO readable_fts(rowid,title,markdown) SELECT m.id,i.title,(SELECT p.markdown FROM representations p WHERE p.revision_id=m.revision_id AND p.kind='markdown' AND p.markdown IS NOT NULL ORDER BY p.rowid LIMIT 1) FROM readable_fts_rows m JOIN items i ON i.id=m.item_id ORDER BY m.id;
+`;

package/dist/migrations/007.d.ts

@@ -0,0 +1,3 @@
+/** Editorial derivations: a reviewed source note (Jungle Reading) kept losslessly beside Digest v1. SQLite cannot widen a CHECK in place, so the table is rebuilt; rows and ids are unchanged. */
+export declare const version = 7;
+export declare const sql = "\nCREATE TABLE derivations_v7(id TEXT PRIMARY KEY,item_id TEXT NOT NULL REFERENCES items(id),revision_id TEXT NOT NULL REFERENCES revisions(id),kind TEXT NOT NULL CHECK(kind IN ('digest','editorial')),schema_version INTEGER NOT NULL CHECK(schema_version>0),producer_json TEXT NOT NULL,input_hash TEXT NOT NULL CHECK(length(input_hash)=64),output_json TEXT NOT NULL,review_json TEXT NOT NULL DEFAULT '{}',created_at TEXT NOT NULL) STRICT;\nINSERT INTO derivations_v7(rowid,id,item_id,revision_id,kind,schema_version,producer_json,input_hash,output_json,review_json,created_at) SELECT rowid,id,item_id,revision_id,kind,schema_version,producer_json,input_hash,output_json,review_json,created_at FROM derivations ORDER BY rowid;\nDROP TABLE derivations;\nALTER TABLE derivations_v7 RENAME TO derivations;\nCREATE INDEX derivations_revision ON derivations(revision_id,kind,input_hash);\nCREATE INDEX derivations_item ON derivations(item_id,kind,created_at);\n";

package/dist/migrations/007.js

@@ -0,0 +1,10 @@
+/** Editorial derivations: a reviewed source note (Jungle Reading) kept losslessly beside Digest v1. SQLite cannot widen a CHECK in place, so the table is rebuilt; rows and ids are unchanged. */
+export const version = 7;
+export const sql = `
+CREATE TABLE derivations_v7(id TEXT PRIMARY KEY,item_id TEXT NOT NULL REFERENCES items(id),revision_id TEXT NOT NULL REFERENCES revisions(id),kind TEXT NOT NULL CHECK(kind IN ('digest','editorial')),schema_version INTEGER NOT NULL CHECK(schema_version>0),producer_json TEXT NOT NULL,input_hash TEXT NOT NULL CHECK(length(input_hash)=64),output_json TEXT NOT NULL,review_json TEXT NOT NULL DEFAULT '{}',created_at TEXT NOT NULL) STRICT;
+INSERT INTO derivations_v7(rowid,id,item_id,revision_id,kind,schema_version,producer_json,input_hash,output_json,review_json,created_at) SELECT rowid,id,item_id,revision_id,kind,schema_version,producer_json,input_hash,output_json,review_json,created_at FROM derivations ORDER BY rowid;
+DROP TABLE derivations;
+ALTER TABLE derivations_v7 RENAME TO derivations;
+CREATE INDEX derivations_revision ON derivations(revision_id,kind,input_hash);
+CREATE INDEX derivations_item ON derivations(item_id,kind,created_at);
+`;