@hraness/sponge 0.2.5 → 0.3.0

package/dist/src/digest/pipeline.js

@@ -1,37 +1,43 @@
-import { createHash } from 'node:crypto';
 import { foldText, locateAll, locateLoose, words } from '../annotations/quote.js';
-import { checkDigest, digestLimits as L, forms, ideaKinds, parseDigest } from './schema.js';
-import { StoreError } from '../store/types.js';
+import { checkDigest, digestLimits as L, forms, parseDigest, } from './schema.js';
+import { SpongeError } from '../errors.js';
+import { sha256 } from '../util/hash.js';
 /**
- * Model-agnostic Digest v1 drafting. The model sees numbered passages and cites a passage id plus a short exact
- * span; code snaps, orders, prunes and checks. Nothing leaves this module except through the injected model.
+ * Model-agnostic Digest v2 drafting. The model sees numbered passages and cites a passage id plus a short exact
+ * span; code snaps, prunes and checks. Nothing leaves this module except through the injected model.
  */
-export const PROMPT_VERSION = 'sponge-digest-prompt-v1';
+export const PROMPT_VERSION = 'sponge-digest-prompt-v4';
 /** 24k characters is roughly 6k tokens of passages, leaving room for instructions and output in an 8k context on an 8 GB Mac. */
-export const defaultDigestParams = { temperature: 0, seed: 1, maxTokens: 2048, contextChars: 24000, maxSections: 24, timeoutMs: 300000, deadlineMs: 1800000, maxOutputBytes: 65536 };
-export class DigestError extends Error {
-    code;
-    issues;
-    constructor(code, issues) {
-        super(code);
-        this.code = code;
-        this.issues = issues;
-        this.name = 'DigestError';
-    }
-}
+const defaultDigestParams = {
+    temperature: 0,
+    seed: 1,
+    maxTokens: 2048,
+    contextChars: 24000,
+    maxSections: 24,
+    timeoutMs: 300000,
+    deadlineMs: 1800000,
+    maxOutputBytes: 65536,
+};
 /** Quotes under this many words carry no evidence; the checker is gaining the same floor. */
-export const minQuoteWords = 4;
-const sha256 = (v) => createHash('sha256').update(v).digest('hex');
+const minQuoteWords = 4;
 const collapse = (v) => v.replace(/\s+/g, ' ').trim();
 const object = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
-/** Sorted-key JSON; the only serialization that enters input hashes. */
+/**
+ * Sorted-key JSON; the only serialization that enters input hashes. Deliberately not `util/canonical.ts`: this one
+ * accepts finite non-integer numbers (temperature) and writes `undefined` array entries as null, so merging would
+ * change digest input hashes.
+ */
 export function canonicalJson(value) {
     if (Array.isArray(value))
         return `[${value.map(canonicalJson).join(',')}]`;
     if (object(value))
-        return `{${Object.keys(value).filter(k => value[k] !== undefined).sort().map(k => `${JSON.stringify(k)}:${canonicalJson(value[k])}`).join(',')}}`;
+        return `{${Object.keys(value)
+            .filter((k) => value[k] !== undefined)
+            .sort()
+            .map((k) => `${JSON.stringify(k)}:${canonicalJson(value[k])}`)
+            .join(',')}}`;
     if (typeof value === 'number' && !Number.isFinite(value))
-        throw new DigestError('invalid-input');
+        throw new SpongeError('invalid-input');
     return JSON.stringify(value ?? null);
 }
 /** Paragraph-level passages with raw offsets. Headings label passages; code fences and image-only blocks are not quotable. */
@@ -85,8 +91,12 @@ export function passages(markdown, maxChars = 1200) {
     }
     return out;
 }
-function lastSentenceEnd(window) { let best = -1; for (const m of window.matchAll(/[.!?]["')\]]*(?=\s)/g))
-    best = m.index + m[0].length; return best; }
+function lastSentenceEnd(window) {
+    let best = -1;
+    for (const m of window.matchAll(/[.!?]["')\]]*(?=\s)/g))
+        best = m.index + m[0].length;
+    return best;
+}
 const display = (p) => `[${p.id}] ${collapse(p.text)}`;
 /** Pack passages into sections of at most `budget` display characters, preferring heading boundaries past half a budget. */
 export function sectionsOf(list, budget) {
@@ -94,7 +104,8 @@ export function sectionsOf(list, budget) {
     let current = [], size = 0;
     for (const p of list) {
         const n = display(p).length + 1;
-        if (current.length && (size + n > budget || p.heading !== current[current.length - 1].heading && size > budget / 2)) {
+        if (current.length &&
+            (size + n > budget || (p.heading !== current[current.length - 1].heading && size > budget / 2))) {
             out.push(current);
             current = [];
             size = 0;
@@ -107,29 +118,74 @@ export function sectionsOf(list, budget) {
     return out;
 }
 const sourceWords = (markdown) => words(markdown);
-export const quoteBudget = (markdown) => Math.min(L.totalQuoteWords, Math.max(L.quoteWords, Math.floor(sourceWords(markdown) * L.sourceQuoteShare)));
+const quoteBudget = (markdown) => Math.min(L.totalQuoteWords, Math.max(L.quoteWords, Math.floor(sourceWords(markdown) * L.sourceQuoteShare)));
 const str = (maxLength) => ({ type: 'string', minLength: 1, maxLength });
 /** JSON Schema for the model's draft. Tags become an enum when a vocabulary is given. */
 export function draftSchema(passageCount, vocabulary) {
-    const quote = { type: 'object', additionalProperties: false, required: ['passage', 'text'], properties: { passage: { type: 'integer', minimum: 1, maximum: Math.max(1, passageCount) }, text: str(300), speaker: str(120) } };
-    const idea = { type: 'object', additionalProperties: false, required: ['claim', 'detail', 'kind', 'voice', 'quotes'], properties: { claim: str(200), detail: str(600), kind: { type: 'string', enum: [...ideaKinds] }, voice: { type: 'object', additionalProperties: false, required: ['type'], properties: { type: { type: 'string', enum: ['author', 'speaker', 'reported'] }, name: str(120) } }, quotes: { type: 'array', minItems: 1, maxItems: L.maxQuotesPerIdea, items: quote } } };
-    const tag = vocabulary?.length ? { type: 'string', enum: [...vocabulary] } : { type: 'string', pattern: '^[a-z0-9][a-z0-9-]{0,63}$' };
-    return { type: 'object', additionalProperties: false, required: ['lead', 'ideas', 'tags'], properties: { lead: str(L.leadChars), context: str(400), ideas: { type: 'array', minItems: L.minIdeas, maxItems: L.maxIdeas, items: idea }, tags: { type: 'array', minItems: 1, maxItems: L.maxTags, items: tag } } };
+    const quote = {
+        type: 'object',
+        additionalProperties: false,
+        required: ['passage', 'text'],
+        properties: {
+            passage: { type: 'integer', minimum: 1, maximum: Math.max(1, passageCount) },
+            text: str(300),
+            attribution: str(200),
+        },
+    };
+    const idea = {
+        type: 'object',
+        additionalProperties: false,
+        required: ['claim', 'detail', 'quotes'],
+        properties: {
+            claim: str(200),
+            detail: str(600),
+            quotes: { type: 'array', minItems: 0, maxItems: L.maxQuotesPerIdea, items: quote },
+        },
+    };
+    const tag = vocabulary?.length
+        ? { type: 'string', enum: [...vocabulary] }
+        : { type: 'string', pattern: '^[a-z0-9][a-z0-9-]{0,63}$' };
+    return {
+        type: 'object',
+        additionalProperties: false,
+        required: ['lead', 'ideas', 'tags'],
+        properties: {
+            lead: str(L.leadChars),
+            context: str(600),
+            ideas: { type: 'array', minItems: L.minIdeas, maxItems: L.maxIdeas, items: idea },
+            tags: { type: 'array', minItems: 0, maxItems: L.maxTags, items: tag },
+        },
+    };
 }
-export function notesSchema(first, last) {
-    return { type: 'object', additionalProperties: false, required: ['notes'], properties: { notes: { type: 'array', minItems: 1, maxItems: 4, items: { type: 'object', additionalProperties: false, required: ['point', 'passage', 'quote'], properties: { point: str(300), passage: { type: 'integer', minimum: first, maximum: last }, quote: str(300) } } } } };
+function notesSchema(first, last) {
+    return {
+        type: 'object',
+        additionalProperties: false,
+        required: ['notes'],
+        properties: {
+            notes: {
+                type: 'array',
+                minItems: 1,
+                maxItems: 4,
+                items: {
+                    type: 'object',
+                    additionalProperties: false,
+                    required: ['point', 'passage', 'quote'],
+                    properties: { point: str(300), passage: { type: 'integer', minimum: first, maximum: last }, quote: str(300) },
+                },
+            },
+        },
+    };
 }
-const system = (budget, vocabulary) => `You write a Digest v1 of one source for a private reading library. Reply with JSON only, matching the schema.
-- lead: one sentence of at most ${L.leadChars} characters that names the author or organization when known and states the main finding.
-- context: optional, at most ${L.contextWords} words of essential background.
-- ideas: ${L.minIdeas} to ${L.maxIdeas}, in the order they appear in the source. Each idea has:
-  - claim: one sentence of at most ${L.claimWords} words that asserts something; never a topic label.
-  - detail: at most ${L.detailWords} words in at most ${L.detailSentences} sentences: evidence, mechanism or limits.
-  - kind: ${ideaKinds.join(', ')}.
-  - voice: "author" when the author makes the claim; "speaker" with name for a speaker in a talk or interview; "reported" with name when the author reports someone else's claim.
-  - quotes: 1 or ${L.maxQuotesPerIdea}, each copied character for character from one numbered passage, ${minQuoteWords} to ${L.quoteWords} words, with that passage number. No added quotation marks or ellipses. Give speaker when the voice is not the author. Prefer reader highlights.
-- All quotes together stay within ${budget} words and never overlap.
-- tags: 1 to ${L.maxTags} ${vocabulary?.length ? `from: ${vocabulary.join(', ')}` : 'lowercase kebab-case topics'}.
+const system = (budget, vocabulary) => `You write a Digest v2 of one source for a private reading library. Reply with JSON only, matching the schema.
+- lead: one dense sentence of at most ${L.leadChars} characters: who (author, organization or product by name) did or argues what, with the concrete specifics that make it this source (names, numbers, mechanism). Not a teaser.
+- context: optional, at most ${L.contextWords} words: the key evidence, numbers or limits the lead could not hold. Lead and context together stay within ${L.gistWords} words and ${L.gistSentences} sentences.
+- ideas: ${L.minIdeas} to ${L.maxIdeas} (prefer ${L.maxIdeas} when the source has them), in the order the author makes them, covering the whole source including its caveats and limits. Each idea is a claim with detail:
+  - claim: one short sentence of at most ${L.claimWords} words (often 4 to 10) that asserts something; never a topic label.
+  - detail: at most ${L.detailWords} words in at most ${L.detailSentences} sentences: the specific evidence, mechanism, numbers or limits from the source.
+  - quotes: 0 to ${L.maxQuotesPerIdea} that support this idea, each a contiguous span copied character for character from one numbered passage, ${minQuoteWords + 2} to 20 words, with that passage number. A quote is a fragment, not a whole long sentence: when a sentence runs past 20 words, copy only its strongest part. No added quotation marks or ellipses. Give attribution (who said it) when it is not the author. Prefer reader highlights.
+- Attach at least ${L.minQuotes} and at most ${L.maxQuotes} quotes in all; ${L.minQuotes + 1} to 3 is typical. All quotes together stay within ${budget} words and never overlap.
+- tags: 0 to ${L.maxTags} ${vocabulary?.length ? `from: ${vocabulary.join(', ')}` : 'lowercase kebab-case topics'}; usually one subject tag.
 Use only what the source says.`;
 function sourceBlock(title, form, list, highlights) {
     let heading;
@@ -145,12 +201,28 @@ function sourceBlock(title, form, list, highlights) {
     const marks = highlights.map((h, i) => `[h${i + 1}${h.passage ? `, passage ${h.passage}` : ''}] ${collapse(h.text).slice(0, 400)}`);
     return `Form: ${form}${title ? `\nTitle: ${collapse(title).slice(0, 300)}` : ''}${marks.length ? `\n\nReader highlights (preferred quotes):\n${marks.join('\n')}` : ''}\n\nPassages:\n${lines.join('\n')}`;
 }
-const rule = { lead: `one sentence, at most ${L.leadChars} characters`, context: `at most ${L.contextWords} words`, claim: `one distinct sentence, at most ${L.claimWords} words`, detail: `at most ${L.detailWords} words and ${L.detailSentences} sentences`, quotes: `1-${L.maxQuotesPerIdea} quotes, each copied exactly from its passage, ${minQuoteWords} to ${L.quoteWords} words, not overlapping another quote`, ideas: `${L.minIdeas}-${L.maxIdeas} ideas; quotes within the total budget`, tags: `1-${L.maxTags} distinct allowed tags` };
+const rule = {
+    gist: `lead and context within ${L.gistWords} words and ${L.gistSentences} sentences`,
+    lead: `one sentence, at most ${L.leadChars} characters`,
+    context: `at most ${L.contextWords} words`,
+    claim: `one distinct sentence, at most ${L.claimWords} words`,
+    detail: `at most ${L.detailWords} words and ${L.detailSentences} sentences`,
+    quotes: `0-${L.maxQuotesPerIdea} quotes per idea, each copied exactly from its passage, ${minQuoteWords} to ${L.quoteWords} words, not overlapping another quote`,
+    attribution: `at most ${L.attributionWords} words`,
+    ideas: `${L.minIdeas}-${L.maxIdeas} ideas; ${L.minQuotes}-${L.maxQuotes} quotes in all, within the total budget`,
+    tags: `0-${L.maxTags} distinct allowed tags`,
+};
 function repairPrompt(issues) {
-    const lines = issues.slice(0, 40).map(i => { const last = i.path.split('.').filter(s => !/^\d+$/.test(s)).pop() ?? ''; return `- ${i.path || 'output'}: ${i.code}${rule[last] ? ` (rule: ${rule[last]})` : ''}`; });
-    return `That draft broke these rules:\n${lines.join('\n')}\nReturn the complete corrected JSON. Copy quotes exactly from the numbered passages.`;
+    const lines = issues.slice(0, 40).map((i) => {
+        const last = i.path
+            .split('.')
+            .filter((s) => !/^\d+$/.test(s))
+            .pop() ?? '';
+        return `- ${i.path || 'output'}: ${i.code}${rule[last] ? ` (rule: ${rule[last]})` : ''}`;
+    });
+    return `That draft broke these rules:\n${lines.join('\n')}\nReturn the complete corrected JSON. Copy quotes exactly from the numbered passages; shorten a too-long quote to its strongest contiguous 6 to 20 words.`;
 }
-/** Map the model's draft shape into DigestV1; parseDigest then decides whether the shape is valid. */
+/** Map the model's draft shape into Digest v2; parseDigest then decides whether the shape is valid. Ideas keep the model's order. */
 function fromModel(text, form) {
     let v;
     try {
@@ -161,20 +233,32 @@ function fromModel(text, form) {
     }
     if (!object(v) || !Array.isArray(v.ideas) || !Array.isArray(v.tags))
         return { issues: [{ path: '', code: 'shape' }] };
-    const trim = (s) => typeof s === 'string' ? s.trim() : s, cites = [];
+    const trim = (s) => (typeof s === 'string' ? s.trim() : s), cites = [];
     const ideas = v.ideas.map((idea) => {
         if (!object(idea)) {
             cites.push([]);
             return idea;
         }
-        const voice = object(idea.voice) && idea.voice.type !== 'author' ? { type: idea.voice.type, name: trim(idea.voice.name) } : { type: 'author' };
         const quotes = Array.isArray(idea.quotes) ? idea.quotes : [];
-        cites.push(quotes.map((q) => Number.isSafeInteger(q?.passage) ? q.passage : undefined));
-        return { claim: trim(idea.claim), detail: trim(idea.detail), kind: idea.kind, voice, quotes: quotes.map((q) => { const speaker = trim(q?.speaker) || (voice.type !== 'author' ? voice.name : undefined); return { text: trim(q?.text), ...(speaker ? { speaker } : {}) }; }) };
+        cites.push(quotes.map((q) => (Number.isSafeInteger(q?.passage) ? q.passage : undefined)));
+        return {
+            claim: trim(idea.claim),
+            detail: trim(idea.detail),
+            quotes: quotes.map((q) => {
+                const attribution = trim(q?.attribution);
+                return { text: trim(q?.text), ...(attribution ? { attribution } : {}) };
+            }),
+        };
     });
     const context = trim(v.context);
-    const parsed = parseDigest({ schemaVersion: 1, form, gist: { lead: trim(v.lead), ...(context ? { context } : {}) }, ideas, tags: v.tags.map(trim) });
-    return parsed.digest ? { digest: parsed.digest, cites } : { issues: parsed.issues };
+    const parsed = parseDigest({
+        schemaVersion: 2,
+        form,
+        gist: { lead: trim(v.lead), ...(context ? { context } : {}) },
+        ideas,
+        tags: v.tags.map(trim),
+    });
+    return parsed.digest?.schemaVersion === 2 ? { digest: parsed.digest, cites } : { issues: parsed.issues };
 }
 /**
  * Make a quote exactly verbatim when that is unambiguous: a unique folded or Markdown-insensitive match, inside
@@ -184,7 +268,10 @@ export function snapQuote(source, text, within) {
     if (source.includes(text))
         return { text, match: 'exact' };
     const inside = (m) => !!within && m.start >= within.start && m.end <= within.end;
-    const pick = (all) => { const local = all.filter(inside); return local.length === 1 ? local[0] : all.length === 1 ? all[0] : undefined; };
+    const pick = (all) => {
+        const local = all.filter(inside);
+        return local.length === 1 ? local[0] : all.length === 1 ? all[0] : undefined;
+    };
     const folded = locateAll(source, text);
     if (folded.length) {
         const m = pick(folded);
@@ -201,34 +288,68 @@ export function snapQuote(source, text, within) {
         end++;
     return { text: source.slice(start, end), match: 'snapped' };
 }
-/** Snap every quote in place and drop quotes under `minQuoteWords`. */
-export function snapQuotes(draft, source, list) {
-    const stats = { exact: 0, folded: 0, snapped: 0, missing: 0, short: 0 }, byId = new Map(list.map(p => [p.id, p]));
+/**
+ * A located quote over the word limit keeps its leading words: a prefix of a source span is still verbatim. Trailing
+ * clause punctuation goes; unlocated quotes are left for the checker.
+ */
+export function trimQuote(text, max = L.quoteWords) {
+    if (words(text) <= max)
+        return text;
+    const ends = [...text.matchAll(/\S+/g)].map((m) => m.index + m[0].length);
+    let n = Math.min(max, ends.length), out = text;
+    do {
+        out = text.slice(0, ends[n - 1]).replace(/[\s,;:(\[\u2014\u2013-]+$/, '');
+        n--;
+    } while (n > 0 && words(out) > max);
+    return out;
+}
+/** Snap every quote in place, trim located quotes over the word limit, and drop quotes under `minQuoteWords`. */
+function snapQuotes(draft, source, list) {
+    const stats = { exact: 0, folded: 0, snapped: 0, missing: 0, short: 0, trimmed: 0 }, byId = new Map(list.map((p) => [p.id, p]));
     draft.digest.ideas.forEach((idea, i) => {
-        idea.quotes.forEach((q, j) => { const r = snapQuote(source, q.text, byId.get(draft.cites[i]?.[j] ?? 0)); q.text = r.text; stats[r.match]++; });
-        const keep = idea.quotes.map(q => words(q.text) >= minQuoteWords);
-        stats.short += keep.filter(k => !k).length;
+        idea.quotes.forEach((q, j) => {
+            const r = snapQuote(source, q.text, byId.get(draft.cites[i]?.[j] ?? 0));
+            q.text = r.text;
+            stats[r.match]++;
+            if (r.match !== 'missing') {
+                const t = trimQuote(q.text);
+                if (t !== q.text) {
+                    q.text = t;
+                    stats.trimmed++;
+                }
+            }
+        });
+        const keep = idea.quotes.map((q) => words(q.text) >= minQuoteWords);
+        stats.short += keep.filter((k) => !k).length;
         idea.quotes = idea.quotes.filter((_, j) => keep[j]);
         draft.cites[i] = (draft.cites[i] ?? []).filter((_, j) => keep[j]);
     });
     return stats;
 }
-/** Ideas and their quotes in source order, by first located occurrence: the checker requires non-decreasing idea starts. */
-function order(draft, source) {
-    const at = (text) => locateAll(source, text)[0]?.start ?? Number.MAX_SAFE_INTEGER;
-    const rows = draft.digest.ideas.map((idea, i) => { const qs = idea.quotes.map((q, j) => ({ q, cite: draft.cites[i]?.[j], at: at(q.text) })).sort((a, b) => a.at - b.at); return { idea: { ...idea, quotes: qs.map(x => x.q) }, cites: qs.map(x => x.cite), at: qs[0]?.at ?? Number.MAX_SAFE_INTEGER }; });
-    rows.sort((a, b) => a.at - b.at);
-    draft.digest.ideas = rows.map(r => r.idea);
-    draft.cites = rows.map(r => r.cites);
-}
 const pruneable = new Set(['not-verbatim', 'overlap', 'unsupported', 'too-long', 'speaker-required']);
-/** Deterministic last resort: drop failing quotes, then quoteless or duplicate ideas, invalid tags, and second quotes over budget. */
+/**
+ * Deterministic last resort: drop failing quotes, then duplicate ideas and invalid tags, then quotes over the count or word
+ * budget (second quotes first, longest first), never the last quote. Quoteless ideas stay: v2 allows them.
+ */
 function prune(draft, issues, source, options) {
-    const dropped = { quotes: 0, ideas: 0, tags: 0 }, d = draft.digest, bad = new Set(issues.filter(i => /^ideas\.\d+\.quotes\.\d+$/.test(i.path) && pruneable.has(i.code)).map(i => i.path));
-    d.ideas.forEach((idea, i) => { const keep = idea.quotes.map((_, j) => !bad.has(`ideas.${i}.quotes.${j}`)).map((k, j) => k && j < L.maxQuotesPerIdea); dropped.quotes += keep.filter(k => !k).length; idea.quotes = idea.quotes.filter((_, j) => keep[j]); draft.cites[i] = (draft.cites[i] ?? []).filter((_, j) => keep[j]); });
+    const dropped = { quotes: 0, ideas: 0, tags: 0 }, d = draft.digest, bad = new Set(issues
+        .filter((i) => /^ideas\.\d+\.quotes\.\d+(?:\.attribution)?$/.test(i.path) && pruneable.has(i.code))
+        .map((i) => i.path.replace(/\.attribution$/, '')));
+    d.ideas.forEach((idea, i) => {
+        const keep = idea.quotes
+            .map((_, j) => !bad.has(`ideas.${i}.quotes.${j}`))
+            .map((k, j) => k && j < L.maxQuotesPerIdea);
+        dropped.quotes += keep.filter((k) => !k).length;
+        idea.quotes = idea.quotes.filter((_, j) => keep[j]);
+        draft.cites[i] = (draft.cites[i] ?? []).filter((_, j) => keep[j]);
+    });
     const seen = new Set();
-    const keepIdea = d.ideas.map(idea => { const key = foldText(idea.claim).toLowerCase(), ok = idea.quotes.length > 0 && !seen.has(key); seen.add(key); return ok; });
-    dropped.ideas = keepIdea.filter(k => !k).length;
+    const keepIdea = d.ideas.map((idea) => {
+        const key = foldText(idea.claim).toLowerCase(), ok = !seen.has(key);
+        seen.add(key);
+        return ok;
+    });
+    dropped.ideas = keepIdea.filter((k) => !k).length;
     d.ideas = d.ideas.filter((_, i) => keepIdea[i]);
     draft.cites = draft.cites.filter((_, i) => keepIdea[i]);
     if (d.ideas.length > L.maxIdeas) {
@@ -236,90 +357,146 @@ function prune(draft, issues, source, options) {
         d.ideas = d.ideas.slice(0, L.maxIdeas);
         draft.cites = draft.cites.slice(0, L.maxIdeas);
     }
-    const tags = [...new Set(d.tags)].filter(t => options.tags ? options.tags.includes(t) : /^[a-z0-9][a-z0-9-]{0,63}$/.test(t)).slice(0, L.maxTags);
+    const tags = [...new Set(d.tags)]
+        .filter((t) => (options.tags ? options.tags.includes(t) : /^[a-z0-9][a-z0-9-]{0,63}$/.test(t)))
+        .slice(0, L.maxTags);
     dropped.tags = d.tags.length - tags.length;
     d.tags = tags;
-    const budget = quoteBudget(source), total = () => d.ideas.reduce((n, idea) => n + idea.quotes.reduce((m, q) => m + words(q.text), 0), 0);
-    while (total() > budget) {
-        const i = d.ideas.map((idea, k) => ({ k, n: idea.quotes.length > 1 ? words(idea.quotes[1].text) : -1 })).sort((a, b) => b.n - a.n)[0];
-        if (!i || i.n < 0)
-            break;
-        d.ideas[i.k].quotes.pop();
-        draft.cites[i.k]?.pop();
+    const budget = quoteBudget(source), all = () => d.ideas.flatMap((idea, i) => idea.quotes.map((q, j) => ({ i, j, n: words(q.text) })));
+    for (let q = all(); q.length > 1 && (q.length > L.maxQuotes || q.reduce((n, x) => n + x.n, 0) > budget); q = all()) {
+        const x = q.sort((a, b) => b.j - a.j || b.n - a.n)[0];
+        d.ideas[x.i].quotes.splice(x.j, 1);
+        draft.cites[x.i]?.splice(x.j, 1);
         dropped.quotes++;
     }
     return dropped;
 }
-const hashedParams = (p) => ({ temperature: p.temperature, seed: p.seed, maxTokens: p.maxTokens, contextChars: p.contextChars, maxSections: p.maxSections });
+const hashedParams = (p) => ({
+    temperature: p.temperature,
+    seed: p.seed,
+    maxTokens: p.maxTokens,
+    contextChars: p.contextChars,
+    maxSections: p.maxSections,
+});
 function resolveParams(params = {}) {
     const p = { ...defaultDigestParams, ...params }, int = (v, min, max) => Number.isSafeInteger(v) && v >= min && v <= max;
-    if (!(p.temperature >= 0 && p.temperature <= 2) || !int(p.seed, 0, 2 ** 31) || !int(p.maxTokens, 256, 32768) || !int(p.contextChars, 2000, 1_000_000) || !int(p.maxSections, 1, 256) || !int(p.timeoutMs, 1, 3_600_000) || !int(p.deadlineMs, 1, 14_400_000) || !int(p.maxOutputBytes, 1024, 4 * 1024 * 1024))
-        throw new DigestError('invalid-input');
+    if (!(p.temperature >= 0 && p.temperature <= 2) ||
+        !int(p.seed, 0, 2 ** 31) ||
+        !int(p.maxTokens, 256, 32768) ||
+        !int(p.contextChars, 2000, 1_000_000) ||
+        !int(p.maxSections, 1, 256) ||
+        !int(p.timeoutMs, 1, 3_600_000) ||
+        !int(p.deadlineMs, 1, 14_400_000) ||
+        !int(p.maxOutputBytes, 1024, 4 * 1024 * 1024))
+        throw new SpongeError('invalid-input');
     return p;
 }
 function validIdentity(id) {
     const ok = (v) => typeof v === 'string' && v.length > 0 && v.length <= 256 && !/[\u0000-\u001f\u007f]/.test(v);
     if (!object(id) || !ok(id.model) || !ok(id.quantization) || !ok(id.runtime))
-        throw new DigestError('invalid-input');
+        throw new SpongeError('invalid-input');
     return { model: id.model, quantization: id.quantization, runtime: id.runtime };
 }
 /** sha256 over canonical JSON of the revision, its Markdown, the prompt version, the model identity and every output-shaping input. */
 export function digestInputHash(input) {
-    return sha256(canonicalJson({ revisionId: input.revisionId, markdownSha256: sha256(input.markdown), promptVersion: PROMPT_VERSION, model: validIdentity(input.model), params: { ...hashedParams(resolveParams(input.params)), form: input.form, vocabulary: input.vocabulary ? [...input.vocabulary] : null, highlights: (input.highlights ?? []).map(h => h.text), support: input.embed ? input.supportThreshold ?? defaultSupportThreshold : null } }));
+    return sha256(canonicalJson({
+        revisionId: input.revisionId,
+        markdownSha256: sha256(input.markdown),
+        promptVersion: PROMPT_VERSION,
+        model: validIdentity(input.model),
+        params: {
+            ...hashedParams(resolveParams(input.params)),
+            form: input.form,
+            vocabulary: input.vocabulary ? [...input.vocabulary] : null,
+            highlights: (input.highlights ?? []).map((h) => h.text),
+            support: input.embed ? (input.supportThreshold ?? defaultSupportThreshold) : null,
+        },
+    }));
 }
 /** Unqualified on real claims: a deliberately low bar that only catches quotes unrelated to their claim. */
-export const defaultSupportThreshold = 0.35;
+const defaultSupportThreshold = 0.35;
 export async function draftDigest(input) {
     const p = resolveParams(input.params), identity = validIdentity(input.model.identity);
     if (typeof input.markdown !== 'string' || !input.markdown.trim() || !forms.includes(input.form))
-        throw new DigestError('invalid-input');
-    if (input.vocabulary && (!input.vocabulary.length || input.vocabulary.length > 256 || input.vocabulary.some(t => !/^[a-z0-9][a-z0-9-]{0,63}$/.test(t))))
-        throw new DigestError('invalid-input');
+        throw new SpongeError('invalid-input');
+    if (input.vocabulary &&
+        (!input.vocabulary.length ||
+            input.vocabulary.length > 256 ||
+            input.vocabulary.some((t) => !/^[a-z0-9][a-z0-9-]{0,63}$/.test(t))))
+        throw new SpongeError('invalid-input');
     const inputHash = digestInputHash({ ...input, model: identity }), source = input.markdown, list = passages(source);
     if (list.length === 0)
-        throw new DigestError('invalid-input');
+        throw new SpongeError('invalid-input');
     const userSignal = input.signal ?? new AbortController().signal, overall = AbortSignal.any([userSignal, AbortSignal.timeout(p.deadlineMs)]);
-    const trace = { strategy: 'single', sections: 1, calls: 0, repaired: false, attempts: [], dropped: { quotes: 0, ideas: 0, tags: 0 } };
+    const trace = {
+        strategy: 'single',
+        sections: 1,
+        calls: 0,
+        repaired: false,
+        attempts: [],
+        dropped: { quotes: 0, ideas: 0, tags: 0 },
+    };
     const call = async (messages, schema) => {
         if (userSignal.aborted)
-            throw new DigestError('cancelled');
+            throw new SpongeError('cancelled');
         if (overall.aborted)
-            throw new DigestError('limit-exceeded');
+            throw new SpongeError('limit-exceeded');
         const signal = AbortSignal.any([overall, AbortSignal.timeout(p.timeoutMs)]);
         trace.calls++;
         let text;
         try {
-            text = await input.model.generate(messages, schema, { signal, maxOutputBytes: p.maxOutputBytes, maxTokens: p.maxTokens, temperature: p.temperature, seed: p.seed });
+            text = await input.model.generate(messages, schema, {
+                signal,
+                maxOutputBytes: p.maxOutputBytes,
+                maxTokens: p.maxTokens,
+                temperature: p.temperature,
+                seed: p.seed,
+            });
         }
         catch (error) {
             if (userSignal.aborted)
-                throw new DigestError('cancelled');
+                throw new SpongeError('cancelled');
             if (signal.aborted)
-                throw new DigestError('limit-exceeded');
-            if (error instanceof DigestError)
+                throw new SpongeError('limit-exceeded');
+            if (error instanceof SpongeError)
                 throw error;
-            throw new DigestError('unavailable');
+            throw new SpongeError('unavailable');
         }
         if (userSignal.aborted)
-            throw new DigestError('cancelled');
+            throw new SpongeError('cancelled');
         if (typeof text !== 'string' || Buffer.byteLength(text) > p.maxOutputBytes)
-            throw new DigestError('limit-exceeded');
+            throw new SpongeError('limit-exceeded');
         return text;
     };
-    const highlights = (input.highlights ?? []).filter(h => typeof h?.text === 'string' && h.text.trim()).slice(0, 32).map(h => { const at = h.start ?? locateAll(source, h.text)[0]?.start; return { text: h.text, passage: at === undefined ? undefined : list.find(x => at >= x.start && at < x.end)?.id }; });
+    const highlights = (input.highlights ?? [])
+        .filter((h) => typeof h?.text === 'string' && h.text.trim())
+        .slice(0, 32)
+        .map((h) => {
+        const at = h.start ?? locateAll(source, h.text)[0]?.start;
+        return {
+            text: h.text,
+            passage: at === undefined ? undefined : list.find((x) => at >= x.start && at < x.end)?.id,
+        };
+    });
     let shown = list, notesText;
     const total = list.reduce((n, x) => n + display(x).length + 1, 0);
     if (total > p.contextChars) {
         // Long source: notes per section, then one combined pass over the cited and highlighted passages only.
         const sections = sectionsOf(list, p.contextChars);
         if (sections.length > p.maxSections)
-            throw new DigestError('limit-exceeded');
+            throw new SpongeError('limit-exceeded');
         trace.strategy = 'sections';
         trace.sections = sections.length;
         const notes = [];
         for (const [k, section] of sections.entries()) {
             const first = section[0].id, last = section[section.length - 1].id;
-            const text = await call([{ role: 'system', content: `You take notes on part ${k + 1} of ${sections.length} of a longer source. Reply with JSON only. Give 1 to 4 notes, each a point of at most 25 words that the section asserts, with one passage number from this part and a short quote copied exactly from that passage.` }, { role: 'user', content: sourceBlock(input.title, input.form, section, []) }], notesSchema(first, last));
+            const text = await call([
+                {
+                    role: 'system',
+                    content: `You take notes on part ${k + 1} of ${sections.length} of a longer source. Reply with JSON only. Give 1 to 4 notes, each a point of at most 25 words that the section asserts, with one passage number from this part and a short quote copied exactly from that passage.`,
+                },
+                { role: 'user', content: sourceBlock(input.title, input.form, section, []) },
+            ], notesSchema(first, last));
             let v;
             try {
                 v = JSON.parse(text);
@@ -329,11 +506,23 @@ export async function draftDigest(input) {
             }
             if (object(v) && Array.isArray(v.notes))
                 for (const n of v.notes.slice(0, 4))
-                    if (object(n) && typeof n.point === 'string' && Number.isSafeInteger(n.passage) && n.passage >= first && n.passage <= last && typeof n.quote === 'string')
-                        notes.push({ point: collapse(n.point).slice(0, 300), passage: n.passage, quote: collapse(n.quote).slice(0, 300) });
+                    if (object(n) &&
+                        typeof n.point === 'string' &&
+                        Number.isSafeInteger(n.passage) &&
+                        n.passage >= first &&
+                        n.passage <= last &&
+                        typeof n.quote === 'string')
+                        notes.push({
+                            point: collapse(n.point).slice(0, 300),
+                            passage: n.passage,
+                            quote: collapse(n.quote).slice(0, 300),
+                        });
         }
-        const wanted = new Set([...notes.map(n => n.passage), ...highlights.flatMap(h => h.passage ? [h.passage] : [])]);
-        notesText = notes.map(n => `- [${n.passage}] ${n.point}`).join('\n');
+        const wanted = new Set([
+            ...notes.map((n) => n.passage),
+            ...highlights.flatMap((h) => (h.passage ? [h.passage] : [])),
+        ]);
+        notesText = notes.map((n) => `- [${n.passage}] ${n.point}`).join('\n');
         shown = [];
         let size = notesText.length;
         for (const x of list)
@@ -342,21 +531,36 @@ export async function draftDigest(input) {
                 size += display(x).length + 1;
             }
         if (!shown.length)
-            throw new DigestError('unavailable', [{ path: '', code: 'no-notes' }]);
+            throw new SpongeError('unavailable', { issues: [{ path: '', code: 'no-notes' }] });
     }
     const budget = quoteBudget(source), schema = draftSchema(list.length, input.vocabulary);
-    const messages = [{ role: 'system', content: system(budget, input.vocabulary) }, { role: 'user', content: (notesText ? `Notes from each part of the source, in order:\n${notesText}\n\n` : '') + sourceBlock(input.title, input.form, shown, highlights) }];
+    const messages = [
+        { role: 'system', content: system(budget, input.vocabulary) },
+        {
+            role: 'user',
+            content: (notesText ? `Notes from each part of the source, in order:\n${notesText}\n\n` : '') +
+                sourceBlock(input.title, input.form, shown, highlights),
+        },
+    ];
     const cache = new Map(), options = { ...(input.vocabulary ? { tags: input.vocabulary } : {}) };
     const embedKey = (t) => Buffer.from(foldText(t)).subarray(0, 900).toString();
     if (input.embed)
-        Object.assign(options, { supportThreshold: input.supportThreshold ?? defaultSupportThreshold, supports: (claim, quote) => { const a = cache.get(embedKey(claim)), b = cache.get(embedKey(quote)); return a && b ? a.reduce((n, x, i) => n + x * b[i], 0) : 0; } });
+        Object.assign(options, {
+            supportThreshold: input.supportThreshold ?? defaultSupportThreshold,
+            supports: (claim, quote) => {
+                const a = cache.get(embedKey(claim)), b = cache.get(embedKey(quote));
+                return a && b ? a.reduce((n, x, i) => n + x * b[i], 0) : 0;
+            },
+        });
     const check = async (draft) => {
         if (input.embed) {
-            const missing = [...new Set(draft.digest.ideas.flatMap(i => [i.claim, ...i.quotes.map(q => q.text)]).map(embedKey))].filter(t => !cache.has(t));
+            const missing = [
+                ...new Set(draft.digest.ideas.flatMap((i) => [i.claim, ...i.quotes.map((q) => q.text)]).map(embedKey)),
+            ].filter((t) => !cache.has(t));
             if (missing.length) {
                 const vectors = await input.embed(missing, overall);
                 if (!Array.isArray(vectors) || vectors.length !== missing.length)
-                    throw new DigestError('unavailable');
+                    throw new SpongeError('unavailable');
                 missing.forEach((t, i) => cache.set(t, vectors[i]));
             }
         }
@@ -369,7 +573,6 @@ export async function draftDigest(input) {
             return { issues: draft.issues };
         }
         const quotes = snapQuotes(draft, source, list);
-        order(draft, source);
         const issues = await check(draft);
         trace.attempts.push({ issues, quotes });
         return { draft, issues };
@@ -381,30 +584,59 @@ export async function draftDigest(input) {
         result = await attempt(await call([...messages, { role: 'assistant', content: first }, { role: 'user', content: repairPrompt(result.issues) }], schema));
     }
     if (!result.draft)
-        throw new DigestError('unavailable', result.issues);
+        throw new SpongeError('unavailable', { issues: result.issues });
     let issues = result.issues;
     if (issues.length) {
         trace.dropped = prune(result.draft, issues, source, options);
         issues = await check(result.draft);
         if (result.draft.digest.ideas.length < L.minIdeas)
-            issues = [...issues.filter(i => !(i.path === 'ideas' && i.code === 'count')), { path: 'ideas', code: 'count' }];
+            issues = [...issues.filter((i) => !(i.path === 'ideas' && i.code === 'count')), { path: 'ideas', code: 'count' }];
         if (issues.length)
-            throw new DigestError('unavailable', issues);
+            throw new SpongeError('unavailable', { issues });
     }
-    const producer = { pipeline: 'sponge-digest', promptVersion: PROMPT_VERSION, model: identity, params: hashedParams(p), strategy: trace.strategy, sections: trace.sections, calls: trace.calls, repaired: trace.repaired, dropped: trace.dropped, ...(input.embed ? { supportThreshold: options.supportThreshold } : {}) };
+    const producer = {
+        pipeline: 'sponge-digest',
+        promptVersion: PROMPT_VERSION,
+        model: identity,
+        params: hashedParams(p),
+        strategy: trace.strategy,
+        sections: trace.sections,
+        calls: trace.calls,
+        repaired: trace.repaired,
+        dropped: trace.dropped,
+        ...(input.embed ? { supportThreshold: options.supportThreshold } : {}),
+    };
     return { output: result.draft.digest, inputHash, producer, options, trace };
 }
-const formFor = (kind) => kind === 'pdf' ? 'pdf' : kind === 'media' ? 'video' : 'article';
+const formFor = (kind) => (kind === 'pdf' ? 'pdf' : kind === 'media' ? 'video' : 'article');
 /** Draft and store a digest for the item's current revision. A matching input hash returns the stored draft without calling the model. */
 export async function createDigest(store, itemId, model, options = {}) {
     const item = store.read(itemId);
     if (!item)
-        throw new StoreError('unavailable');
-    const highlights = store.listAnnotations(itemId).filter(a => a.kind === 'highlight' && a.revisionId === item.revisionId && a.anchored !== false).map(a => ({ text: a.anchor.exact, start: a.anchor.start }));
-    const input = { ...options, revisionId: item.revisionId, markdown: item.markdown, title: item.title, form: formFor(item.kind), model, highlights };
-    const prior = store.findDigest(item.revisionId, digestInputHash({ ...input, model: model.identity }));
+        throw new SpongeError('unavailable');
+    const highlights = store.annotations
+        .list(itemId)
+        .filter((a) => a.kind === 'highlight' && a.revisionId === item.revisionId && a.anchored !== false)
+        .map((a) => ({ text: a.anchor.exact, start: a.anchor.start }));
+    const input = {
+        ...options,
+        revisionId: item.revisionId,
+        markdown: item.markdown,
+        title: item.title,
+        form: formFor(item.kind),
+        model,
+        highlights,
+    };
+    const prior = store.derivations.find('digest', item.revisionId, digestInputHash({ ...input, model: model.identity }));
     if (prior)
         return { derivation: prior, duplicate: true };
-    const draft = await draftDigest(input), stored = store.putDigest({ itemId, revisionId: item.revisionId, producer: draft.producer, inputHash: draft.inputHash, output: draft.output, options: draft.options });
-    return { derivation: store.readDigest(stored.id), duplicate: stored.duplicate, trace: draft.trace };
+    const draft = await draftDigest(input), stored = store.derivations.put('digest', {
+        itemId,
+        revisionId: item.revisionId,
+        producer: draft.producer,
+        inputHash: draft.inputHash,
+        output: draft.output,
+        options: draft.options,
+    });
+    return { derivation: store.derivations.read('digest', stored.id), duplicate: stored.duplicate, trace: draft.trace };
 }