@hraness/sponge 0.2.3 → 0.2.5

package/dist/src/digest/pipeline.js

@@ -0,0 +1,410 @@
+import { createHash } from 'node:crypto';
+import { foldText, locateAll, locateLoose, words } from '../annotations/quote.js';
+import { checkDigest, digestLimits as L, forms, ideaKinds, parseDigest } from './schema.js';
+import { StoreError } from '../store/types.js';
+/**
+ * Model-agnostic Digest v1 drafting. The model sees numbered passages and cites a passage id plus a short exact
+ * span; code snaps, orders, prunes and checks. Nothing leaves this module except through the injected model.
+ */
+export const PROMPT_VERSION = 'sponge-digest-prompt-v1';
+/** 24k characters is roughly 6k tokens of passages, leaving room for instructions and output in an 8k context on an 8 GB Mac. */
+export const defaultDigestParams = { temperature: 0, seed: 1, maxTokens: 2048, contextChars: 24000, maxSections: 24, timeoutMs: 300000, deadlineMs: 1800000, maxOutputBytes: 65536 };
+export class DigestError extends Error {
+    code;
+    issues;
+    constructor(code, issues) {
+        super(code);
+        this.code = code;
+        this.issues = issues;
+        this.name = 'DigestError';
+    }
+}
+/** Quotes under this many words carry no evidence; the checker is gaining the same floor. */
+export const minQuoteWords = 4;
+const sha256 = (v) => createHash('sha256').update(v).digest('hex');
+const collapse = (v) => v.replace(/\s+/g, ' ').trim();
+const object = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
+/** Sorted-key JSON; the only serialization that enters input hashes. */
+export function canonicalJson(value) {
+    if (Array.isArray(value))
+        return `[${value.map(canonicalJson).join(',')}]`;
+    if (object(value))
+        return `{${Object.keys(value).filter(k => value[k] !== undefined).sort().map(k => `${JSON.stringify(k)}:${canonicalJson(value[k])}`).join(',')}}`;
+    if (typeof value === 'number' && !Number.isFinite(value))
+        throw new DigestError('invalid-input');
+    return JSON.stringify(value ?? null);
+}
+/** Paragraph-level passages with raw offsets. Headings label passages; code fences and image-only blocks are not quotable. */
+export function passages(markdown, maxChars = 1200) {
+    const out = [], blocks = [];
+    let at = 0, start = -1, fence = false, fenced = false;
+    for (const line of markdown.split('\n')) {
+        const end = at + line.length, blank = !line.trim(), marker = /^\s{0,3}(```|~~~)/.test(line);
+        if (marker)
+            fence = !fence;
+        if (blank && !fence && !marker) {
+            if (start >= 0)
+                blocks.push({ start, end: at - 1, fence: fenced });
+            start = -1;
+            fenced = false;
+        }
+        else {
+            if (start < 0)
+                start = at;
+            if (marker || fence)
+                fenced = true;
+        }
+        at = end + 1;
+    }
+    if (start >= 0)
+        blocks.push({ start, end: markdown.length, fence: fenced });
+    let heading = '';
+    for (const b of blocks) {
+        const raw = markdown.slice(b.start, b.end).trimEnd(), end = b.start + raw.length;
+        if (b.fence || !raw.trim())
+            continue;
+        const h = /^\s{0,3}#{1,6}\s+(.+)$/.exec(raw);
+        if (h && !raw.includes('\n')) {
+            heading = collapse(h[1]).slice(0, 200);
+            continue;
+        }
+        if (/^(\s*!\[[^\]]*\]\([^)]*\)\s*)+$/.test(raw))
+            continue;
+        // Long blocks split at sentence ends, then at whitespace, so every passage fits a prompt line.
+        let from = b.start;
+        while (end - from > maxChars) {
+            const window = markdown.slice(from, from + maxChars), sentence = lastSentenceEnd(window), space = window.lastIndexOf(' ');
+            const stop = from + (sentence > maxChars / 4 ? sentence : space > maxChars / 4 ? space : maxChars);
+            out.push({ id: out.length + 1, start: from, end: stop, heading, text: markdown.slice(from, stop) });
+            from = stop;
+            while (from < end && /\s/.test(markdown[from]))
+                from++;
+        }
+        if (end > from)
+            out.push({ id: out.length + 1, start: from, end, heading, text: markdown.slice(from, end) });
+    }
+    return out;
+}
+function lastSentenceEnd(window) { let best = -1; for (const m of window.matchAll(/[.!?]["')\]]*(?=\s)/g))
+    best = m.index + m[0].length; return best; }
+const display = (p) => `[${p.id}] ${collapse(p.text)}`;
+/** Pack passages into sections of at most `budget` display characters, preferring heading boundaries past half a budget. */
+export function sectionsOf(list, budget) {
+    const out = [];
+    let current = [], size = 0;
+    for (const p of list) {
+        const n = display(p).length + 1;
+        if (current.length && (size + n > budget || p.heading !== current[current.length - 1].heading && size > budget / 2)) {
+            out.push(current);
+            current = [];
+            size = 0;
+        }
+        current.push(p);
+        size += n;
+    }
+    if (current.length)
+        out.push(current);
+    return out;
+}
+const sourceWords = (markdown) => words(markdown);
+export const quoteBudget = (markdown) => Math.min(L.totalQuoteWords, Math.max(L.quoteWords, Math.floor(sourceWords(markdown) * L.sourceQuoteShare)));
+const str = (maxLength) => ({ type: 'string', minLength: 1, maxLength });
+/** JSON Schema for the model's draft. Tags become an enum when a vocabulary is given. */
+export function draftSchema(passageCount, vocabulary) {
+    const quote = { type: 'object', additionalProperties: false, required: ['passage', 'text'], properties: { passage: { type: 'integer', minimum: 1, maximum: Math.max(1, passageCount) }, text: str(300), speaker: str(120) } };
+    const idea = { type: 'object', additionalProperties: false, required: ['claim', 'detail', 'kind', 'voice', 'quotes'], properties: { claim: str(200), detail: str(600), kind: { type: 'string', enum: [...ideaKinds] }, voice: { type: 'object', additionalProperties: false, required: ['type'], properties: { type: { type: 'string', enum: ['author', 'speaker', 'reported'] }, name: str(120) } }, quotes: { type: 'array', minItems: 1, maxItems: L.maxQuotesPerIdea, items: quote } } };
+    const tag = vocabulary?.length ? { type: 'string', enum: [...vocabulary] } : { type: 'string', pattern: '^[a-z0-9][a-z0-9-]{0,63}$' };
+    return { type: 'object', additionalProperties: false, required: ['lead', 'ideas', 'tags'], properties: { lead: str(L.leadChars), context: str(400), ideas: { type: 'array', minItems: L.minIdeas, maxItems: L.maxIdeas, items: idea }, tags: { type: 'array', minItems: 1, maxItems: L.maxTags, items: tag } } };
+}
+export function notesSchema(first, last) {
+    return { type: 'object', additionalProperties: false, required: ['notes'], properties: { notes: { type: 'array', minItems: 1, maxItems: 4, items: { type: 'object', additionalProperties: false, required: ['point', 'passage', 'quote'], properties: { point: str(300), passage: { type: 'integer', minimum: first, maximum: last }, quote: str(300) } } } } };
+}
+const system = (budget, vocabulary) => `You write a Digest v1 of one source for a private reading library. Reply with JSON only, matching the schema.
+- lead: one sentence of at most ${L.leadChars} characters that names the author or organization when known and states the main finding.
+- context: optional, at most ${L.contextWords} words of essential background.
+- ideas: ${L.minIdeas} to ${L.maxIdeas}, in the order they appear in the source. Each idea has:
+  - claim: one sentence of at most ${L.claimWords} words that asserts something; never a topic label.
+  - detail: at most ${L.detailWords} words in at most ${L.detailSentences} sentences: evidence, mechanism or limits.
+  - kind: ${ideaKinds.join(', ')}.
+  - voice: "author" when the author makes the claim; "speaker" with name for a speaker in a talk or interview; "reported" with name when the author reports someone else's claim.
+  - quotes: 1 or ${L.maxQuotesPerIdea}, each copied character for character from one numbered passage, ${minQuoteWords} to ${L.quoteWords} words, with that passage number. No added quotation marks or ellipses. Give speaker when the voice is not the author. Prefer reader highlights.
+- All quotes together stay within ${budget} words and never overlap.
+- tags: 1 to ${L.maxTags} ${vocabulary?.length ? `from: ${vocabulary.join(', ')}` : 'lowercase kebab-case topics'}.
+Use only what the source says.`;
+function sourceBlock(title, form, list, highlights) {
+    let heading;
+    const lines = [];
+    for (const p of list) {
+        if (p.heading !== heading) {
+            heading = p.heading;
+            if (heading)
+                lines.push(`## ${heading}`);
+        }
+        lines.push(display(p));
+    }
+    const marks = highlights.map((h, i) => `[h${i + 1}${h.passage ? `, passage ${h.passage}` : ''}] ${collapse(h.text).slice(0, 400)}`);
+    return `Form: ${form}${title ? `\nTitle: ${collapse(title).slice(0, 300)}` : ''}${marks.length ? `\n\nReader highlights (preferred quotes):\n${marks.join('\n')}` : ''}\n\nPassages:\n${lines.join('\n')}`;
+}
+const rule = { lead: `one sentence, at most ${L.leadChars} characters`, context: `at most ${L.contextWords} words`, claim: `one distinct sentence, at most ${L.claimWords} words`, detail: `at most ${L.detailWords} words and ${L.detailSentences} sentences`, quotes: `1-${L.maxQuotesPerIdea} quotes, each copied exactly from its passage, ${minQuoteWords} to ${L.quoteWords} words, not overlapping another quote`, ideas: `${L.minIdeas}-${L.maxIdeas} ideas; quotes within the total budget`, tags: `1-${L.maxTags} distinct allowed tags` };
+function repairPrompt(issues) {
+    const lines = issues.slice(0, 40).map(i => { const last = i.path.split('.').filter(s => !/^\d+$/.test(s)).pop() ?? ''; return `- ${i.path || 'output'}: ${i.code}${rule[last] ? ` (rule: ${rule[last]})` : ''}`; });
+    return `That draft broke these rules:\n${lines.join('\n')}\nReturn the complete corrected JSON. Copy quotes exactly from the numbered passages.`;
+}
+/** Map the model's draft shape into DigestV1; parseDigest then decides whether the shape is valid. */
+function fromModel(text, form) {
+    let v;
+    try {
+        v = JSON.parse(text);
+    }
+    catch {
+        return { issues: [{ path: '', code: 'json' }] };
+    }
+    if (!object(v) || !Array.isArray(v.ideas) || !Array.isArray(v.tags))
+        return { issues: [{ path: '', code: 'shape' }] };
+    const trim = (s) => typeof s === 'string' ? s.trim() : s, cites = [];
+    const ideas = v.ideas.map((idea) => {
+        if (!object(idea)) {
+            cites.push([]);
+            return idea;
+        }
+        const voice = object(idea.voice) && idea.voice.type !== 'author' ? { type: idea.voice.type, name: trim(idea.voice.name) } : { type: 'author' };
+        const quotes = Array.isArray(idea.quotes) ? idea.quotes : [];
+        cites.push(quotes.map((q) => Number.isSafeInteger(q?.passage) ? q.passage : undefined));
+        return { claim: trim(idea.claim), detail: trim(idea.detail), kind: idea.kind, voice, quotes: quotes.map((q) => { const speaker = trim(q?.speaker) || (voice.type !== 'author' ? voice.name : undefined); return { text: trim(q?.text), ...(speaker ? { speaker } : {}) }; }) };
+    });
+    const context = trim(v.context);
+    const parsed = parseDigest({ schemaVersion: 1, form, gist: { lead: trim(v.lead), ...(context ? { context } : {}) }, ideas, tags: v.tags.map(trim) });
+    return parsed.digest ? { digest: parsed.digest, cites } : { issues: parsed.issues };
+}
+/**
+ * Make a quote exactly verbatim when that is unambiguous: a unique folded or Markdown-insensitive match, inside
+ * `within` first, becomes the raw source span. Ambiguous folded matches keep their text for the checker; absent ones are 'missing'.
+ */
+export function snapQuote(source, text, within) {
+    if (source.includes(text))
+        return { text, match: 'exact' };
+    const inside = (m) => !!within && m.start >= within.start && m.end <= within.end;
+    const pick = (all) => { const local = all.filter(inside); return local.length === 1 ? local[0] : all.length === 1 ? all[0] : undefined; };
+    const folded = locateAll(source, text);
+    if (folded.length) {
+        const m = pick(folded);
+        return { text: m ? source.slice(m.start, m.end) : text, match: 'folded' };
+    }
+    const loose = (within && pick(locateLoose(source, text, within))) || pick(locateLoose(source, text));
+    if (!loose)
+        return { text, match: 'missing' };
+    // Keep emphasis and code markers balanced: a span that starts or ends inside `**bold**` takes the adjacent markers.
+    let { start, end } = loose;
+    while (start > 0 && /[*_`]/.test(source[start - 1]))
+        start--;
+    while (end < source.length && /[*_`]/.test(source[end]))
+        end++;
+    return { text: source.slice(start, end), match: 'snapped' };
+}
+/** Snap every quote in place and drop quotes under `minQuoteWords`. */
+export function snapQuotes(draft, source, list) {
+    const stats = { exact: 0, folded: 0, snapped: 0, missing: 0, short: 0 }, byId = new Map(list.map(p => [p.id, p]));
+    draft.digest.ideas.forEach((idea, i) => {
+        idea.quotes.forEach((q, j) => { const r = snapQuote(source, q.text, byId.get(draft.cites[i]?.[j] ?? 0)); q.text = r.text; stats[r.match]++; });
+        const keep = idea.quotes.map(q => words(q.text) >= minQuoteWords);
+        stats.short += keep.filter(k => !k).length;
+        idea.quotes = idea.quotes.filter((_, j) => keep[j]);
+        draft.cites[i] = (draft.cites[i] ?? []).filter((_, j) => keep[j]);
+    });
+    return stats;
+}
+/** Ideas and their quotes in source order, by first located occurrence: the checker requires non-decreasing idea starts. */
+function order(draft, source) {
+    const at = (text) => locateAll(source, text)[0]?.start ?? Number.MAX_SAFE_INTEGER;
+    const rows = draft.digest.ideas.map((idea, i) => { const qs = idea.quotes.map((q, j) => ({ q, cite: draft.cites[i]?.[j], at: at(q.text) })).sort((a, b) => a.at - b.at); return { idea: { ...idea, quotes: qs.map(x => x.q) }, cites: qs.map(x => x.cite), at: qs[0]?.at ?? Number.MAX_SAFE_INTEGER }; });
+    rows.sort((a, b) => a.at - b.at);
+    draft.digest.ideas = rows.map(r => r.idea);
+    draft.cites = rows.map(r => r.cites);
+}
+const pruneable = new Set(['not-verbatim', 'overlap', 'unsupported', 'too-long', 'speaker-required']);
+/** Deterministic last resort: drop failing quotes, then quoteless or duplicate ideas, invalid tags, and second quotes over budget. */
+function prune(draft, issues, source, options) {
+    const dropped = { quotes: 0, ideas: 0, tags: 0 }, d = draft.digest, bad = new Set(issues.filter(i => /^ideas\.\d+\.quotes\.\d+$/.test(i.path) && pruneable.has(i.code)).map(i => i.path));
+    d.ideas.forEach((idea, i) => { const keep = idea.quotes.map((_, j) => !bad.has(`ideas.${i}.quotes.${j}`)).map((k, j) => k && j < L.maxQuotesPerIdea); dropped.quotes += keep.filter(k => !k).length; idea.quotes = idea.quotes.filter((_, j) => keep[j]); draft.cites[i] = (draft.cites[i] ?? []).filter((_, j) => keep[j]); });
+    const seen = new Set();
+    const keepIdea = d.ideas.map(idea => { const key = foldText(idea.claim).toLowerCase(), ok = idea.quotes.length > 0 && !seen.has(key); seen.add(key); return ok; });
+    dropped.ideas = keepIdea.filter(k => !k).length;
+    d.ideas = d.ideas.filter((_, i) => keepIdea[i]);
+    draft.cites = draft.cites.filter((_, i) => keepIdea[i]);
+    if (d.ideas.length > L.maxIdeas) {
+        dropped.ideas += d.ideas.length - L.maxIdeas;
+        d.ideas = d.ideas.slice(0, L.maxIdeas);
+        draft.cites = draft.cites.slice(0, L.maxIdeas);
+    }
+    const tags = [...new Set(d.tags)].filter(t => options.tags ? options.tags.includes(t) : /^[a-z0-9][a-z0-9-]{0,63}$/.test(t)).slice(0, L.maxTags);
+    dropped.tags = d.tags.length - tags.length;
+    d.tags = tags;
+    const budget = quoteBudget(source), total = () => d.ideas.reduce((n, idea) => n + idea.quotes.reduce((m, q) => m + words(q.text), 0), 0);
+    while (total() > budget) {
+        const i = d.ideas.map((idea, k) => ({ k, n: idea.quotes.length > 1 ? words(idea.quotes[1].text) : -1 })).sort((a, b) => b.n - a.n)[0];
+        if (!i || i.n < 0)
+            break;
+        d.ideas[i.k].quotes.pop();
+        draft.cites[i.k]?.pop();
+        dropped.quotes++;
+    }
+    return dropped;
+}
+const hashedParams = (p) => ({ temperature: p.temperature, seed: p.seed, maxTokens: p.maxTokens, contextChars: p.contextChars, maxSections: p.maxSections });
+function resolveParams(params = {}) {
+    const p = { ...defaultDigestParams, ...params }, int = (v, min, max) => Number.isSafeInteger(v) && v >= min && v <= max;
+    if (!(p.temperature >= 0 && p.temperature <= 2) || !int(p.seed, 0, 2 ** 31) || !int(p.maxTokens, 256, 32768) || !int(p.contextChars, 2000, 1_000_000) || !int(p.maxSections, 1, 256) || !int(p.timeoutMs, 1, 3_600_000) || !int(p.deadlineMs, 1, 14_400_000) || !int(p.maxOutputBytes, 1024, 4 * 1024 * 1024))
+        throw new DigestError('invalid-input');
+    return p;
+}
+function validIdentity(id) {
+    const ok = (v) => typeof v === 'string' && v.length > 0 && v.length <= 256 && !/[\u0000-\u001f\u007f]/.test(v);
+    if (!object(id) || !ok(id.model) || !ok(id.quantization) || !ok(id.runtime))
+        throw new DigestError('invalid-input');
+    return { model: id.model, quantization: id.quantization, runtime: id.runtime };
+}
+/** sha256 over canonical JSON of the revision, its Markdown, the prompt version, the model identity and every output-shaping input. */
+export function digestInputHash(input) {
+    return sha256(canonicalJson({ revisionId: input.revisionId, markdownSha256: sha256(input.markdown), promptVersion: PROMPT_VERSION, model: validIdentity(input.model), params: { ...hashedParams(resolveParams(input.params)), form: input.form, vocabulary: input.vocabulary ? [...input.vocabulary] : null, highlights: (input.highlights ?? []).map(h => h.text), support: input.embed ? input.supportThreshold ?? defaultSupportThreshold : null } }));
+}
+/** Unqualified on real claims: a deliberately low bar that only catches quotes unrelated to their claim. */
+export const defaultSupportThreshold = 0.35;
+export async function draftDigest(input) {
+    const p = resolveParams(input.params), identity = validIdentity(input.model.identity);
+    if (typeof input.markdown !== 'string' || !input.markdown.trim() || !forms.includes(input.form))
+        throw new DigestError('invalid-input');
+    if (input.vocabulary && (!input.vocabulary.length || input.vocabulary.length > 256 || input.vocabulary.some(t => !/^[a-z0-9][a-z0-9-]{0,63}$/.test(t))))
+        throw new DigestError('invalid-input');
+    const inputHash = digestInputHash({ ...input, model: identity }), source = input.markdown, list = passages(source);
+    if (list.length === 0)
+        throw new DigestError('invalid-input');
+    const userSignal = input.signal ?? new AbortController().signal, overall = AbortSignal.any([userSignal, AbortSignal.timeout(p.deadlineMs)]);
+    const trace = { strategy: 'single', sections: 1, calls: 0, repaired: false, attempts: [], dropped: { quotes: 0, ideas: 0, tags: 0 } };
+    const call = async (messages, schema) => {
+        if (userSignal.aborted)
+            throw new DigestError('cancelled');
+        if (overall.aborted)
+            throw new DigestError('limit-exceeded');
+        const signal = AbortSignal.any([overall, AbortSignal.timeout(p.timeoutMs)]);
+        trace.calls++;
+        let text;
+        try {
+            text = await input.model.generate(messages, schema, { signal, maxOutputBytes: p.maxOutputBytes, maxTokens: p.maxTokens, temperature: p.temperature, seed: p.seed });
+        }
+        catch (error) {
+            if (userSignal.aborted)
+                throw new DigestError('cancelled');
+            if (signal.aborted)
+                throw new DigestError('limit-exceeded');
+            if (error instanceof DigestError)
+                throw error;
+            throw new DigestError('unavailable');
+        }
+        if (userSignal.aborted)
+            throw new DigestError('cancelled');
+        if (typeof text !== 'string' || Buffer.byteLength(text) > p.maxOutputBytes)
+            throw new DigestError('limit-exceeded');
+        return text;
+    };
+    const highlights = (input.highlights ?? []).filter(h => typeof h?.text === 'string' && h.text.trim()).slice(0, 32).map(h => { const at = h.start ?? locateAll(source, h.text)[0]?.start; return { text: h.text, passage: at === undefined ? undefined : list.find(x => at >= x.start && at < x.end)?.id }; });
+    let shown = list, notesText;
+    const total = list.reduce((n, x) => n + display(x).length + 1, 0);
+    if (total > p.contextChars) {
+        // Long source: notes per section, then one combined pass over the cited and highlighted passages only.
+        const sections = sectionsOf(list, p.contextChars);
+        if (sections.length > p.maxSections)
+            throw new DigestError('limit-exceeded');
+        trace.strategy = 'sections';
+        trace.sections = sections.length;
+        const notes = [];
+        for (const [k, section] of sections.entries()) {
+            const first = section[0].id, last = section[section.length - 1].id;
+            const text = await call([{ role: 'system', content: `You take notes on part ${k + 1} of ${sections.length} of a longer source. Reply with JSON only. Give 1 to 4 notes, each a point of at most 25 words that the section asserts, with one passage number from this part and a short quote copied exactly from that passage.` }, { role: 'user', content: sourceBlock(input.title, input.form, section, []) }], notesSchema(first, last));
+            let v;
+            try {
+                v = JSON.parse(text);
+            }
+            catch {
+                continue;
+            }
+            if (object(v) && Array.isArray(v.notes))
+                for (const n of v.notes.slice(0, 4))
+                    if (object(n) && typeof n.point === 'string' && Number.isSafeInteger(n.passage) && n.passage >= first && n.passage <= last && typeof n.quote === 'string')
+                        notes.push({ point: collapse(n.point).slice(0, 300), passage: n.passage, quote: collapse(n.quote).slice(0, 300) });
+        }
+        const wanted = new Set([...notes.map(n => n.passage), ...highlights.flatMap(h => h.passage ? [h.passage] : [])]);
+        notesText = notes.map(n => `- [${n.passage}] ${n.point}`).join('\n');
+        shown = [];
+        let size = notesText.length;
+        for (const x of list)
+            if (wanted.has(x.id) && size + display(x).length + 1 <= p.contextChars) {
+                shown.push(x);
+                size += display(x).length + 1;
+            }
+        if (!shown.length)
+            throw new DigestError('unavailable', [{ path: '', code: 'no-notes' }]);
+    }
+    const budget = quoteBudget(source), schema = draftSchema(list.length, input.vocabulary);
+    const messages = [{ role: 'system', content: system(budget, input.vocabulary) }, { role: 'user', content: (notesText ? `Notes from each part of the source, in order:\n${notesText}\n\n` : '') + sourceBlock(input.title, input.form, shown, highlights) }];
+    const cache = new Map(), options = { ...(input.vocabulary ? { tags: input.vocabulary } : {}) };
+    const embedKey = (t) => Buffer.from(foldText(t)).subarray(0, 900).toString();
+    if (input.embed)
+        Object.assign(options, { supportThreshold: input.supportThreshold ?? defaultSupportThreshold, supports: (claim, quote) => { const a = cache.get(embedKey(claim)), b = cache.get(embedKey(quote)); return a && b ? a.reduce((n, x, i) => n + x * b[i], 0) : 0; } });
+    const check = async (draft) => {
+        if (input.embed) {
+            const missing = [...new Set(draft.digest.ideas.flatMap(i => [i.claim, ...i.quotes.map(q => q.text)]).map(embedKey))].filter(t => !cache.has(t));
+            if (missing.length) {
+                const vectors = await input.embed(missing, overall);
+                if (!Array.isArray(vectors) || vectors.length !== missing.length)
+                    throw new DigestError('unavailable');
+                missing.forEach((t, i) => cache.set(t, vectors[i]));
+            }
+        }
+        return checkDigest(draft.digest, source, options).issues;
+    };
+    const attempt = async (text) => {
+        const draft = fromModel(text, input.form);
+        if ('issues' in draft) {
+            trace.attempts.push({ issues: draft.issues });
+            return { issues: draft.issues };
+        }
+        const quotes = snapQuotes(draft, source, list);
+        order(draft, source);
+        const issues = await check(draft);
+        trace.attempts.push({ issues, quotes });
+        return { draft, issues };
+    };
+    const first = await call(messages, schema);
+    let result = await attempt(first);
+    if (result.issues.length) {
+        trace.repaired = true;
+        result = await attempt(await call([...messages, { role: 'assistant', content: first }, { role: 'user', content: repairPrompt(result.issues) }], schema));
+    }
+    if (!result.draft)
+        throw new DigestError('unavailable', result.issues);
+    let issues = result.issues;
+    if (issues.length) {
+        trace.dropped = prune(result.draft, issues, source, options);
+        issues = await check(result.draft);
+        if (result.draft.digest.ideas.length < L.minIdeas)
+            issues = [...issues.filter(i => !(i.path === 'ideas' && i.code === 'count')), { path: 'ideas', code: 'count' }];
+        if (issues.length)
+            throw new DigestError('unavailable', issues);
+    }
+    const producer = { pipeline: 'sponge-digest', promptVersion: PROMPT_VERSION, model: identity, params: hashedParams(p), strategy: trace.strategy, sections: trace.sections, calls: trace.calls, repaired: trace.repaired, dropped: trace.dropped, ...(input.embed ? { supportThreshold: options.supportThreshold } : {}) };
+    return { output: result.draft.digest, inputHash, producer, options, trace };
+}
+const formFor = (kind) => kind === 'pdf' ? 'pdf' : kind === 'media' ? 'video' : 'article';
+/** Draft and store a digest for the item's current revision. A matching input hash returns the stored draft without calling the model. */
+export async function createDigest(store, itemId, model, options = {}) {
+    const item = store.read(itemId);
+    if (!item)
+        throw new StoreError('unavailable');
+    const highlights = store.listAnnotations(itemId).filter(a => a.kind === 'highlight' && a.revisionId === item.revisionId && a.anchored !== false).map(a => ({ text: a.anchor.exact, start: a.anchor.start }));
+    const input = { ...options, revisionId: item.revisionId, markdown: item.markdown, title: item.title, form: formFor(item.kind), model, highlights };
+    const prior = store.findDigest(item.revisionId, digestInputHash({ ...input, model: model.identity }));
+    if (prior)
+        return { derivation: prior, duplicate: true };
+    const draft = await draftDigest(input), stored = store.putDigest({ itemId, revisionId: item.revisionId, producer: draft.producer, inputHash: draft.inputHash, output: draft.output, options: draft.options });
+    return { derivation: store.readDigest(stored.id), duplicate: stored.duplicate, trace: draft.trace };
+}

package/dist/src/digest/schema.d.ts

@@ -0,0 +1,85 @@
+import { type TextQuote } from '../annotations/quote.js';
+/**
+ * Digest v1: a one-sentence gist and 3-5 ideas, each a claim with detail and 1-2 verbatim quotes.
+ * The model proposes; this validator decides. Nothing here calls a model.
+ */
+export declare const ideaKinds: readonly ["finding", "argument", "method", "evidence", "recommendation", "counterpoint"];
+export declare const forms: readonly ["article", "video", "paper", "thread", "pdf"];
+export type Voice = {
+    type: 'author';
+} | {
+    type: 'speaker' | 'reported';
+    name: string;
+};
+export type QuoteLocator = {
+    type: 'timestamp';
+    ms: number;
+} | {
+    type: 'page';
+    page: number;
+};
+export interface DigestQuote {
+    text: string;
+    speaker?: string;
+    locator?: QuoteLocator;
+}
+export interface DigestIdea {
+    claim: string;
+    detail: string;
+    kind: typeof ideaKinds[number];
+    voice: Voice;
+    quotes: DigestQuote[];
+}
+export interface DigestV1 {
+    schemaVersion: 1;
+    form: typeof forms[number];
+    gist: {
+        lead: string;
+        context?: string;
+    };
+    ideas: DigestIdea[];
+    tags: string[];
+}
+export declare const digestLimits: {
+    readonly leadChars: 155;
+    readonly contextWords: 50;
+    readonly minIdeas: 3;
+    readonly maxIdeas: 5;
+    readonly claimWords: 20;
+    readonly detailWords: 60;
+    readonly detailSentences: 3;
+    readonly maxQuotesPerIdea: 2;
+    readonly minQuoteWords: 4;
+    readonly quoteWords: 25;
+    readonly totalQuoteWords: 150;
+    readonly sourceQuoteShare: 0.1;
+    readonly maxTags: 3;
+};
+export interface DigestIssue {
+    path: string;
+    code: string;
+}
+export interface LocatedQuote {
+    idea: number;
+    quote: number;
+    anchor: TextQuote;
+}
+export interface DigestCheck {
+    ok: boolean;
+    issues: DigestIssue[];
+    quotes: LocatedQuote[];
+}
+export interface DigestOptions {
+    /** Controlled tag vocabulary; omitted means any 1-3 lowercase tags. */
+    tags?: readonly string[];
+    /** Optional semantic check that a quote supports its claim, e.g. embedding cosine. */
+    supports?: (claim: string, quote: string) => number;
+    supportThreshold?: number;
+}
+/** Parse an untrusted model output into a DigestV1 shape, or report why not. */
+export declare function parseDigest(value: unknown): {
+    digest?: DigestV1;
+    issues: DigestIssue[];
+};
+/** Check every v1 rule that code can decide against the captured Markdown. */
+export declare function checkDigest(digest: DigestV1, source: string, options?: DigestOptions): DigestCheck;

package/dist/src/digest/schema.js

@@ -0,0 +1,118 @@
+import { foldText, locateAll, textQuote, words } from '../annotations/quote.js';
+/**
+ * Digest v1: a one-sentence gist and 3-5 ideas, each a claim with detail and 1-2 verbatim quotes.
+ * The model proposes; this validator decides. Nothing here calls a model.
+ */
+export const ideaKinds = ['finding', 'argument', 'method', 'evidence', 'recommendation', 'counterpoint'];
+export const forms = ['article', 'video', 'paper', 'thread', 'pdf'];
+export const digestLimits = { leadChars: 155, contextWords: 50, minIdeas: 3, maxIdeas: 5, claimWords: 20, detailWords: 60, detailSentences: 3, maxQuotesPerIdea: 2, minQuoteWords: 4, quoteWords: 25, totalQuoteWords: 150, sourceQuoteShare: 0.1, maxTags: 3 };
+const object = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
+const keys = (v, allowed) => Object.keys(v).every(k => allowed.includes(k));
+const str = (v, max = 2000) => typeof v === 'string' && v.trim().length > 0 && v.length <= max && !/[\u0000-\u0008\u000b-\u001f\u007f]/.test(v);
+/** Sentence terminators followed by more text; abbreviations are rare enough in a 155-character lead to reject. */
+const sentences = (v) => foldText(v).split(/(?<=[.!?])\s+(?=\S)/).length;
+/** Parse an untrusted model output into a DigestV1 shape, or report why not. */
+export function parseDigest(value) {
+    const issues = [], bad = (path, code = 'shape') => { issues.push({ path, code }); };
+    if (!object(value) || !keys(value, ['schemaVersion', 'form', 'gist', 'ideas', 'tags']) || value.schemaVersion !== 1)
+        return { issues: [{ path: '', code: 'shape' }] };
+    if (!forms.includes(value.form))
+        bad('form');
+    const gist = value.gist;
+    if (!object(gist) || !keys(gist, ['lead', 'context']) || !str(gist.lead) || gist.context !== undefined && !str(gist.context))
+        bad('gist');
+    if (!Array.isArray(value.ideas) || value.ideas.length > 8)
+        bad('ideas');
+    else
+        value.ideas.forEach((idea, i) => {
+            const at = `ideas.${i}`;
+            if (!object(idea) || !keys(idea, ['claim', 'detail', 'kind', 'voice', 'quotes']) || !str(idea.claim) || !str(idea.detail) || !ideaKinds.includes(idea.kind))
+                return bad(at);
+            const voice = idea.voice;
+            if (!object(voice) || !(voice.type === 'author' && keys(voice, ['type']) || (voice.type === 'speaker' || voice.type === 'reported') && keys(voice, ['type', 'name']) && str(voice.name, 200)))
+                bad(at + '.voice');
+            if (!Array.isArray(idea.quotes) || idea.quotes.length > 4)
+                return bad(at + '.quotes');
+            idea.quotes.forEach((quote, j) => {
+                if (!object(quote) || !keys(quote, ['text', 'speaker', 'locator']) || !str(quote.text, 600) || quote.speaker !== undefined && !str(quote.speaker, 200))
+                    return bad(`${at}.quotes.${j}`);
+                const loc = quote.locator;
+                if (loc !== undefined && !(object(loc) && (loc.type === 'timestamp' && keys(loc, ['type', 'ms']) && Number.isSafeInteger(loc.ms) && loc.ms >= 0 || loc.type === 'page' && keys(loc, ['type', 'page']) && Number.isSafeInteger(loc.page) && loc.page >= 1)))
+                    bad(`${at}.quotes.${j}.locator`);
+            });
+        });
+    if (!Array.isArray(value.tags) || value.tags.some(t => !str(t, 64)))
+        bad('tags');
+    return issues.length ? { issues } : { digest: value, issues };
+}
+/** Check every v1 rule that code can decide against the captured Markdown. */
+export function checkDigest(digest, source, options = {}) {
+    const issues = [], located = [], issue = (path, code) => { issues.push({ path, code }); }, L = digestLimits;
+    if (digest.gist.lead.length > L.leadChars)
+        issue('gist.lead', 'too-long');
+    if (sentences(digest.gist.lead) !== 1)
+        issue('gist.lead', 'one-sentence');
+    if (digest.gist.context !== undefined && words(digest.gist.context) > L.contextWords)
+        issue('gist.context', 'too-long');
+    if (digest.ideas.length < L.minIdeas || digest.ideas.length > L.maxIdeas)
+        issue('ideas', 'count');
+    const claims = new Set();
+    let quoteWords = 0;
+    digest.ideas.forEach((idea, i) => {
+        const at = `ideas.${i}`;
+        if (words(idea.claim) > L.claimWords)
+            issue(at + '.claim', 'too-long');
+        if (sentences(idea.claim) !== 1)
+            issue(at + '.claim', 'one-sentence');
+        if (claims.has(foldText(idea.claim).toLowerCase()))
+            issue(at + '.claim', 'duplicate');
+        claims.add(foldText(idea.claim).toLowerCase());
+        if (words(idea.detail) > L.detailWords)
+            issue(at + '.detail', 'too-long');
+        if (sentences(idea.detail) > L.detailSentences)
+            issue(at + '.detail', 'too-many-sentences');
+        if (idea.quotes.length < 1 || idea.quotes.length > L.maxQuotesPerIdea)
+            issue(at + '.quotes', 'count');
+        idea.quotes.forEach((quote, j) => {
+            const path = `${at}.quotes.${j}`, count = words(quote.text);
+            quoteWords += count;
+            if (count > L.quoteWords)
+                issue(path, 'too-long');
+            if (count < L.minQuoteWords)
+                issue(path, 'too-short');
+            if (idea.voice.type !== 'author' && !quote.speaker)
+                issue(path, 'speaker-required');
+            const matches = locateAll(source, quote.text);
+            if (!matches.length)
+                return issue(path, 'not-verbatim');
+            // Prefer the first occurrence not already claimed by an earlier quote.
+            const match = matches.find(m => !located.some(q => m.start < q.anchor.end && q.anchor.start < m.end));
+            if (!match)
+                return issue(path, 'overlap');
+            located.push({ idea: i, quote: j, anchor: textQuote(source, match.start, match.end) });
+            if (options.supports && options.supports(idea.claim, quote.text) < (options.supportThreshold ?? 0.5))
+                issue(path, 'unsupported');
+        });
+    });
+    // Ideas follow the source: each idea's earliest located quote starts no earlier than the previous idea's.
+    let previous = -1;
+    digest.ideas.forEach((_, i) => {
+        const starts = located.filter(q => q.idea === i).map(q => q.anchor.start);
+        if (!starts.length)
+            return;
+        const first = Math.min(...starts);
+        if (first < previous)
+            issue(`ideas.${i}`, 'out-of-order');
+        previous = first;
+    });
+    if (quoteWords > L.totalQuoteWords)
+        issue('ideas', 'quote-budget');
+    // The 10% share has a 25-word floor: an exemption so a short source can still carry one full quote.
+    if (quoteWords > Math.max(L.quoteWords, Math.floor(words(source) * L.sourceQuoteShare)))
+        issue('ideas', 'quote-share');
+    if (digest.tags.length < 1 || digest.tags.length > L.maxTags || new Set(digest.tags).size !== digest.tags.length)
+        issue('tags', 'count');
+    digest.tags.forEach((tag, i) => { if (options.tags ? !options.tags.includes(tag) : !/^[a-z0-9][a-z0-9-]{0,63}$/.test(tag))
+        issue(`tags.${i}`, 'vocabulary'); });
+    return { ok: issues.length === 0, issues, quotes: located };
+}