@hraness/sponge 0.2.4 → 0.2.6

package/dist/src/digest/pipeline.d.ts

@@ -0,0 +1,148 @@
+import { forms, type DigestIssue, type DigestOptions, type DigestV2 } from './schema.js';
+import type { LibraryStore } from '../store/index.js';
+import type { Derivation } from '../store/foundation.js';
+/**
+ * Model-agnostic Digest v2 drafting. The model sees numbered passages and cites a passage id plus a short exact
+ * span; code snaps, prunes and checks. Nothing leaves this module except through the injected model.
+ */
+export declare const PROMPT_VERSION = "sponge-digest-prompt-v2";
+export type ChatMessage = {
+    role: 'system' | 'user' | 'assistant';
+    content: string;
+};
+export type JsonSchema = Record<string, unknown>;
+/** Stable identity recorded in the producer and the input hash. Unknown parts are the literal 'unknown'. */
+export interface ModelIdentity {
+    model: string;
+    quantization: string;
+    runtime: string;
+}
+export interface GenerateOptions {
+    signal: AbortSignal;
+    maxOutputBytes: number;
+    maxTokens: number;
+    temperature: number;
+    seed: number;
+}
+export interface DigestModel {
+    identity: ModelIdentity;
+    generate(messages: ChatMessage[], schema: JsonSchema, options: GenerateOptions): Promise<string>;
+}
+/** Output-shaping parameters are hashed; time and byte bounds are not, because they only decide whether a run finishes. */
+export interface DigestParams {
+    temperature: number;
+    seed: number;
+    maxTokens: number;
+    contextChars: number;
+    maxSections: number;
+    timeoutMs: number;
+    deadlineMs: number;
+    maxOutputBytes: number;
+}
+/** 24k characters is roughly 6k tokens of passages, leaving room for instructions and output in an 8k context on an 8 GB Mac. */
+export declare const defaultDigestParams: DigestParams;
+export type DigestErrorCode = 'invalid-input' | 'unavailable' | 'cancelled' | 'limit-exceeded';
+export declare class DigestError extends Error {
+    readonly code: DigestErrorCode;
+    readonly issues?: DigestIssue[] | undefined;
+    constructor(code: DigestErrorCode, issues?: DigestIssue[] | undefined);
+}
+export type Form = typeof forms[number];
+/** Quotes under this many words carry no evidence; the checker is gaining the same floor. */
+export declare const minQuoteWords = 4;
+export interface Passage {
+    id: number;
+    start: number;
+    end: number;
+    heading: string;
+    text: string;
+}
+export interface Highlight {
+    text: string;
+    start?: number;
+}
+/** Sorted-key JSON; the only serialization that enters input hashes. */
+export declare function canonicalJson(value: unknown): string;
+/** Paragraph-level passages with raw offsets. Headings label passages; code fences and image-only blocks are not quotable. */
+export declare function passages(markdown: string, maxChars?: number): Passage[];
+/** Pack passages into sections of at most `budget` display characters, preferring heading boundaries past half a budget. */
+export declare function sectionsOf(list: Passage[], budget: number): Passage[][];
+export declare const quoteBudget: (markdown: string) => number;
+/** JSON Schema for the model's draft. Tags become an enum when a vocabulary is given. */
+export declare function draftSchema(passageCount: number, vocabulary?: readonly string[]): JsonSchema;
+export declare function notesSchema(first: number, last: number): JsonSchema;
+interface Draft {
+    digest: DigestV2;
+    cites: (number | undefined)[][];
+}
+export interface QuoteStats {
+    exact: number;
+    folded: number;
+    snapped: number;
+    missing: number;
+    short: number;
+}
+export type QuoteMatch = 'exact' | 'folded' | 'snapped' | 'missing';
+/**
+ * Make a quote exactly verbatim when that is unambiguous: a unique folded or Markdown-insensitive match, inside
+ * `within` first, becomes the raw source span. Ambiguous folded matches keep their text for the checker; absent ones are 'missing'.
+ */
+export declare function snapQuote(source: string, text: string, within?: {
+    start: number;
+    end: number;
+}): {
+    text: string;
+    match: QuoteMatch;
+};
+/** Snap every quote in place and drop quotes under `minQuoteWords`. */
+export declare function snapQuotes(draft: Draft, source: string, list: Passage[]): QuoteStats;
+export interface DraftInput {
+    revisionId: string;
+    markdown: string;
+    title?: string;
+    form: Form;
+    model: DigestModel;
+    highlights?: readonly Highlight[];
+    vocabulary?: readonly string[];
+    params?: Partial<DigestParams>;
+    signal?: AbortSignal;
+    /** Unit-normalized embeddings for short texts; when given, quotes must support their claims by cosine. */
+    embed?: (texts: string[], signal: AbortSignal) => Promise<number[][]>;
+    supportThreshold?: number;
+}
+export interface DraftTrace {
+    strategy: 'single' | 'sections';
+    sections: number;
+    calls: number;
+    repaired: boolean;
+    attempts: {
+        issues: DigestIssue[];
+        quotes?: QuoteStats;
+    }[];
+    dropped: {
+        quotes: number;
+        ideas: number;
+        tags: number;
+    };
+}
+export interface DraftResult {
+    output: DigestV2;
+    inputHash: string;
+    producer: Record<string, unknown>;
+    options: DigestOptions;
+    trace: DraftTrace;
+}
+/** sha256 over canonical JSON of the revision, its Markdown, the prompt version, the model identity and every output-shaping input. */
+export declare function digestInputHash(input: Pick<DraftInput, 'revisionId' | 'markdown' | 'form' | 'highlights' | 'vocabulary' | 'params' | 'supportThreshold' | 'embed'> & {
+    model: ModelIdentity;
+}): string;
+/** Unqualified on real claims: a deliberately low bar that only catches quotes unrelated to their claim. */
+export declare const defaultSupportThreshold = 0.35;
+export declare function draftDigest(input: DraftInput): Promise<DraftResult>;
+/** Draft and store a digest for the item's current revision. A matching input hash returns the stored draft without calling the model. */
+export declare function createDigest(store: LibraryStore, itemId: string, model: DigestModel, options?: Omit<DraftInput, 'revisionId' | 'markdown' | 'title' | 'form' | 'model' | 'highlights'>): Promise<{
+    derivation: Derivation;
+    duplicate: boolean;
+    trace?: DraftTrace;
+}>;
+export {};

package/dist/src/digest/pipeline.js

@@ -0,0 +1,399 @@
+import { createHash } from 'node:crypto';
+import { foldText, locateAll, locateLoose, words } from '../annotations/quote.js';
+import { checkDigest, digestLimits as L, forms, parseDigest } from './schema.js';
+import { StoreError } from '../store/types.js';
+/**
+ * Model-agnostic Digest v2 drafting. The model sees numbered passages and cites a passage id plus a short exact
+ * span; code snaps, prunes and checks. Nothing leaves this module except through the injected model.
+ */
+export const PROMPT_VERSION = 'sponge-digest-prompt-v2';
+/** 24k characters is roughly 6k tokens of passages, leaving room for instructions and output in an 8k context on an 8 GB Mac. */
+export const defaultDigestParams = { temperature: 0, seed: 1, maxTokens: 2048, contextChars: 24000, maxSections: 24, timeoutMs: 300000, deadlineMs: 1800000, maxOutputBytes: 65536 };
+export class DigestError extends Error {
+    code;
+    issues;
+    constructor(code, issues) {
+        super(code);
+        this.code = code;
+        this.issues = issues;
+        this.name = 'DigestError';
+    }
+}
+/** Quotes under this many words carry no evidence; the checker is gaining the same floor. */
+export const minQuoteWords = 4;
+const sha256 = (v) => createHash('sha256').update(v).digest('hex');
+const collapse = (v) => v.replace(/\s+/g, ' ').trim();
+const object = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
+/** Sorted-key JSON; the only serialization that enters input hashes. */
+export function canonicalJson(value) {
+    if (Array.isArray(value))
+        return `[${value.map(canonicalJson).join(',')}]`;
+    if (object(value))
+        return `{${Object.keys(value).filter(k => value[k] !== undefined).sort().map(k => `${JSON.stringify(k)}:${canonicalJson(value[k])}`).join(',')}}`;
+    if (typeof value === 'number' && !Number.isFinite(value))
+        throw new DigestError('invalid-input');
+    return JSON.stringify(value ?? null);
+}
+/** Paragraph-level passages with raw offsets. Headings label passages; code fences and image-only blocks are not quotable. */
+export function passages(markdown, maxChars = 1200) {
+    const out = [], blocks = [];
+    let at = 0, start = -1, fence = false, fenced = false;
+    for (const line of markdown.split('\n')) {
+        const end = at + line.length, blank = !line.trim(), marker = /^\s{0,3}(```|~~~)/.test(line);
+        if (marker)
+            fence = !fence;
+        if (blank && !fence && !marker) {
+            if (start >= 0)
+                blocks.push({ start, end: at - 1, fence: fenced });
+            start = -1;
+            fenced = false;
+        }
+        else {
+            if (start < 0)
+                start = at;
+            if (marker || fence)
+                fenced = true;
+        }
+        at = end + 1;
+    }
+    if (start >= 0)
+        blocks.push({ start, end: markdown.length, fence: fenced });
+    let heading = '';
+    for (const b of blocks) {
+        const raw = markdown.slice(b.start, b.end).trimEnd(), end = b.start + raw.length;
+        if (b.fence || !raw.trim())
+            continue;
+        const h = /^\s{0,3}#{1,6}\s+(.+)$/.exec(raw);
+        if (h && !raw.includes('\n')) {
+            heading = collapse(h[1]).slice(0, 200);
+            continue;
+        }
+        if (/^(\s*!\[[^\]]*\]\([^)]*\)\s*)+$/.test(raw))
+            continue;
+        // Long blocks split at sentence ends, then at whitespace, so every passage fits a prompt line.
+        let from = b.start;
+        while (end - from > maxChars) {
+            const window = markdown.slice(from, from + maxChars), sentence = lastSentenceEnd(window), space = window.lastIndexOf(' ');
+            const stop = from + (sentence > maxChars / 4 ? sentence : space > maxChars / 4 ? space : maxChars);
+            out.push({ id: out.length + 1, start: from, end: stop, heading, text: markdown.slice(from, stop) });
+            from = stop;
+            while (from < end && /\s/.test(markdown[from]))
+                from++;
+        }
+        if (end > from)
+            out.push({ id: out.length + 1, start: from, end, heading, text: markdown.slice(from, end) });
+    }
+    return out;
+}
+function lastSentenceEnd(window) { let best = -1; for (const m of window.matchAll(/[.!?]["')\]]*(?=\s)/g))
+    best = m.index + m[0].length; return best; }
+const display = (p) => `[${p.id}] ${collapse(p.text)}`;
+/** Pack passages into sections of at most `budget` display characters, preferring heading boundaries past half a budget. */
+export function sectionsOf(list, budget) {
+    const out = [];
+    let current = [], size = 0;
+    for (const p of list) {
+        const n = display(p).length + 1;
+        if (current.length && (size + n > budget || p.heading !== current[current.length - 1].heading && size > budget / 2)) {
+            out.push(current);
+            current = [];
+            size = 0;
+        }
+        current.push(p);
+        size += n;
+    }
+    if (current.length)
+        out.push(current);
+    return out;
+}
+const sourceWords = (markdown) => words(markdown);
+export const quoteBudget = (markdown) => Math.min(L.totalQuoteWords, Math.max(L.quoteWords, Math.floor(sourceWords(markdown) * L.sourceQuoteShare)));
+const str = (maxLength) => ({ type: 'string', minLength: 1, maxLength });
+/** JSON Schema for the model's draft. Tags become an enum when a vocabulary is given. */
+export function draftSchema(passageCount, vocabulary) {
+    const quote = { type: 'object', additionalProperties: false, required: ['passage', 'text'], properties: { passage: { type: 'integer', minimum: 1, maximum: Math.max(1, passageCount) }, text: str(300), attribution: str(200) } };
+    const idea = { type: 'object', additionalProperties: false, required: ['claim', 'detail', 'quotes'], properties: { claim: str(200), detail: str(600), quotes: { type: 'array', minItems: 0, maxItems: L.maxQuotesPerIdea, items: quote } } };
+    const tag = vocabulary?.length ? { type: 'string', enum: [...vocabulary] } : { type: 'string', pattern: '^[a-z0-9][a-z0-9-]{0,63}$' };
+    return { type: 'object', additionalProperties: false, required: ['lead', 'ideas', 'tags'], properties: { lead: str(L.leadChars), context: str(600), ideas: { type: 'array', minItems: L.minIdeas, maxItems: L.maxIdeas, items: idea }, tags: { type: 'array', minItems: 0, maxItems: L.maxTags, items: tag } } };
+}
+export function notesSchema(first, last) {
+    return { type: 'object', additionalProperties: false, required: ['notes'], properties: { notes: { type: 'array', minItems: 1, maxItems: 4, items: { type: 'object', additionalProperties: false, required: ['point', 'passage', 'quote'], properties: { point: str(300), passage: { type: 'integer', minimum: first, maximum: last }, quote: str(300) } } } } };
+}
+const system = (budget, vocabulary) => `You write a Digest v2 of one source for a private reading library. Reply with JSON only, matching the schema.
+- lead: one sentence of at most ${L.leadChars} characters that names the author or organization when known and states the main finding.
+- context: optional, at most ${L.contextWords} words of essential background. Lead and context together stay within ${L.gistWords} words and ${L.gistSentences} sentences.
+- ideas: ${L.minIdeas} to ${L.maxIdeas}, in the order the author makes them. Each idea is a bold one-sentence claim with detail:
+  - claim: one sentence of at most ${L.claimWords} words that asserts something; never a topic label.
+  - detail: at most ${L.detailWords} words in at most ${L.detailSentences} sentences: evidence, mechanism or limits.
+  - quotes: 0 to ${L.maxQuotesPerIdea} that support this idea, each copied character for character from one numbered passage, ${minQuoteWords} to ${L.quoteWords} words, with that passage number. No added quotation marks or ellipses. Give attribution (who said it) when it is not the author. Prefer reader highlights.
+- Attach at least ${L.minQuotes} and at most ${L.maxQuotes} quotes in all. All quotes together stay within ${budget} words and never overlap.
+- tags: 0 to ${L.maxTags} ${vocabulary?.length ? `from: ${vocabulary.join(', ')}` : 'lowercase kebab-case topics'}.
+Use only what the source says.`;
+function sourceBlock(title, form, list, highlights) {
+    let heading;
+    const lines = [];
+    for (const p of list) {
+        if (p.heading !== heading) {
+            heading = p.heading;
+            if (heading)
+                lines.push(`## ${heading}`);
+        }
+        lines.push(display(p));
+    }
+    const marks = highlights.map((h, i) => `[h${i + 1}${h.passage ? `, passage ${h.passage}` : ''}] ${collapse(h.text).slice(0, 400)}`);
+    return `Form: ${form}${title ? `\nTitle: ${collapse(title).slice(0, 300)}` : ''}${marks.length ? `\n\nReader highlights (preferred quotes):\n${marks.join('\n')}` : ''}\n\nPassages:\n${lines.join('\n')}`;
+}
+const rule = { gist: `lead and context within ${L.gistWords} words and ${L.gistSentences} sentences`, lead: `one sentence, at most ${L.leadChars} characters`, context: `at most ${L.contextWords} words`, claim: `one distinct sentence, at most ${L.claimWords} words`, detail: `at most ${L.detailWords} words and ${L.detailSentences} sentences`, quotes: `0-${L.maxQuotesPerIdea} quotes per idea, each copied exactly from its passage, ${minQuoteWords} to ${L.quoteWords} words, not overlapping another quote`, attribution: `at most ${L.attributionWords} words`, ideas: `${L.minIdeas}-${L.maxIdeas} ideas; ${L.minQuotes}-${L.maxQuotes} quotes in all, within the total budget`, tags: `0-${L.maxTags} distinct allowed tags` };
+function repairPrompt(issues) {
+    const lines = issues.slice(0, 40).map(i => { const last = i.path.split('.').filter(s => !/^\d+$/.test(s)).pop() ?? ''; return `- ${i.path || 'output'}: ${i.code}${rule[last] ? ` (rule: ${rule[last]})` : ''}`; });
+    return `That draft broke these rules:\n${lines.join('\n')}\nReturn the complete corrected JSON. Copy quotes exactly from the numbered passages.`;
+}
+/** Map the model's draft shape into Digest v2; parseDigest then decides whether the shape is valid. Ideas keep the model's order. */
+function fromModel(text, form) {
+    let v;
+    try {
+        v = JSON.parse(text);
+    }
+    catch {
+        return { issues: [{ path: '', code: 'json' }] };
+    }
+    if (!object(v) || !Array.isArray(v.ideas) || !Array.isArray(v.tags))
+        return { issues: [{ path: '', code: 'shape' }] };
+    const trim = (s) => typeof s === 'string' ? s.trim() : s, cites = [];
+    const ideas = v.ideas.map((idea) => {
+        if (!object(idea)) {
+            cites.push([]);
+            return idea;
+        }
+        const quotes = Array.isArray(idea.quotes) ? idea.quotes : [];
+        cites.push(quotes.map((q) => Number.isSafeInteger(q?.passage) ? q.passage : undefined));
+        return { claim: trim(idea.claim), detail: trim(idea.detail), quotes: quotes.map((q) => { const attribution = trim(q?.attribution); return { text: trim(q?.text), ...(attribution ? { attribution } : {}) }; }) };
+    });
+    const context = trim(v.context);
+    const parsed = parseDigest({ schemaVersion: 2, form, gist: { lead: trim(v.lead), ...(context ? { context } : {}) }, ideas, tags: v.tags.map(trim) });
+    return parsed.digest?.schemaVersion === 2 ? { digest: parsed.digest, cites } : { issues: parsed.issues };
+}
+/**
+ * Make a quote exactly verbatim when that is unambiguous: a unique folded or Markdown-insensitive match, inside
+ * `within` first, becomes the raw source span. Ambiguous folded matches keep their text for the checker; absent ones are 'missing'.
+ */
+export function snapQuote(source, text, within) {
+    if (source.includes(text))
+        return { text, match: 'exact' };
+    const inside = (m) => !!within && m.start >= within.start && m.end <= within.end;
+    const pick = (all) => { const local = all.filter(inside); return local.length === 1 ? local[0] : all.length === 1 ? all[0] : undefined; };
+    const folded = locateAll(source, text);
+    if (folded.length) {
+        const m = pick(folded);
+        return { text: m ? source.slice(m.start, m.end) : text, match: 'folded' };
+    }
+    const loose = (within && pick(locateLoose(source, text, within))) || pick(locateLoose(source, text));
+    if (!loose)
+        return { text, match: 'missing' };
+    // Keep emphasis and code markers balanced: a span that starts or ends inside `**bold**` takes the adjacent markers.
+    let { start, end } = loose;
+    while (start > 0 && /[*_`]/.test(source[start - 1]))
+        start--;
+    while (end < source.length && /[*_`]/.test(source[end]))
+        end++;
+    return { text: source.slice(start, end), match: 'snapped' };
+}
+/** Snap every quote in place and drop quotes under `minQuoteWords`. */
+export function snapQuotes(draft, source, list) {
+    const stats = { exact: 0, folded: 0, snapped: 0, missing: 0, short: 0 }, byId = new Map(list.map(p => [p.id, p]));
+    draft.digest.ideas.forEach((idea, i) => {
+        idea.quotes.forEach((q, j) => { const r = snapQuote(source, q.text, byId.get(draft.cites[i]?.[j] ?? 0)); q.text = r.text; stats[r.match]++; });
+        const keep = idea.quotes.map(q => words(q.text) >= minQuoteWords);
+        stats.short += keep.filter(k => !k).length;
+        idea.quotes = idea.quotes.filter((_, j) => keep[j]);
+        draft.cites[i] = (draft.cites[i] ?? []).filter((_, j) => keep[j]);
+    });
+    return stats;
+}
+const pruneable = new Set(['not-verbatim', 'overlap', 'unsupported', 'too-long', 'speaker-required']);
+/**
+ * Deterministic last resort: drop failing quotes, then duplicate ideas and invalid tags, then quotes over the count or word
+ * budget (second quotes first, longest first), never the last quote. Quoteless ideas stay: v2 allows them.
+ */
+function prune(draft, issues, source, options) {
+    const dropped = { quotes: 0, ideas: 0, tags: 0 }, d = draft.digest, bad = new Set(issues.filter(i => /^ideas\.\d+\.quotes\.\d+(?:\.attribution)?$/.test(i.path) && pruneable.has(i.code)).map(i => i.path.replace(/\.attribution$/, '')));
+    d.ideas.forEach((idea, i) => { const keep = idea.quotes.map((_, j) => !bad.has(`ideas.${i}.quotes.${j}`)).map((k, j) => k && j < L.maxQuotesPerIdea); dropped.quotes += keep.filter(k => !k).length; idea.quotes = idea.quotes.filter((_, j) => keep[j]); draft.cites[i] = (draft.cites[i] ?? []).filter((_, j) => keep[j]); });
+    const seen = new Set();
+    const keepIdea = d.ideas.map(idea => { const key = foldText(idea.claim).toLowerCase(), ok = !seen.has(key); seen.add(key); return ok; });
+    dropped.ideas = keepIdea.filter(k => !k).length;
+    d.ideas = d.ideas.filter((_, i) => keepIdea[i]);
+    draft.cites = draft.cites.filter((_, i) => keepIdea[i]);
+    if (d.ideas.length > L.maxIdeas) {
+        dropped.ideas += d.ideas.length - L.maxIdeas;
+        d.ideas = d.ideas.slice(0, L.maxIdeas);
+        draft.cites = draft.cites.slice(0, L.maxIdeas);
+    }
+    const tags = [...new Set(d.tags)].filter(t => options.tags ? options.tags.includes(t) : /^[a-z0-9][a-z0-9-]{0,63}$/.test(t)).slice(0, L.maxTags);
+    dropped.tags = d.tags.length - tags.length;
+    d.tags = tags;
+    const budget = quoteBudget(source), all = () => d.ideas.flatMap((idea, i) => idea.quotes.map((q, j) => ({ i, j, n: words(q.text) })));
+    for (let q = all(); q.length > 1 && (q.length > L.maxQuotes || q.reduce((n, x) => n + x.n, 0) > budget); q = all()) {
+        const x = q.sort((a, b) => b.j - a.j || b.n - a.n)[0];
+        d.ideas[x.i].quotes.splice(x.j, 1);
+        draft.cites[x.i]?.splice(x.j, 1);
+        dropped.quotes++;
+    }
+    return dropped;
+}
+const hashedParams = (p) => ({ temperature: p.temperature, seed: p.seed, maxTokens: p.maxTokens, contextChars: p.contextChars, maxSections: p.maxSections });
+function resolveParams(params = {}) {
+    const p = { ...defaultDigestParams, ...params }, int = (v, min, max) => Number.isSafeInteger(v) && v >= min && v <= max;
+    if (!(p.temperature >= 0 && p.temperature <= 2) || !int(p.seed, 0, 2 ** 31) || !int(p.maxTokens, 256, 32768) || !int(p.contextChars, 2000, 1_000_000) || !int(p.maxSections, 1, 256) || !int(p.timeoutMs, 1, 3_600_000) || !int(p.deadlineMs, 1, 14_400_000) || !int(p.maxOutputBytes, 1024, 4 * 1024 * 1024))
+        throw new DigestError('invalid-input');
+    return p;
+}
+function validIdentity(id) {
+    const ok = (v) => typeof v === 'string' && v.length > 0 && v.length <= 256 && !/[\u0000-\u001f\u007f]/.test(v);
+    if (!object(id) || !ok(id.model) || !ok(id.quantization) || !ok(id.runtime))
+        throw new DigestError('invalid-input');
+    return { model: id.model, quantization: id.quantization, runtime: id.runtime };
+}
+/** sha256 over canonical JSON of the revision, its Markdown, the prompt version, the model identity and every output-shaping input. */
+export function digestInputHash(input) {
+    return sha256(canonicalJson({ revisionId: input.revisionId, markdownSha256: sha256(input.markdown), promptVersion: PROMPT_VERSION, model: validIdentity(input.model), params: { ...hashedParams(resolveParams(input.params)), form: input.form, vocabulary: input.vocabulary ? [...input.vocabulary] : null, highlights: (input.highlights ?? []).map(h => h.text), support: input.embed ? input.supportThreshold ?? defaultSupportThreshold : null } }));
+}
+/** Unqualified on real claims: a deliberately low bar that only catches quotes unrelated to their claim. */
+export const defaultSupportThreshold = 0.35;
+export async function draftDigest(input) {
+    const p = resolveParams(input.params), identity = validIdentity(input.model.identity);
+    if (typeof input.markdown !== 'string' || !input.markdown.trim() || !forms.includes(input.form))
+        throw new DigestError('invalid-input');
+    if (input.vocabulary && (!input.vocabulary.length || input.vocabulary.length > 256 || input.vocabulary.some(t => !/^[a-z0-9][a-z0-9-]{0,63}$/.test(t))))
+        throw new DigestError('invalid-input');
+    const inputHash = digestInputHash({ ...input, model: identity }), source = input.markdown, list = passages(source);
+    if (list.length === 0)
+        throw new DigestError('invalid-input');
+    const userSignal = input.signal ?? new AbortController().signal, overall = AbortSignal.any([userSignal, AbortSignal.timeout(p.deadlineMs)]);
+    const trace = { strategy: 'single', sections: 1, calls: 0, repaired: false, attempts: [], dropped: { quotes: 0, ideas: 0, tags: 0 } };
+    const call = async (messages, schema) => {
+        if (userSignal.aborted)
+            throw new DigestError('cancelled');
+        if (overall.aborted)
+            throw new DigestError('limit-exceeded');
+        const signal = AbortSignal.any([overall, AbortSignal.timeout(p.timeoutMs)]);
+        trace.calls++;
+        let text;
+        try {
+            text = await input.model.generate(messages, schema, { signal, maxOutputBytes: p.maxOutputBytes, maxTokens: p.maxTokens, temperature: p.temperature, seed: p.seed });
+        }
+        catch (error) {
+            if (userSignal.aborted)
+                throw new DigestError('cancelled');
+            if (signal.aborted)
+                throw new DigestError('limit-exceeded');
+            if (error instanceof DigestError)
+                throw error;
+            throw new DigestError('unavailable');
+        }
+        if (userSignal.aborted)
+            throw new DigestError('cancelled');
+        if (typeof text !== 'string' || Buffer.byteLength(text) > p.maxOutputBytes)
+            throw new DigestError('limit-exceeded');
+        return text;
+    };
+    const highlights = (input.highlights ?? []).filter(h => typeof h?.text === 'string' && h.text.trim()).slice(0, 32).map(h => { const at = h.start ?? locateAll(source, h.text)[0]?.start; return { text: h.text, passage: at === undefined ? undefined : list.find(x => at >= x.start && at < x.end)?.id }; });
+    let shown = list, notesText;
+    const total = list.reduce((n, x) => n + display(x).length + 1, 0);
+    if (total > p.contextChars) {
+        // Long source: notes per section, then one combined pass over the cited and highlighted passages only.
+        const sections = sectionsOf(list, p.contextChars);
+        if (sections.length > p.maxSections)
+            throw new DigestError('limit-exceeded');
+        trace.strategy = 'sections';
+        trace.sections = sections.length;
+        const notes = [];
+        for (const [k, section] of sections.entries()) {
+            const first = section[0].id, last = section[section.length - 1].id;
+            const text = await call([{ role: 'system', content: `You take notes on part ${k + 1} of ${sections.length} of a longer source. Reply with JSON only. Give 1 to 4 notes, each a point of at most 25 words that the section asserts, with one passage number from this part and a short quote copied exactly from that passage.` }, { role: 'user', content: sourceBlock(input.title, input.form, section, []) }], notesSchema(first, last));
+            let v;
+            try {
+                v = JSON.parse(text);
+            }
+            catch {
+                continue;
+            }
+            if (object(v) && Array.isArray(v.notes))
+                for (const n of v.notes.slice(0, 4))
+                    if (object(n) && typeof n.point === 'string' && Number.isSafeInteger(n.passage) && n.passage >= first && n.passage <= last && typeof n.quote === 'string')
+                        notes.push({ point: collapse(n.point).slice(0, 300), passage: n.passage, quote: collapse(n.quote).slice(0, 300) });
+        }
+        const wanted = new Set([...notes.map(n => n.passage), ...highlights.flatMap(h => h.passage ? [h.passage] : [])]);
+        notesText = notes.map(n => `- [${n.passage}] ${n.point}`).join('\n');
+        shown = [];
+        let size = notesText.length;
+        for (const x of list)
+            if (wanted.has(x.id) && size + display(x).length + 1 <= p.contextChars) {
+                shown.push(x);
+                size += display(x).length + 1;
+            }
+        if (!shown.length)
+            throw new DigestError('unavailable', [{ path: '', code: 'no-notes' }]);
+    }
+    const budget = quoteBudget(source), schema = draftSchema(list.length, input.vocabulary);
+    const messages = [{ role: 'system', content: system(budget, input.vocabulary) }, { role: 'user', content: (notesText ? `Notes from each part of the source, in order:\n${notesText}\n\n` : '') + sourceBlock(input.title, input.form, shown, highlights) }];
+    const cache = new Map(), options = { ...(input.vocabulary ? { tags: input.vocabulary } : {}) };
+    const embedKey = (t) => Buffer.from(foldText(t)).subarray(0, 900).toString();
+    if (input.embed)
+        Object.assign(options, { supportThreshold: input.supportThreshold ?? defaultSupportThreshold, supports: (claim, quote) => { const a = cache.get(embedKey(claim)), b = cache.get(embedKey(quote)); return a && b ? a.reduce((n, x, i) => n + x * b[i], 0) : 0; } });
+    const check = async (draft) => {
+        if (input.embed) {
+            const missing = [...new Set(draft.digest.ideas.flatMap(i => [i.claim, ...i.quotes.map(q => q.text)]).map(embedKey))].filter(t => !cache.has(t));
+            if (missing.length) {
+                const vectors = await input.embed(missing, overall);
+                if (!Array.isArray(vectors) || vectors.length !== missing.length)
+                    throw new DigestError('unavailable');
+                missing.forEach((t, i) => cache.set(t, vectors[i]));
+            }
+        }
+        return checkDigest(draft.digest, source, options).issues;
+    };
+    const attempt = async (text) => {
+        const draft = fromModel(text, input.form);
+        if ('issues' in draft) {
+            trace.attempts.push({ issues: draft.issues });
+            return { issues: draft.issues };
+        }
+        const quotes = snapQuotes(draft, source, list);
+        const issues = await check(draft);
+        trace.attempts.push({ issues, quotes });
+        return { draft, issues };
+    };
+    const first = await call(messages, schema);
+    let result = await attempt(first);
+    if (result.issues.length) {
+        trace.repaired = true;
+        result = await attempt(await call([...messages, { role: 'assistant', content: first }, { role: 'user', content: repairPrompt(result.issues) }], schema));
+    }
+    if (!result.draft)
+        throw new DigestError('unavailable', result.issues);
+    let issues = result.issues;
+    if (issues.length) {
+        trace.dropped = prune(result.draft, issues, source, options);
+        issues = await check(result.draft);
+        if (result.draft.digest.ideas.length < L.minIdeas)
+            issues = [...issues.filter(i => !(i.path === 'ideas' && i.code === 'count')), { path: 'ideas', code: 'count' }];
+        if (issues.length)
+            throw new DigestError('unavailable', issues);
+    }
+    const producer = { pipeline: 'sponge-digest', promptVersion: PROMPT_VERSION, model: identity, params: hashedParams(p), strategy: trace.strategy, sections: trace.sections, calls: trace.calls, repaired: trace.repaired, dropped: trace.dropped, ...(input.embed ? { supportThreshold: options.supportThreshold } : {}) };
+    return { output: result.draft.digest, inputHash, producer, options, trace };
+}
+const formFor = (kind) => kind === 'pdf' ? 'pdf' : kind === 'media' ? 'video' : 'article';
+/** Draft and store a digest for the item's current revision. A matching input hash returns the stored draft without calling the model. */
+export async function createDigest(store, itemId, model, options = {}) {
+    const item = store.read(itemId);
+    if (!item)
+        throw new StoreError('unavailable');
+    const highlights = store.listAnnotations(itemId).filter(a => a.kind === 'highlight' && a.revisionId === item.revisionId && a.anchored !== false).map(a => ({ text: a.anchor.exact, start: a.anchor.start }));
+    const input = { ...options, revisionId: item.revisionId, markdown: item.markdown, title: item.title, form: formFor(item.kind), model, highlights };
+    const prior = store.findDigest(item.revisionId, digestInputHash({ ...input, model: model.identity }));
+    if (prior)
+        return { derivation: prior, duplicate: true };
+    const draft = await draftDigest(input), stored = store.putDigest({ itemId, revisionId: item.revisionId, producer: draft.producer, inputHash: draft.inputHash, output: draft.output, options: draft.options });
+    return { derivation: store.readDigest(stored.id), duplicate: stored.duplicate, trace: draft.trace };
+}