@hraness/sponge 0.2.5 → 0.2.6

package/README.md

@@ -110,7 +110,8 @@ pruned generation needs a rebuild.
 
 ### Digests
 
-A digest is a one-sentence gist and 3–5 ideas, each with 1–2 verbatim quotes.
+A digest (Digest v2) is a short gist led by one sentence and 3–5 ideas, each a
+claim with detail and 0–2 verbatim quotes, with 1–6 quotes in all.
 A local model proposes it; Sponge checks every rule against the captured text
 and stores only a passing draft. Drafting is opt-in and needs a model server you
 start yourself, such as llama.cpp `llama-server`, LM Studio or Ollama:

package/dist/src/cli/digest.js

@@ -55,9 +55,9 @@ export function digestText(d) {
     if (o.gist.context)
         lines.push(`  ${o.gist.context}`);
     o.ideas.forEach((idea, i) => {
-        lines.push('', `idea:${i} ${mark(d.review.ideas?.[i])} ${idea.claim}`, `  ${idea.kind}; ${idea.voice.type === 'author' ? 'author' : `${idea.voice.type}: ${idea.voice.name}`}`, `  ${idea.detail}`);
-        idea.quotes.forEach((q, j) => lines.push(`  quote:${i}.${j} ${mark(d.review.quotes?.[i]?.[j])} “${q.text.replace(/\s+/g, ' ')}”${q.speaker ? ` (${q.speaker})` : ''}`));
+        lines.push('', `idea:${i} ${mark(d.review.ideas?.[i])} ${idea.claim}`, ...(idea.kind || idea.voice ? [`  ${[idea.kind, idea.voice && (idea.voice.type === 'author' ? 'author' : `${idea.voice.type}: ${idea.voice.name}`)].filter(Boolean).join('; ')}`] : []), `  ${idea.detail}`);
+        idea.quotes.forEach((q, j) => lines.push(`  quote:${i}.${j} ${mark(d.review.quotes?.[i]?.[j])} “${q.text.replace(/\s+/g, ' ')}”${q.attribution ?? q.speaker ? ` (${q.attribution ?? q.speaker})` : ''}`));
     });
-    lines.push('', `Tags: ${o.tags.join(', ')}`);
+    lines.push('', `Tags: ${o.tags.join(', ') || 'none'}`);
     return lines.join('\n');
 }

package/dist/src/cli/import.js

@@ -9,5 +9,5 @@ export function jungleAction(store, path, options) {
 }
 const tally = (m) => Object.entries(m).sort().map(([k, n]) => `${k} ${n}`).join(', ') || 'none';
 export function jungleText(r) {
-    return [`Jungle ${r.dryRun ? 'dry run (captures verified on --apply)' : 'import'}: ${r.entries.length} of ${r.notes} notes`, `Items: ${r.counts.newItems} new, ${r.counts.existingItems} existing`, `Editorial: ${tally(r.counts.editorial)}`, `Digest v1: ${tally(r.counts.v1)}`, `Visibility: ${tally(r.counts.visibility)}`, `Skipped: ${tally(r.counts.skipped)}`, ...(r.nextOffset !== null ? [`Next offset: ${r.nextOffset}`] : [])].join('\n');
+    return [`Jungle ${r.dryRun ? 'dry run (captures verified on --apply)' : 'import'}: ${r.entries.length} of ${r.notes} notes`, `Items: ${r.counts.newItems} new, ${r.counts.existingItems} existing`, `Editorial: ${tally(r.counts.editorial)}`, `Digest v2: ${tally(r.counts.digest)}`, ...(Object.keys(r.counts.digestIssues).length ? [`Not convertible because: ${tally(r.counts.digestIssues)}`] : []), `Quotes: ${r.counts.quotes.assigned} assigned, dropped ${tally(r.counts.quotes.dropped)}`, `Visibility: ${tally(r.counts.visibility)}`, `Skipped: ${tally(r.counts.skipped)}`, ...(r.nextOffset !== null ? [`Next offset: ${r.nextOffset}`] : [])].join('\n');
 }

package/dist/src/digest/editorial.d.ts

@@ -3,7 +3,7 @@ import { type JungleEditorial } from '../importers/jungle-digest.js';
 import { type ReviewStatus } from '../store/changes.js';
 /**
  * An editorial derivation keeps a reviewed source note whole (today: one Jungle Reading note of any form), without
- * Digest v1's limits. It is immutable data addressed like a digest; its review is one logged path, `review.note`.
+ * Digest's limits. It is immutable data addressed like a digest; its review is one logged path, `review.note`.
  */
 export interface Editorial {
     id: string;
@@ -39,9 +39,10 @@ export declare function listEditorials(db: DatabaseSync, itemId: string): Editor
 /** One logged change, and only when the status differs. */
 export declare function reviewEditorial(db: DatabaseSync, id: string, status: ReviewStatus | null): Editorial;
 /**
- * The public subset of an accepted editorial note, in the public digest shape: a lead only when the note's first
- * sentence fits the v1 lead limit (never cut), claims with detail, and quotes that occur verbatim in the capture,
- * each at most 25 words, within the v1 per-entry budget (150 words, and the 10% source share with its 25-word floor).
+ * The public subset of an accepted editorial note, in the public digest shape and within Digest v2's limits: a lead only
+ * when the note's first sentence fits the lead limit (never cut), claims with detail, and quotes that occur verbatim in the
+ * capture with 4-25 words, assigned to ideas as the importer assigns them (at most 2 per idea and 6 in all), within the
+ * per-entry budget (150 words, and the 10% source share with its 25-word floor).
  */
 export declare function publicEditorial(db: DatabaseSync, e: Editorial): {
     lead?: string;

package/dist/src/digest/editorial.js

@@ -51,9 +51,10 @@ export function reviewEditorial(db, id, status) {
     return readEditorial(db, id);
 }
 /**
- * The public subset of an accepted editorial note, in the public digest shape: a lead only when the note's first
- * sentence fits the v1 lead limit (never cut), claims with detail, and quotes that occur verbatim in the capture,
- * each at most 25 words, within the v1 per-entry budget (150 words, and the 10% source share with its 25-word floor).
+ * The public subset of an accepted editorial note, in the public digest shape and within Digest v2's limits: a lead only
+ * when the note's first sentence fits the lead limit (never cut), claims with detail, and quotes that occur verbatim in the
+ * capture with 4-25 words, assigned to ideas as the importer assigns them (at most 2 per idea and 6 in all), within the
+ * per-entry budget (150 words, and the 10% source share with its 25-word floor).
  */
 export function publicEditorial(db, e) {
     if (e.review.note !== 'accepted')
@@ -61,20 +62,25 @@ export function publicEditorial(db, e) {
     const o = e.output, [first] = firstSentence(o.gist ?? ''), lead = first && first.length <= L.leadChars ? first : undefined;
     const ideas = (o.ideas ?? []).map(i => ({ claim: i.claim, detail: i.detail, quotes: [] }));
     if (ideas.length && o.quotes?.length) {
-        const source = markdownFor(db, e.revisionId), budget = Math.min(L.totalQuoteWords, Math.max(L.quoteWords, Math.floor(words(source) * L.sourceQuoteShare))), taken = [];
-        let used = 0;
-        assignQuotes(ideas, o.quotes).forEach((idea, k) => {
-            if (idea === null || ideas[idea].quotes.length >= L.maxQuotesPerIdea)
-                return;
-            const match = locateAll(source, o.quotes[k].text).find(m => !taken.some(t => m.start < t.end && t.start < m.end));
+        const source = markdownFor(db, e.revisionId), budget = Math.min(L.totalQuoteWords, Math.max(L.quoteWords, Math.floor(words(source) * L.sourceQuoteShare))), taken = [], texts = [];
+        for (const q of o.quotes) {
+            const match = locateAll(source, q.text).find(m => !taken.some(t => m.start < t.end && t.start < m.end));
             if (!match)
-                return;
+                continue;
             const text = source.slice(match.start, match.end), n = words(text);
-            if (n < L.minQuoteWords || n > L.quoteWords || used + n > budget)
+            if (n >= L.minQuoteWords && n <= L.quoteWords) {
+                taken.push(match);
+                texts.push(text);
+            }
+        }
+        let used = 0, count = 0;
+        assignQuotes(ideas, texts.map(text => ({ text }))).forEach((idea, k) => {
+            const n = words(texts[k]);
+            if (idea === null || count >= L.maxQuotes || used + n > budget)
                 return;
             used += n;
-            taken.push(match);
-            ideas[idea].quotes.push(text);
+            count++;
+            ideas[idea].quotes.push(texts[k]);
         });
     }
     return lead || ideas.length ? { ...(lead ? { lead } : {}), ideas } : undefined;

package/dist/src/digest/pipeline.d.ts

@@ -1,11 +1,11 @@
-import { forms, type DigestIssue, type DigestOptions, type DigestV1 } from './schema.js';
+import { forms, type DigestIssue, type DigestOptions, type DigestV2 } from './schema.js';
 import type { LibraryStore } from '../store/index.js';
 import type { Derivation } from '../store/foundation.js';
 /**
- * Model-agnostic Digest v1 drafting. The model sees numbered passages and cites a passage id plus a short exact
- * span; code snaps, orders, prunes and checks. Nothing leaves this module except through the injected model.
+ * Model-agnostic Digest v2 drafting. The model sees numbered passages and cites a passage id plus a short exact
+ * span; code snaps, prunes and checks. Nothing leaves this module except through the injected model.
  */
-export declare const PROMPT_VERSION = "sponge-digest-prompt-v1";
+export declare const PROMPT_VERSION = "sponge-digest-prompt-v2";
 export type ChatMessage = {
     role: 'system' | 'user' | 'assistant';
     content: string;
@@ -72,7 +72,7 @@ export declare const quoteBudget: (markdown: string) => number;
 export declare function draftSchema(passageCount: number, vocabulary?: readonly string[]): JsonSchema;
 export declare function notesSchema(first: number, last: number): JsonSchema;
 interface Draft {
-    digest: DigestV1;
+    digest: DigestV2;
     cites: (number | undefined)[][];
 }
 export interface QuoteStats {
@@ -126,7 +126,7 @@ export interface DraftTrace {
     };
 }
 export interface DraftResult {
-    output: DigestV1;
+    output: DigestV2;
     inputHash: string;
     producer: Record<string, unknown>;
     options: DigestOptions;

package/dist/src/digest/pipeline.js

@@ -1,12 +1,12 @@
 import { createHash } from 'node:crypto';
 import { foldText, locateAll, locateLoose, words } from '../annotations/quote.js';
-import { checkDigest, digestLimits as L, forms, ideaKinds, parseDigest } from './schema.js';
+import { checkDigest, digestLimits as L, forms, parseDigest } from './schema.js';
 import { StoreError } from '../store/types.js';
 /**
- * Model-agnostic Digest v1 drafting. The model sees numbered passages and cites a passage id plus a short exact
- * span; code snaps, orders, prunes and checks. Nothing leaves this module except through the injected model.
+ * Model-agnostic Digest v2 drafting. The model sees numbered passages and cites a passage id plus a short exact
+ * span; code snaps, prunes and checks. Nothing leaves this module except through the injected model.
  */
-export const PROMPT_VERSION = 'sponge-digest-prompt-v1';
+export const PROMPT_VERSION = 'sponge-digest-prompt-v2';
 /** 24k characters is roughly 6k tokens of passages, leaving room for instructions and output in an 8k context on an 8 GB Mac. */
 export const defaultDigestParams = { temperature: 0, seed: 1, maxTokens: 2048, contextChars: 24000, maxSections: 24, timeoutMs: 300000, deadlineMs: 1800000, maxOutputBytes: 65536 };
 export class DigestError extends Error {
@@ -111,25 +111,23 @@ export const quoteBudget = (markdown) => Math.min(L.totalQuoteWords, Math.max(L.
 const str = (maxLength) => ({ type: 'string', minLength: 1, maxLength });
 /** JSON Schema for the model's draft. Tags become an enum when a vocabulary is given. */
 export function draftSchema(passageCount, vocabulary) {
-    const quote = { type: 'object', additionalProperties: false, required: ['passage', 'text'], properties: { passage: { type: 'integer', minimum: 1, maximum: Math.max(1, passageCount) }, text: str(300), speaker: str(120) } };
-    const idea = { type: 'object', additionalProperties: false, required: ['claim', 'detail', 'kind', 'voice', 'quotes'], properties: { claim: str(200), detail: str(600), kind: { type: 'string', enum: [...ideaKinds] }, voice: { type: 'object', additionalProperties: false, required: ['type'], properties: { type: { type: 'string', enum: ['author', 'speaker', 'reported'] }, name: str(120) } }, quotes: { type: 'array', minItems: 1, maxItems: L.maxQuotesPerIdea, items: quote } } };
+    const quote = { type: 'object', additionalProperties: false, required: ['passage', 'text'], properties: { passage: { type: 'integer', minimum: 1, maximum: Math.max(1, passageCount) }, text: str(300), attribution: str(200) } };
+    const idea = { type: 'object', additionalProperties: false, required: ['claim', 'detail', 'quotes'], properties: { claim: str(200), detail: str(600), quotes: { type: 'array', minItems: 0, maxItems: L.maxQuotesPerIdea, items: quote } } };
     const tag = vocabulary?.length ? { type: 'string', enum: [...vocabulary] } : { type: 'string', pattern: '^[a-z0-9][a-z0-9-]{0,63}$' };
-    return { type: 'object', additionalProperties: false, required: ['lead', 'ideas', 'tags'], properties: { lead: str(L.leadChars), context: str(400), ideas: { type: 'array', minItems: L.minIdeas, maxItems: L.maxIdeas, items: idea }, tags: { type: 'array', minItems: 1, maxItems: L.maxTags, items: tag } } };
+    return { type: 'object', additionalProperties: false, required: ['lead', 'ideas', 'tags'], properties: { lead: str(L.leadChars), context: str(600), ideas: { type: 'array', minItems: L.minIdeas, maxItems: L.maxIdeas, items: idea }, tags: { type: 'array', minItems: 0, maxItems: L.maxTags, items: tag } } };
 }
 export function notesSchema(first, last) {
     return { type: 'object', additionalProperties: false, required: ['notes'], properties: { notes: { type: 'array', minItems: 1, maxItems: 4, items: { type: 'object', additionalProperties: false, required: ['point', 'passage', 'quote'], properties: { point: str(300), passage: { type: 'integer', minimum: first, maximum: last }, quote: str(300) } } } } };
 }
-const system = (budget, vocabulary) => `You write a Digest v1 of one source for a private reading library. Reply with JSON only, matching the schema.
+const system = (budget, vocabulary) => `You write a Digest v2 of one source for a private reading library. Reply with JSON only, matching the schema.
 - lead: one sentence of at most ${L.leadChars} characters that names the author or organization when known and states the main finding.
-- context: optional, at most ${L.contextWords} words of essential background.
-- ideas: ${L.minIdeas} to ${L.maxIdeas}, in the order they appear in the source. Each idea has:
+- context: optional, at most ${L.contextWords} words of essential background. Lead and context together stay within ${L.gistWords} words and ${L.gistSentences} sentences.
+- ideas: ${L.minIdeas} to ${L.maxIdeas}, in the order the author makes them. Each idea is a bold one-sentence claim with detail:
   - claim: one sentence of at most ${L.claimWords} words that asserts something; never a topic label.
   - detail: at most ${L.detailWords} words in at most ${L.detailSentences} sentences: evidence, mechanism or limits.
-  - kind: ${ideaKinds.join(', ')}.
-  - voice: "author" when the author makes the claim; "speaker" with name for a speaker in a talk or interview; "reported" with name when the author reports someone else's claim.
-  - quotes: 1 or ${L.maxQuotesPerIdea}, each copied character for character from one numbered passage, ${minQuoteWords} to ${L.quoteWords} words, with that passage number. No added quotation marks or ellipses. Give speaker when the voice is not the author. Prefer reader highlights.
-- All quotes together stay within ${budget} words and never overlap.
-- tags: 1 to ${L.maxTags} ${vocabulary?.length ? `from: ${vocabulary.join(', ')}` : 'lowercase kebab-case topics'}.
+  - quotes: 0 to ${L.maxQuotesPerIdea} that support this idea, each copied character for character from one numbered passage, ${minQuoteWords} to ${L.quoteWords} words, with that passage number. No added quotation marks or ellipses. Give attribution (who said it) when it is not the author. Prefer reader highlights.
+- Attach at least ${L.minQuotes} and at most ${L.maxQuotes} quotes in all. All quotes together stay within ${budget} words and never overlap.
+- tags: 0 to ${L.maxTags} ${vocabulary?.length ? `from: ${vocabulary.join(', ')}` : 'lowercase kebab-case topics'}.
 Use only what the source says.`;
 function sourceBlock(title, form, list, highlights) {
     let heading;
@@ -145,12 +143,12 @@ function sourceBlock(title, form, list, highlights) {
     const marks = highlights.map((h, i) => `[h${i + 1}${h.passage ? `, passage ${h.passage}` : ''}] ${collapse(h.text).slice(0, 400)}`);
     return `Form: ${form}${title ? `\nTitle: ${collapse(title).slice(0, 300)}` : ''}${marks.length ? `\n\nReader highlights (preferred quotes):\n${marks.join('\n')}` : ''}\n\nPassages:\n${lines.join('\n')}`;
 }
-const rule = { lead: `one sentence, at most ${L.leadChars} characters`, context: `at most ${L.contextWords} words`, claim: `one distinct sentence, at most ${L.claimWords} words`, detail: `at most ${L.detailWords} words and ${L.detailSentences} sentences`, quotes: `1-${L.maxQuotesPerIdea} quotes, each copied exactly from its passage, ${minQuoteWords} to ${L.quoteWords} words, not overlapping another quote`, ideas: `${L.minIdeas}-${L.maxIdeas} ideas; quotes within the total budget`, tags: `1-${L.maxTags} distinct allowed tags` };
+const rule = { gist: `lead and context within ${L.gistWords} words and ${L.gistSentences} sentences`, lead: `one sentence, at most ${L.leadChars} characters`, context: `at most ${L.contextWords} words`, claim: `one distinct sentence, at most ${L.claimWords} words`, detail: `at most ${L.detailWords} words and ${L.detailSentences} sentences`, quotes: `0-${L.maxQuotesPerIdea} quotes per idea, each copied exactly from its passage, ${minQuoteWords} to ${L.quoteWords} words, not overlapping another quote`, attribution: `at most ${L.attributionWords} words`, ideas: `${L.minIdeas}-${L.maxIdeas} ideas; ${L.minQuotes}-${L.maxQuotes} quotes in all, within the total budget`, tags: `0-${L.maxTags} distinct allowed tags` };
 function repairPrompt(issues) {
     const lines = issues.slice(0, 40).map(i => { const last = i.path.split('.').filter(s => !/^\d+$/.test(s)).pop() ?? ''; return `- ${i.path || 'output'}: ${i.code}${rule[last] ? ` (rule: ${rule[last]})` : ''}`; });
     return `That draft broke these rules:\n${lines.join('\n')}\nReturn the complete corrected JSON. Copy quotes exactly from the numbered passages.`;
 }
-/** Map the model's draft shape into DigestV1; parseDigest then decides whether the shape is valid. */
+/** Map the model's draft shape into Digest v2; parseDigest then decides whether the shape is valid. Ideas keep the model's order. */
 function fromModel(text, form) {
     let v;
     try {
@@ -167,14 +165,13 @@ function fromModel(text, form) {
             cites.push([]);
             return idea;
         }
-        const voice = object(idea.voice) && idea.voice.type !== 'author' ? { type: idea.voice.type, name: trim(idea.voice.name) } : { type: 'author' };
         const quotes = Array.isArray(idea.quotes) ? idea.quotes : [];
         cites.push(quotes.map((q) => Number.isSafeInteger(q?.passage) ? q.passage : undefined));
-        return { claim: trim(idea.claim), detail: trim(idea.detail), kind: idea.kind, voice, quotes: quotes.map((q) => { const speaker = trim(q?.speaker) || (voice.type !== 'author' ? voice.name : undefined); return { text: trim(q?.text), ...(speaker ? { speaker } : {}) }; }) };
+        return { claim: trim(idea.claim), detail: trim(idea.detail), quotes: quotes.map((q) => { const attribution = trim(q?.attribution); return { text: trim(q?.text), ...(attribution ? { attribution } : {}) }; }) };
     });
     const context = trim(v.context);
-    const parsed = parseDigest({ schemaVersion: 1, form, gist: { lead: trim(v.lead), ...(context ? { context } : {}) }, ideas, tags: v.tags.map(trim) });
-    return parsed.digest ? { digest: parsed.digest, cites } : { issues: parsed.issues };
+    const parsed = parseDigest({ schemaVersion: 2, form, gist: { lead: trim(v.lead), ...(context ? { context } : {}) }, ideas, tags: v.tags.map(trim) });
+    return parsed.digest?.schemaVersion === 2 ? { digest: parsed.digest, cites } : { issues: parsed.issues };
 }
 /**
  * Make a quote exactly verbatim when that is unambiguous: a unique folded or Markdown-insensitive match, inside
@@ -213,21 +210,16 @@ export function snapQuotes(draft, source, list) {
     });
     return stats;
 }
-/** Ideas and their quotes in source order, by first located occurrence: the checker requires non-decreasing idea starts. */
-function order(draft, source) {
-    const at = (text) => locateAll(source, text)[0]?.start ?? Number.MAX_SAFE_INTEGER;
-    const rows = draft.digest.ideas.map((idea, i) => { const qs = idea.quotes.map((q, j) => ({ q, cite: draft.cites[i]?.[j], at: at(q.text) })).sort((a, b) => a.at - b.at); return { idea: { ...idea, quotes: qs.map(x => x.q) }, cites: qs.map(x => x.cite), at: qs[0]?.at ?? Number.MAX_SAFE_INTEGER }; });
-    rows.sort((a, b) => a.at - b.at);
-    draft.digest.ideas = rows.map(r => r.idea);
-    draft.cites = rows.map(r => r.cites);
-}
 const pruneable = new Set(['not-verbatim', 'overlap', 'unsupported', 'too-long', 'speaker-required']);
-/** Deterministic last resort: drop failing quotes, then quoteless or duplicate ideas, invalid tags, and second quotes over budget. */
+/**
+ * Deterministic last resort: drop failing quotes, then duplicate ideas and invalid tags, then quotes over the count or word
+ * budget (second quotes first, longest first), never the last quote. Quoteless ideas stay: v2 allows them.
+ */
 function prune(draft, issues, source, options) {
-    const dropped = { quotes: 0, ideas: 0, tags: 0 }, d = draft.digest, bad = new Set(issues.filter(i => /^ideas\.\d+\.quotes\.\d+$/.test(i.path) && pruneable.has(i.code)).map(i => i.path));
+    const dropped = { quotes: 0, ideas: 0, tags: 0 }, d = draft.digest, bad = new Set(issues.filter(i => /^ideas\.\d+\.quotes\.\d+(?:\.attribution)?$/.test(i.path) && pruneable.has(i.code)).map(i => i.path.replace(/\.attribution$/, '')));
     d.ideas.forEach((idea, i) => { const keep = idea.quotes.map((_, j) => !bad.has(`ideas.${i}.quotes.${j}`)).map((k, j) => k && j < L.maxQuotesPerIdea); dropped.quotes += keep.filter(k => !k).length; idea.quotes = idea.quotes.filter((_, j) => keep[j]); draft.cites[i] = (draft.cites[i] ?? []).filter((_, j) => keep[j]); });
     const seen = new Set();
-    const keepIdea = d.ideas.map(idea => { const key = foldText(idea.claim).toLowerCase(), ok = idea.quotes.length > 0 && !seen.has(key); seen.add(key); return ok; });
+    const keepIdea = d.ideas.map(idea => { const key = foldText(idea.claim).toLowerCase(), ok = !seen.has(key); seen.add(key); return ok; });
     dropped.ideas = keepIdea.filter(k => !k).length;
     d.ideas = d.ideas.filter((_, i) => keepIdea[i]);
     draft.cites = draft.cites.filter((_, i) => keepIdea[i]);
@@ -239,13 +231,11 @@ function prune(draft, issues, source, options) {
     const tags = [...new Set(d.tags)].filter(t => options.tags ? options.tags.includes(t) : /^[a-z0-9][a-z0-9-]{0,63}$/.test(t)).slice(0, L.maxTags);
     dropped.tags = d.tags.length - tags.length;
     d.tags = tags;
-    const budget = quoteBudget(source), total = () => d.ideas.reduce((n, idea) => n + idea.quotes.reduce((m, q) => m + words(q.text), 0), 0);
-    while (total() > budget) {
-        const i = d.ideas.map((idea, k) => ({ k, n: idea.quotes.length > 1 ? words(idea.quotes[1].text) : -1 })).sort((a, b) => b.n - a.n)[0];
-        if (!i || i.n < 0)
-            break;
-        d.ideas[i.k].quotes.pop();
-        draft.cites[i.k]?.pop();
+    const budget = quoteBudget(source), all = () => d.ideas.flatMap((idea, i) => idea.quotes.map((q, j) => ({ i, j, n: words(q.text) })));
+    for (let q = all(); q.length > 1 && (q.length > L.maxQuotes || q.reduce((n, x) => n + x.n, 0) > budget); q = all()) {
+        const x = q.sort((a, b) => b.j - a.j || b.n - a.n)[0];
+        d.ideas[x.i].quotes.splice(x.j, 1);
+        draft.cites[x.i]?.splice(x.j, 1);
         dropped.quotes++;
     }
     return dropped;
@@ -369,7 +359,6 @@ export async function draftDigest(input) {
             return { issues: draft.issues };
         }
         const quotes = snapQuotes(draft, source, list);
-        order(draft, source);
         const issues = await check(draft);
         trace.attempts.push({ issues, quotes });
         return { draft, issues };

package/dist/src/digest/schema.d.ts

@@ -1,7 +1,9 @@
 import { type TextQuote } from '../annotations/quote.js';
 /**
- * Digest v1: a one-sentence gist and 3-5 ideas, each a claim with detail and 1-2 verbatim quotes.
- * The model proposes; this validator decides. Nothing here calls a model.
+ * Digest v2: a gist led by one sentence, and 3-5 ideas in the author's order, each a claim with detail and 0-2 verbatim
+ * quotes; the digest carries 1-6 quotes in all. Digest v1 (one short lead, 1-2 quotes on every idea, ideas in source
+ * order, kind and voice required) stays readable and checkable; new digests are v2. The model proposes; this validator
+ * decides. Nothing here calls a model.
  */
 export declare const ideaKinds: readonly ["finding", "argument", "method", "evidence", "recommendation", "counterpoint"];
 export declare const forms: readonly ["article", "video", "paper", "thread", "pdf"];
@@ -18,17 +20,23 @@ export type QuoteLocator = {
     type: 'page';
     page: number;
 };
+/** `attribution` is v2 only: who said it and where, as the reviewer wrote it. */
 export interface DigestQuote {
     text: string;
     speaker?: string;
+    attribution?: string;
     locator?: QuoteLocator;
 }
 export interface DigestIdea {
     claim: string;
     detail: string;
+    kind?: typeof ideaKinds[number];
+    voice?: Voice;
+    quotes: DigestQuote[];
+}
+export interface DigestIdeaV1 extends DigestIdea {
     kind: typeof ideaKinds[number];
     voice: Voice;
-    quotes: DigestQuote[];
 }
 export interface DigestV1 {
     schemaVersion: 1;
@@ -37,10 +45,22 @@ export interface DigestV1 {
         lead: string;
         context?: string;
     };
+    ideas: DigestIdeaV1[];
+    tags: string[];
+}
+export interface DigestV2 {
+    schemaVersion: 2;
+    form: typeof forms[number];
+    gist: {
+        lead: string;
+        context?: string;
+    };
     ideas: DigestIdea[];
     tags: string[];
 }
-export declare const digestLimits: {
+export type Digest = DigestV1 | DigestV2;
+export declare const digestSchemaVersion = 2;
+export declare const digestLimitsV1: {
     readonly leadChars: 155;
     readonly contextWords: 50;
     readonly minIdeas: 3;
@@ -55,6 +75,27 @@ export declare const digestLimits: {
     readonly sourceQuoteShare: 0.1;
     readonly maxTags: 3;
 };
+/** The current (v2) limits. A gist is its lead plus context; quotes are counted per idea and per digest. */
+export declare const digestLimits: {
+    readonly leadChars: 240;
+    readonly contextWords: 80;
+    readonly gistWords: 100;
+    readonly gistSentences: 5;
+    readonly minIdeas: 3;
+    readonly maxIdeas: 5;
+    readonly claimWords: 20;
+    readonly detailWords: 60;
+    readonly detailSentences: 3;
+    readonly maxQuotesPerIdea: 2;
+    readonly minQuotes: 1;
+    readonly maxQuotes: 6;
+    readonly minQuoteWords: 4;
+    readonly quoteWords: 25;
+    readonly attributionWords: 25;
+    readonly totalQuoteWords: 150;
+    readonly sourceQuoteShare: 0.1;
+    readonly maxTags: 3;
+};
 export interface DigestIssue {
     path: string;
     code: string;
@@ -70,16 +111,16 @@ export interface DigestCheck {
     quotes: LocatedQuote[];
 }
 export interface DigestOptions {
-    /** Controlled tag vocabulary; omitted means any 1-3 lowercase tags. */
+    /** Controlled tag vocabulary; omitted means any lowercase kebab-case tags (1-3 in v1, 0-3 in v2). */
     tags?: readonly string[];
     /** Optional semantic check that a quote supports its claim, e.g. embedding cosine. */
     supports?: (claim: string, quote: string) => number;
     supportThreshold?: number;
 }
-/** Parse an untrusted model output into a DigestV1 shape, or report why not. */
+/** Parse an untrusted digest (v1 or v2) into its shape, or report why not. */
 export declare function parseDigest(value: unknown): {
-    digest?: DigestV1;
+    digest?: Digest;
     issues: DigestIssue[];
 };
-/** Check every v1 rule that code can decide against the captured Markdown. */
-export declare function checkDigest(digest: DigestV1, source: string, options?: DigestOptions): DigestCheck;
+/** Check every rule of the digest's own version that code can decide against the captured Markdown. */
+export declare function checkDigest(digest: Digest, source: string, options?: DigestOptions): DigestCheck;

package/dist/src/digest/schema.js

@@ -1,21 +1,29 @@
 import { foldText, locateAll, textQuote, words } from '../annotations/quote.js';
 /**
- * Digest v1: a one-sentence gist and 3-5 ideas, each a claim with detail and 1-2 verbatim quotes.
- * The model proposes; this validator decides. Nothing here calls a model.
+ * Digest v2: a gist led by one sentence, and 3-5 ideas in the author's order, each a claim with detail and 0-2 verbatim
+ * quotes; the digest carries 1-6 quotes in all. Digest v1 (one short lead, 1-2 quotes on every idea, ideas in source
+ * order, kind and voice required) stays readable and checkable; new digests are v2. The model proposes; this validator
+ * decides. Nothing here calls a model.
  */
 export const ideaKinds = ['finding', 'argument', 'method', 'evidence', 'recommendation', 'counterpoint'];
 export const forms = ['article', 'video', 'paper', 'thread', 'pdf'];
-export const digestLimits = { leadChars: 155, contextWords: 50, minIdeas: 3, maxIdeas: 5, claimWords: 20, detailWords: 60, detailSentences: 3, maxQuotesPerIdea: 2, minQuoteWords: 4, quoteWords: 25, totalQuoteWords: 150, sourceQuoteShare: 0.1, maxTags: 3 };
+export const digestSchemaVersion = 2;
+export const digestLimitsV1 = { leadChars: 155, contextWords: 50, minIdeas: 3, maxIdeas: 5, claimWords: 20, detailWords: 60, detailSentences: 3, maxQuotesPerIdea: 2, minQuoteWords: 4, quoteWords: 25, totalQuoteWords: 150, sourceQuoteShare: 0.1, maxTags: 3 };
+/** The current (v2) limits. A gist is its lead plus context; quotes are counted per idea and per digest. */
+export const digestLimits = { leadChars: 240, contextWords: 80, gistWords: 100, gistSentences: 5, minIdeas: 3, maxIdeas: 5, claimWords: 20, detailWords: 60, detailSentences: 3, maxQuotesPerIdea: 2, minQuotes: 1, maxQuotes: 6, minQuoteWords: 4, quoteWords: 25, attributionWords: 25, totalQuoteWords: 150, sourceQuoteShare: 0.1, maxTags: 3 };
 const object = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
 const keys = (v, allowed) => Object.keys(v).every(k => allowed.includes(k));
 const str = (v, max = 2000) => typeof v === 'string' && v.trim().length > 0 && v.length <= max && !/[\u0000-\u0008\u000b-\u001f\u007f]/.test(v);
-/** Sentence terminators followed by more text; abbreviations are rare enough in a 155-character lead to reject. */
-const sentences = (v) => foldText(v).split(/(?<=[.!?])\s+(?=\S)/).length;
-/** Parse an untrusted model output into a DigestV1 shape, or report why not. */
+/** v1: sentence terminators followed by more text; abbreviations are rare enough in a 155-character lead to reject. */
+const sentencesV1 = (v) => foldText(v).split(/(?<=[.!?])\s+(?=\S)/).length;
+/** v2: a terminator (and closing quotes) followed by a capital or digit, the same split Jungle's first-sentence lead uses, so `e.g. this` stays one sentence. */
+const sentences = (v) => foldText(v).split(/(?<=[.!?]["')]*)\s+(?=["(]?[A-Z0-9])/).length;
+/** Parse an untrusted digest (v1 or v2) into its shape, or report why not. */
 export function parseDigest(value) {
     const issues = [], bad = (path, code = 'shape') => { issues.push({ path, code }); };
-    if (!object(value) || !keys(value, ['schemaVersion', 'form', 'gist', 'ideas', 'tags']) || value.schemaVersion !== 1)
+    if (!object(value) || !keys(value, ['schemaVersion', 'form', 'gist', 'ideas', 'tags']) || value.schemaVersion !== 1 && value.schemaVersion !== 2)
         return { issues: [{ path: '', code: 'shape' }] };
+    const v2 = value.schemaVersion === 2;
     if (!forms.includes(value.form))
         bad('form');
     const gist = value.gist;
@@ -26,15 +34,15 @@ export function parseDigest(value) {
     else
         value.ideas.forEach((idea, i) => {
             const at = `ideas.${i}`;
-            if (!object(idea) || !keys(idea, ['claim', 'detail', 'kind', 'voice', 'quotes']) || !str(idea.claim) || !str(idea.detail) || !ideaKinds.includes(idea.kind))
+            if (!object(idea) || !keys(idea, ['claim', 'detail', 'kind', 'voice', 'quotes']) || !str(idea.claim) || !str(idea.detail) || !(v2 && idea.kind === undefined || ideaKinds.includes(idea.kind)))
                 return bad(at);
             const voice = idea.voice;
-            if (!object(voice) || !(voice.type === 'author' && keys(voice, ['type']) || (voice.type === 'speaker' || voice.type === 'reported') && keys(voice, ['type', 'name']) && str(voice.name, 200)))
+            if (!(v2 && voice === undefined) && !(object(voice) && (voice.type === 'author' && keys(voice, ['type']) || (voice.type === 'speaker' || voice.type === 'reported') && keys(voice, ['type', 'name']) && str(voice.name, 200))))
                 bad(at + '.voice');
             if (!Array.isArray(idea.quotes) || idea.quotes.length > 4)
                 return bad(at + '.quotes');
             idea.quotes.forEach((quote, j) => {
-                if (!object(quote) || !keys(quote, ['text', 'speaker', 'locator']) || !str(quote.text, 600) || quote.speaker !== undefined && !str(quote.speaker, 200))
+                if (!object(quote) || !keys(quote, v2 ? ['text', 'speaker', 'attribution', 'locator'] : ['text', 'speaker', 'locator']) || !str(quote.text, 600) || quote.speaker !== undefined && !str(quote.speaker, 200) || quote.attribution !== undefined && !str(quote.attribution, 600))
                     return bad(`${at}.quotes.${j}`);
                 const loc = quote.locator;
                 if (loc !== undefined && !(object(loc) && (loc.type === 'timestamp' && keys(loc, ['type', 'ms']) && Number.isSafeInteger(loc.ms) && loc.ms >= 0 || loc.type === 'page' && keys(loc, ['type', 'page']) && Number.isSafeInteger(loc.page) && loc.page >= 1)))
@@ -45,42 +53,51 @@ export function parseDigest(value) {
         bad('tags');
     return issues.length ? { issues } : { digest: value, issues };
 }
-/** Check every v1 rule that code can decide against the captured Markdown. */
+/** Check every rule of the digest's own version that code can decide against the captured Markdown. */
 export function checkDigest(digest, source, options = {}) {
-    const issues = [], located = [], issue = (path, code) => { issues.push({ path, code }); }, L = digestLimits;
-    if (digest.gist.lead.length > L.leadChars)
+    const issues = [], located = [], issue = (path, code) => { issues.push({ path, code }); }, v2 = digest.schemaVersion === 2, L = v2 ? digestLimits : digestLimitsV1, count = v2 ? sentences : sentencesV1;
+    const { lead, context } = digest.gist;
+    if (lead.length > L.leadChars)
         issue('gist.lead', 'too-long');
-    if (sentences(digest.gist.lead) !== 1)
+    if (count(lead) !== 1)
         issue('gist.lead', 'one-sentence');
-    if (digest.gist.context !== undefined && words(digest.gist.context) > L.contextWords)
+    if (context !== undefined && words(context) > L.contextWords)
         issue('gist.context', 'too-long');
+    if (v2 && words(lead) + (context === undefined ? 0 : words(context)) > digestLimits.gistWords)
+        issue('gist', 'too-long');
+    if (v2 && count(lead) + (context === undefined ? 0 : count(context)) > digestLimits.gistSentences)
+        issue('gist', 'too-many-sentences');
     if (digest.ideas.length < L.minIdeas || digest.ideas.length > L.maxIdeas)
         issue('ideas', 'count');
     const claims = new Set();
-    let quoteWords = 0;
+    let quoteWords = 0, quoteCount = 0;
     digest.ideas.forEach((idea, i) => {
         const at = `ideas.${i}`;
         if (words(idea.claim) > L.claimWords)
             issue(at + '.claim', 'too-long');
-        if (sentences(idea.claim) !== 1)
+        if (count(idea.claim) !== 1)
             issue(at + '.claim', 'one-sentence');
         if (claims.has(foldText(idea.claim).toLowerCase()))
             issue(at + '.claim', 'duplicate');
         claims.add(foldText(idea.claim).toLowerCase());
         if (words(idea.detail) > L.detailWords)
             issue(at + '.detail', 'too-long');
-        if (sentences(idea.detail) > L.detailSentences)
+        if (count(idea.detail) > L.detailSentences)
             issue(at + '.detail', 'too-many-sentences');
-        if (idea.quotes.length < 1 || idea.quotes.length > L.maxQuotesPerIdea)
+        // v1 needs a quote on every idea; v2 allows quoteless ideas and counts quotes per digest below.
+        if (idea.quotes.length < (v2 ? 0 : 1) || idea.quotes.length > L.maxQuotesPerIdea)
             issue(at + '.quotes', 'count');
         idea.quotes.forEach((quote, j) => {
-            const path = `${at}.quotes.${j}`, count = words(quote.text);
-            quoteWords += count;
-            if (count > L.quoteWords)
+            const path = `${at}.quotes.${j}`, n = words(quote.text);
+            quoteWords += n;
+            quoteCount++;
+            if (n > L.quoteWords)
                 issue(path, 'too-long');
-            if (count < L.minQuoteWords)
+            if (n < L.minQuoteWords)
                 issue(path, 'too-short');
-            if (idea.voice.type !== 'author' && !quote.speaker)
+            if (quote.attribution !== undefined && words(quote.attribution) > digestLimits.attributionWords)
+                issue(path + '.attribution', 'too-long');
+            if (idea.voice && idea.voice.type !== 'author' && !quote.speaker)
                 issue(path, 'speaker-required');
             const matches = locateAll(source, quote.text);
             if (!matches.length)
@@ -94,23 +111,26 @@ export function checkDigest(digest, source, options = {}) {
                 issue(path, 'unsupported');
         });
     });
-    // Ideas follow the source: each idea's earliest located quote starts no earlier than the previous idea's.
+    if (v2 && (quoteCount < digestLimits.minQuotes || quoteCount > digestLimits.maxQuotes))
+        issue('ideas', 'quote-count');
+    // v1 only: ideas follow the source, each idea's earliest located quote starting no earlier than the previous idea's.
     let previous = -1;
-    digest.ideas.forEach((_, i) => {
-        const starts = located.filter(q => q.idea === i).map(q => q.anchor.start);
-        if (!starts.length)
-            return;
-        const first = Math.min(...starts);
-        if (first < previous)
-            issue(`ideas.${i}`, 'out-of-order');
-        previous = first;
-    });
+    if (!v2)
+        digest.ideas.forEach((_, i) => {
+            const starts = located.filter(q => q.idea === i).map(q => q.anchor.start);
+            if (!starts.length)
+                return;
+            const first = Math.min(...starts);
+            if (first < previous)
+                issue(`ideas.${i}`, 'out-of-order');
+            previous = first;
+        });
     if (quoteWords > L.totalQuoteWords)
         issue('ideas', 'quote-budget');
     // The 10% share has a 25-word floor: an exemption so a short source can still carry one full quote.
     if (quoteWords > Math.max(L.quoteWords, Math.floor(words(source) * L.sourceQuoteShare)))
         issue('ideas', 'quote-share');
-    if (digest.tags.length < 1 || digest.tags.length > L.maxTags || new Set(digest.tags).size !== digest.tags.length)
+    if (digest.tags.length < (v2 ? 0 : 1) || digest.tags.length > L.maxTags || new Set(digest.tags).size !== digest.tags.length)
         issue('tags', 'count');
     digest.tags.forEach((tag, i) => { if (options.tags ? !options.tags.includes(tag) : !/^[a-z0-9][a-z0-9-]{0,63}$/.test(tag))
         issue(`tags.${i}`, 'vocabulary'); });

package/dist/src/digest/store.d.ts

@@ -1,5 +1,5 @@
 import type { DatabaseSync } from 'node:sqlite';
-import { type DigestOptions, type DigestV1 } from './schema.js';
+import { type Digest, type DigestOptions } from './schema.js';
 import { type ReviewStatus } from '../store/changes.js';
 export type { ReviewStatus };
 /** Per-field review of a digest. Only accepted fields can ever be published. */
@@ -15,17 +15,17 @@ export interface Derivation {
     kind: 'digest';
     producer: unknown;
     inputHash: string;
-    output: DigestV1;
+    output: Digest;
     review: DigestReview;
     createdAt: string;
 }
-/** A parsed digest serialized in fixed key order, so key order never makes a different object. */
-export declare function canonicalDigest(d: DigestV1): string;
+/** A parsed digest serialized in fixed key order, so key order never makes a different object. Optional fields appear only when present. */
+export declare function canonicalDigest(d: Digest): string;
 /** Content address of a derivation: the same output from the same input is the same immutable object on every device. */
 export declare const derivationId: (revisionId: string, kind: string, inputHash: string, output: string) => string;
 /**
- * Store a draft digest only when every v1 rule passes against the revision it was made from.
- * Drafts are immutable data, not change-log entries; a different output is a new object.
+ * Store a draft digest only when it is Digest v2 and every v2 rule passes against the revision it was made from.
+ * Stored v1 digests stay readable; new ones are v2. Drafts are immutable data, not change-log entries; a different output is a new object.
  */
 export declare function putDigest(db: DatabaseSync, input: {
     itemId: string;
@@ -45,13 +45,13 @@ export declare function readDigest(db: DatabaseSync, id: string): Derivation | u
 export declare function findDigest(db: DatabaseSync, revisionId: string, inputHash: string): Derivation | undefined;
 /** Review fields of a digest. Each changed path is its own change: `review.gist`, `review.ideas.<i>`, `review.quotes.<i>.<j>`. */
 export declare function reviewDigest(db: DatabaseSync, id: string, review: DigestReview): Derivation;
-/** The accepted subset of a digest. Rejected or unreviewed fields never leave the device. */
+/** The accepted subset of a digest (v1 or v2). Rejected or unreviewed fields never leave the device; v1 ideas keep their kind. */
 export declare function acceptedDigest(d: Derivation): {
     lead?: string;
     ideas: {
         claim: string;
         detail: string;
-        kind: string;
+        kind?: string;
         quotes: string[];
     }[];
 } | undefined;