@hraness/sponge 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cli/commands.js +8 -1
- package/dist/src/digest/pipeline.d.ts +29 -4
- package/dist/src/digest/pipeline.js +125 -21
- package/dist/src/errors.d.ts +1 -0
- package/dist/src/errors.js +2 -0
- package/dist/src/store/digest-jobs.d.ts +5 -2
- package/dist/src/store/digest-jobs.js +7 -3
- package/dist/src/store/revisions.js +1 -1
- package/native/runtime-manifest.json +4 -4
- package/package.json +1 -1
package/dist/src/cli/commands.js
CHANGED
|
@@ -18,7 +18,7 @@ import { pruneText } from '../indexing/prune-run.js';
|
|
|
18
18
|
import { importAction, jungleAction, jungleText } from './import.js';
|
|
19
19
|
import { digestText, reviewFromSpec, reviewSpec } from './digest.js';
|
|
20
20
|
import { connectEndpoint } from '../digest/endpoint.js';
|
|
21
|
-
import { createDigest } from '../digest/pipeline.js';
|
|
21
|
+
import { createDigest, undigestible } from '../digest/pipeline.js';
|
|
22
22
|
import { createQueryEmbedder, createTextEmbedder } from '../search/query.js';
|
|
23
23
|
import { runtimeGcAction, runtimeGcText, setupRuntimeGc } from './runtimes.js';
|
|
24
24
|
import { drainLibrary } from '../worker/basic.js';
|
|
@@ -742,6 +742,13 @@ export const commands = [
|
|
|
742
742
|
return 0;
|
|
743
743
|
}
|
|
744
744
|
if (args.flags.draft) {
|
|
745
|
+
// Refuse an unknown item or a thin capture before the model server spends its startup on it.
|
|
746
|
+
const item = store.read(args.positional[0]);
|
|
747
|
+
if (!item)
|
|
748
|
+
throw new SpongeError('unavailable');
|
|
749
|
+
const thin = undigestible(item.markdown);
|
|
750
|
+
if (thin)
|
|
751
|
+
throw new SpongeError(thin);
|
|
745
752
|
const runtime = await inspectDigest(c.nativeRoot, args.flags.model ?? store.digests.settings().model ?? undefined), result = await withDigestServer(store.root, runtime, (model) => createDigest(store, args.positional[0], model, { signal }), { signal });
|
|
746
753
|
c.emit({ digest: result.derivation, duplicate: result.duplicate, ...(result.trace ? { trace: result.trace } : {}) }, digestText(result.derivation));
|
|
747
754
|
return 0;
|
|
@@ -4,7 +4,7 @@ import type { Derivation, DigestCandidate } from './derivation.js';
|
|
|
4
4
|
* Model-agnostic Digest v2 drafting. The model sees numbered passages and cites a passage id plus a short exact
|
|
5
5
|
* span; code snaps, prunes and checks. Nothing leaves this module except through the injected model.
|
|
6
6
|
*/
|
|
7
|
-
export declare const PROMPT_VERSION = "sponge-digest-prompt-
|
|
7
|
+
export declare const PROMPT_VERSION = "sponge-digest-prompt-v6";
|
|
8
8
|
export type ChatMessage = {
|
|
9
9
|
role: 'system' | 'user' | 'assistant';
|
|
10
10
|
content: string;
|
|
@@ -56,8 +56,30 @@ interface Highlight {
|
|
|
56
56
|
* change digest input hashes.
|
|
57
57
|
*/
|
|
58
58
|
export declare function canonicalJson(value: unknown): string;
|
|
59
|
-
/**
|
|
60
|
-
export
|
|
59
|
+
/** Where a post starts in a captured thread, and who wrote it as the model is told. */
|
|
60
|
+
export interface Post {
|
|
61
|
+
start: number;
|
|
62
|
+
label: string;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* The posts of an X thread capture: top-level list items that carry X's post links. The poster's handle comes from
|
|
66
|
+
* the item's avatar or name link; the thread's author from the title (`Name (@handle) on X`), else the first post. A
|
|
67
|
+
* later post without a handle is still a reply. Other list items (an X article's bullets) are not posts, and a thread
|
|
68
|
+
* with no reply has none.
|
|
69
|
+
*/
|
|
70
|
+
export declare function threadPosts(markdown: string): Post[];
|
|
71
|
+
/** A thread's passages without its replies, which are readers' comments rather than the source; renumbered from 1. */
|
|
72
|
+
export declare function authorPassages(markdown: string): Passage[];
|
|
73
|
+
/**
|
|
74
|
+
* Paragraph-level passages with raw offsets. Headings label passages; code fences and image-only blocks are not
|
|
75
|
+
* quotable. `posts` (a thread's) also break blocks, and their labels head the passages that follow.
|
|
76
|
+
*/
|
|
77
|
+
export declare function passages(markdown: string, maxChars?: number, posts?: readonly Post[]): Passage[];
|
|
78
|
+
/**
|
|
79
|
+
* Why a capture cannot be digested, as a detail code, or undefined when it can: its body (past the frontmatter, images
|
|
80
|
+
* dropped, link text kept) has under `minProseWords` words with letters, or only an archive's metadata was saved.
|
|
81
|
+
*/
|
|
82
|
+
export declare function undigestible(markdown: string): 'source-too-thin' | undefined;
|
|
61
83
|
/** Pack passages into sections of at most `budget` display characters, preferring heading boundaries past half a budget. */
|
|
62
84
|
export declare function sectionsOf(list: Passage[], budget: number): Passage[][];
|
|
63
85
|
/** JSON Schema for the model's draft. Tags become an enum when a vocabulary is given. */
|
|
@@ -160,8 +182,11 @@ export interface DigestStore {
|
|
|
160
182
|
read(kind: 'digest', id: string): Derivation | undefined;
|
|
161
183
|
};
|
|
162
184
|
}
|
|
185
|
+
type DraftOptions = Omit<DraftInput, 'revisionId' | 'markdown' | 'title' | 'form' | 'model' | 'highlights'>;
|
|
186
|
+
/** What the item's current revision is drafted from: its Markdown, title, form and anchored highlights. */
|
|
187
|
+
export declare function digestInputFor(store: Pick<DigestStore, 'read' | 'annotations'>, itemId: string, model: DigestModel, options?: DraftOptions): DraftInput;
|
|
163
188
|
/** Draft and store a digest for the item's current revision. A matching input hash returns the stored draft without calling the model. */
|
|
164
|
-
export declare function createDigest(store: DigestStore, itemId: string, model: DigestModel, options?:
|
|
189
|
+
export declare function createDigest(store: DigestStore, itemId: string, model: DigestModel, options?: DraftOptions): Promise<{
|
|
165
190
|
derivation: Derivation;
|
|
166
191
|
duplicate: boolean;
|
|
167
192
|
trace?: DraftTrace;
|
|
@@ -6,7 +6,7 @@ import { sha256 } from '../util/hash.js';
|
|
|
6
6
|
* Model-agnostic Digest v2 drafting. The model sees numbered passages and cites a passage id plus a short exact
|
|
7
7
|
* span; code snaps, prunes and checks. Nothing leaves this module except through the injected model.
|
|
8
8
|
*/
|
|
9
|
-
export const PROMPT_VERSION = 'sponge-digest-prompt-
|
|
9
|
+
export const PROMPT_VERSION = 'sponge-digest-prompt-v6';
|
|
10
10
|
/** 24k characters is roughly 6k tokens of passages, leaving room for instructions and output in an 8k context on an 8 GB Mac. */
|
|
11
11
|
const defaultDigestParams = {
|
|
12
12
|
temperature: 0,
|
|
@@ -40,14 +40,62 @@ export function canonicalJson(value) {
|
|
|
40
40
|
throw new SpongeError('invalid-input');
|
|
41
41
|
return JSON.stringify(value ?? null);
|
|
42
42
|
}
|
|
43
|
-
|
|
44
|
-
|
|
43
|
+
const postHandle = [
|
|
44
|
+
/^- \[!\[@(\w+)\]/,
|
|
45
|
+
/^- \[\[remote image: @(\w+)\]/,
|
|
46
|
+
/^-\s+\[[^\]]+\]\(https:\/\/x\.com\/(\w+)\)\s*$/,
|
|
47
|
+
];
|
|
48
|
+
/**
|
|
49
|
+
* The posts of an X thread capture: top-level list items that carry X's post links. The poster's handle comes from
|
|
50
|
+
* the item's avatar or name link; the thread's author from the title (`Name (@handle) on X`), else the first post. A
|
|
51
|
+
* later post without a handle is still a reply. Other list items (an X article's bullets) are not posts, and a thread
|
|
52
|
+
* with no reply has none.
|
|
53
|
+
*/
|
|
54
|
+
export function threadPosts(markdown) {
|
|
55
|
+
const author = /^title: ".*\(@(\w+)\) on X"/m.exec(frontmatter(markdown))?.[1], items = [];
|
|
56
|
+
let at = 0;
|
|
57
|
+
for (const line of markdown.split('\n')) {
|
|
58
|
+
if (/^- /.test(line) || /^-\s+\[/.test(line))
|
|
59
|
+
items.push({ start: at, handle: postHandle.map((r) => r.exec(line)?.[1]).find(Boolean), post: false });
|
|
60
|
+
const item = items.at(-1);
|
|
61
|
+
if (item && /x\.com\/(compose\/post\?in_reply_to=|\w+\/status\/\d)/.test(line))
|
|
62
|
+
item.post = true;
|
|
63
|
+
at += line.length + 1;
|
|
64
|
+
}
|
|
65
|
+
const posts = items.filter((i) => i.post || i.handle), by = author ?? posts[0]?.handle, same = (h) => h.toLowerCase() === by?.toLowerCase(), labelled = posts.map((p, i) => ({
|
|
66
|
+
start: p.start,
|
|
67
|
+
label: p.handle && !same(p.handle)
|
|
68
|
+
? `Reply by @${p.handle}`
|
|
69
|
+
: p.handle || i === 0
|
|
70
|
+
? `Post by ${by ? `@${by}, ` : ''}the author`
|
|
71
|
+
: 'Reply (poster not captured)',
|
|
72
|
+
}));
|
|
73
|
+
// With no reply there is nobody to tell apart, so the passages stay as they are.
|
|
74
|
+
return labelled.some((p) => p.label !== labelled[0].label) ? labelled : [];
|
|
75
|
+
}
|
|
76
|
+
/** A thread's passages without its replies, which are readers' comments rather than the source; renumbered from 1. */
|
|
77
|
+
export function authorPassages(markdown) {
|
|
78
|
+
return passages(markdown, undefined, threadPosts(markdown))
|
|
79
|
+
.filter((p) => !p.heading?.startsWith('Reply'))
|
|
80
|
+
.map((p, i) => ({ ...p, id: i + 1 }));
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Paragraph-level passages with raw offsets. Headings label passages; code fences and image-only blocks are not
|
|
84
|
+
* quotable. `posts` (a thread's) also break blocks, and their labels head the passages that follow.
|
|
85
|
+
*/
|
|
86
|
+
export function passages(markdown, maxChars = 1200, posts = []) {
|
|
45
87
|
const out = [], blocks = [];
|
|
46
88
|
let at = 0, start = -1, fence = false, fenced = false;
|
|
89
|
+
const breaks = new Set(posts.map((p) => p.start));
|
|
47
90
|
for (const line of markdown.split('\n')) {
|
|
48
91
|
const end = at + line.length, blank = !line.trim(), marker = /^\s{0,3}(```|~~~)/.test(line);
|
|
49
92
|
if (marker)
|
|
50
93
|
fence = !fence;
|
|
94
|
+
if (!fence && !marker && start >= 0 && breaks.has(at)) {
|
|
95
|
+
blocks.push({ start, end: at - 1, fence: fenced });
|
|
96
|
+
start = -1;
|
|
97
|
+
fenced = false;
|
|
98
|
+
}
|
|
51
99
|
if (blank && !fence && !marker) {
|
|
52
100
|
if (start >= 0)
|
|
53
101
|
blocks.push({ start, end: at - 1, fence: fenced });
|
|
@@ -64,14 +112,17 @@ export function passages(markdown, maxChars = 1200) {
|
|
|
64
112
|
}
|
|
65
113
|
if (start >= 0)
|
|
66
114
|
blocks.push({ start, end: markdown.length, fence: fenced });
|
|
67
|
-
let heading = '';
|
|
115
|
+
let heading = '', post = '';
|
|
116
|
+
const label = (start) => posts.findLast((p) => p.start <= start)?.label ?? '';
|
|
68
117
|
for (const b of blocks) {
|
|
69
118
|
const raw = markdown.slice(b.start, b.end).trimEnd(), end = b.start + raw.length;
|
|
70
119
|
if (b.fence || !raw.trim())
|
|
71
120
|
continue;
|
|
121
|
+
if (label(b.start) !== post)
|
|
122
|
+
heading = post = label(b.start);
|
|
72
123
|
const h = /^\s{0,3}#{1,6}\s+(.+)$/.exec(raw);
|
|
73
124
|
if (h && !raw.includes('\n')) {
|
|
74
|
-
heading = collapse(h[1]).slice(0, 200);
|
|
125
|
+
heading = (post ? `${post}: ` : '') + collapse(h[1]).slice(0, 200);
|
|
75
126
|
continue;
|
|
76
127
|
}
|
|
77
128
|
if (/^(\s*!\[[^\]]*\]\([^)]*\)\s*)+$/.test(raw))
|
|
@@ -91,6 +142,32 @@ export function passages(markdown, maxChars = 1200) {
|
|
|
91
142
|
}
|
|
92
143
|
return out;
|
|
93
144
|
}
|
|
145
|
+
const frontmatter = (markdown) => /^---\r?\n[\s\S]*?\r?\n---(?:\r?\n|$)/u.exec(markdown)?.[0] ?? '';
|
|
146
|
+
/** A `key: "value"` frontmatter field, as captures write them. */
|
|
147
|
+
function captureField(markdown, key) {
|
|
148
|
+
const match = new RegExp(`^${key}: (".*")\\r?$`, 'm').exec(frontmatter(markdown));
|
|
149
|
+
try {
|
|
150
|
+
const value = match ? JSON.parse(match[1]) : undefined;
|
|
151
|
+
return typeof value === 'string' ? value : undefined;
|
|
152
|
+
}
|
|
153
|
+
catch {
|
|
154
|
+
return undefined;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
/** Prose words under which a capture holds nothing to digest: a gallery of numbered images, a bare link list. */
|
|
158
|
+
const minProseWords = 20;
|
|
159
|
+
/**
|
|
160
|
+
* Why a capture cannot be digested, as a detail code, or undefined when it can: its body (past the frontmatter, images
|
|
161
|
+
* dropped, link text kept) has under `minProseWords` words with letters, or only an archive's metadata was saved.
|
|
162
|
+
*/
|
|
163
|
+
export function undigestible(markdown) {
|
|
164
|
+
if (captureField(markdown, 'capture_method') === 'archive-metadata')
|
|
165
|
+
return 'source-too-thin';
|
|
166
|
+
const prose = passages(markdown.slice(frontmatter(markdown).length))
|
|
167
|
+
.map((p) => p.text.replace(/(!?)\[([^\]]*)\]\([^)]*\)/g, (_, image, text) => (image ? ' ' : text)))
|
|
168
|
+
.join(' ');
|
|
169
|
+
return (prose.match(/\p{L}[\p{L}\p{M}'’-]*/gu) ?? []).length < minProseWords ? 'source-too-thin' : undefined;
|
|
170
|
+
}
|
|
94
171
|
function lastSentenceEnd(window) {
|
|
95
172
|
let best = -1;
|
|
96
173
|
for (const m of window.matchAll(/[.!?]["')\]]*(?=\s)/g))
|
|
@@ -196,7 +273,12 @@ Source: an essay arguing that taste is a skill.
|
|
|
196
273
|
claim: Volume beats polish early. | detail: Side-by-side comparison of many rough drafts sharpens judgment faster than finishing one.
|
|
197
274
|
Bad claim: The essay explores how practice over time shapes a person's sense of taste.
|
|
198
275
|
Good claim: Taste is trained.`;
|
|
199
|
-
|
|
276
|
+
/** Who speaks in each form, so claims are credited to the person who made them. */
|
|
277
|
+
const voices = {
|
|
278
|
+
thread: 'This source is an X thread. Its replies are left out. When the author quotes or answers someone, that view stays theirs.',
|
|
279
|
+
video: 'This source is a recording: credit each claim to the person who says it. In an interview or podcast, the guest’s views are the guest’s, not the host’s.',
|
|
280
|
+
};
|
|
281
|
+
const system = (budget, form, vocabulary) => `You write a Digest v2 of one source for a private reading library, in the owner's voice: terse, declarative, specific. Reply with JSON only, matching the schema.
|
|
200
282
|
- lead: one sentence of at most ${L.leadChars} characters stating this source's specific thesis or result: who (author, organization or product by name) shows or argues what, with the number, name or mechanism that makes it this source. Do not restate the title, describe the source ("This article discusses"), or tease.
|
|
201
283
|
- context: 1 or 2 sentences, at most 45 words: the evidence, method, numbers or limits the lead could not hold. Lead and context together stay within ${L.gistWords} words and ${L.gistSentences} sentences.
|
|
202
284
|
- ideas: ${L.minIdeas} to ${L.maxIdeas} (prefer ${L.maxIdeas} when the source has them), in the order the author makes them, covering the whole source including its caveats and limits. Each idea is a claim with detail:
|
|
@@ -205,7 +287,12 @@ const system = (budget, vocabulary) => `You write a Digest v2 of one source for
|
|
|
205
287
|
- quotes: 0 to ${L.maxQuotesPerIdea} that support this idea, each a contiguous span copied character for character from one numbered passage, ${minQuoteWords + 2} to 20 words, with that passage number. A quote is a fragment, not a whole long sentence: when a sentence runs past 20 words, copy only its strongest part. No added quotation marks or ellipses. Give attribution (who said it) when it is not the author. Prefer reader highlights.
|
|
206
288
|
- Attach at least ${L.minQuotes} and at most ${L.maxQuotes} quotes in all; ${L.minQuotes + 1} to 3 is typical. All quotes together stay within ${budget} words and never overlap.
|
|
207
289
|
- tags: 0 to ${L.maxTags} ${vocabulary?.length ? `from: ${vocabulary.join(', ')}` : 'lowercase kebab-case topics'}; usually one subject tag.
|
|
208
|
-
|
|
290
|
+
Faithfulness comes before style:
|
|
291
|
+
- Credit each view to who holds it. When the source quotes, interviews or reports someone, their claims are theirs: name them in the detail and the quote attribution, never pass them off as the author's.${voices[form] ? `\n- ${voices[form]}` : ''}
|
|
292
|
+
- Keep each number with its own subject, unit and meaning (an area is not a height, a share is not a count, a cost is not a price) and never compute one the source does not give.
|
|
293
|
+
- Keep the source's direction: never invert a claim, comparison or cause.
|
|
294
|
+
- Keep the source's certainty: a forecast, plan, proposal, rumor or question stays one ("plans to", "expects", "asks whether"); never state it as done or settled. Hedge only where the source does.
|
|
295
|
+
Plain words. Name things and use numbers. No filler: various, significantly, key, crucial, notably, highlights, explores, discusses, "the author argues", "the article".
|
|
209
296
|
Use only what the source says.
|
|
210
297
|
|
|
211
298
|
${styleExamples}`;
|
|
@@ -255,7 +342,9 @@ function fromModel(text, form) {
|
|
|
255
342
|
}
|
|
256
343
|
if (!object(v) || !Array.isArray(v.ideas) || !Array.isArray(v.tags))
|
|
257
344
|
return { issues: [{ path: '', code: 'shape' }] };
|
|
258
|
-
const trim = (s) => (typeof s === 'string' ? s.trim() : s),
|
|
345
|
+
const trim = (s) => (typeof s === 'string' ? s.trim() : s),
|
|
346
|
+
// Models sometimes end a sentence with the passage it came from ("… market. [2]").
|
|
347
|
+
prose = (s) => (typeof s === 'string' ? s.replace(/(\s*\[\d+(?:,\s*\d+)*\])+$/, '').trim() : s), cites = [];
|
|
259
348
|
const ideas = v.ideas.map((idea) => {
|
|
260
349
|
if (!object(idea)) {
|
|
261
350
|
cites.push([]);
|
|
@@ -264,19 +353,20 @@ function fromModel(text, form) {
|
|
|
264
353
|
const quotes = Array.isArray(idea.quotes) ? idea.quotes : [];
|
|
265
354
|
cites.push(quotes.map((q) => (Number.isSafeInteger(q?.passage) ? q.passage : undefined)));
|
|
266
355
|
return {
|
|
267
|
-
claim:
|
|
268
|
-
detail:
|
|
356
|
+
claim: prose(idea.claim),
|
|
357
|
+
detail: prose(idea.detail),
|
|
269
358
|
quotes: quotes.map((q) => {
|
|
270
359
|
const attribution = trim(q?.attribution);
|
|
271
360
|
return { text: trim(q?.text), ...(attribution ? { attribution } : {}) };
|
|
272
361
|
}),
|
|
273
362
|
};
|
|
274
363
|
});
|
|
275
|
-
|
|
364
|
+
// A context the model stopped mid-sentence is dropped; context is optional.
|
|
365
|
+
const context = ((c) => (typeof c === 'string' && !/[.!?…。!?)"”’]$/.test(c) ? undefined : c))(prose(v.context));
|
|
276
366
|
const parsed = parseDigest({
|
|
277
367
|
schemaVersion: 2,
|
|
278
368
|
form,
|
|
279
|
-
gist: { lead:
|
|
369
|
+
gist: { lead: prose(v.lead), ...(context ? { context } : {}) },
|
|
280
370
|
ideas,
|
|
281
371
|
tags: v.tags.map(trim),
|
|
282
372
|
});
|
|
@@ -453,7 +543,10 @@ export async function draftDigest(input) {
|
|
|
453
543
|
input.vocabulary.length > 256 ||
|
|
454
544
|
input.vocabulary.some((t) => !/^[a-z0-9][a-z0-9-]{0,63}$/.test(t))))
|
|
455
545
|
throw new SpongeError('invalid-input');
|
|
456
|
-
const
|
|
546
|
+
const thin = undigestible(input.markdown);
|
|
547
|
+
if (thin)
|
|
548
|
+
throw new SpongeError(thin);
|
|
549
|
+
const inputHash = digestInputHash({ ...input, model: identity }), source = input.markdown, list = input.form === 'thread' ? authorPassages(source) : passages(source);
|
|
457
550
|
if (list.length === 0)
|
|
458
551
|
throw new SpongeError('invalid-input');
|
|
459
552
|
const userSignal = input.signal ?? new AbortController().signal, overall = AbortSignal.any([userSignal, AbortSignal.timeout(p.deadlineMs)]);
|
|
@@ -564,7 +657,7 @@ export async function draftDigest(input) {
|
|
|
564
657
|
}
|
|
565
658
|
const budget = quoteBudget(source), schema = draftSchema(list.length, input.vocabulary);
|
|
566
659
|
const messages = [
|
|
567
|
-
{ role: 'system', content: system(budget, input.vocabulary) },
|
|
660
|
+
{ role: 'system', content: system(budget, input.form, input.vocabulary) },
|
|
568
661
|
{
|
|
569
662
|
role: 'user',
|
|
570
663
|
content: (notesText ? `Notes from each part of the source, in order:\n${notesText}\n\n` : '') +
|
|
@@ -637,9 +730,16 @@ export async function draftDigest(input) {
|
|
|
637
730
|
};
|
|
638
731
|
return { output: result.draft.digest, inputHash, producer, options, trace };
|
|
639
732
|
}
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
733
|
+
/** X thread captures draft as `thread`, whose prompt rules keep replies apart from the author's posts. */
|
|
734
|
+
const formFor = (kind, markdown) => kind === 'pdf'
|
|
735
|
+
? 'pdf'
|
|
736
|
+
: kind === 'media'
|
|
737
|
+
? 'video'
|
|
738
|
+
: captureField(markdown, 'platform') === 'x' && captureField(markdown, 'capture_scope') === 'thread'
|
|
739
|
+
? 'thread'
|
|
740
|
+
: 'article';
|
|
741
|
+
/** What the item's current revision is drafted from: its Markdown, title, form and anchored highlights. */
|
|
742
|
+
export function digestInputFor(store, itemId, model, options = {}) {
|
|
643
743
|
const item = store.read(itemId);
|
|
644
744
|
if (!item)
|
|
645
745
|
throw new SpongeError('unavailable');
|
|
@@ -647,21 +747,25 @@ export async function createDigest(store, itemId, model, options = {}) {
|
|
|
647
747
|
.list(itemId)
|
|
648
748
|
.filter((a) => a.kind === 'highlight' && a.revisionId === item.revisionId && a.anchored !== false)
|
|
649
749
|
.map((a) => ({ text: a.anchor.exact, start: a.anchor.start }));
|
|
650
|
-
|
|
750
|
+
return {
|
|
651
751
|
...options,
|
|
652
752
|
revisionId: item.revisionId,
|
|
653
753
|
markdown: item.markdown,
|
|
654
754
|
title: item.title,
|
|
655
|
-
form: formFor(item.kind),
|
|
755
|
+
form: formFor(item.kind, item.markdown),
|
|
656
756
|
model,
|
|
657
757
|
highlights,
|
|
658
758
|
};
|
|
659
|
-
|
|
759
|
+
}
|
|
760
|
+
/** Draft and store a digest for the item's current revision. A matching input hash returns the stored draft without calling the model. */
|
|
761
|
+
export async function createDigest(store, itemId, model, options = {}) {
|
|
762
|
+
const input = digestInputFor(store, itemId, model, options);
|
|
763
|
+
const prior = store.derivations.find('digest', input.revisionId, digestInputHash({ ...input, model: model.identity }));
|
|
660
764
|
if (prior)
|
|
661
765
|
return { derivation: prior, duplicate: true };
|
|
662
766
|
const draft = await draftDigest(input), stored = store.derivations.put('digest', {
|
|
663
767
|
itemId,
|
|
664
|
-
revisionId:
|
|
768
|
+
revisionId: input.revisionId,
|
|
665
769
|
producer: draft.producer,
|
|
666
770
|
inputHash: draft.inputHash,
|
|
667
771
|
output: draft.output,
|
package/dist/src/errors.d.ts
CHANGED
|
@@ -19,6 +19,7 @@ declare const parents: {
|
|
|
19
19
|
readonly 'draft-rejected': "unavailable";
|
|
20
20
|
readonly 'model-unavailable': "unavailable";
|
|
21
21
|
readonly 'model-busy': "unavailable";
|
|
22
|
+
readonly 'source-too-thin': "unavailable";
|
|
22
23
|
readonly 'parent-missing': "invalid-input";
|
|
23
24
|
readonly 'target-exists': "invalid-input";
|
|
24
25
|
};
|
package/dist/src/errors.js
CHANGED
|
@@ -33,6 +33,7 @@ const parents = {
|
|
|
33
33
|
'draft-rejected': 'unavailable',
|
|
34
34
|
'model-unavailable': 'unavailable',
|
|
35
35
|
'model-busy': 'unavailable',
|
|
36
|
+
'source-too-thin': 'unavailable',
|
|
36
37
|
'parent-missing': 'invalid-input',
|
|
37
38
|
'target-exists': 'invalid-input',
|
|
38
39
|
};
|
|
@@ -84,6 +85,7 @@ export const hints = {
|
|
|
84
85
|
export const detailHints = {
|
|
85
86
|
'draft-rejected': 'The model draft failed the digest checks listed above, even after one repair. Drafting is deterministic, so the same capture and model fail again; refresh the item or use another model.',
|
|
86
87
|
'model-unavailable': 'The model server did not answer or returned an error. Check that it is still running and has free GPU memory (another loaded model can exhaust it), then retry.',
|
|
88
|
+
'source-too-thin': 'The capture holds too little text to digest: under 20 words of prose, or only an archive’s metadata. Refresh the item when a fuller capture is possible.',
|
|
87
89
|
'model-busy': 'Another llama-server is running, and two loaded models can exhaust GPU memory. Wait for it to exit, then retry.',
|
|
88
90
|
};
|
|
89
91
|
/** The one error type. `message` is the code, so bare `String(error)` never carries detail, paths or causes. */
|
|
@@ -20,8 +20,11 @@ export type DigestOutcome = {
|
|
|
20
20
|
notBefore?: string;
|
|
21
21
|
countAttempt?: boolean;
|
|
22
22
|
};
|
|
23
|
-
/**
|
|
24
|
-
|
|
23
|
+
/**
|
|
24
|
+
* Queue a digest of a newly captured revision when auto-digest is on and the capture has enough text to digest. Runs
|
|
25
|
+
* inside the capture commit's transaction.
|
|
26
|
+
*/
|
|
27
|
+
export declare function enqueueDigest(db: DatabaseSync, itemId: string, revisionId: string, markdown: string): void;
|
|
25
28
|
/**
|
|
26
29
|
* Digest jobs under the shared worker lease. A job names one revision; when its item moves on or is deleted before the
|
|
27
30
|
* worker reaches it, the job is cancelled as `superseded`, since the newer revision has a job of its own. The drafting
|
|
@@ -3,12 +3,16 @@ import { text } from './files.js';
|
|
|
3
3
|
import { checkOwner } from './lease.js';
|
|
4
4
|
import { now, transaction } from './context.js';
|
|
5
5
|
import { SpongeError } from '../errors.js';
|
|
6
|
+
import { undigestible } from '../digest/pipeline.js';
|
|
6
7
|
/** Automatic retries a retryable failure gets before it stays failed. */
|
|
7
8
|
const maxAttempts = 3;
|
|
8
9
|
const alias = /^[\w.:/@+-]{1,256}$/;
|
|
9
|
-
/**
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
/**
|
|
11
|
+
* Queue a digest of a newly captured revision when auto-digest is on and the capture has enough text to digest. Runs
|
|
12
|
+
* inside the capture commit's transaction.
|
|
13
|
+
*/
|
|
14
|
+
export function enqueueDigest(db, itemId, revisionId, markdown) {
|
|
15
|
+
if (db.prepare('SELECT auto FROM digest_settings WHERE id=1').get()?.auto !== 1 || undigestible(markdown))
|
|
12
16
|
return;
|
|
13
17
|
db.prepare("INSERT INTO digest_jobs(id,item_id,revision_id,state,created_at) VALUES(?,?,?,'queued',?) ON CONFLICT(revision_id) DO NOTHING").run(randomUUID(), itemId, revisionId, now());
|
|
14
18
|
}
|
|
@@ -154,7 +154,7 @@ export class Revisions {
|
|
|
154
154
|
trackLibrary(db);
|
|
155
155
|
// A capture, not an import: imported libraries are digested on purpose, not as a side effect.
|
|
156
156
|
if (!imported)
|
|
157
|
-
enqueueDigest(db, input.itemId, revisionId);
|
|
157
|
+
enqueueDigest(db, input.itemId, revisionId, String(stored.markdown));
|
|
158
158
|
}
|
|
159
159
|
if (imported) {
|
|
160
160
|
const receipt = importReceipt(db, imported.namespace, imported.sourceKey, imported.sourceHash);
|
|
@@ -27,14 +27,14 @@
|
|
|
27
27
|
}
|
|
28
28
|
],
|
|
29
29
|
"build": {
|
|
30
|
-
"asrSourceSha256": "
|
|
30
|
+
"asrSourceSha256": "6a84b1b2040f17cb510a71548c1453442df15f820ceb0ac8cec3086b4ccefbdc",
|
|
31
31
|
"clangVersion": "Apple clang version 21.0.0 (clang-2100.1.1.101)",
|
|
32
|
-
"embeddingSourceSha256": "
|
|
32
|
+
"embeddingSourceSha256": "b029dadccaeba029fb9ef302daaefc6a134e8d46440c1e8c227f374d97202ce3",
|
|
33
33
|
"swiftVersion": "Apple Swift version 6.3.3 (swiftlang-6.3.3.1.3 clang-2100.1.1.101)"
|
|
34
34
|
},
|
|
35
35
|
"minimumMacOS": "26.0",
|
|
36
|
-
"packageVersion": "0.4.
|
|
36
|
+
"packageVersion": "0.4.1",
|
|
37
37
|
"platform": "darwin",
|
|
38
38
|
"schemaVersion": 1,
|
|
39
|
-
"sourceCommit": "
|
|
39
|
+
"sourceCommit": "82fefa4ca18700e2e7390e1372c7c672ddac11fa"
|
|
40
40
|
}
|