@hraness/sponge 0.4.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -18,7 +18,7 @@ import { pruneText } from '../indexing/prune-run.js';
18
18
  import { importAction, jungleAction, jungleText } from './import.js';
19
19
  import { digestText, reviewFromSpec, reviewSpec } from './digest.js';
20
20
  import { connectEndpoint } from '../digest/endpoint.js';
21
- import { createDigest } from '../digest/pipeline.js';
21
+ import { createDigest, undigestible } from '../digest/pipeline.js';
22
22
  import { createQueryEmbedder, createTextEmbedder } from '../search/query.js';
23
23
  import { runtimeGcAction, runtimeGcText, setupRuntimeGc } from './runtimes.js';
24
24
  import { drainLibrary } from '../worker/basic.js';
@@ -742,6 +742,13 @@ export const commands = [
742
742
  return 0;
743
743
  }
744
744
  if (args.flags.draft) {
745
+ // Refuse an unknown item or a thin capture before the model server spends its startup on it.
746
+ const item = store.read(args.positional[0]);
747
+ if (!item)
748
+ throw new SpongeError('unavailable');
749
+ const thin = undigestible(item.markdown);
750
+ if (thin)
751
+ throw new SpongeError(thin);
745
752
  const runtime = await inspectDigest(c.nativeRoot, args.flags.model ?? store.digests.settings().model ?? undefined), result = await withDigestServer(store.root, runtime, (model) => createDigest(store, args.positional[0], model, { signal }), { signal });
746
753
  c.emit({ digest: result.derivation, duplicate: result.duplicate, ...(result.trace ? { trace: result.trace } : {}) }, digestText(result.derivation));
747
754
  return 0;
@@ -4,7 +4,7 @@ import type { Derivation, DigestCandidate } from './derivation.js';
4
4
  * Model-agnostic Digest v2 drafting. The model sees numbered passages and cites a passage id plus a short exact
5
5
  * span; code snaps, prunes and checks. Nothing leaves this module except through the injected model.
6
6
  */
7
- export declare const PROMPT_VERSION = "sponge-digest-prompt-v5";
7
+ export declare const PROMPT_VERSION = "sponge-digest-prompt-v6";
8
8
  export type ChatMessage = {
9
9
  role: 'system' | 'user' | 'assistant';
10
10
  content: string;
@@ -56,8 +56,30 @@ interface Highlight {
56
56
  * change digest input hashes.
57
57
  */
58
58
  export declare function canonicalJson(value: unknown): string;
59
- /** Paragraph-level passages with raw offsets. Headings label passages; code fences and image-only blocks are not quotable. */
60
- export declare function passages(markdown: string, maxChars?: number): Passage[];
59
+ /** Where a post starts in a captured thread, and who wrote it as the model is told. */
60
+ export interface Post {
61
+ start: number;
62
+ label: string;
63
+ }
64
+ /**
65
+ * The posts of an X thread capture: top-level list items that carry X's post links. The poster's handle comes from
66
+ * the item's avatar or name link; the thread's author from the title (`Name (@handle) on X`), else the first post. A
67
+ * later post without a handle is still a reply. Other list items (an X article's bullets) are not posts, and a thread
68
+ * with no reply has none.
69
+ */
70
+ export declare function threadPosts(markdown: string): Post[];
71
+ /** A thread's passages without its replies, which are readers' comments rather than the source; renumbered from 1. */
72
+ export declare function authorPassages(markdown: string): Passage[];
73
+ /**
74
+ * Paragraph-level passages with raw offsets. Headings label passages; code fences and image-only blocks are not
75
+ * quotable. `posts` (a thread's) also break blocks, and their labels head the passages that follow.
76
+ */
77
+ export declare function passages(markdown: string, maxChars?: number, posts?: readonly Post[]): Passage[];
78
+ /**
79
+ * Why a capture cannot be digested, as a detail code, or undefined when it can: its body (past the frontmatter, images
80
+ * dropped, link text kept) has under `minProseWords` words with letters, or only an archive's metadata was saved.
81
+ */
82
+ export declare function undigestible(markdown: string): 'source-too-thin' | undefined;
61
83
  /** Pack passages into sections of at most `budget` display characters, preferring heading boundaries past half a budget. */
62
84
  export declare function sectionsOf(list: Passage[], budget: number): Passage[][];
63
85
  /** JSON Schema for the model's draft. Tags become an enum when a vocabulary is given. */
@@ -160,8 +182,11 @@ export interface DigestStore {
160
182
  read(kind: 'digest', id: string): Derivation | undefined;
161
183
  };
162
184
  }
185
+ type DraftOptions = Omit<DraftInput, 'revisionId' | 'markdown' | 'title' | 'form' | 'model' | 'highlights'>;
186
+ /** What the item's current revision is drafted from: its Markdown, title, form and anchored highlights. */
187
+ export declare function digestInputFor(store: Pick<DigestStore, 'read' | 'annotations'>, itemId: string, model: DigestModel, options?: DraftOptions): DraftInput;
163
188
  /** Draft and store a digest for the item's current revision. A matching input hash returns the stored draft without calling the model. */
164
- export declare function createDigest(store: DigestStore, itemId: string, model: DigestModel, options?: Omit<DraftInput, 'revisionId' | 'markdown' | 'title' | 'form' | 'model' | 'highlights'>): Promise<{
189
+ export declare function createDigest(store: DigestStore, itemId: string, model: DigestModel, options?: DraftOptions): Promise<{
165
190
  derivation: Derivation;
166
191
  duplicate: boolean;
167
192
  trace?: DraftTrace;
@@ -6,7 +6,7 @@ import { sha256 } from '../util/hash.js';
6
6
  * Model-agnostic Digest v2 drafting. The model sees numbered passages and cites a passage id plus a short exact
7
7
  * span; code snaps, prunes and checks. Nothing leaves this module except through the injected model.
8
8
  */
9
- export const PROMPT_VERSION = 'sponge-digest-prompt-v5';
9
+ export const PROMPT_VERSION = 'sponge-digest-prompt-v6';
10
10
  /** 24k characters is roughly 6k tokens of passages, leaving room for instructions and output in an 8k context on an 8 GB Mac. */
11
11
  const defaultDigestParams = {
12
12
  temperature: 0,
@@ -40,14 +40,62 @@ export function canonicalJson(value) {
40
40
  throw new SpongeError('invalid-input');
41
41
  return JSON.stringify(value ?? null);
42
42
  }
43
- /** Paragraph-level passages with raw offsets. Headings label passages; code fences and image-only blocks are not quotable. */
44
- export function passages(markdown, maxChars = 1200) {
43
+ const postHandle = [
44
+ /^- \[!\[@(\w+)\]/,
45
+ /^- \[\[remote image: @(\w+)\]/,
46
+ /^-\s+\[[^\]]+\]\(https:\/\/x\.com\/(\w+)\)\s*$/,
47
+ ];
48
+ /**
49
+ * The posts of an X thread capture: top-level list items that carry X's post links. The poster's handle comes from
50
+ * the item's avatar or name link; the thread's author from the title (`Name (@handle) on X`), else the first post. A
51
+ * later post without a handle is still a reply. Other list items (an X article's bullets) are not posts, and a thread
52
+ * with no reply has none.
53
+ */
54
+ export function threadPosts(markdown) {
55
+ const author = /^title: ".*\(@(\w+)\) on X"/m.exec(frontmatter(markdown))?.[1], items = [];
56
+ let at = 0;
57
+ for (const line of markdown.split('\n')) {
58
+ if (/^- /.test(line) || /^-\s+\[/.test(line))
59
+ items.push({ start: at, handle: postHandle.map((r) => r.exec(line)?.[1]).find(Boolean), post: false });
60
+ const item = items.at(-1);
61
+ if (item && /x\.com\/(compose\/post\?in_reply_to=|\w+\/status\/\d)/.test(line))
62
+ item.post = true;
63
+ at += line.length + 1;
64
+ }
65
+ const posts = items.filter((i) => i.post || i.handle), by = author ?? posts[0]?.handle, same = (h) => h.toLowerCase() === by?.toLowerCase(), labelled = posts.map((p, i) => ({
66
+ start: p.start,
67
+ label: p.handle && !same(p.handle)
68
+ ? `Reply by @${p.handle}`
69
+ : p.handle || i === 0
70
+ ? `Post by ${by ? `@${by}, ` : ''}the author`
71
+ : 'Reply (poster not captured)',
72
+ }));
73
+ // With no reply there is nobody to tell apart, so the passages stay as they are.
74
+ return labelled.some((p) => p.label !== labelled[0].label) ? labelled : [];
75
+ }
76
+ /** A thread's passages without its replies, which are readers' comments rather than the source; renumbered from 1. */
77
+ export function authorPassages(markdown) {
78
+ return passages(markdown, undefined, threadPosts(markdown))
79
+ .filter((p) => !p.heading?.startsWith('Reply'))
80
+ .map((p, i) => ({ ...p, id: i + 1 }));
81
+ }
82
+ /**
83
+ * Paragraph-level passages with raw offsets. Headings label passages; code fences and image-only blocks are not
84
+ * quotable. `posts` (a thread's) also break blocks, and their labels head the passages that follow.
85
+ */
86
+ export function passages(markdown, maxChars = 1200, posts = []) {
45
87
  const out = [], blocks = [];
46
88
  let at = 0, start = -1, fence = false, fenced = false;
89
+ const breaks = new Set(posts.map((p) => p.start));
47
90
  for (const line of markdown.split('\n')) {
48
91
  const end = at + line.length, blank = !line.trim(), marker = /^\s{0,3}(```|~~~)/.test(line);
49
92
  if (marker)
50
93
  fence = !fence;
94
+ if (!fence && !marker && start >= 0 && breaks.has(at)) {
95
+ blocks.push({ start, end: at - 1, fence: fenced });
96
+ start = -1;
97
+ fenced = false;
98
+ }
51
99
  if (blank && !fence && !marker) {
52
100
  if (start >= 0)
53
101
  blocks.push({ start, end: at - 1, fence: fenced });
@@ -64,14 +112,17 @@ export function passages(markdown, maxChars = 1200) {
64
112
  }
65
113
  if (start >= 0)
66
114
  blocks.push({ start, end: markdown.length, fence: fenced });
67
- let heading = '';
115
+ let heading = '', post = '';
116
+ const label = (start) => posts.findLast((p) => p.start <= start)?.label ?? '';
68
117
  for (const b of blocks) {
69
118
  const raw = markdown.slice(b.start, b.end).trimEnd(), end = b.start + raw.length;
70
119
  if (b.fence || !raw.trim())
71
120
  continue;
121
+ if (label(b.start) !== post)
122
+ heading = post = label(b.start);
72
123
  const h = /^\s{0,3}#{1,6}\s+(.+)$/.exec(raw);
73
124
  if (h && !raw.includes('\n')) {
74
- heading = collapse(h[1]).slice(0, 200);
125
+ heading = (post ? `${post}: ` : '') + collapse(h[1]).slice(0, 200);
75
126
  continue;
76
127
  }
77
128
  if (/^(\s*!\[[^\]]*\]\([^)]*\)\s*)+$/.test(raw))
@@ -91,6 +142,32 @@ export function passages(markdown, maxChars = 1200) {
91
142
  }
92
143
  return out;
93
144
  }
145
+ const frontmatter = (markdown) => /^---\r?\n[\s\S]*?\r?\n---(?:\r?\n|$)/u.exec(markdown)?.[0] ?? '';
146
+ /** A `key: "value"` frontmatter field, as captures write them. */
147
+ function captureField(markdown, key) {
148
+ const match = new RegExp(`^${key}: (".*")\\r?$`, 'm').exec(frontmatter(markdown));
149
+ try {
150
+ const value = match ? JSON.parse(match[1]) : undefined;
151
+ return typeof value === 'string' ? value : undefined;
152
+ }
153
+ catch {
154
+ return undefined;
155
+ }
156
+ }
157
+ /** Prose words under which a capture holds nothing to digest: a gallery of numbered images, a bare link list. */
158
+ const minProseWords = 20;
159
+ /**
160
+ * Why a capture cannot be digested, as a detail code, or undefined when it can: its body (past the frontmatter, images
161
+ * dropped, link text kept) has under `minProseWords` words with letters, or only an archive's metadata was saved.
162
+ */
163
+ export function undigestible(markdown) {
164
+ if (captureField(markdown, 'capture_method') === 'archive-metadata')
165
+ return 'source-too-thin';
166
+ const prose = passages(markdown.slice(frontmatter(markdown).length))
167
+ .map((p) => p.text.replace(/(!?)\[([^\]]*)\]\([^)]*\)/g, (_, image, text) => (image ? ' ' : text)))
168
+ .join(' ');
169
+ return (prose.match(/\p{L}[\p{L}\p{M}'’-]*/gu) ?? []).length < minProseWords ? 'source-too-thin' : undefined;
170
+ }
94
171
  function lastSentenceEnd(window) {
95
172
  let best = -1;
96
173
  for (const m of window.matchAll(/[.!?]["')\]]*(?=\s)/g))
@@ -196,7 +273,12 @@ Source: an essay arguing that taste is a skill.
196
273
  claim: Volume beats polish early. | detail: Side-by-side comparison of many rough drafts sharpens judgment faster than finishing one.
197
274
  Bad claim: The essay explores how practice over time shapes a person's sense of taste.
198
275
  Good claim: Taste is trained.`;
199
- const system = (budget, vocabulary) => `You write a Digest v2 of one source for a private reading library, in the owner's voice: terse, declarative, specific. Reply with JSON only, matching the schema.
276
+ /** Who speaks in each form, so claims are credited to the person who made them. */
277
+ const voices = {
278
+ thread: 'This source is an X thread. Its replies are left out. When the author quotes or answers someone, that view stays theirs.',
279
+ video: 'This source is a recording: credit each claim to the person who says it. In an interview or podcast, the guest’s views are the guest’s, not the host’s.',
280
+ };
281
+ const system = (budget, form, vocabulary) => `You write a Digest v2 of one source for a private reading library, in the owner's voice: terse, declarative, specific. Reply with JSON only, matching the schema.
200
282
  - lead: one sentence of at most ${L.leadChars} characters stating this source's specific thesis or result: who (author, organization or product by name) shows or argues what, with the number, name or mechanism that makes it this source. Do not restate the title, describe the source ("This article discusses"), or tease.
201
283
  - context: 1 or 2 sentences, at most 45 words: the evidence, method, numbers or limits the lead could not hold. Lead and context together stay within ${L.gistWords} words and ${L.gistSentences} sentences.
202
284
  - ideas: ${L.minIdeas} to ${L.maxIdeas} (prefer ${L.maxIdeas} when the source has them), in the order the author makes them, covering the whole source including its caveats and limits. Each idea is a claim with detail:
@@ -205,7 +287,12 @@ const system = (budget, vocabulary) => `You write a Digest v2 of one source for
205
287
  - quotes: 0 to ${L.maxQuotesPerIdea} that support this idea, each a contiguous span copied character for character from one numbered passage, ${minQuoteWords + 2} to 20 words, with that passage number. A quote is a fragment, not a whole long sentence: when a sentence runs past 20 words, copy only its strongest part. No added quotation marks or ellipses. Give attribution (who said it) when it is not the author. Prefer reader highlights.
206
288
  - Attach at least ${L.minQuotes} and at most ${L.maxQuotes} quotes in all; ${L.minQuotes + 1} to 3 is typical. All quotes together stay within ${budget} words and never overlap.
207
289
  - tags: 0 to ${L.maxTags} ${vocabulary?.length ? `from: ${vocabulary.join(', ')}` : 'lowercase kebab-case topics'}; usually one subject tag.
208
- Plain words. Name things and use numbers. No hedges or filler: may, might, potentially, various, significantly, key, crucial, notably, highlights, explores, discusses, "the author argues", "the article".
290
+ Faithfulness comes before style:
291
+ - Credit each view to who holds it. When the source quotes, interviews or reports someone, their claims are theirs: name them in the detail and the quote attribution, never pass them off as the author's.${voices[form] ? `\n- ${voices[form]}` : ''}
292
+ - Keep each number with its own subject, unit and meaning (an area is not a height, a share is not a count, a cost is not a price) and never compute one the source does not give.
293
+ - Keep the source's direction: never invert a claim, comparison or cause.
294
+ - Keep the source's certainty: a forecast, plan, proposal, rumor or question stays one ("plans to", "expects", "asks whether"); never state it as done or settled. Hedge only where the source does.
295
+ Plain words. Name things and use numbers. No filler: various, significantly, key, crucial, notably, highlights, explores, discusses, "the author argues", "the article".
209
296
  Use only what the source says.
210
297
 
211
298
  ${styleExamples}`;
@@ -255,7 +342,9 @@ function fromModel(text, form) {
255
342
  }
256
343
  if (!object(v) || !Array.isArray(v.ideas) || !Array.isArray(v.tags))
257
344
  return { issues: [{ path: '', code: 'shape' }] };
258
- const trim = (s) => (typeof s === 'string' ? s.trim() : s), cites = [];
345
+ const trim = (s) => (typeof s === 'string' ? s.trim() : s),
346
+ // Models sometimes end a sentence with the passage it came from ("… market. [2]").
347
+ prose = (s) => (typeof s === 'string' ? s.replace(/(\s*\[\d+(?:,\s*\d+)*\])+$/, '').trim() : s), cites = [];
259
348
  const ideas = v.ideas.map((idea) => {
260
349
  if (!object(idea)) {
261
350
  cites.push([]);
@@ -264,19 +353,20 @@ function fromModel(text, form) {
264
353
  const quotes = Array.isArray(idea.quotes) ? idea.quotes : [];
265
354
  cites.push(quotes.map((q) => (Number.isSafeInteger(q?.passage) ? q.passage : undefined)));
266
355
  return {
267
- claim: trim(idea.claim),
268
- detail: trim(idea.detail),
356
+ claim: prose(idea.claim),
357
+ detail: prose(idea.detail),
269
358
  quotes: quotes.map((q) => {
270
359
  const attribution = trim(q?.attribution);
271
360
  return { text: trim(q?.text), ...(attribution ? { attribution } : {}) };
272
361
  }),
273
362
  };
274
363
  });
275
- const context = trim(v.context);
364
+ // A context the model stopped mid-sentence is dropped; context is optional.
365
+ const context = ((c) => (typeof c === 'string' && !/[.!?…。!?)"”’]$/.test(c) ? undefined : c))(prose(v.context));
276
366
  const parsed = parseDigest({
277
367
  schemaVersion: 2,
278
368
  form,
279
- gist: { lead: trim(v.lead), ...(context ? { context } : {}) },
369
+ gist: { lead: prose(v.lead), ...(context ? { context } : {}) },
280
370
  ideas,
281
371
  tags: v.tags.map(trim),
282
372
  });
@@ -453,7 +543,10 @@ export async function draftDigest(input) {
453
543
  input.vocabulary.length > 256 ||
454
544
  input.vocabulary.some((t) => !/^[a-z0-9][a-z0-9-]{0,63}$/.test(t))))
455
545
  throw new SpongeError('invalid-input');
456
- const inputHash = digestInputHash({ ...input, model: identity }), source = input.markdown, list = passages(source);
546
+ const thin = undigestible(input.markdown);
547
+ if (thin)
548
+ throw new SpongeError(thin);
549
+ const inputHash = digestInputHash({ ...input, model: identity }), source = input.markdown, list = input.form === 'thread' ? authorPassages(source) : passages(source);
457
550
  if (list.length === 0)
458
551
  throw new SpongeError('invalid-input');
459
552
  const userSignal = input.signal ?? new AbortController().signal, overall = AbortSignal.any([userSignal, AbortSignal.timeout(p.deadlineMs)]);
@@ -564,7 +657,7 @@ export async function draftDigest(input) {
564
657
  }
565
658
  const budget = quoteBudget(source), schema = draftSchema(list.length, input.vocabulary);
566
659
  const messages = [
567
- { role: 'system', content: system(budget, input.vocabulary) },
660
+ { role: 'system', content: system(budget, input.form, input.vocabulary) },
568
661
  {
569
662
  role: 'user',
570
663
  content: (notesText ? `Notes from each part of the source, in order:\n${notesText}\n\n` : '') +
@@ -637,9 +730,16 @@ export async function draftDigest(input) {
637
730
  };
638
731
  return { output: result.draft.digest, inputHash, producer, options, trace };
639
732
  }
640
- const formFor = (kind) => (kind === 'pdf' ? 'pdf' : kind === 'media' ? 'video' : 'article');
641
- /** Draft and store a digest for the item's current revision. A matching input hash returns the stored draft without calling the model. */
642
- export async function createDigest(store, itemId, model, options = {}) {
733
+ /** X thread captures draft as `thread`, whose prompt rules keep replies apart from the author's posts. */
734
+ const formFor = (kind, markdown) => kind === 'pdf'
735
+ ? 'pdf'
736
+ : kind === 'media'
737
+ ? 'video'
738
+ : captureField(markdown, 'platform') === 'x' && captureField(markdown, 'capture_scope') === 'thread'
739
+ ? 'thread'
740
+ : 'article';
741
+ /** What the item's current revision is drafted from: its Markdown, title, form and anchored highlights. */
742
+ export function digestInputFor(store, itemId, model, options = {}) {
643
743
  const item = store.read(itemId);
644
744
  if (!item)
645
745
  throw new SpongeError('unavailable');
@@ -647,21 +747,25 @@ export async function createDigest(store, itemId, model, options = {}) {
647
747
  .list(itemId)
648
748
  .filter((a) => a.kind === 'highlight' && a.revisionId === item.revisionId && a.anchored !== false)
649
749
  .map((a) => ({ text: a.anchor.exact, start: a.anchor.start }));
650
- const input = {
750
+ return {
651
751
  ...options,
652
752
  revisionId: item.revisionId,
653
753
  markdown: item.markdown,
654
754
  title: item.title,
655
- form: formFor(item.kind),
755
+ form: formFor(item.kind, item.markdown),
656
756
  model,
657
757
  highlights,
658
758
  };
659
- const prior = store.derivations.find('digest', item.revisionId, digestInputHash({ ...input, model: model.identity }));
759
+ }
760
+ /** Draft and store a digest for the item's current revision. A matching input hash returns the stored draft without calling the model. */
761
+ export async function createDigest(store, itemId, model, options = {}) {
762
+ const input = digestInputFor(store, itemId, model, options);
763
+ const prior = store.derivations.find('digest', input.revisionId, digestInputHash({ ...input, model: model.identity }));
660
764
  if (prior)
661
765
  return { derivation: prior, duplicate: true };
662
766
  const draft = await draftDigest(input), stored = store.derivations.put('digest', {
663
767
  itemId,
664
- revisionId: item.revisionId,
768
+ revisionId: input.revisionId,
665
769
  producer: draft.producer,
666
770
  inputHash: draft.inputHash,
667
771
  output: draft.output,
@@ -19,6 +19,7 @@ declare const parents: {
19
19
  readonly 'draft-rejected': "unavailable";
20
20
  readonly 'model-unavailable': "unavailable";
21
21
  readonly 'model-busy': "unavailable";
22
+ readonly 'source-too-thin': "unavailable";
22
23
  readonly 'parent-missing': "invalid-input";
23
24
  readonly 'target-exists': "invalid-input";
24
25
  };
@@ -33,6 +33,7 @@ const parents = {
33
33
  'draft-rejected': 'unavailable',
34
34
  'model-unavailable': 'unavailable',
35
35
  'model-busy': 'unavailable',
36
+ 'source-too-thin': 'unavailable',
36
37
  'parent-missing': 'invalid-input',
37
38
  'target-exists': 'invalid-input',
38
39
  };
@@ -84,6 +85,7 @@ export const hints = {
84
85
  export const detailHints = {
85
86
  'draft-rejected': 'The model draft failed the digest checks listed above, even after one repair. Drafting is deterministic, so the same capture and model fail again; refresh the item or use another model.',
86
87
  'model-unavailable': 'The model server did not answer or returned an error. Check that it is still running and has free GPU memory (another loaded model can exhaust it), then retry.',
88
+ 'source-too-thin': 'The capture holds too little text to digest: under 20 words of prose, or only an archive’s metadata. Refresh the item when a fuller capture is possible.',
87
89
  'model-busy': 'Another llama-server is running, and two loaded models can exhaust GPU memory. Wait for it to exit, then retry.',
88
90
  };
89
91
  /** The one error type. `message` is the code, so bare `String(error)` never carries detail, paths or causes. */
@@ -20,8 +20,11 @@ export type DigestOutcome = {
20
20
  notBefore?: string;
21
21
  countAttempt?: boolean;
22
22
  };
23
- /** Queue a digest of a newly captured revision when auto-digest is on. Runs inside the capture commit's transaction. */
24
- export declare function enqueueDigest(db: DatabaseSync, itemId: string, revisionId: string): void;
23
+ /**
24
+ * Queue a digest of a newly captured revision when auto-digest is on and the capture has enough text to digest. Runs
25
+ * inside the capture commit's transaction.
26
+ */
27
+ export declare function enqueueDigest(db: DatabaseSync, itemId: string, revisionId: string, markdown: string): void;
25
28
  /**
26
29
  * Digest jobs under the shared worker lease. A job names one revision; when its item moves on or is deleted before the
27
30
  * worker reaches it, the job is cancelled as `superseded`, since the newer revision has a job of its own. The drafting
@@ -3,12 +3,16 @@ import { text } from './files.js';
3
3
  import { checkOwner } from './lease.js';
4
4
  import { now, transaction } from './context.js';
5
5
  import { SpongeError } from '../errors.js';
6
+ import { undigestible } from '../digest/pipeline.js';
6
7
  /** Automatic retries a retryable failure gets before it stays failed. */
7
8
  const maxAttempts = 3;
8
9
  const alias = /^[\w.:/@+-]{1,256}$/;
9
- /** Queue a digest of a newly captured revision when auto-digest is on. Runs inside the capture commit's transaction. */
10
- export function enqueueDigest(db, itemId, revisionId) {
11
- if (db.prepare('SELECT auto FROM digest_settings WHERE id=1').get()?.auto !== 1)
10
+ /**
11
+ * Queue a digest of a newly captured revision when auto-digest is on and the capture has enough text to digest. Runs
12
+ * inside the capture commit's transaction.
13
+ */
14
+ export function enqueueDigest(db, itemId, revisionId, markdown) {
15
+ if (db.prepare('SELECT auto FROM digest_settings WHERE id=1').get()?.auto !== 1 || undigestible(markdown))
12
16
  return;
13
17
  db.prepare("INSERT INTO digest_jobs(id,item_id,revision_id,state,created_at) VALUES(?,?,?,'queued',?) ON CONFLICT(revision_id) DO NOTHING").run(randomUUID(), itemId, revisionId, now());
14
18
  }
@@ -154,7 +154,7 @@ export class Revisions {
154
154
  trackLibrary(db);
155
155
  // A capture, not an import: imported libraries are digested on purpose, not as a side effect.
156
156
  if (!imported)
157
- enqueueDigest(db, input.itemId, revisionId);
157
+ enqueueDigest(db, input.itemId, revisionId, String(stored.markdown));
158
158
  }
159
159
  if (imported) {
160
160
  const receipt = importReceipt(db, imported.namespace, imported.sourceKey, imported.sourceHash);
@@ -27,14 +27,14 @@
27
27
  }
28
28
  ],
29
29
  "build": {
30
- "asrSourceSha256": "21a284b38eca324015bc9d03efb14335a5a73f8b3761b0f547180bf851fce2a3",
30
+ "asrSourceSha256": "6a84b1b2040f17cb510a71548c1453442df15f820ceb0ac8cec3086b4ccefbdc",
31
31
  "clangVersion": "Apple clang version 21.0.0 (clang-2100.1.1.101)",
32
- "embeddingSourceSha256": "ff0a27cefa0ddc23e1d8bea7a70a49a112865ee730500a1da69ca9b8e6104adc",
32
+ "embeddingSourceSha256": "b029dadccaeba029fb9ef302daaefc6a134e8d46440c1e8c227f374d97202ce3",
33
33
  "swiftVersion": "Apple Swift version 6.3.3 (swiftlang-6.3.3.1.3 clang-2100.1.1.101)"
34
34
  },
35
35
  "minimumMacOS": "26.0",
36
- "packageVersion": "0.4.0",
36
+ "packageVersion": "0.4.1",
37
37
  "platform": "darwin",
38
38
  "schemaVersion": 1,
39
- "sourceCommit": "307210823c8523059f896fb64ff92dade3760b59"
39
+ "sourceCommit": "82fefa4ca18700e2e7390e1372c7c672ddac11fa"
40
40
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@hraness/sponge",
3
- "version": "0.4.0",
3
+ "version": "0.4.1",
4
4
  "description": "A local library for captured articles and verified offline assets.",
5
5
  "repository": {
6
6
  "type": "git",