@hraness/sponge 0.4.0 → 0.5.0

package/dist/src/acquisition/runtime.js

@@ -12,7 +12,7 @@ const installDeadlineMs = 30 * 60 * 1000;
 function checkPins(pins) {
     if (!/^[a-z0-9][a-z0-9.-]{0,99}$/.test(pins.id) || hash(pins.lockfile) !== pins.lockSha256)
         throw new SpongeError('invalid-input');
-    for (const pin of [pins.bun, pins.ghostget, pins.wordcell]) {
+    for (const pin of [pins.bun, pins.wordcell]) {
         let url;
         try {
             url = new URL(pin.url);
@@ -36,7 +36,7 @@ function checkPins(pins) {
 }
 export function captureSetupPlan(pins) {
     checkPins(pins);
-    const artifacts = [pins.bun, pins.ghostget, pins.wordcell], downloadBytes = artifacts.reduce((n, a) => n + a.bytes, 0);
+    const artifacts = [pins.bun, pins.wordcell], downloadBytes = artifacts.reduce((n, a) => n + a.bytes, 0);
     return {
         artifacts,
         downloadBytes,
@@ -46,7 +46,7 @@ export function captureSetupPlan(pins) {
         installationDeadlineMs: installDeadlineMs,
         dependencyDownloads: 'frozen-lock',
         requiredFreeBytes: 2 * 1024 ** 3 + budget + stagingBudget + downloadBytes,
-        capabilities: ['article-http', 'selected-ghostget-account', 'selected-attached-browser', 'verified-offline-images'],
+        capabilities: ['article-http', 'selected-attached-browser', 'verified-offline-images'],
     };
 }
 async function download(pin, path, signal) {
@@ -116,7 +116,7 @@ export const runtimeVerification = { fullWalks: 0 };
 async function inspectRuntime(root, pins) {
     checkPins(pins);
     await privateRoot(root);
-    const key = JSON.stringify([root, pins.id, pins.lockSha256, pins.bun.archiveExecutable, pins.bun.executableSha256]), bun = join(root, pins.bun.archiveExecutable), ghostget = join(root, 'node_modules/@hraness/ghostget/src/cli.ts');
+    const key = JSON.stringify([root, pins.id, pins.lockSha256, pins.bun.archiveExecutable, pins.bun.executableSha256]), bun = join(root, pins.bun.archiveExecutable);
     // Stat receipt of the root, its receipt file and every inventoried entry. Each stat precedes any read of its entry, so a later change always shows.
     const head = [
         statLine('.', await lstat(root, { bigint: true })),
@@ -124,7 +124,7 @@ async function inspectRuntime(root, pins) {
     ];
     const cached = verified.get(key);
     if (cached !== undefined && cached === hash([...head, ...(await inventory(root, false)).stats].join('\n')))
-        return { root, bun, ghostget, pins };
+        return { root, bun, pins };
     verified.delete(key);
     runtimeVerification.fullWalks++;
     const receipt = JSON.parse((await readConfined(root, 'runtime-receipt.json', 4096)).toString()), walked = await inventory(root);
@@ -136,7 +136,7 @@ async function inspectRuntime(root, pins) {
     if (hash(await readConfined(root, pins.bun.archiveExecutable, 128 * 1024 ** 2)) !== pins.bun.executableSha256)
         throw new SpongeError('integrity');
     verified.set(key, hash([...head, ...walked.stats].join('\n')));
-    return { root, bun, ghostget, pins };
+    return { root, bun, pins };
 }
 async function provisionRuntime(options) {
     const { pins, signal } = options, plan = captureSetupPlan(pins);
@@ -188,7 +188,7 @@ async function provisionRuntime(options) {
         await download(pins.bun, join(stage, 'bun.zip'), controller.signal);
         const unpack = await runOwned('/usr/bin/unzip', ['-q', join(stage, 'bun.zip'), '-d', stage], {
             cwd: stage,
-            env: isolatedEnvironment(home, home, home),
+            env: isolatedEnvironment(home, home),
             signal: controller.signal,
             deadlineMs: 30000,
             maxOutputBytes: 65536,
@@ -200,16 +200,15 @@ async function provisionRuntime(options) {
         if (hash(await readConfined(stage, pins.bun.archiveExecutable, 128 * 1024 ** 2)) !== pins.bun.executableSha256)
             throw new SpongeError('integrity');
         await chmod(bun, 0o700);
-        await download(pins.ghostget, join(stage, `hraness-ghostget-${pins.ghostget.version}.tgz`), controller.signal);
         await download(pins.wordcell, join(stage, `hraness-wordcell-${pins.wordcell.version}.tgz`), controller.signal);
         await writeFile(join(stage, 'package.json'), JSON.stringify({
             private: true,
-            dependencies: { '@hraness/ghostget': `./hraness-ghostget-${pins.ghostget.version}.tgz` },
+            dependencies: { '@hraness/wordcell': `./hraness-wordcell-${pins.wordcell.version}.tgz` },
         }), { mode: 0o600 });
         await writeFile(join(stage, 'bun.lock'), pins.lockfile, { mode: 0o600 });
         const install = await runOwned(bun, ['install', '--frozen-lockfile', '--ignore-scripts'], {
             cwd: stage,
-            env: isolatedEnvironment(home, home, home),
+            env: isolatedEnvironment(home, home),
             signal: controller.signal,
             deadlineMs: installDeadlineMs,
             maxOutputBytes: 65536,
@@ -219,14 +218,9 @@ async function provisionRuntime(options) {
             throw new SpongeError(limitExceeded ? 'limit-exceeded' : (install.stopped ?? 'unavailable'));
         await rm(home, { recursive: true, force: true });
         await rm(join(stage, 'bun.zip'));
-        for (const [name, pin] of [
-            ['ghostget', pins.ghostget],
-            ['wordcell', pins.wordcell],
-        ]) {
-            const pkg = JSON.parse((await readConfined(stage, `node_modules/@hraness/${name}/package.json`, 65536)).toString());
-            if (pkg.version !== pin.version)
-                throw new SpongeError('integrity');
-        }
+        const pkg = JSON.parse((await readConfined(stage, 'node_modules/@hraness/wordcell/package.json', 65536)).toString());
+        if (pkg.version !== pins.wordcell.version)
+            throw new SpongeError('integrity');
         await writeFile(join(stage, 'runtime-receipt.json'), JSON.stringify({
             schemaVersion: 1,
             pinId: pins.id,

package/dist/src/cli/commands.js

@@ -8,17 +8,18 @@ import { queueAction } from './queue.js';
 import { portabilityAction } from './portability.js';
 import { restoreLibrary } from '../portability/backup.js';
 import { workerAction, serveWorker, hasDueAcquisition, workerDoctor } from './worker.js';
-import { nativeSetupPlan, provisionNative, inspectNative } from '../models/runtime.js';
+import { nativeSetupPlan, provisionNative, inspectNative, optionalNative } from '../models/runtime.js';
 import { digestModel, digestSetupPlan, inspectDigest, provisionDigest } from '../models/digest.js';
 import { withDigestServer } from '../digest/server.js';
 import { nativeDownloadAuthGuidance } from '../models/download.js';
+import { nativePlatformSupported } from '../models/catalog.js';
 import { drainIndexQueue } from '../indexing/runner.js';
 import { indexAction, overviewText } from './indexing.js';
 import { pruneText } from '../indexing/prune-run.js';
 import { importAction, jungleAction, jungleText } from './import.js';
 import { digestText, reviewFromSpec, reviewSpec } from './digest.js';
 import { connectEndpoint } from '../digest/endpoint.js';
-import { createDigest } from '../digest/pipeline.js';
+import { createDigest, undigestible } from '../digest/pipeline.js';
 import { createQueryEmbedder, createTextEmbedder } from '../search/query.js';
 import { runtimeGcAction, runtimeGcText, setupRuntimeGc } from './runtimes.js';
 import { drainLibrary } from '../worker/basic.js';
@@ -55,6 +56,8 @@ const resolveConnection = (connections) => (realm) => {
     const connection = connections.find((c) => c.realm === realm);
     if (!connection)
         throw new SpongeError('repair-auth');
+    if (!connection.capture)
+        throw new SpongeError('connection-retired');
     return connection.capture;
 };
 /** add/refresh with --wait and run --drain: drain acquisition (one job when given), then queued indexes for run. */
@@ -67,16 +70,9 @@ async function drain(c, jobId, connections) {
             throw jobFailure(store, jobId);
     }
     const runtime = await captureRuntime(c);
-    let asrRuntime;
-    try {
-        asrRuntime = await inspectNative('asr', c.nativeRoot, c.version);
-    }
-    catch (error) {
-        if (!['runtime-missing', 'unsupported'].includes(publicCode(error, 'integrity')))
-            throw error;
-    }
     const results = await drainLibrary(store, {
-        asrRuntime,
+        asrRuntime: await optionalNative('asr', c.nativeRoot, c.version),
+        mediaRuntime: await optionalNative('media', c.nativeRoot, c.version),
         language: args.flags.language,
         forceLocal: args.flags['force-local'] === true,
         runtime,
@@ -154,8 +150,8 @@ export const commands = [
                 c.emit({ digest: 'ready', model: pin.alias, auto: settings.auto, gc }, `Digest model ready: ${pin.alias}. New captures are digested in the background; turn that off with \`sponge digest --auto off\`.`);
                 return 0;
             }
-            if (!c.args.flags.capture) {
-                const kind = c.args.flags.asr ? 'asr' : 'embedding', plan = await nativeSetupPlan(kind, c.version);
+            const installNative = async (kind) => {
+                const plan = await nativeSetupPlan(kind, c.version);
                 process.stderr.write(JSON.stringify({ setup: kind, ...plan }) + '\n');
                 try {
                     await provisionNative(kind, c.nativeRoot, c.version, c.signal);
@@ -165,10 +161,20 @@ export const commands = [
                         process.stderr.write(nativeDownloadAuthGuidance + '\n');
                     throw error;
                 }
+            };
+            if (!c.args.flags.capture) {
+                const kind = c.args.flags.asr ? 'asr' : 'embedding';
+                await installNative(kind);
                 const gc = await setupRuntimeGc(c.store, c.version);
                 c.emit({ [kind]: 'ready', gc });
                 return 0;
             }
+            // Media goes first so a running worker never drains media jobs against a half-finished setup.
+            let media = nativePlatformSupported() ? 'ready' : 'unsupported';
+            if (media === 'ready')
+                await installNative('media').catch((error) => {
+                    media = codeOf(error);
+                });
             const plan = captureSetupPlan(defaultCapturePins);
             process.stderr.write(`Capture setup: ${plan.downloadBytes} pinned artifact bytes, plus frozen dependency downloads. Temporary installation limit: ${plan.stagingBudgetBytes} bytes; retained runtime limit: ${plan.installationBudgetBytes} bytes; dependency installation deadline: ${plan.installationDeadlineMs / 60000} minutes. ${plan.requiredFreeBytes} free bytes required.\n`);
             await provisionCapture({
@@ -176,8 +182,13 @@ export const commands = [
                 pins: defaultCapturePins,
                 signal: c.signal,
             });
-            const gc = await setupRuntimeGc(c.store, c.version);
-            c.emit({ capture: 'ready', gc }, 'skipped' in gc ? `Capture ready\nRuntime GC skipped: ${gc.skipped}` : `Capture ready\n${runtimeGcText(gc)}`);
+            // Articles and PDFs run anywhere; the media helper is a native release asset and is best-effort.
+            const gc = await setupRuntimeGc(c.store, c.version), ready = media === 'ready'
+                ? 'Capture ready'
+                : media === 'unsupported'
+                    ? 'Capture ready (media capture needs macOS 26 on Apple Silicon)'
+                    : `Capture ready; media helper not installed (${media}). Articles and PDFs work; rerun \`sponge setup --capture\` for media.`;
+            c.emit({ capture: 'ready', media, gc }, 'skipped' in gc ? `${ready}\nRuntime GC skipped: ${gc.skipped}` : `${ready}\n${runtimeGcText(gc)}`);
             return 0;
         },
     },
@@ -436,7 +447,7 @@ export const commands = [
                 capture = publicCode(error, 'runtime-missing');
             }
             const states = {};
-            for (const kind of ['asr', 'embedding']) {
+            for (const kind of ['asr', 'embedding', 'media']) {
                 try {
                     await inspectNative(kind, c.nativeRoot, c.version);
                     states[kind] = 'ready';
@@ -742,6 +753,13 @@ export const commands = [
                 return 0;
             }
             if (args.flags.draft) {
+                // Refuse an unknown item or a thin capture before the model server spends its startup on it.
+                const item = store.read(args.positional[0]);
+                if (!item)
+                    throw new SpongeError('unavailable');
+                const thin = undigestible(item.markdown);
+                if (thin)
+                    throw new SpongeError(thin);
                 const runtime = await inspectDigest(c.nativeRoot, args.flags.model ?? store.digests.settings().model ?? undefined), result = await withDigestServer(store.root, runtime, (model) => createDigest(store, args.positional[0], model, { signal }), { signal });
                 c.emit({ digest: result.derivation, duplicate: result.duplicate, ...(result.trace ? { trace: result.trace } : {}) }, digestText(result.derivation));
                 return 0;

package/dist/src/cli/config.d.ts

@@ -2,7 +2,8 @@ import type { CaptureConnection } from '../acquisition/index.js';
 export interface Connection {
     id: string;
     realm: string;
-    capture: CaptureConnection;
+    /** Absent for a retired GhostGet account connection: it still names its items' realm, but cannot capture. */
+    capture?: CaptureConnection;
 }
 /** Local references only. Tokens, cookies and passwords never belong in this file. */
 export declare function readConnections(root: string): Promise<Connection[]>;

package/dist/src/cli/config.js

@@ -32,17 +32,17 @@ export async function readConnections(root) {
         if (!entry || typeof entry.id !== 'string' || !/^[a-zA-Z0-9_-]{1,64}$/.test(entry.id) || ids.has(entry.id))
             throw new SpongeError('invalid-input');
         ids.add(entry.id);
-        let capture;
         if (entry.kind === 'account' &&
             typeof entry.stateHome === 'string' &&
             entry.stateHome.startsWith('/') &&
-            typeof entry.accountId === 'string')
-            capture = { kind: 'account', stateHome: entry.stateHome, accountId: entry.accountId };
-        else if (entry.kind === 'cdp' && typeof entry.endpoint === 'string')
-            capture = { kind: 'cdp', endpoint: entry.endpoint };
-        else
+            typeof entry.accountId === 'string') {
+            // The realm keeps its original derivation, so items captured through it keep their access class.
+            const legacy = { kind: 'account', stateHome: entry.stateHome, accountId: entry.accountId };
+            return { id: entry.id, realm: 'connection:' + sha256(JSON.stringify([entry.id, legacy])) };
+        }
+        if (entry.kind !== 'cdp' || typeof entry.endpoint !== 'string')
             throw new SpongeError('invalid-input');
-        const realm = 'connection:' + sha256(JSON.stringify([entry.id, capture]));
-        return { id: entry.id, realm, capture };
+        const capture = { kind: 'cdp', endpoint: entry.endpoint };
+        return { id: entry.id, realm: 'connection:' + sha256(JSON.stringify([entry.id, capture])), capture };
     });
 }

package/dist/src/cli/runtimes.js

@@ -11,7 +11,7 @@ async function installedReferences(version) {
         const catalog = await loadNativeCatalog(version);
         models = {
             fingerprints: [
-                ...['asr', 'embedding'].map((kind) => modelSetupPlan(nativeModelPackage(catalog, kind)).fingerprint),
+                ...['asr', 'embedding', 'media'].map((kind) => modelSetupPlan(nativeModelPackage(catalog, kind)).fingerprint),
                 ...digestFingerprints(),
             ],
         };

package/dist/src/cli/worker.js

@@ -1,5 +1,5 @@
 import { join } from 'node:path';
-import { inspectNative } from '../models/runtime.js';
+import { inspectNative, optionalNative } from '../models/runtime.js';
 import { drainIndexQueue } from '../indexing/runner.js';
 import { drainDigestQueue } from '../digest/runner.js';
 import { drainLibrary } from '../worker/basic.js';
@@ -63,15 +63,18 @@ export async function drainServiceQueues(store, options, version, signal) {
             const runtime = options.runtime ?? (await options.resolveCaptureRuntime?.());
             if (!runtime)
                 throw new SpongeError('runtime-missing');
-            let asrRuntime = options.asrRuntime;
-            try {
-                asrRuntime ??= await inspectNative('asr', join(store.root, 'runtime', 'models'), version);
-            }
-            catch (error) {
-                if (!['runtime-missing', 'unsupported'].includes(codeOf(error)))
-                    throw error;
-            }
-            results.push(...(await drainLibrary(store, { ...options, runtime, asrRuntime, signal, allowPrivateNetwork: false })));
+            const models = join(store.root, 'runtime', 'models');
+            // Media jobs report a missing helper themselves; articles and PDFs still drain.
+            const asrRuntime = options.asrRuntime ?? (await optionalNative('asr', models, version));
+            const mediaRuntime = options.mediaRuntime ?? (await optionalNative('media', models, version));
+            results.push(...(await drainLibrary(store, {
+                ...options,
+                runtime,
+                asrRuntime,
+                mediaRuntime,
+                signal,
+                allowPrivateNetwork: false,
+            })));
         }
         catch (error) {
             acquisitionError = error;

package/dist/src/digest/pipeline.d.ts

@@ -4,7 +4,7 @@ import type { Derivation, DigestCandidate } from './derivation.js';
  * Model-agnostic Digest v2 drafting. The model sees numbered passages and cites a passage id plus a short exact
  * span; code snaps, prunes and checks. Nothing leaves this module except through the injected model.
  */
-export declare const PROMPT_VERSION = "sponge-digest-prompt-v5";
+export declare const PROMPT_VERSION = "sponge-digest-prompt-v6";
 export type ChatMessage = {
     role: 'system' | 'user' | 'assistant';
     content: string;
@@ -56,8 +56,30 @@ interface Highlight {
  * change digest input hashes.
  */
 export declare function canonicalJson(value: unknown): string;
-/** Paragraph-level passages with raw offsets. Headings label passages; code fences and image-only blocks are not quotable. */
-export declare function passages(markdown: string, maxChars?: number): Passage[];
+/** Where a post starts in a captured thread, and who wrote it as the model is told. */
+export interface Post {
+    start: number;
+    label: string;
+}
+/**
+ * The posts of an X thread capture: top-level list items that carry X's post links. The poster's handle comes from
+ * the item's avatar or name link; the thread's author from the title (`Name (@handle) on X`), else the first post. A
+ * later post without a handle is still a reply. Other list items (an X article's bullets) are not posts, and a thread
+ * with no reply has none.
+ */
+export declare function threadPosts(markdown: string): Post[];
+/** A thread's passages without its replies, which are readers' comments rather than the source; renumbered from 1. */
+export declare function authorPassages(markdown: string): Passage[];
+/**
+ * Paragraph-level passages with raw offsets. Headings label passages; code fences and image-only blocks are not
+ * quotable. `posts` (a thread's) also break blocks, and their labels head the passages that follow.
+ */
+export declare function passages(markdown: string, maxChars?: number, posts?: readonly Post[]): Passage[];
+/**
+ * Why a capture cannot be digested, as a detail code, or undefined when it can: its body (past the frontmatter, images
+ * dropped, link text kept) has under `minProseWords` words with letters, or only an archive's metadata was saved.
+ */
+export declare function undigestible(markdown: string): 'source-too-thin' | undefined;
 /** Pack passages into sections of at most `budget` display characters, preferring heading boundaries past half a budget. */
 export declare function sectionsOf(list: Passage[], budget: number): Passage[][];
 /** JSON Schema for the model's draft. Tags become an enum when a vocabulary is given. */
@@ -160,8 +182,11 @@ export interface DigestStore {
         read(kind: 'digest', id: string): Derivation | undefined;
     };
 }
+type DraftOptions = Omit<DraftInput, 'revisionId' | 'markdown' | 'title' | 'form' | 'model' | 'highlights'>;
+/** What the item's current revision is drafted from: its Markdown, title, form and anchored highlights. */
+export declare function digestInputFor(store: Pick<DigestStore, 'read' | 'annotations'>, itemId: string, model: DigestModel, options?: DraftOptions): DraftInput;
 /** Draft and store a digest for the item's current revision. A matching input hash returns the stored draft without calling the model. */
-export declare function createDigest(store: DigestStore, itemId: string, model: DigestModel, options?: Omit<DraftInput, 'revisionId' | 'markdown' | 'title' | 'form' | 'model' | 'highlights'>): Promise<{
+export declare function createDigest(store: DigestStore, itemId: string, model: DigestModel, options?: DraftOptions): Promise<{
     derivation: Derivation;
     duplicate: boolean;
     trace?: DraftTrace;

package/dist/src/digest/pipeline.js

@@ -6,7 +6,7 @@ import { sha256 } from '../util/hash.js';
  * Model-agnostic Digest v2 drafting. The model sees numbered passages and cites a passage id plus a short exact
  * span; code snaps, prunes and checks. Nothing leaves this module except through the injected model.
  */
-export const PROMPT_VERSION = 'sponge-digest-prompt-v5';
+export const PROMPT_VERSION = 'sponge-digest-prompt-v6';
 /** 24k characters is roughly 6k tokens of passages, leaving room for instructions and output in an 8k context on an 8 GB Mac. */
 const defaultDigestParams = {
     temperature: 0,
@@ -40,14 +40,62 @@ export function canonicalJson(value) {
         throw new SpongeError('invalid-input');
     return JSON.stringify(value ?? null);
 }
-/** Paragraph-level passages with raw offsets. Headings label passages; code fences and image-only blocks are not quotable. */
-export function passages(markdown, maxChars = 1200) {
+const postHandle = [
+    /^- \[!\[@(\w+)\]/,
+    /^- \[\[remote image: @(\w+)\]/,
+    /^-\s+\[[^\]]+\]\(https:\/\/x\.com\/(\w+)\)\s*$/,
+];
+/**
+ * The posts of an X thread capture: top-level list items that carry X's post links. The poster's handle comes from
+ * the item's avatar or name link; the thread's author from the title (`Name (@handle) on X`), else the first post. A
+ * later post without a handle is still a reply. Other list items (an X article's bullets) are not posts, and a thread
+ * with no reply has none.
+ */
+export function threadPosts(markdown) {
+    const author = /^title: ".*\(@(\w+)\) on X"/m.exec(frontmatter(markdown))?.[1], items = [];
+    let at = 0;
+    for (const line of markdown.split('\n')) {
+        if (/^- /.test(line) || /^-\s+\[/.test(line))
+            items.push({ start: at, handle: postHandle.map((r) => r.exec(line)?.[1]).find(Boolean), post: false });
+        const item = items.at(-1);
+        if (item && /x\.com\/(compose\/post\?in_reply_to=|\w+\/status\/\d)/.test(line))
+            item.post = true;
+        at += line.length + 1;
+    }
+    const posts = items.filter((i) => i.post || i.handle), by = author ?? posts[0]?.handle, same = (h) => h.toLowerCase() === by?.toLowerCase(), labelled = posts.map((p, i) => ({
+        start: p.start,
+        label: p.handle && !same(p.handle)
+            ? `Reply by @${p.handle}`
+            : p.handle || i === 0
+                ? `Post by ${by ? `@${by}, ` : ''}the author`
+                : 'Reply (poster not captured)',
+    }));
+    // With no reply there is nobody to tell apart, so the passages stay as they are.
+    return labelled.some((p) => p.label !== labelled[0].label) ? labelled : [];
+}
+/** A thread's passages without its replies, which are readers' comments rather than the source; renumbered from 1. */
+export function authorPassages(markdown) {
+    return passages(markdown, undefined, threadPosts(markdown))
+        .filter((p) => !p.heading?.startsWith('Reply'))
+        .map((p, i) => ({ ...p, id: i + 1 }));
+}
+/**
+ * Paragraph-level passages with raw offsets. Headings label passages; code fences and image-only blocks are not
+ * quotable. `posts` (a thread's) also break blocks, and their labels head the passages that follow.
+ */
+export function passages(markdown, maxChars = 1200, posts = []) {
     const out = [], blocks = [];
     let at = 0, start = -1, fence = false, fenced = false;
+    const breaks = new Set(posts.map((p) => p.start));
     for (const line of markdown.split('\n')) {
         const end = at + line.length, blank = !line.trim(), marker = /^\s{0,3}(```|~~~)/.test(line);
         if (marker)
             fence = !fence;
+        if (!fence && !marker && start >= 0 && breaks.has(at)) {
+            blocks.push({ start, end: at - 1, fence: fenced });
+            start = -1;
+            fenced = false;
+        }
         if (blank && !fence && !marker) {
             if (start >= 0)
                 blocks.push({ start, end: at - 1, fence: fenced });
@@ -64,14 +112,17 @@ export function passages(markdown, maxChars = 1200) {
     }
     if (start >= 0)
         blocks.push({ start, end: markdown.length, fence: fenced });
-    let heading = '';
+    let heading = '', post = '';
+    const label = (start) => posts.findLast((p) => p.start <= start)?.label ?? '';
     for (const b of blocks) {
         const raw = markdown.slice(b.start, b.end).trimEnd(), end = b.start + raw.length;
         if (b.fence || !raw.trim())
             continue;
+        if (label(b.start) !== post)
+            heading = post = label(b.start);
         const h = /^\s{0,3}#{1,6}\s+(.+)$/.exec(raw);
         if (h && !raw.includes('\n')) {
-            heading = collapse(h[1]).slice(0, 200);
+            heading = (post ? `${post}: ` : '') + collapse(h[1]).slice(0, 200);
             continue;
         }
         if (/^(\s*!\[[^\]]*\]\([^)]*\)\s*)+$/.test(raw))
@@ -91,6 +142,32 @@ export function passages(markdown, maxChars = 1200) {
     }
     return out;
 }
+const frontmatter = (markdown) => /^---\r?\n[\s\S]*?\r?\n---(?:\r?\n|$)/u.exec(markdown)?.[0] ?? '';
+/** A `key: "value"` frontmatter field, as captures write them. */
+function captureField(markdown, key) {
+    const match = new RegExp(`^${key}: (".*")\\r?$`, 'm').exec(frontmatter(markdown));
+    try {
+        const value = match ? JSON.parse(match[1]) : undefined;
+        return typeof value === 'string' ? value : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/** Prose words under which a capture holds nothing to digest: a gallery of numbered images, a bare link list. */
+const minProseWords = 20;
+/**
+ * Why a capture cannot be digested, as a detail code, or undefined when it can: its body (past the frontmatter, images
+ * dropped, link text kept) has under `minProseWords` words with letters, or only an archive's metadata was saved.
+ */
+export function undigestible(markdown) {
+    if (captureField(markdown, 'capture_method') === 'archive-metadata')
+        return 'source-too-thin';
+    const prose = passages(markdown.slice(frontmatter(markdown).length))
+        .map((p) => p.text.replace(/(!?)\[([^\]]*)\]\([^)]*\)/g, (_, image, text) => (image ? ' ' : text)))
+        .join(' ');
+    return (prose.match(/\p{L}[\p{L}\p{M}'’-]*/gu) ?? []).length < minProseWords ? 'source-too-thin' : undefined;
+}
 function lastSentenceEnd(window) {
     let best = -1;
     for (const m of window.matchAll(/[.!?]["')\]]*(?=\s)/g))
@@ -196,7 +273,12 @@ Source: an essay arguing that taste is a skill.
   claim: Volume beats polish early. | detail: Side-by-side comparison of many rough drafts sharpens judgment faster than finishing one.
 Bad claim: The essay explores how practice over time shapes a person's sense of taste.
 Good claim: Taste is trained.`;
-const system = (budget, vocabulary) => `You write a Digest v2 of one source for a private reading library, in the owner's voice: terse, declarative, specific. Reply with JSON only, matching the schema.
+/** Who speaks in each form, so claims are credited to the person who made them. */
+const voices = {
+    thread: 'This source is an X thread. Its replies are left out. When the author quotes or answers someone, that view stays theirs.',
+    video: 'This source is a recording: credit each claim to the person who says it. In an interview or podcast, the guest’s views are the guest’s, not the host’s.',
+};
+const system = (budget, form, vocabulary) => `You write a Digest v2 of one source for a private reading library, in the owner's voice: terse, declarative, specific. Reply with JSON only, matching the schema.
 - lead: one sentence of at most ${L.leadChars} characters stating this source's specific thesis or result: who (author, organization or product by name) shows or argues what, with the number, name or mechanism that makes it this source. Do not restate the title, describe the source ("This article discusses"), or tease.
 - context: 1 or 2 sentences, at most 45 words: the evidence, method, numbers or limits the lead could not hold. Lead and context together stay within ${L.gistWords} words and ${L.gistSentences} sentences.
 - ideas: ${L.minIdeas} to ${L.maxIdeas} (prefer ${L.maxIdeas} when the source has them), in the order the author makes them, covering the whole source including its caveats and limits. Each idea is a claim with detail:
@@ -205,7 +287,12 @@ const system = (budget, vocabulary) => `You write a Digest v2 of one source for
   - quotes: 0 to ${L.maxQuotesPerIdea} that support this idea, each a contiguous span copied character for character from one numbered passage, ${minQuoteWords + 2} to 20 words, with that passage number. A quote is a fragment, not a whole long sentence: when a sentence runs past 20 words, copy only its strongest part. No added quotation marks or ellipses. Give attribution (who said it) when it is not the author. Prefer reader highlights.
 - Attach at least ${L.minQuotes} and at most ${L.maxQuotes} quotes in all; ${L.minQuotes + 1} to 3 is typical. All quotes together stay within ${budget} words and never overlap.
 - tags: 0 to ${L.maxTags} ${vocabulary?.length ? `from: ${vocabulary.join(', ')}` : 'lowercase kebab-case topics'}; usually one subject tag.
-Plain words. Name things and use numbers. No hedges or filler: may, might, potentially, various, significantly, key, crucial, notably, highlights, explores, discusses, "the author argues", "the article".
+Faithfulness comes before style:
+- Credit each view to who holds it. When the source quotes, interviews or reports someone, their claims are theirs: name them in the detail and the quote attribution, never pass them off as the author's.${voices[form] ? `\n- ${voices[form]}` : ''}
+- Keep each number with its own subject, unit and meaning (an area is not a height, a share is not a count, a cost is not a price) and never compute one the source does not give.
+- Keep the source's direction: never invert a claim, comparison or cause.
+- Keep the source's certainty: a forecast, plan, proposal, rumor or question stays one ("plans to", "expects", "asks whether"); never state it as done or settled. Hedge only where the source does.
+Plain words. Name things and use numbers. No filler: various, significantly, key, crucial, notably, highlights, explores, discusses, "the author argues", "the article".
 Use only what the source says.
 
 ${styleExamples}`;
@@ -255,7 +342,9 @@ function fromModel(text, form) {
     }
     if (!object(v) || !Array.isArray(v.ideas) || !Array.isArray(v.tags))
         return { issues: [{ path: '', code: 'shape' }] };
-    const trim = (s) => (typeof s === 'string' ? s.trim() : s), cites = [];
+    const trim = (s) => (typeof s === 'string' ? s.trim() : s), 
+    // Models sometimes end a sentence with the passage it came from ("… market. [2]").
+    prose = (s) => (typeof s === 'string' ? s.replace(/(\s*\[\d+(?:,\s*\d+)*\])+$/, '').trim() : s), cites = [];
     const ideas = v.ideas.map((idea) => {
         if (!object(idea)) {
             cites.push([]);
@@ -264,19 +353,20 @@ function fromModel(text, form) {
         const quotes = Array.isArray(idea.quotes) ? idea.quotes : [];
         cites.push(quotes.map((q) => (Number.isSafeInteger(q?.passage) ? q.passage : undefined)));
         return {
-            claim: trim(idea.claim),
-            detail: trim(idea.detail),
+            claim: prose(idea.claim),
+            detail: prose(idea.detail),
             quotes: quotes.map((q) => {
                 const attribution = trim(q?.attribution);
                 return { text: trim(q?.text), ...(attribution ? { attribution } : {}) };
             }),
         };
     });
-    const context = trim(v.context);
+    // A context the model stopped mid-sentence is dropped; context is optional.
+    const context = ((c) => (typeof c === 'string' && !/[.!?…。！？)"”’]$/.test(c) ? undefined : c))(prose(v.context));
     const parsed = parseDigest({
         schemaVersion: 2,
         form,
-        gist: { lead: trim(v.lead), ...(context ? { context } : {}) },
+        gist: { lead: prose(v.lead), ...(context ? { context } : {}) },
         ideas,
         tags: v.tags.map(trim),
     });
@@ -453,7 +543,10 @@ export async function draftDigest(input) {
             input.vocabulary.length > 256 ||
             input.vocabulary.some((t) => !/^[a-z0-9][a-z0-9-]{0,63}$/.test(t))))
         throw new SpongeError('invalid-input');
-    const inputHash = digestInputHash({ ...input, model: identity }), source = input.markdown, list = passages(source);
+    const thin = undigestible(input.markdown);
+    if (thin)
+        throw new SpongeError(thin);
+    const inputHash = digestInputHash({ ...input, model: identity }), source = input.markdown, list = input.form === 'thread' ? authorPassages(source) : passages(source);
     if (list.length === 0)
         throw new SpongeError('invalid-input');
     const userSignal = input.signal ?? new AbortController().signal, overall = AbortSignal.any([userSignal, AbortSignal.timeout(p.deadlineMs)]);
@@ -564,7 +657,7 @@ export async function draftDigest(input) {
     }
     const budget = quoteBudget(source), schema = draftSchema(list.length, input.vocabulary);
     const messages = [
-        { role: 'system', content: system(budget, input.vocabulary) },
+        { role: 'system', content: system(budget, input.form, input.vocabulary) },
         {
             role: 'user',
             content: (notesText ? `Notes from each part of the source, in order:\n${notesText}\n\n` : '') +
@@ -637,9 +730,16 @@ export async function draftDigest(input) {
     };
     return { output: result.draft.digest, inputHash, producer, options, trace };
 }
-const formFor = (kind) => (kind === 'pdf' ? 'pdf' : kind === 'media' ? 'video' : 'article');
-/** Draft and store a digest for the item's current revision. A matching input hash returns the stored draft without calling the model. */
-export async function createDigest(store, itemId, model, options = {}) {
+/** X thread captures draft as `thread`, whose prompt rules keep replies apart from the author's posts. */
+const formFor = (kind, markdown) => kind === 'pdf'
+    ? 'pdf'
+    : kind === 'media'
+        ? 'video'
+        : captureField(markdown, 'platform') === 'x' && captureField(markdown, 'capture_scope') === 'thread'
+            ? 'thread'
+            : 'article';
+/** What the item's current revision is drafted from: its Markdown, title, form and anchored highlights. */
+export function digestInputFor(store, itemId, model, options = {}) {
     const item = store.read(itemId);
     if (!item)
         throw new SpongeError('unavailable');
@@ -647,21 +747,25 @@ export async function createDigest(store, itemId, model, options = {}) {
         .list(itemId)
         .filter((a) => a.kind === 'highlight' && a.revisionId === item.revisionId && a.anchored !== false)
         .map((a) => ({ text: a.anchor.exact, start: a.anchor.start }));
-    const input = {
+    return {
         ...options,
         revisionId: item.revisionId,
         markdown: item.markdown,
         title: item.title,
-        form: formFor(item.kind),
+        form: formFor(item.kind, item.markdown),
         model,
         highlights,
     };
-    const prior = store.derivations.find('digest', item.revisionId, digestInputHash({ ...input, model: model.identity }));
+}
+/** Draft and store a digest for the item's current revision. A matching input hash returns the stored draft without calling the model. */
+export async function createDigest(store, itemId, model, options = {}) {
+    const input = digestInputFor(store, itemId, model, options);
+    const prior = store.derivations.find('digest', input.revisionId, digestInputHash({ ...input, model: model.identity }));
     if (prior)
         return { derivation: prior, duplicate: true };
     const draft = await draftDigest(input), stored = store.derivations.put('digest', {
         itemId,
-        revisionId: item.revisionId,
+        revisionId: input.revisionId,
         producer: draft.producer,
         inputHash: draft.inputHash,
         output: draft.output,

package/dist/src/errors.d.ts

@@ -19,8 +19,10 @@ declare const parents: {
     readonly 'draft-rejected': "unavailable";
     readonly 'model-unavailable': "unavailable";
     readonly 'model-busy': "unavailable";
+    readonly 'source-too-thin': "unavailable";
     readonly 'parent-missing': "invalid-input";
     readonly 'target-exists': "invalid-input";
+    readonly 'connection-retired': "unsupported";
 };
 export type DetailCode = keyof typeof parents;
 export type SpongeCode = ErrorCode | DetailCode;