@hraness/sponge 0.3.5 → 0.4.1

package/dist/src/models/digest.js

@@ -0,0 +1,122 @@
+import { totalmem } from 'node:os';
+import { mkdir } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import { privateRoot } from '../acquisition/files.js';
+import { inspectInstalledModelPackage, modelSetupPlan, provisionModelPackage, recoverModelPackage, } from './artifacts.js';
+import { nativePlatformSupported } from './catalog.js';
+import { SpongeError } from '../errors.js';
+const GiB = 1024 ** 3;
+/** llama.cpp b11669 (0.7.0, commit 620d2225c): self-contained, `@loader_path` dylibs, ad hoc signed. */
+export const llamaRuntimePin = {
+    name: 'llama-b11669-bin-macos-arm64.tar.gz',
+    url: 'https://github.com/ggml-org/llama.cpp/releases/download/b11669/llama-b11669-bin-macos-arm64.tar.gz',
+    bytes: 12_191_582,
+    sha256: 'dbed55493d72bc1ba2babc8770d669c2289b5bc4ee9fb2a4dfc158d45b7d13fd',
+    directory: 'llama-b11669',
+    build: 11669,
+    commit: '620d2225c',
+};
+/**
+ * Gemma 4 26B-A4B (Apache 2.0) at Unsloth's dynamic Q4_K_M is the default: on the 40-item Jungle eval it matched the
+ * 9B's idea coverage with no failures, a tenth of its copied details and about 60% of its time per item (see
+ * src/digest/README.md). Qwen3.5-9B is the pin for machines with 16-32 GiB.
+ */
+export const digestModels = [
+    {
+        alias: 'gemma-4-26b-a4b',
+        file: 'gemma-4-26B-A4B-it-UD-Q4_K_M.gguf',
+        bytes: 16_947_541_728,
+        sha256: 'f2c28b3dc4776931ac6f879e11f203dec637ea0f14267a86ec8f6165f63f293f',
+        repo: 'unsloth/gemma-4-26B-A4B-it-GGUF',
+        revision: 'c099eb48e663fd284577b04978a94ffccb261841',
+        base: 'google/gemma-4-26B-A4B-it',
+        minimumMemoryBytes: 32 * GiB,
+    },
+    {
+        alias: 'qwen3.5-9b',
+        file: 'Qwen3.5-9B-Q4_K_M.gguf',
+        bytes: 5_627_044_256,
+        sha256: 'cd76ec205963b3b33350093e6904d9de16c4e666fd104e1f632d25c7f15f2a13',
+        repo: 'lmstudio-community/Qwen3.5-9B-GGUF',
+        revision: '1379f25c6b505a3fc737bd7818cb09389cf807c1',
+        base: 'Qwen/Qwen3.5-9B',
+        minimumMemoryBytes: 16 * GiB,
+    },
+];
+export const modelUrl = (m) => `https://huggingface.co/${m.repo}/resolve/${m.revision}/${m.file}`;
+/** An explicit alias, or the largest pin this machine's memory allows; `unsupported` below every pin. */
+export function digestModel(alias, memory = totalmem()) {
+    if (alias !== undefined) {
+        const pin = digestModels.find((m) => m.alias === alias);
+        if (!pin)
+            throw new SpongeError('invalid-input');
+        return pin;
+    }
+    const pin = digestModels.find((m) => memory >= m.minimumMemoryBytes);
+    if (!pin)
+        throw new SpongeError('unsupported');
+    return pin;
+}
+export function digestModelPackage(pin) {
+    return {
+        id: `sponge-digest-b${llamaRuntimePin.build}-${pin.alias}`,
+        license: 'llama.cpp: MIT; model: see its repository license (Gemma 4 and Qwen3.5 are Apache-2.0)',
+        maximumBytes: 20 * GiB,
+        artifacts: [
+            {
+                name: llamaRuntimePin.name,
+                url: llamaRuntimePin.url,
+                sha256: llamaRuntimePin.sha256,
+                bytes: llamaRuntimePin.bytes,
+            },
+            { name: pin.file, url: modelUrl(pin), sha256: pin.sha256, bytes: pin.bytes },
+        ],
+    };
+}
+/** Every pin's fingerprint, so runtime GC keeps any installed digest package. */
+export const digestFingerprints = () => digestModels.map((pin) => modelSetupPlan(digestModelPackage(pin)).fingerprint);
+export function digestSetupPlan(pin) {
+    return {
+        model: pin.alias,
+        memoryBytes: totalmem(),
+        supported: nativePlatformSupported(),
+        ...modelSetupPlan(digestModelPackage(pin)),
+    };
+}
+/** Explicit setup: download (or adopt `modelFile`, a copy of the pinned GGUF the user already has) and verify. */
+export async function provisionDigest(cacheRoot, pin, options = {}) {
+    if (!nativePlatformSupported())
+        throw new SpongeError('unsupported');
+    const manifest = digestModelPackage(pin);
+    await privateRoot(dirname(cacheRoot));
+    await mkdir(cacheRoot, { recursive: true, mode: 0o700 });
+    await privateRoot(cacheRoot);
+    await recoverModelPackage(cacheRoot, manifest);
+    const paths = await provisionModelPackage(cacheRoot, manifest, {
+        signal: options.signal,
+        fetch: options.fetch,
+        ...(options.modelFile ? { adopt: { [pin.file]: options.modelFile } } : {}),
+    });
+    return { archive: paths[llamaRuntimePin.name], model: paths[pin.file], pin };
+}
+/** The installed runtime for `alias`, or for this machine's default pin, else the first installed pin. */
+export async function inspectDigest(cacheRoot, alias) {
+    if (!nativePlatformSupported())
+        throw new SpongeError('unsupported');
+    let preferred;
+    try {
+        preferred = digestModel(alias);
+    }
+    catch (error) {
+        if (alias !== undefined)
+            throw error;
+    }
+    for (const pin of preferred
+        ? [preferred, ...(alias ? [] : digestModels.filter((m) => m !== preferred))]
+        : digestModels) {
+        const paths = await inspectInstalledModelPackage(cacheRoot, digestModelPackage(pin));
+        if (paths)
+            return { archive: paths[llamaRuntimePin.name], model: paths[pin.file], pin };
+    }
+    throw new SpongeError('runtime-missing');
+}

package/dist/src/portability/database.js

@@ -96,6 +96,8 @@ export function neutralizeRestoredDatabase(path) {
             db.exec("UPDATE index_jobs SET state='failed',reason='restore-reconciliation-required' WHERE state NOT IN ('succeeded','failed','cancelled')");
             db.prepare("UPDATE index_attempts SET stage='restored-unexecuted',cleanup='settled',finished_at=COALESCE(finished_at,?) WHERE cleanup='pending' OR finished_at IS NULL").run(new Date().toISOString());
         }
+        if (inspected.schemaVersion >= 11)
+            db.exec("UPDATE digest_jobs SET state='queued',owner_incarnation=NULL,fence=NULL WHERE state='running'; UPDATE digest_jobs SET state='cancelled' WHERE state='queued' AND cancel_requested=1");
         // A restored copy is a new device of the same library: a fresh device id keeps its HLCs distinct; clock_ms only moves forward.
         if (inspected.schemaVersion >= 5)
             db.exec('UPDATE device SET device_id=lower(hex(randomblob(16))) WHERE id=1');

package/dist/src/search/query.js

@@ -1,66 +1,19 @@
-import { randomUUID } from 'node:crypto';
-import { join } from 'node:path';
-import { rm } from 'node:fs/promises';
-import { acquireWorkerCustody, proveAttemptCleanup } from '../worker/custody.js';
-import { directory, writeJournal, readJournal } from '../worker/journal.js';
-import { syncDirectoryAsync } from '../util/fsync.js';
-import { supervisedRunner } from '../worker/supervisor.js';
+import { withTransientAttempt } from '../worker/transient.js';
 import { embedBatch, embeddingFingerprint, embeddingTextBytes, resolveIndexSpace, projectVector, encodeEmbeddingRequest, } from '../inference/embedding/index.js';
-import { SpongeError, codeOf } from '../errors.js';
+import { SpongeError } from '../errors.js';
 /** One custody-held helper run. Mutates coordination journals only, never library content or job tables. */
-async function custodyEmbed(store, runtime, inputs, signal, options) {
-    const custody = await acquireWorkerCustody(store.root), attemptId = randomUUID();
-    let scratchRoot, cleanupUnknown = false;
-    try {
-        const attempts = await directory(join(custody.directory, 'attempts', attemptId));
-        await writeJournal(join(attempts, 'query.json'), {
-            schemaVersion: 1,
-            kind: 'query',
-            attemptId,
-            ownerIncarnation: custody.incarnation,
-        });
-        scratchRoot = await directory(join(store.root, 'staging', 'query-' + attemptId));
-        await writeJournal(join(scratchRoot, 'ownership.json'), {
-            schemaVersion: 1,
-            kind: 'query',
-            requestId: attemptId,
-            ownerIncarnation: custody.incarnation,
-        });
-        await options.onReady?.({ attemptId, ownerDirectory: custody.directory, scratchRoot });
+function custodyEmbed(store, runtime, inputs, signal, options) {
+    return withTransientAttempt(store.root, 'query', async ({ attemptId, ownerDirectory, scratchRoot, runProcess }) => {
+        await options.onReady?.({ attemptId, ownerDirectory, scratchRoot });
         const expected = encodeEmbeddingRequest(inputs).inputHashes;
-        const result = await (options.embed ?? embedBatch)(inputs, {
-            runtime,
-            scratchRoot,
-            signal,
-            runProcess: supervisedRunner(custody.directory, custody.incarnation, attemptId),
-        });
+        const result = await (options.embed ?? embedBatch)(inputs, { runtime, scratchRoot, signal, runProcess });
         if (result.fingerprint !== embeddingFingerprint(runtime.helperSha256) ||
             result.inputHashes.length !== expected.length ||
             result.inputHashes.some((h, i) => h !== expected[i]) ||
             result.vectors.length !== expected.length)
             throw new SpongeError('integrity');
         return result.vectors;
-    }
-    catch (error) {
-        cleanupUnknown = codeOf(error) === 'cleanup-required';
-        throw error;
-    }
-    finally {
-        const proved = !cleanupUnknown && (await proveAttemptCleanup(custody.directory, custody.incarnation, attemptId));
-        if (!proved)
-            throw new SpongeError('cleanup-required');
-        if (scratchRoot) {
-            const marker = await readJournal(join(scratchRoot, 'ownership.json'));
-            if (marker.schemaVersion !== 1 ||
-                marker.kind !== 'query' ||
-                marker.requestId !== attemptId ||
-                marker.ownerIncarnation !== custody.incarnation)
-                throw new SpongeError('cleanup-required');
-            await rm(scratchRoot, { recursive: true });
-            await syncDirectoryAsync(join(store.root, 'staging'));
-        }
-        await custody.release();
-    }
+    });
 }
 /** Queries mutate coordination journals only, never library content or job tables. */
 export function createQueryEmbedder(store, runtime, options = {}) {

package/dist/src/store/digest-jobs.d.ts

@@ -0,0 +1,58 @@
+import type { DatabaseSync } from 'node:sqlite';
+import { type Context } from './context.js';
+import type { JobKind } from './queue.js';
+import type { OwnerToken } from './types.js';
+export interface DigestSettings {
+    auto: boolean;
+    /** The model alias `sponge setup --digest` installed, or null for the memory-based default. */
+    model: string | null;
+}
+export interface DigestClaim extends OwnerToken {
+    jobId: string;
+    itemId: string;
+    revisionId: string;
+}
+export type DigestOutcome = {
+    derivationId: string;
+} | {
+    reason: string;
+    retryable?: boolean;
+    notBefore?: string;
+    countAttempt?: boolean;
+};
+/**
+ * Queue a digest of a newly captured revision when auto-digest is on and the capture has enough text to digest. Runs
+ * inside the capture commit's transaction.
+ */
+export declare function enqueueDigest(db: DatabaseSync, itemId: string, revisionId: string, markdown: string): void;
+/**
+ * Digest jobs under the shared worker lease. A job names one revision; when its item moves on or is deleted before the
+ * worker reaches it, the job is cancelled as `superseded`, since the newer revision has a job of its own. The drafting
+ * server's process custody is the worker's filesystem journal (`withDigestServer`), so the database holds no attempt
+ * rows or staging, and owner recovery returns a crashed owner's running job to the queue.
+ */
+export declare class DigestJobs {
+    private readonly c;
+    readonly jobs: JobKind;
+    constructor(c: Context);
+    settings(): DigestSettings;
+    /** Turn auto-digest on or off. Turning it on queues nothing already captured; turning it off leaves queued jobs. */
+    configure(settings: Partial<DigestSettings>): DigestSettings;
+    /** Whether a job is ready for `claim` now. */
+    due(): boolean;
+    /**
+     * Delay every ready job to `notBefore` with `reason`, without counting an attempt: the drafting runtime is missing or
+     * could not start, so no job reached the model. Running jobs are untouched. Returns how many were delayed.
+     */
+    postpone(reason: string, notBefore: string): number;
+    /** The oldest ready job whose revision is still its item's current one; jobs passed over on the way are superseded. */
+    claim(owner: OwnerToken): DigestClaim | null;
+    /**
+     * Settle a claimed job. A retryable failure under the attempt limit is delayed until `notBefore` (default: a backoff
+     * from the attempt count); `countAttempt: false` gives the attempt back, for a refusal that never reached the model.
+     */
+    finish(claim: DigestClaim, outcome: DigestOutcome): void;
+    /** Whether the user asked to cancel the claimed job while it ran. */
+    cancelRequested(claim: DigestClaim): boolean;
+    private row;
+}

package/dist/src/store/digest-jobs.js

@@ -0,0 +1,168 @@
+import { randomUUID } from 'node:crypto';
+import { text } from './files.js';
+import { checkOwner } from './lease.js';
+import { now, transaction } from './context.js';
+import { SpongeError } from '../errors.js';
+import { undigestible } from '../digest/pipeline.js';
+/** Automatic retries a retryable failure gets before it stays failed. */
+const maxAttempts = 3;
+const alias = /^[\w.:/@+-]{1,256}$/;
+/**
+ * Queue a digest of a newly captured revision when auto-digest is on and the capture has enough text to digest. Runs
+ * inside the capture commit's transaction.
+ */
+export function enqueueDigest(db, itemId, revisionId, markdown) {
+    if (db.prepare('SELECT auto FROM digest_settings WHERE id=1').get()?.auto !== 1 || undigestible(markdown))
+        return;
+    db.prepare("INSERT INTO digest_jobs(id,item_id,revision_id,state,created_at) VALUES(?,?,?,'queued',?) ON CONFLICT(revision_id) DO NOTHING").run(randomUUID(), itemId, revisionId, now());
+}
+/**
+ * Digest jobs under the shared worker lease. A job names one revision; when its item moves on or is deleted before the
+ * worker reaches it, the job is cancelled as `superseded`, since the newer revision has a job of its own. The drafting
+ * server's process custody is the worker's filesystem journal (`withDigestServer`), so the database holds no attempt
+ * rows or staging, and owner recovery returns a crashed owner's running job to the queue.
+ */
+export class DigestJobs {
+    c;
+    jobs;
+    constructor(c) {
+        this.c = c;
+        const db = c.db;
+        this.jobs = {
+            owns: (id) => !!db.prepare('SELECT 1 FROM digest_jobs WHERE id=?').get(id),
+            list: () => db
+                .prepare("SELECT d.*,'digest' AS operation FROM digest_jobs d JOIN items i ON i.id=d.item_id WHERE i.deleted_at IS NULL ORDER BY d.created_at,d.id")
+                .all(),
+            attempts: () => [],
+            cancel: (id) => transaction(db, () => {
+                const job = this.row(id);
+                if (['succeeded', 'cancelled'].includes(String(job.state)))
+                    return;
+                db.prepare("UPDATE digest_jobs SET cancel_requested=1,state=CASE WHEN state='running' THEN state ELSE 'cancelled' END,finished_at=CASE WHEN state='running' THEN finished_at ELSE ? END WHERE id=?").run(now(), id);
+            }),
+            retry: (id, options) => {
+                if (options.automatic || options.notBefore !== undefined)
+                    throw new SpongeError('invalid-input');
+                transaction(db, () => {
+                    const job = this.row(id);
+                    if (job.state !== 'failed' || job.cancel_requested)
+                        throw new SpongeError('unavailable');
+                    db.prepare("UPDATE digest_jobs SET state='queued',reason=NULL,retryable=0,not_before=NULL,attempt_count=0,finished_at=NULL WHERE id=?").run(id);
+                });
+            },
+            cancelRequested: (id) => this.row(id).cancel_requested === 1,
+            recover: () => {
+                throw new SpongeError('integrity');
+            },
+            release: (owner) => {
+                db.prepare("UPDATE digest_jobs SET state=CASE WHEN cancel_requested=1 THEN 'cancelled' ELSE 'queued' END,owner_incarnation=NULL,fence=NULL WHERE state='running' AND owner_incarnation=? AND fence=?").run(owner.ownerIncarnation, owner.fence);
+            },
+        };
+    }
+    settings() {
+        const row = this.c.db.prepare('SELECT auto,model FROM digest_settings WHERE id=1').get();
+        return { auto: row.auto === 1, model: row.model === null ? null : String(row.model) };
+    }
+    /** Turn auto-digest on or off. Turning it on queues nothing already captured; turning it off leaves queued jobs. */
+    configure(settings) {
+        if (settings.auto !== undefined && typeof settings.auto !== 'boolean')
+            throw new SpongeError('invalid-input');
+        if (settings.model !== undefined && settings.model !== null && !alias.test(String(settings.model)))
+            throw new SpongeError('invalid-input');
+        transaction(this.c.db, () => {
+            if (settings.auto !== undefined)
+                this.c.db.prepare('UPDATE digest_settings SET auto=? WHERE id=1').run(Number(settings.auto));
+            if (settings.model !== undefined)
+                this.c.db.prepare('UPDATE digest_settings SET model=? WHERE id=1').run(settings.model);
+        });
+        return this.settings();
+    }
+    /** Whether a job is ready for `claim` now. */
+    due() {
+        return !!this.c.db
+            .prepare("SELECT 1 FROM digest_jobs WHERE state IN ('queued','delayed') AND cancel_requested=0 AND (not_before IS NULL OR not_before<=?) LIMIT 1")
+            .get(now());
+    }
+    /**
+     * Delay every ready job to `notBefore` with `reason`, without counting an attempt: the drafting runtime is missing or
+     * could not start, so no job reached the model. Running jobs are untouched. Returns how many were delayed.
+     */
+    postpone(reason, notBefore) {
+        if (!/^[a-z][a-z-]{0,63}$/.test(reason) || !Number.isFinite(Date.parse(notBefore)))
+            throw new SpongeError('invalid-input');
+        return Number(this.c.db
+            .prepare("UPDATE digest_jobs SET state='delayed',reason=?,retryable=1,not_before=? WHERE state IN ('queued','delayed') AND cancel_requested=0 AND (not_before IS NULL OR not_before<=?)")
+            .run(reason, notBefore, now()).changes);
+    }
+    /** The oldest ready job whose revision is still its item's current one; jobs passed over on the way are superseded. */
+    claim(owner) {
+        const db = this.c.db;
+        return transaction(db, () => {
+            checkOwner(db, owner);
+            for (;;) {
+                const job = db
+                    .prepare("SELECT d.id,d.item_id,d.revision_id,i.current_revision,i.deleted_at FROM digest_jobs d JOIN items i ON i.id=d.item_id WHERE d.state IN ('queued','delayed') AND d.cancel_requested=0 AND (d.not_before IS NULL OR d.not_before<=?) ORDER BY d.created_at,d.id LIMIT 1")
+                    .get(now());
+                if (!job)
+                    return null;
+                if (job.deleted_at !== null || job.current_revision !== job.revision_id) {
+                    db.prepare("UPDATE digest_jobs SET state='cancelled',reason='superseded',finished_at=? WHERE id=?").run(now(), String(job.id));
+                    continue;
+                }
+                db.prepare("UPDATE digest_jobs SET state='running',attempt_count=attempt_count+1,owner_incarnation=?,fence=? WHERE id=?").run(owner.ownerIncarnation, owner.fence, String(job.id));
+                return {
+                    ...owner,
+                    jobId: String(job.id),
+                    itemId: String(job.item_id),
+                    revisionId: String(job.revision_id),
+                };
+            }
+        });
+    }
+    /**
+     * Settle a claimed job. A retryable failure under the attempt limit is delayed until `notBefore` (default: a backoff
+     * from the attempt count); `countAttempt: false` gives the attempt back, for a refusal that never reached the model.
+     */
+    finish(claim, outcome) {
+        const db = this.c.db;
+        if ('reason' in outcome && !/^[a-z][a-z-]{0,63}$/.test(outcome.reason))
+            throw new SpongeError('invalid-input');
+        if ('reason' in outcome && outcome.notBefore !== undefined && !Number.isFinite(Date.parse(outcome.notBefore)))
+            throw new SpongeError('invalid-input');
+        transaction(db, () => {
+            checkOwner(db, claim);
+            const job = this.row(claim.jobId);
+            if (job.state !== 'running' || job.owner_incarnation !== claim.ownerIncarnation || job.fence !== claim.fence)
+                throw new SpongeError('owner-busy');
+            const release = 'UPDATE digest_jobs SET owner_incarnation=NULL,fence=NULL WHERE id=?';
+            if (job.cancel_requested) {
+                db.prepare("UPDATE digest_jobs SET state='cancelled',derivation_id=?,finished_at=? WHERE id=?").run('derivationId' in outcome ? outcome.derivationId : null, now(), claim.jobId);
+            }
+            else if ('derivationId' in outcome) {
+                text(outcome.derivationId, 256);
+                db.prepare("UPDATE digest_jobs SET state='succeeded',reason=NULL,retryable=0,not_before=NULL,derivation_id=?,finished_at=? WHERE id=?").run(outcome.derivationId, now(), claim.jobId);
+            }
+            else {
+                const attempts = Number(job.attempt_count) - (outcome.countAttempt === false ? 1 : 0);
+                if (outcome.retryable && (attempts < maxAttempts || outcome.countAttempt === false)) {
+                    const due = outcome.notBefore ?? new Date(Date.now() + 60000 * 2 ** Math.max(0, attempts - 1)).toISOString();
+                    db.prepare("UPDATE digest_jobs SET state='delayed',reason=?,retryable=1,not_before=?,attempt_count=? WHERE id=?").run(outcome.reason, due, attempts, claim.jobId);
+                }
+                else
+                    db.prepare("UPDATE digest_jobs SET state='failed',reason=?,retryable=?,not_before=NULL,attempt_count=?,finished_at=? WHERE id=?").run(outcome.reason, Number(!!outcome.retryable), attempts, now(), claim.jobId);
+            }
+            db.prepare(release).run(claim.jobId);
+        });
+    }
+    /** Whether the user asked to cancel the claimed job while it ran. */
+    cancelRequested(claim) {
+        return this.row(claim.jobId).cancel_requested === 1;
+    }
+    row(id) {
+        text(id, 256);
+        const job = this.c.db.prepare('SELECT * FROM digest_jobs WHERE id=?').get(id);
+        if (!job)
+            throw new SpongeError('unavailable');
+        return job;
+    }
+}

package/dist/src/store/library.d.ts

@@ -6,6 +6,7 @@ import { Revisions } from './revisions.js';
 import { Derivations } from './derivations.js';
 import { Annotations } from './annotations.js';
 import { SearchIndex } from './search-index.js';
+import { DigestJobs } from './digest-jobs.js';
 import { Backups } from './backups.js';
 import { Imports } from './imports.js';
 import type { ReadableItem, StoreOptions } from './types.js';
@@ -32,6 +33,8 @@ export declare class LibraryStore {
     readonly annotations: Annotations;
     /** Embedding index generations, index jobs, pruning and hybrid search. */
     readonly index: SearchIndex;
+    /** Digest jobs and the auto-digest settings. */
+    readonly digests: DigestJobs;
     readonly backups: Backups;
     readonly imports: Imports;
     constructor(root: string, db: DatabaseSync, options: StoreOptions);

package/dist/src/store/library.js

@@ -11,6 +11,7 @@ import { Revisions } from './revisions.js';
 import { Derivations } from './derivations.js';
 import { Annotations } from './annotations.js';
 import { SearchIndex } from './search-index.js';
+import { DigestJobs } from './digest-jobs.js';
 import { Backups } from './backups.js';
 import { Imports } from './imports.js';
 import { validateMetadataWarnings } from '../../contracts/validate.js';
@@ -32,6 +33,8 @@ export class LibraryStore {
     annotations;
     /** Embedding index generations, index jobs, pruning and hybrid search. */
     index;
+    /** Digest jobs and the auto-digest settings. */
+    digests;
     backups;
     imports;
     constructor(root, db, options) {
@@ -39,7 +42,8 @@ export class LibraryStore {
         const c = { root, db, options };
         this.db = db;
         this.index = new SearchIndex(c);
-        this.queue = new Queue(c, this.index.jobs);
+        this.digests = new DigestJobs(c);
+        this.queue = new Queue(c, [this.index.jobs, this.digests.jobs]);
         this.blobs = new Blobs(c);
         this.revisions = new Revisions(c);
         this.derivations = new Derivations(c);

package/dist/src/store/queue.d.ts

@@ -6,7 +6,8 @@ export interface RetryOptions {
     automatic?: boolean;
 }
 /**
- * One kind of job under the shared worker lease. Capture jobs live here; index jobs in `search-index.ts`. The queue dispatches
+ * One kind of job under the shared worker lease. Capture jobs live here; index jobs in `search-index.ts`, digest jobs in
+ * `digest-jobs.ts`. The queue dispatches
  * listing, cancellation, retry and owner recovery through this interface, so a new job kind plugs in without new branches.
  */
 export interface JobKind {
@@ -19,19 +20,22 @@ export interface JobKind {
     cancelRequested(jobId: string): boolean;
     /** Inside `recoverOwner`'s transaction, once custody of the attempt is proved: remove its staging and settle it. */
     recover(attempt: OwnedAttempt): void;
+    /** Inside `recoverOwner`'s transaction: return work the recovered owner held without attempt rows. */
+    release?(owner: OwnerToken): void;
 }
 /** Capture jobs (acquire, extract, commit one source) and the owner lease shared with every other job kind. */
 export declare class Queue {
     private readonly c;
-    private readonly index;
+    private readonly kinds;
     private readonly capture;
-    constructor(c: Context, index: JobKind);
-    /** Index jobs are checked first; any other id is a capture job (which reports `unavailable` when it does not exist). */
+    /** `kinds` are every other job kind, in dispatch order. */
+    constructor(c: Context, kinds: readonly JobKind[]);
+    /** The other kinds are checked first; any other id is a capture job (which reports `unavailable` when it does not exist). */
     private kindOf;
     acquireOwner(ownerIncarnation: string): OwnerToken;
     releaseOwner(token: OwnerToken): void;
     ownerSnapshot(): OwnerToken | null;
-    /** Every attempt the owner claimed, capture first, then index. */
+    /** Every attempt the owner claimed, capture first, then the other kinds in order. */
     ownerAttempts(token: OwnerToken): OwnedAttempt[];
     /** Requires external custody proof for EVERY pending attempt, even after its job committed.
      * IDs are acknowledgements from the custody owner, never evidence inferred from PID/timeouts here. */

package/dist/src/store/queue.js

@@ -12,11 +12,12 @@ const activeStates = "('queued','running','delayed','cleanup-pending')";
 /** Capture jobs (acquire, extract, commit one source) and the owner lease shared with every other job kind. */
 export class Queue {
     c;
-    index;
+    kinds;
     capture;
-    constructor(c, index) {
+    /** `kinds` are every other job kind, in dispatch order. */
+    constructor(c, kinds) {
         this.c = c;
-        this.index = index;
+        this.kinds = kinds;
         const db = c.db;
         this.capture = {
             owns: (id) => !!db.prepare('SELECT 1 FROM jobs WHERE id=?').get(id),
@@ -99,9 +100,9 @@ export class Queue {
             },
         };
     }
-    /** Index jobs are checked first; any other id is a capture job (which reports `unavailable` when it does not exist). */
+    /** The other kinds are checked first; any other id is a capture job (which reports `unavailable` when it does not exist). */
     kindOf(jobId) {
-        return this.index.owns(jobId) ? this.index : this.capture;
+        return this.kinds.find((kind) => kind.owns(jobId)) ?? this.capture;
     }
     acquireOwner(ownerIncarnation) {
         text(ownerIncarnation, 256);
@@ -127,10 +128,10 @@ export class Queue {
     ownerSnapshot() {
         return ownerSnapshot(this.c.db);
     }
-    /** Every attempt the owner claimed, capture first, then index. */
+    /** Every attempt the owner claimed, capture first, then the other kinds in order. */
     ownerAttempts(token) {
         checkOwner(this.c.db, token);
-        return [...this.capture.attempts(token), ...this.index.attempts(token)];
+        return [this.capture, ...this.kinds].flatMap((kind) => kind.attempts(token));
     }
     /** Requires external custody proof for EVERY pending attempt, even after its job committed.
      * IDs are acknowledgements from the custody owner, never evidence inferred from PID/timeouts here. */
@@ -146,7 +147,9 @@ export class Queue {
             if (proof.size !== pending.length || pending.some((a) => !proof.has(a.attemptId)))
                 throw new SpongeError('owner-busy');
             for (const attempt of pending)
-                (attempt.kind === 'index' ? this.index : this.capture).recover(attempt);
+                this.kindOf(attempt.jobId).recover(attempt);
+            for (const kind of this.kinds)
+                kind.release?.(expected);
             this.c.db.prepare('UPDATE worker_owner SET incarnation=NULL,fence=fence+1 WHERE id=1').run();
         });
     }
@@ -217,7 +220,9 @@ export class Queue {
     }
     /** Jobs of every kind, oldest first. */
     list() {
-        return [...this.capture.list(), ...this.index.list()].sort((a, b) => String(a.created_at).localeCompare(String(b.created_at)) || String(a.id).localeCompare(String(b.id)));
+        return [this.capture, ...this.kinds]
+            .flatMap((kind) => kind.list())
+            .sort((a, b) => String(a.created_at).localeCompare(String(b.created_at)) || String(a.id).localeCompare(String(b.id)));
     }
     /** Acquisition data for a capture job. A local file's locator is its path on this device (`local_sources`), never the synced form. */
     source(jobId) {

package/dist/src/store/revisions.js

@@ -11,6 +11,7 @@ import { indexReadable } from '../search/lexical.js';
 import { trackLibrary } from '../indexing/index.js';
 import { checkAttempt } from './lease.js';
 import { importReceipt } from './imports.js';
+import { enqueueDigest } from './digest-jobs.js';
 import { bumpEpoch, now, transaction } from './context.js';
 import { SpongeError } from '../errors.js';
 /** Captured revisions: content-addressed, immutable once committed, and the item's readable pointer. */
@@ -151,6 +152,9 @@ export class Revisions {
                 bumpEpoch(db);
                 // Active indexes follow the new revision; only its new chunks are queued for embedding.
                 trackLibrary(db);
+                // A capture, not an import: imported libraries are digested on purpose, not as a side effect.
+                if (!imported)
+                    enqueueDigest(db, input.itemId, revisionId, String(stored.markdown));
             }
             if (imported) {
                 const receipt = importReceipt(db, imported.namespace, imported.sourceKey, imported.sourceHash);

package/dist/src/worker/custody.d.ts

@@ -7,6 +7,10 @@ export interface WorkerCustody {
 export declare function acquireWorkerCustody(root: string): Promise<WorkerCustody>;
 /** Receipts are specific to every pre-launch ticket. Missing receipts never mean absent children. */
 export declare function proveAttemptCleanup(ownerDirectory: string, incarnation: string, attemptId: string): Promise<boolean>;
+/** Attempts outside the job tables: a search query, an import verification, a digest server. Each writes `KIND.json`
+ * under its attempt and `staging/KIND-ID/ownership.json`. */
+declare const transientKinds: readonly ["query", "import-verification", "digest-server"];
+export type TransientKind = (typeof transientKinds)[number];
 export type RecoveryFault = 'guarded' | 'recovered';
 export declare function recoverWorker(store: LibraryStore, options?: {
     fault?: (point: RecoveryFault) => void;
@@ -14,3 +18,4 @@ export declare function recoverWorker(store: LibraryStore, options?: {
     recovered: boolean;
     attempts: number;
 }>;
+export {};