@capybara-hq/kernel 0.1.0-rc.23 → 0.1.0-rc.24

package/dist/runtime/run-orphan-check.job.d.ts

@@ -0,0 +1,142 @@
+import { EventLog } from "../conversations/event-log.js";
+import type { ConversationKind, KernelSchemaDb } from "../db/kernel-schema.js";
+import { KernelConfigService } from "../infra/config/kernel-config.service.js";
+import { KernelDb } from "../infra/database/kernel-db.js";
+import { TransactionHost } from "../infra/database/transaction-host.js";
+import { KernelEvents } from "../infra/events/kernel-events.js";
+import { LockService } from "../infra/lock/lock.service.js";
+import type { KernelLogger } from "../log.js";
+import { KyselyCheckpointSaver } from "./checkpointer.js";
+import { ConversationLock } from "./conversation-lock.js";
+import { QueueLostJobs } from "./queue-lost-jobs.js";
+import { RunFinish } from "./run-finish.js";
+import { RunJobs } from "./run-jobs.js";
+import { RunQueue } from "./run-queue.js";
+import { Waits } from "./waits.js";
+/** The check's `@LockedInterval` job name (its run lock key is the default `cron:run-orphan-check`). */
+export declare const RUN_ORPHAN_CHECK_JOB = "run-orphan-check";
+/** The check's period: a fixed constant, not a config field. */
+export declare const RUN_ORPHAN_CHECK_PERIOD_MS = 60000;
+/**
+ * How long a run must have been in its state (queued, or running since its last start) before it
+ * can count as an orphan: a transition in flight is never repaired.
+ */
+export declare const RUN_ORPHAN_MIN_AGE_MS = 60000;
+/** The candidates one page of a pass reads (a keyset page over the state's start time and id). */
+export declare const RUN_ORPHAN_CHECK_BATCH = 500;
+/**
+ * The most pages one pass reads: a bound on its work. Healthy long runs and queued runs waiting
+ * for a quota slot are passed over page by page, so a real orphan behind them is still reached;
+ * beyond this many candidates the rest wait for the next minute.
+ */
+export declare const RUN_ORPHAN_CHECK_MAX_PAGES = 10;
+/** The error the canary logs (transactional outbox spec §8.3). */
+export declare const QUEUE_LOST_JOBS_MESSAGE = "queued run without a job: the queue lost jobs";
+/** What one pass of {@link RunOrphanCheck.check} repaired. */
+export interface RunOrphanCheckResult {
+    /** `running` runs nobody runs, failed `run.orphaned`. */
+    runningRepaired: number;
+    /** `queued` runs whose job was lost, owed again (each one is the canary). */
+    queuedReadded: number;
+}
+/** An active run that may be an orphan, with where its job lives. */
+export interface OrphanCandidate {
+    runId: string;
+    conversationId: string;
+    orgId: string;
+    kind: ConversationKind;
+    status: "queued" | "running";
+    /** The run's job id (a resume's `runs.job_id`, else its trigger's) and queue. */
+    jobId: string;
+    queue: string;
+    /** When the run entered its state, as PG prints it (compared again in the repair). */
+    since: string;
+}
+/**
+ * One page of the active runs of chat and sub-agent conversations in `statuses` that entered their
+ * state more than `minAgeMs` ago, by (state start, id), after `after` (an eval's hidden
+ * conversation drives its runs itself).
+ */
+export declare function orphanCandidates(db: KernelSchemaDb, statuses: readonly ("queued" | "running")[], minAgeMs: number, limit: number, after?: Pick<OrphanCandidate, "since" | "runId">): Promise<OrphanCandidate[]>;
+/**
+ * Whether the outbox holds the job `jobId` on `queue`: a pending row (it is still owed) or a dead
+ * one (the outbox gave up on it; the super admin sees it under `outbox_dead` and retries or
+ * discards it, so it is not a job the queue lost).
+ */
+export declare function jobInOutbox(db: KernelSchemaDb, queue: string, jobId: string): Promise<boolean>;
+/**
+ * The `queued` runs (in their state for over `minAgeMs`) whose job is neither in the outbox
+ * (owed, or dead) nor in BullMQ (`jobExists`), and that are still queued after both looks: the run
+ * queue lost their jobs. The outbox is read before BullMQ (a row is deleted only after its job was added) and the
+ * run again after both (a job that finished took its run out of `queued` first), so a run in
+ * flight is never reported. The `run-orphan-check` uses it; so does the dual-replica test harness,
+ * which asserts at its teardown that the kernel left none (`minAgeMs` 0).
+ */
+export declare function queuedRunsWithoutJob(db: KernelSchemaDb, jobExists: (queue: string, jobId: string) => Promise<boolean>, opts: {
+    minAgeMs: number;
+}): Promise<OrphanCandidate[]>;
+/**
+ * The `run-orphan-check` (transactional outbox spec §6 "Replica death during a run", §8.3): once a
+ * minute across replicas (worker role), the one repair of a run nobody runs. A run counts when it
+ * has been in its state for over {@link RUN_ORPHAN_MIN_AGE_MS} with neither its BullMQ job nor a
+ * conversation lock holder:
+ * - `running`: its replica died while running it (BullMQ's stalled re-deliveries ran out, or the
+ *   process died inside `onFailed`; run jobs are `attempts: 1, removeOnFail`). It is failed
+ *   `run.orphaned` (its terminal transaction cancels its waits and owes what every run end owes),
+ *   then, after the commit, its checkpoints are dropped, its fork child stopped (`run.orphaned`), a
+ *   sub-agent's failure delivered to its parent at once and the conversation's next run started.
+ * - `queued`, with no outbox row either (pending or dead): a canary. Every queued run has its job owed in its
+ *   own transaction, so this means Redis lost data. It is logged as an error, counted for the admin
+ *   overview ({@link QueueLostJobs}) and its job owed again so the run is not stuck.
+ * Redis is read before each repair's transaction, which then re-reads the run under its
+ * conversation row lock and acts only if it is still in the same state since the same time. Read
+ * paths never repair runs.
+ */
+export declare class RunOrphanCheck {
+    private readonly kdb;
+    private readonly jobs;
+    private readonly lock;
+    private readonly finisher;
+    private readonly eventLog;
+    private readonly checkpoints;
+    /** A failed sub-agent run reports to its parent at once ({@link Waits.childFinished}). */
+    private readonly waits;
+    /** The conversation's next run, after a repaired run's end. */
+    private readonly queue;
+    private readonly lostJobs;
+    private readonly events;
+    readonly txHost: TransactionHost;
+    readonly config: KernelConfigService;
+    readonly lockService: LockService;
+    readonly log: KernelLogger;
+    constructor(kdb: KernelDb, jobs: Pick<RunJobs, "getJob" | "owe">, lock: Pick<ConversationLock, "holder">, finisher: Pick<RunFinish, "finishIn">, eventLog: Pick<EventLog, "notify" | "publishChanged">, checkpoints: Pick<KyselyCheckpointSaver, "deleteThread">, 
+    /** A failed sub-agent run reports to its parent at once ({@link Waits.childFinished}). */
+    waits: Pick<Waits, "childFinished">, 
+    /** The conversation's next run, after a repaired run's end. */
+    queue: Pick<RunQueue, "ensureRun">, lostJobs: Pick<QueueLostJobs, "report">, events: KernelEvents, txHost: TransactionHost, config: KernelConfigService, lockService: LockService, log: KernelLogger);
+    private get db();
+    run(): Promise<void>;
+    /** One pass: repairs the orphans it finds (each on its own; a failure is logged). */
+    check(): Promise<RunOrphanCheckResult>;
+    /**
+     * Neither a conversation lock holder nor the run's job in BullMQ (nor, queued, in the outbox: an
+     * owed row is delivered by the relay, a dead one waits for the super admin).
+     */
+    private withoutJobOrHolder;
+    /**
+     * Fails a `running` orphan (`run.orphaned`), then does what its end causes after the commit:
+     * the open pages hear it, its checkpoints go, its fork child stops, a sub-agent reports to its
+     * parent and the conversation's next run starts (each also owed by the terminal transaction).
+     */
+    private failRunning;
+    /** {@link failRunning}'s transaction: the terminal state, if the run is still the same orphan. */
+    private failIn;
+    /** Owes a lost queued run's job again under its own id, if it is still the same orphan. */
+    private oweAgainIn;
+    /**
+     * Under the conversation row lock (the first lock of every path that changes a run's state): the
+     * run is still in the state and since the time the pass read. Not `FOR UPDATE` on the run: the
+     * terminal transaction locks it after the run's wait rows.
+     */
+    private stillIn;
+}

package/dist/runtime/run-orphan-check.job.js

@@ -0,0 +1,339 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+import { Inject, Injectable } from "@nestjs/common";
+import { sql } from "kysely";
+import { EventLog } from "../conversations/event-log.js";
+import { KernelConfigService } from "../infra/config/kernel-config.service.js";
+import { KernelDb } from "../infra/database/kernel-db.js";
+import { TransactionHost } from "../infra/database/transaction-host.js";
+import { Transactional } from "../infra/database/transactional.decorator.js";
+import { KernelEvents } from "../infra/events/kernel-events.js";
+import { LockService } from "../infra/lock/lock.service.js";
+import { LockedInterval } from "../infra/schedule/locked-interval.js";
+import { KERNEL_LOGGER } from "../infra/tokens.js";
+import { KyselyCheckpointSaver } from "./checkpointer.js";
+import { ConversationLock } from "./conversation-lock.js";
+import { QueueLostJobs } from "./queue-lost-jobs.js";
+import { notifyFinished, RunFinish } from "./run-finish.js";
+import { RunJobs, runJobId, runQueueNameFor } from "./run-jobs.js";
+import { RunQueue } from "./run-queue.js";
+import { Waits } from "./waits.js";
+/** The check's `@LockedInterval` job name (its run lock key is the default `cron:run-orphan-check`). */
+export const RUN_ORPHAN_CHECK_JOB = "run-orphan-check";
+/** The check's period: a fixed constant, not a config field. */
+export const RUN_ORPHAN_CHECK_PERIOD_MS = 60_000;
+/**
+ * How long a run must have been in its state (queued, or running since its last start) before it
+ * can count as an orphan: a transition in flight is never repaired.
+ */
+export const RUN_ORPHAN_MIN_AGE_MS = 60_000;
+/** The candidates one page of a pass reads (a keyset page over the state's start time and id). */
+export const RUN_ORPHAN_CHECK_BATCH = 500;
+/**
+ * The most pages one pass reads: a bound on its work. Healthy long runs and queued runs waiting
+ * for a quota slot are passed over page by page, so a real orphan behind them is still reached;
+ * beyond this many candidates the rest wait for the next minute.
+ */
+export const RUN_ORPHAN_CHECK_MAX_PAGES = 10;
+/** The error the canary logs (transactional outbox spec §8.3). */
+export const QUEUE_LOST_JOBS_MESSAGE = "queued run without a job: the queue lost jobs";
+/** The SQL of the time a run entered its current state (queued, or its last start). */
+const SINCE = sql `GREATEST(COALESCE(r.queued_at, r.created_at), COALESCE(r.started_at, r.created_at))`;
+/**
+ * One page of the active runs of chat and sub-agent conversations in `statuses` that entered their
+ * state more than `minAgeMs` ago, by (state start, id), after `after` (an eval's hidden
+ * conversation drives its runs itself).
+ */
+export async function orphanCandidates(db, statuses, minAgeMs, limit, after) {
+    let q = db
+        .selectFrom("runs as r")
+        .innerJoin("conversations as c", "c.id", "r.conversation_id")
+        .select([
+        "r.id",
+        "r.conversation_id",
+        "r.status",
+        "r.job_id",
+        "r.trigger_seq",
+        "c.kind",
+        "c.org_id",
+        sql `${SINCE}::text`.as("since"),
+    ])
+        .where("r.status", "in", statuses)
+        .where("c.kind", "in", ["chat", "subagent"])
+        .where(SINCE, "<", sql `now() - ${minAgeMs}::integer * interval '1 millisecond'`);
+    if (after !== undefined) {
+        q = q.where(sql `(${SINCE}, r.id) > (${after.since}::timestamptz, ${after.runId})`);
+    }
+    const rows = await q.orderBy(SINCE).orderBy("r.id").limit(limit).execute();
+    return rows.map((r) => ({
+        runId: r.id,
+        conversationId: r.conversation_id,
+        orgId: r.org_id,
+        kind: r.kind,
+        status: r.status,
+        jobId: r.job_id ?? runJobId(r.conversation_id, Number(r.trigger_seq)),
+        queue: runQueueNameFor(r.kind),
+        since: r.since,
+    }));
+}
+/**
+ * Whether the outbox holds the job `jobId` on `queue`: a pending row (it is still owed) or a dead
+ * one (the outbox gave up on it; the super admin sees it under `outbox_dead` and retries or
+ * discards it, so it is not a job the queue lost).
+ */
+export async function jobInOutbox(db, queue, jobId) {
+    const row = await db
+        .selectFrom("outbox")
+        .select("id")
+        .where("queue", "=", queue)
+        .where("job_id", "=", jobId)
+        .executeTakeFirst();
+    return row !== undefined;
+}
+/** Every page of {@link orphanCandidates}, at most {@link RUN_ORPHAN_CHECK_MAX_PAGES}. */
+async function* candidatePages(db, statuses, minAgeMs) {
+    let after;
+    for (let page = 0; page < RUN_ORPHAN_CHECK_MAX_PAGES; page++) {
+        const rows = await orphanCandidates(db, statuses, minAgeMs, RUN_ORPHAN_CHECK_BATCH, after);
+        if (rows.length > 0)
+            yield rows;
+        if (rows.length < RUN_ORPHAN_CHECK_BATCH)
+            return;
+        after = rows[rows.length - 1];
+    }
+}
+/**
+ * The `queued` runs (in their state for over `minAgeMs`) whose job is neither in the outbox
+ * (owed, or dead) nor in BullMQ (`jobExists`), and that are still queued after both looks: the run
+ * queue lost their jobs. The outbox is read before BullMQ (a row is deleted only after its job was added) and the
+ * run again after both (a job that finished took its run out of `queued` first), so a run in
+ * flight is never reported. The `run-orphan-check` uses it; so does the dual-replica test harness,
+ * which asserts at its teardown that the kernel left none (`minAgeMs` 0).
+ */
+export async function queuedRunsWithoutJob(db, jobExists, opts) {
+    const out = [];
+    for await (const page of candidatePages(db, ["queued"], opts.minAgeMs)) {
+        for (const c of page) {
+            if (await jobInOutbox(db, c.queue, c.jobId))
+                continue;
+            if (await jobExists(c.queue, c.jobId))
+                continue;
+            const now = await db
+                .selectFrom("runs")
+                .select("status")
+                .where("id", "=", c.runId)
+                .executeTakeFirst();
+            if (now?.status === "queued")
+                out.push(c);
+        }
+    }
+    return out;
+}
+/**
+ * The `run-orphan-check` (transactional outbox spec §6 "Replica death during a run", §8.3): once a
+ * minute across replicas (worker role), the one repair of a run nobody runs. A run counts when it
+ * has been in its state for over {@link RUN_ORPHAN_MIN_AGE_MS} with neither its BullMQ job nor a
+ * conversation lock holder:
+ * - `running`: its replica died while running it (BullMQ's stalled re-deliveries ran out, or the
+ *   process died inside `onFailed`; run jobs are `attempts: 1, removeOnFail`). It is failed
+ *   `run.orphaned` (its terminal transaction cancels its waits and owes what every run end owes),
+ *   then, after the commit, its checkpoints are dropped, its fork child stopped (`run.orphaned`), a
+ *   sub-agent's failure delivered to its parent at once and the conversation's next run started.
+ * - `queued`, with no outbox row either (pending or dead): a canary. Every queued run has its job owed in its
+ *   own transaction, so this means Redis lost data. It is logged as an error, counted for the admin
+ *   overview ({@link QueueLostJobs}) and its job owed again so the run is not stuck.
+ * Redis is read before each repair's transaction, which then re-reads the run under its
+ * conversation row lock and acts only if it is still in the same state since the same time. Read
+ * paths never repair runs.
+ */
+let RunOrphanCheck = class RunOrphanCheck {
+    kdb;
+    jobs;
+    lock;
+    finisher;
+    eventLog;
+    checkpoints;
+    waits;
+    queue;
+    lostJobs;
+    events;
+    txHost;
+    config;
+    lockService;
+    log;
+    constructor(kdb, jobs, lock, finisher, eventLog, checkpoints, 
+    /** A failed sub-agent run reports to its parent at once ({@link Waits.childFinished}). */
+    waits, 
+    /** The conversation's next run, after a repaired run's end. */
+    queue, lostJobs, events, txHost, config, lockService, log) {
+        this.kdb = kdb;
+        this.jobs = jobs;
+        this.lock = lock;
+        this.finisher = finisher;
+        this.eventLog = eventLog;
+        this.checkpoints = checkpoints;
+        this.waits = waits;
+        this.queue = queue;
+        this.lostJobs = lostJobs;
+        this.events = events;
+        this.txHost = txHost;
+        this.config = config;
+        this.lockService = lockService;
+        this.log = log;
+    }
+    get db() {
+        return this.kdb.db;
+    }
+    async run() {
+        await this.check();
+    }
+    /** One pass: repairs the orphans it finds (each on its own; a failure is logged). */
+    async check() {
+        const out = { runningRepaired: 0, queuedReadded: 0 };
+        const lost = [];
+        const pages = candidatePages(this.db, ["queued", "running"], RUN_ORPHAN_MIN_AGE_MS);
+        for await (const page of pages) {
+            for (const c of page) {
+                try {
+                    if (!(await this.withoutJobOrHolder(c)))
+                        continue;
+                    if (c.status === "running") {
+                        if (await this.failRunning(c))
+                            out.runningRepaired++;
+                    }
+                    else if (await this.oweAgainIn(c)) {
+                        out.queuedReadded++;
+                        lost.push(c);
+                    }
+                }
+                catch (err) {
+                    this.log.error("run-orphan-check: cannot repair a run", { runId: c.runId, err });
+                }
+            }
+        }
+        if (lost.length > 0) {
+            this.log.error(QUEUE_LOST_JOBS_MESSAGE, {
+                count: lost.length,
+                runs: lost.map((c) => ({ runId: c.runId, conversationId: c.conversationId })),
+            });
+            await this.lostJobs.report(lost.length);
+        }
+        return out;
+    }
+    /**
+     * Neither a conversation lock holder nor the run's job in BullMQ (nor, queued, in the outbox: an
+     * owed row is delivered by the relay, a dead one waits for the super admin).
+     */
+    async withoutJobOrHolder(c) {
+        if ((await this.lock.holder(c.conversationId)) !== null)
+            return false;
+        if (c.status === "queued" && (await jobInOutbox(this.db, c.queue, c.jobId)))
+            return false;
+        return (await this.jobs.getJob(c.jobId, c.queue)) === undefined;
+    }
+    /**
+     * Fails a `running` orphan (`run.orphaned`), then does what its end causes after the commit:
+     * the open pages hear it, its checkpoints go, its fork child stops, a sub-agent reports to its
+     * parent and the conversation's next run starts (each also owed by the terminal transaction).
+     */
+    async failRunning(c) {
+        const seq = await this.failIn(c);
+        if (seq === null)
+            return false;
+        this.log.warn("run-orphan-check: failed a running run nobody runs", {
+            runId: c.runId,
+            conversationId: c.conversationId,
+        });
+        await notifyFinished(this.eventLog, c.conversationId, seq, "failed");
+        await this.checkpoints
+            .deleteThread(c.runId)
+            .catch((err) => this.log.warn("checkpoint delete failed", { runId: c.runId, err }));
+        await this.events.emit("run.orphaned", { runId: c.runId });
+        await this.waits.childFinished(c.runId).catch((err) => this.log.error("cannot report an orphaned sub-agent run to its parent", {
+            runId: c.runId,
+            err,
+        }));
+        await this.queue.ensureRun(c.conversationId).catch((err) => this.log.error("cannot start the next run after an orphan", {
+            conversationId: c.conversationId,
+            err,
+        }));
+        return true;
+    }
+    /** {@link failRunning}'s transaction: the terminal state, if the run is still the same orphan. */
+    async failIn(c) {
+        if (!(await this.stillIn(c)))
+            return null;
+        return this.finisher.finishIn({
+            conversationId: c.conversationId,
+            runId: c.runId,
+            status: "failed",
+            consumed: null,
+            event: {
+                type: "run_failed",
+                payload: {
+                    error: { code: "run.orphaned", message: "The run was lost and will not continue" },
+                },
+            },
+        });
+    }
+    /** Owes a lost queued run's job again under its own id, if it is still the same orphan. */
+    async oweAgainIn(c) {
+        if (!(await this.stillIn(c)))
+            return false;
+        const data = { runId: c.runId, conversationId: c.conversationId };
+        return this.jobs.owe(data, c.jobId, c.kind, c.orgId);
+    }
+    /**
+     * Under the conversation row lock (the first lock of every path that changes a run's state): the
+     * run is still in the state and since the time the pass read. Not `FOR UPDATE` on the run: the
+     * terminal transaction locks it after the run's wait rows.
+     */
+    async stillIn(c) {
+        await this.db
+            .selectFrom("conversations")
+            .select("id")
+            .where("id", "=", c.conversationId)
+            .forUpdate()
+            .executeTakeFirst();
+        const r = await this.db
+            .selectFrom("runs as r")
+            .select(["r.status", sql `${SINCE}::text`.as("since")])
+            .where("r.id", "=", c.runId)
+            .executeTakeFirst();
+        return r?.status === c.status && r.since === c.since;
+    }
+};
+__decorate([
+    LockedInterval(RUN_ORPHAN_CHECK_JOB, () => RUN_ORPHAN_CHECK_PERIOD_MS)
+], RunOrphanCheck.prototype, "run", null);
+__decorate([
+    Transactional({ propagation: "requires_new" })
+], RunOrphanCheck.prototype, "failIn", null);
+__decorate([
+    Transactional({ propagation: "requires_new" })
+], RunOrphanCheck.prototype, "oweAgainIn", null);
+RunOrphanCheck = __decorate([
+    Injectable(),
+    __param(0, Inject(KernelDb)),
+    __param(1, Inject(RunJobs)),
+    __param(2, Inject(ConversationLock)),
+    __param(3, Inject(RunFinish)),
+    __param(4, Inject(EventLog)),
+    __param(5, Inject(KyselyCheckpointSaver)),
+    __param(6, Inject(Waits)),
+    __param(7, Inject(RunQueue)),
+    __param(8, Inject(QueueLostJobs)),
+    __param(9, Inject(KernelEvents)),
+    __param(10, Inject(TransactionHost)),
+    __param(11, Inject(KernelConfigService)),
+    __param(12, Inject(LockService)),
+    __param(13, Inject(KERNEL_LOGGER))
+], RunOrphanCheck);
+export { RunOrphanCheck };

package/dist/runtime/run-processors.d.ts

@@ -3,33 +3,40 @@ import { type DynamicModule } from "@nestjs/common";
 import type { Job, Worker } from "bullmq";
 import type { KernelConfig } from "../config/schema.js";
 import { TitleQueue } from "../conversations/titles.js";
-import { type RunJobData } from "./run-jobs.js";
+import { type RUN_JOB, type RunJobData } from "./run-jobs.js";
 import { RunWorker } from "./run-worker.js";
-import { UsageReporter, type UsageReportJob } from "./usage-report.js";
+import { type USAGE_REPORT_JOB, UsageReporter, type UsageReportJob } from "./usage-report.js";
 type RunResult = Awaited<ReturnType<RunWorker["process"]>>;
+/** A run job: both run queues carry only {@link RUN_JOB} jobs (the outbox registry's `run`). */
+type RunJob = Job<RunJobData, RunResult, typeof RUN_JOB>;
 /** The consumer of `cap-kernel-runs` (conversation runs): {@link RunWorker}. */
 export declare class RunsProcessor extends WorkerHost<Worker<RunJobData, RunResult>> {
     private readonly runs;
     constructor(runs: RunWorker);
-    process(job: Job<RunJobData>, token?: string): Promise<RunResult>;
-    completed(job: Job<RunJobData>, result: RunResult): void;
-    failed(job: Job<RunJobData> | undefined, err: Error): void;
+    process(job: RunJob, token?: string): Promise<RunResult>;
+    completed(job: RunJob, result: RunResult): void;
+    failed(job: RunJob | undefined, err: Error): void;
     error(err: Error): void;
 }
 /** The consumer of `cap-kernel-subagent-runs` (sub-agent runs, spec §7.4): {@link RunWorker}. */
 export declare class SubagentRunsProcessor extends WorkerHost<Worker<RunJobData, RunResult>> {
     private readonly runs;
     constructor(runs: RunWorker);
-    process(job: Job<RunJobData>, token?: string): Promise<RunResult>;
-    completed(job: Job<RunJobData>, result: RunResult): void;
-    failed(job: Job<RunJobData> | undefined, err: Error): void;
+    process(job: RunJob, token?: string): Promise<RunResult>;
+    completed(job: RunJob, result: RunResult): void;
+    failed(job: RunJob | undefined, err: Error): void;
     error(err: Error): void;
 }
+/**
+ * A delivery job: the usage report queue carries only {@link USAGE_REPORT_JOB} jobs (the outbox
+ * registry's `usage-report`).
+ */
+type UsageJob = Job<UsageReportJob, void, typeof USAGE_REPORT_JOB>;
 /** The consumer of `cap-kernel-usage-report`: {@link UsageReporter}. */
 export declare class UsageReportProcessor extends WorkerHost<Worker<UsageReportJob>> {
     private readonly reporter;
     constructor(reporter: UsageReporter);
-    process(job: Job<UsageReportJob>, token?: string): Promise<void>;
+    process(job: UsageJob, token?: string): Promise<void>;
     failed(job: Job<UsageReportJob> | undefined, err: Error): void;
     error(err: Error): void;
 }

package/dist/runtime/run-processors.js

@@ -14,7 +14,7 @@ import { Inject, Injectable, Module } from "@nestjs/common";
 import { TITLE_QUEUE, TitleQueue, TitlesModule } from "../conversations/titles.js";
 import { configuredProcessor } from "../infra/queue/configured-processor.js";
 import { runJob } from "../infra/queue/run-job.js";
-import { RUN_QUEUE_NAME, SUBAGENT_RUN_QUEUE_NAME } from "./run-jobs.js";
+import { RUN_QUEUE_NAME, SUBAGENT_RUN_QUEUE_NAME, } from "./run-jobs.js";
 import { MAX_STALLED_COUNT, RunWorker } from "./run-worker.js";
 import { RuntimeModule } from "./runtime.module.js";
 import { USAGE_REPORT_QUEUE, UsageReporter, UsageReportModule, } from "./usage-report.js";