@capybara-hq/kernel 0.1.0-rc.23 → 0.1.0-rc.25

package/dist/runtime/run-orphan-check.job.js

@@ -0,0 +1,339 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+import { Inject, Injectable } from "@nestjs/common";
+import { sql } from "kysely";
+import { EventLog } from "../conversations/event-log.js";
+import { KernelConfigService } from "../infra/config/kernel-config.service.js";
+import { KernelDb } from "../infra/database/kernel-db.js";
+import { TransactionHost } from "../infra/database/transaction-host.js";
+import { Transactional } from "../infra/database/transactional.decorator.js";
+import { KernelEvents } from "../infra/events/kernel-events.js";
+import { LockService } from "../infra/lock/lock.service.js";
+import { LockedInterval } from "../infra/schedule/locked-interval.js";
+import { KERNEL_LOGGER } from "../infra/tokens.js";
+import { KyselyCheckpointSaver } from "./checkpointer.js";
+import { ConversationLock } from "./conversation-lock.js";
+import { QueueLostJobs } from "./queue-lost-jobs.js";
+import { notifyFinished, RunFinish } from "./run-finish.js";
+import { RunJobs, runJobId, runQueueNameFor } from "./run-jobs.js";
+import { RunQueue } from "./run-queue.js";
+import { Waits } from "./waits.js";
+/** The check's `@LockedInterval` job name (its run lock key is the default `cron:run-orphan-check`). */
+export const RUN_ORPHAN_CHECK_JOB = "run-orphan-check";
+/** The check's period: a fixed constant, not a config field. */
+export const RUN_ORPHAN_CHECK_PERIOD_MS = 60_000;
+/**
+ * How long a run must have been in its state (queued, or running since its last start) before it
+ * can count as an orphan: a transition in flight is never repaired.
+ */
+export const RUN_ORPHAN_MIN_AGE_MS = 60_000;
+/** The candidates one page of a pass reads (a keyset page over the state's start time and id). */
+export const RUN_ORPHAN_CHECK_BATCH = 500;
+/**
+ * The most pages one pass reads: a bound on its work. Healthy long runs and queued runs waiting
+ * for a quota slot are passed over page by page, so a real orphan behind them is still reached;
+ * beyond this many candidates the rest wait for the next minute.
+ */
+export const RUN_ORPHAN_CHECK_MAX_PAGES = 10;
+/** The error the canary logs (transactional outbox spec §8.3). */
+export const QUEUE_LOST_JOBS_MESSAGE = "queued run without a job: the queue lost jobs";
+/** The SQL of the time a run entered its current state (queued, or its last start). */
+const SINCE = sql `GREATEST(COALESCE(r.queued_at, r.created_at), COALESCE(r.started_at, r.created_at))`;
+/**
+ * One page of the active runs of chat and sub-agent conversations in `statuses` that entered their
+ * state more than `minAgeMs` ago, by (state start, id), after `after` (an eval's hidden
+ * conversation drives its runs itself).
+ */
+export async function orphanCandidates(db, statuses, minAgeMs, limit, after) {
+    let q = db
+        .selectFrom("runs as r")
+        .innerJoin("conversations as c", "c.id", "r.conversation_id")
+        .select([
+        "r.id",
+        "r.conversation_id",
+        "r.status",
+        "r.job_id",
+        "r.trigger_seq",
+        "c.kind",
+        "c.org_id",
+        sql `${SINCE}::text`.as("since"),
+    ])
+        .where("r.status", "in", statuses)
+        .where("c.kind", "in", ["chat", "subagent"])
+        .where(SINCE, "<", sql `now() - ${minAgeMs}::integer * interval '1 millisecond'`);
+    if (after !== undefined) {
+        q = q.where(sql `(${SINCE}, r.id) > (${after.since}::timestamptz, ${after.runId})`);
+    }
+    const rows = await q.orderBy(SINCE).orderBy("r.id").limit(limit).execute();
+    return rows.map((r) => ({
+        runId: r.id,
+        conversationId: r.conversation_id,
+        orgId: r.org_id,
+        kind: r.kind,
+        status: r.status,
+        jobId: r.job_id ?? runJobId(r.conversation_id, Number(r.trigger_seq)),
+        queue: runQueueNameFor(r.kind),
+        since: r.since,
+    }));
+}
+/**
+ * Whether the outbox holds the job `jobId` on `queue`: a pending row (it is still owed) or a dead
+ * one (the outbox gave up on it; the super admin sees it under `outbox_dead` and retries or
+ * discards it, so it is not a job the queue lost).
+ */
+export async function jobInOutbox(db, queue, jobId) {
+    const row = await db
+        .selectFrom("outbox")
+        .select("id")
+        .where("queue", "=", queue)
+        .where("job_id", "=", jobId)
+        .executeTakeFirst();
+    return row !== undefined;
+}
+/** Every page of {@link orphanCandidates}, at most {@link RUN_ORPHAN_CHECK_MAX_PAGES}. */
+async function* candidatePages(db, statuses, minAgeMs) {
+    let after;
+    for (let page = 0; page < RUN_ORPHAN_CHECK_MAX_PAGES; page++) {
+        const rows = await orphanCandidates(db, statuses, minAgeMs, RUN_ORPHAN_CHECK_BATCH, after);
+        if (rows.length > 0)
+            yield rows;
+        if (rows.length < RUN_ORPHAN_CHECK_BATCH)
+            return;
+        after = rows[rows.length - 1];
+    }
+}
+/**
+ * The `queued` runs (in their state for over `minAgeMs`) whose job is neither in the outbox
+ * (owed, or dead) nor in BullMQ (`jobExists`), and that are still queued after both looks: the run
+ * queue lost their jobs. The outbox is read before BullMQ (a row is deleted only after its job was added) and the
+ * run again after both (a job that finished took its run out of `queued` first), so a run in
+ * flight is never reported. The `run-orphan-check` uses it; so does the dual-replica test harness,
+ * which asserts at its teardown that the kernel left none (`minAgeMs` 0).
+ */
+export async function queuedRunsWithoutJob(db, jobExists, opts) {
+    const out = [];
+    for await (const page of candidatePages(db, ["queued"], opts.minAgeMs)) {
+        for (const c of page) {
+            if (await jobInOutbox(db, c.queue, c.jobId))
+                continue;
+            if (await jobExists(c.queue, c.jobId))
+                continue;
+            const now = await db
+                .selectFrom("runs")
+                .select("status")
+                .where("id", "=", c.runId)
+                .executeTakeFirst();
+            if (now?.status === "queued")
+                out.push(c);
+        }
+    }
+    return out;
+}
+/**
+ * The `run-orphan-check` (transactional outbox spec §6 "Replica death during a run", §8.3): once a
+ * minute across replicas (worker role), the one repair of a run nobody runs. A run counts when it
+ * has been in its state for over {@link RUN_ORPHAN_MIN_AGE_MS} with neither its BullMQ job nor a
+ * conversation lock holder:
+ * - `running`: its replica died while running it (BullMQ's stalled re-deliveries ran out, or the
+ *   process died inside `onFailed`; run jobs are `attempts: 1, removeOnFail`). It is failed
+ *   `run.orphaned` (its terminal transaction cancels its waits and owes what every run end owes),
+ *   then, after the commit, its checkpoints are dropped, its fork child stopped (`run.orphaned`), a
+ *   sub-agent's failure delivered to its parent at once and the conversation's next run started.
+ * - `queued`, with no outbox row either (pending or dead): a canary. Every queued run has its job owed in its
+ *   own transaction, so this means Redis lost data. It is logged as an error, counted for the admin
+ *   overview ({@link QueueLostJobs}) and its job owed again so the run is not stuck.
+ * Redis is read before each repair's transaction, which then re-reads the run under its
+ * conversation row lock and acts only if it is still in the same state since the same time. Read
+ * paths never repair runs.
+ */
+let RunOrphanCheck = class RunOrphanCheck {
+    kdb;
+    jobs;
+    lock;
+    finisher;
+    eventLog;
+    checkpoints;
+    waits;
+    queue;
+    lostJobs;
+    events;
+    txHost;
+    config;
+    lockService;
+    log;
+    constructor(kdb, jobs, lock, finisher, eventLog, checkpoints, 
+    /** A failed sub-agent run reports to its parent at once ({@link Waits.childFinished}). */
+    waits, 
+    /** The conversation's next run, after a repaired run's end. */
+    queue, lostJobs, events, txHost, config, lockService, log) {
+        this.kdb = kdb;
+        this.jobs = jobs;
+        this.lock = lock;
+        this.finisher = finisher;
+        this.eventLog = eventLog;
+        this.checkpoints = checkpoints;
+        this.waits = waits;
+        this.queue = queue;
+        this.lostJobs = lostJobs;
+        this.events = events;
+        this.txHost = txHost;
+        this.config = config;
+        this.lockService = lockService;
+        this.log = log;
+    }
+    get db() {
+        return this.kdb.db;
+    }
+    async run() {
+        await this.check();
+    }
+    /** One pass: repairs the orphans it finds (each on its own; a failure is logged). */
+    async check() {
+        const out = { runningRepaired: 0, queuedReadded: 0 };
+        const lost = [];
+        const pages = candidatePages(this.db, ["queued", "running"], RUN_ORPHAN_MIN_AGE_MS);
+        for await (const page of pages) {
+            for (const c of page) {
+                try {
+                    if (!(await this.withoutJobOrHolder(c)))
+                        continue;
+                    if (c.status === "running") {
+                        if (await this.failRunning(c))
+                            out.runningRepaired++;
+                    }
+                    else if (await this.oweAgainIn(c)) {
+                        out.queuedReadded++;
+                        lost.push(c);
+                    }
+                }
+                catch (err) {
+                    this.log.error("run-orphan-check: cannot repair a run", { runId: c.runId, err });
+                }
+            }
+        }
+        if (lost.length > 0) {
+            this.log.error(QUEUE_LOST_JOBS_MESSAGE, {
+                count: lost.length,
+                runs: lost.map((c) => ({ runId: c.runId, conversationId: c.conversationId })),
+            });
+            await this.lostJobs.report(lost.length);
+        }
+        return out;
+    }
+    /**
+     * Neither a conversation lock holder nor the run's job in BullMQ (nor, queued, in the outbox: an
+     * owed row is delivered by the relay, a dead one waits for the super admin).
+     */
+    async withoutJobOrHolder(c) {
+        if ((await this.lock.holder(c.conversationId)) !== null)
+            return false;
+        if (c.status === "queued" && (await jobInOutbox(this.db, c.queue, c.jobId)))
+            return false;
+        return (await this.jobs.getJob(c.jobId, c.queue)) === undefined;
+    }
+    /**
+     * Fails a `running` orphan (`run.orphaned`), then does what its end causes after the commit:
+     * the open pages hear it, its checkpoints go, its fork child stops, a sub-agent reports to its
+     * parent and the conversation's next run starts (each also owed by the terminal transaction).
+     */
+    async failRunning(c) {
+        const seq = await this.failIn(c);
+        if (seq === null)
+            return false;
+        this.log.warn("run-orphan-check: failed a running run nobody runs", {
+            runId: c.runId,
+            conversationId: c.conversationId,
+        });
+        await notifyFinished(this.eventLog, c.conversationId, seq, "failed");
+        await this.checkpoints
+            .deleteThread(c.runId)
+            .catch((err) => this.log.warn("checkpoint delete failed", { runId: c.runId, err }));
+        await this.events.emit("run.orphaned", { runId: c.runId });
+        await this.waits.childFinished(c.runId).catch((err) => this.log.error("cannot report an orphaned sub-agent run to its parent", {
+            runId: c.runId,
+            err,
+        }));
+        await this.queue.ensureRun(c.conversationId).catch((err) => this.log.error("cannot start the next run after an orphan", {
+            conversationId: c.conversationId,
+            err,
+        }));
+        return true;
+    }
+    /** {@link failRunning}'s transaction: the terminal state, if the run is still the same orphan. */
+    async failIn(c) {
+        if (!(await this.stillIn(c)))
+            return null;
+        return this.finisher.finishIn({
+            conversationId: c.conversationId,
+            runId: c.runId,
+            status: "failed",
+            consumed: null,
+            event: {
+                type: "run_failed",
+                payload: {
+                    error: { code: "run.orphaned", message: "The run was lost and will not continue" },
+                },
+            },
+        });
+    }
+    /** Owes a lost queued run's job again under its own id, if it is still the same orphan. */
+    async oweAgainIn(c) {
+        if (!(await this.stillIn(c)))
+            return false;
+        const data = { runId: c.runId, conversationId: c.conversationId };
+        return this.jobs.owe(data, c.jobId, c.kind, c.orgId);
+    }
+    /**
+     * Under the conversation row lock (the first lock of every path that changes a run's state): the
+     * run is still in the state and since the time the pass read. Not `FOR UPDATE` on the run: the
+     * terminal transaction locks it after the run's wait rows.
+     */
+    async stillIn(c) {
+        await this.db
+            .selectFrom("conversations")
+            .select("id")
+            .where("id", "=", c.conversationId)
+            .forUpdate()
+            .executeTakeFirst();
+        const r = await this.db
+            .selectFrom("runs as r")
+            .select(["r.status", sql `${SINCE}::text`.as("since")])
+            .where("r.id", "=", c.runId)
+            .executeTakeFirst();
+        return r?.status === c.status && r.since === c.since;
+    }
+};
+__decorate([
+    LockedInterval(RUN_ORPHAN_CHECK_JOB, () => RUN_ORPHAN_CHECK_PERIOD_MS)
+], RunOrphanCheck.prototype, "run", null);
+__decorate([
+    Transactional({ propagation: "requires_new" })
+], RunOrphanCheck.prototype, "failIn", null);
+__decorate([
+    Transactional({ propagation: "requires_new" })
+], RunOrphanCheck.prototype, "oweAgainIn", null);
+RunOrphanCheck = __decorate([
+    Injectable(),
+    __param(0, Inject(KernelDb)),
+    __param(1, Inject(RunJobs)),
+    __param(2, Inject(ConversationLock)),
+    __param(3, Inject(RunFinish)),
+    __param(4, Inject(EventLog)),
+    __param(5, Inject(KyselyCheckpointSaver)),
+    __param(6, Inject(Waits)),
+    __param(7, Inject(RunQueue)),
+    __param(8, Inject(QueueLostJobs)),
+    __param(9, Inject(KernelEvents)),
+    __param(10, Inject(TransactionHost)),
+    __param(11, Inject(KernelConfigService)),
+    __param(12, Inject(LockService)),
+    __param(13, Inject(KERNEL_LOGGER))
+], RunOrphanCheck);
+export { RunOrphanCheck };

package/dist/runtime/run-processors.d.ts

@@ -3,33 +3,40 @@ import { type DynamicModule } from "@nestjs/common";
 import type { Job, Worker } from "bullmq";
 import type { KernelConfig } from "../config/schema.js";
 import { TitleQueue } from "../conversations/titles.js";
-import { type RunJobData } from "./run-jobs.js";
+import { type RUN_JOB, type RunJobData } from "./run-jobs.js";
 import { RunWorker } from "./run-worker.js";
-import { UsageReporter, type UsageReportJob } from "./usage-report.js";
+import { type USAGE_REPORT_JOB, UsageReporter, type UsageReportJob } from "./usage-report.js";
 type RunResult = Awaited<ReturnType<RunWorker["process"]>>;
+/** A run job: both run queues carry only {@link RUN_JOB} jobs (the outbox registry's `run`). */
+type RunJob = Job<RunJobData, RunResult, typeof RUN_JOB>;
 /** The consumer of `cap-kernel-runs` (conversation runs): {@link RunWorker}. */
 export declare class RunsProcessor extends WorkerHost<Worker<RunJobData, RunResult>> {
     private readonly runs;
     constructor(runs: RunWorker);
-    process(job: Job<RunJobData>, token?: string): Promise<RunResult>;
-    completed(job: Job<RunJobData>, result: RunResult): void;
-    failed(job: Job<RunJobData> | undefined, err: Error): void;
+    process(job: RunJob, token?: string): Promise<RunResult>;
+    completed(job: RunJob, result: RunResult): void;
+    failed(job: RunJob | undefined, err: Error): void;
     error(err: Error): void;
 }
 /** The consumer of `cap-kernel-subagent-runs` (sub-agent runs, spec §7.4): {@link RunWorker}. */
 export declare class SubagentRunsProcessor extends WorkerHost<Worker<RunJobData, RunResult>> {
     private readonly runs;
     constructor(runs: RunWorker);
-    process(job: Job<RunJobData>, token?: string): Promise<RunResult>;
-    completed(job: Job<RunJobData>, result: RunResult): void;
-    failed(job: Job<RunJobData> | undefined, err: Error): void;
+    process(job: RunJob, token?: string): Promise<RunResult>;
+    completed(job: RunJob, result: RunResult): void;
+    failed(job: RunJob | undefined, err: Error): void;
     error(err: Error): void;
 }
+/**
+ * A delivery job: the usage report queue carries only {@link USAGE_REPORT_JOB} jobs (the outbox
+ * registry's `usage-report`).
+ */
+type UsageJob = Job<UsageReportJob, void, typeof USAGE_REPORT_JOB>;
 /** The consumer of `cap-kernel-usage-report`: {@link UsageReporter}. */
 export declare class UsageReportProcessor extends WorkerHost<Worker<UsageReportJob>> {
     private readonly reporter;
     constructor(reporter: UsageReporter);
-    process(job: Job<UsageReportJob>, token?: string): Promise<void>;
+    process(job: UsageJob, token?: string): Promise<void>;
     failed(job: Job<UsageReportJob> | undefined, err: Error): void;
     error(err: Error): void;
 }

package/dist/runtime/run-processors.js

@@ -14,7 +14,7 @@ import { Inject, Injectable, Module } from "@nestjs/common";
 import { TITLE_QUEUE, TitleQueue, TitlesModule } from "../conversations/titles.js";
 import { configuredProcessor } from "../infra/queue/configured-processor.js";
 import { runJob } from "../infra/queue/run-job.js";
-import { RUN_QUEUE_NAME, SUBAGENT_RUN_QUEUE_NAME } from "./run-jobs.js";
+import { RUN_QUEUE_NAME, SUBAGENT_RUN_QUEUE_NAME, } from "./run-jobs.js";
 import { MAX_STALLED_COUNT, RunWorker } from "./run-worker.js";
 import { RuntimeModule } from "./runtime.module.js";
 import { USAGE_REPORT_QUEUE, UsageReporter, UsageReportModule, } from "./usage-report.js";

package/dist/runtime/run-queue.d.ts

@@ -1,18 +1,21 @@
 import type { Logger } from "@capybara-hq/plugin-sdk";
 import type { Queue } from "bullmq";
 import { EventLog } from "../conversations/event-log.js";
-import type { ConversationKind } from "../db/kernel-schema.js";
 import { KernelConfigService } from "../infra/config/kernel-config.service.js";
 import { KernelDb } from "../infra/database/kernel-db.js";
 import { TransactionHost } from "../infra/database/transaction-host.js";
-import { KernelEvents } from "../infra/events/kernel-events.js";
-import { ToolPipeline } from "../tools/pipeline.js";
 import type { RunAbortDetail } from "./abort.js";
 import { KyselyCheckpointSaver } from "./checkpointer.js";
 import { ConversationLock } from "./conversation-lock.js";
 import { RunFinish } from "./run-finish.js";
-import { type RunJobData, RunJobs } from "./run-jobs.js";
-export { ACTIVE_RUN_STATUSES, RUN_QUEUE_NAME, type RunJobData, runJobId, runQueueNameFor, SUBAGENT_RUN_QUEUE_NAME, } from "./run-jobs.js";
+import { RunJobs } from "./run-jobs.js";
+export { ACTIVE_RUN_STATUSES, ENSURE_RUN_BACKSTOP_MS, ENSURE_RUN_JOB, type EnsureRunJobData, ensureRunJobId, RUN_QUEUE_NAME, type RunJobData, runJobId, runQueueNameFor, SUBAGENT_RUN_QUEUE_NAME, } from "./run-jobs.js";
+/**
+ * Test seam (`KernelOverride`): returns true to make {@link RunQueue.ensureRun} throw before it
+ * does anything (Redis unreachable, or a direct call after a commit that never happened). The
+ * kernel never provides it.
+ */
+export declare const ENSURE_RUN_FAILS: unique symbol;
 /**
  * 运行队列（生产方）：`ensureRun` 在会话有未消费的用户消息、且没有进行中的运行时插入 `runs` 行并投递任务。
  * 「是否已有运行」以 PG 为准（会话行锁内检查 `runs` 中的 queued / running），并参考会话锁的持有者；
@@ -24,32 +27,25 @@ export declare class RunQueue {
     private readonly lock;
     private readonly jobs;
     private readonly finisher;
-    /** 运行以孤儿结束（标记失败 / 直接中止）后删除它的 checkpoint thread（thread_id = runId）。 */
+    /** An orphan aborted directly loses its checkpoint thread (thread_id = runId). */
     private readonly checkpoints;
-    /** The run's open `Agent` call results, in the orphan repair's transaction (sub-agents spec §12). */
-    private readonly pipeline;
-    private readonly events;
     private readonly log;
     readonly txHost: TransactionHost;
     private readonly config;
     /** The producer side of the delayed `bg-wake` job (`MaintenanceProcessor` consumes it). */
     private readonly maintenance;
+    private readonly ensureFails;
     constructor(kdb: KernelDb, eventLog: EventLog, lock: Pick<ConversationLock, "holder">, jobs: RunJobs, finisher: Pick<RunFinish, "finishIn">, 
-    /** 运行以孤儿结束（标记失败 / 直接中止）后删除它的 checkpoint thread（thread_id = runId）。 */
-    checkpoints: Pick<KyselyCheckpointSaver, "deleteThread">, 
-    /** The run's open `Agent` call results, in the orphan repair's transaction (sub-agents spec §12). */
-    pipeline: Pick<ToolPipeline, "stopOpenAgentCalls">, events: KernelEvents, log: Pick<Logger, "error" | "warn">, txHost: TransactionHost, config: KernelConfigService, 
+    /** An orphan aborted directly loses its checkpoint thread (thread_id = runId). */
+    checkpoints: Pick<KyselyCheckpointSaver, "deleteThread">, log: Pick<Logger, "error" | "warn">, txHost: TransactionHost, config: KernelConfigService, 
     /** The producer side of the delayed `bg-wake` job (`MaintenanceProcessor` consumes it). */
-    maintenance: Pick<Queue, "add">);
+    maintenance: Pick<Queue, "add">, ensureFails?: () => boolean);
     private get db();
-    /**
-     * The `beforeTerminal` of an orphan repair: writes the run's open `Agent` call results in the
-     * repair's transaction; `fire()` runs their observe hooks after the commit (review m1).
-     */
-    private stopOpenCalls;
     /**
      * 有未消费的用户消息（最新 user_message 的 seq > `consumed_seq`）、会话锁无人持有、
-     * 且没有 queued / running / waiting 的运行时：插入 `runs`（queued）并投递任务，返回 runId；否则返回 null。
+     * 且没有 queued / running / waiting 的运行时：插入 `runs`（queued），返回 runId；否则返回 null。
+     * The run's job is owed in the same transaction (transactional outbox spec §6): the outbox
+     * delivers it after the commit, so a queued run never lacks its job.
      *
      * The boundary (`trigger_seq`) is that newest message; with none unconsumed, the queue's head
      * once it is sent; with no head either, a wake-eligible finished background task or sub-agent
@@ -58,13 +54,11 @@ export declare class RunQueue {
      * event inside the wake window schedules the delayed `bg-wake` job instead. A sub-agent's
      * conversation is only woken this way ({@link childWakeIn}).
      *
-     * 孤儿清理（R29，按需进行，没有定时器）：已有的 queued / running 运行若 BullMQ 任务已不存在、
-     * 会话锁也无人持有（queued 的还要超过宽限期），说明没有任何 worker 会再执行它——写
-     * `run_failed { code: "run.orphaned" }` 把它标记为失败（consumed_seq 推进到它的触发消息），
-     * 再按常规判断是否起新一轮，并删除它的 checkpoint thread。会话因此不会永远卡在「有运行」。
-     * 除了写入消息后，打开 SSE、读事件、运行结束后也会调用它（幂等），孤儿因此在有人查看时就被清理。
+     * An active run (queued, running or waiting) is left alone: a run nobody runs any more is the
+     * `run-orphan-check`'s to repair (transactional outbox spec §6), never this call's, and reads
+     * never call it.
      *
-     * agent 已软删除的会话不起新运行（孤儿清理照常进行）。
+     * A conversation whose agent is soft-deleted starts no run.
      */
     ensureRun(conversationId: string, 
     /** From the `bg-wake` job: it fired `wakeRetry - 1` times before its window ended. */
@@ -73,6 +67,20 @@ export declare class RunQueue {
     }): Promise<string | null>;
     /** {@link ensureRun}'s transaction: what it decided goes back through `st`. */
     private ensureRunIn;
+    /**
+     * The job id of a new run for `seq` (in {@link ensureRunIn}'s transaction, read from PG only):
+     * the trigger's `runJobId` for its first run; a later run of the same trigger (a run that hit its
+     * deadline leaves the messages it did not absorb unconsumed, R34) gets `<runJobId>-<n>`,
+     * recorded in `runs.job_id`, so its job is never taken for the earlier run's (BullMQ ignores an
+     * add under the id of a job it still holds).
+     */
+    private jobIdFor;
+    /**
+     * Owes the new run's job in {@link ensureRunIn}'s transaction, before its run row is written.
+     * False (logged): the same job id is still owed, so the caller writes no run (a second run of
+     * one trigger; the conversation's unique active run makes this a should-not-happen).
+     */
+    private oweJob;
     /**
      * {@link ensureRunIn} for a sub-agent's conversation (agent map spec §5.3), under its row lock.
      * Its runs from a task, or from a message to an idle sub-agent, are started by the dispatch
@@ -84,8 +92,7 @@ export declare class RunQueue {
      * started again only by its parent. Both read the queue and the trigger account's access on the root
      * of its tree; a message needs no wake setting and has no wake window. The run counts against
      * the root's budget window and runs on the sub-agent queue under `runJobId(child, boundary
-     * seq)`. An active run is left to the orphan repair after the commit (`repairRunId`), never
-     * started past.
+     * seq)`. An active run is never started past (one nobody runs is the `run-orphan-check`'s).
      */
     private childWakeIn;
     /**
@@ -101,10 +108,24 @@ export declare class RunQueue {
      * Re-derives the wakes a paused root held for its tree (agent map spec §5.3: a paused root queue
      * holds the wakes of the whole tree): after the root's queue is unpaused, {@link ensureRun} on
      * every sub-agent conversation below it that has an unconsumed wake event or message and no
-     * active run (the root itself is the caller's own `ensureRun`). Errors are logged; the next
-     * event, run end or page opening of that conversation re-derives its wake.
+     * active run (the root itself is the caller's own `ensureRun`). Errors are logged; the
+     * ensure-run backstop the releasing transaction owed ({@link oweHeldTreeIn}) wakes it later.
      */
     wakeHeldTree(rootId: string): Promise<void>;
+    /**
+     * Owes, in the caller's transaction (the one that unpauses the root's queue or sends its item
+     * now), the ensure-run backstop of every sub-agent {@link wakeHeldTree} would wake after the
+     * commit, under the sub-agent's newest seq (transactional outbox spec §6): a process that dies
+     * before that call leaves no held wake behind. A root whose queue is still paused (or that is
+     * deleted) releases nothing and owes nothing.
+     */
+    oweHeldTreeIn(rootId: string): Promise<void>;
+    /**
+     * The sub-agent conversations below `rootId` a paused root holds: not deleted, with an
+     * unconsumed wake event (a finished background task, a delivery) or message, and no active run;
+     * with their org and newest seq. Reads only (no row locks).
+     */
+    private heldTree;
     /**
      * Schedules the delayed `bg-wake` job at the end of the conversation's wake window (after the
      * commit): one job per conversation and window end, so finishes inside one window share it. It
@@ -113,41 +134,42 @@ export declare class RunQueue {
      */
     private scheduleWake;
     /**
-     * Enqueues a run job under the caller's job id (resumes, dispatches, repairs), with the same
-     * options as {@link ensureRun} (see {@link RunJobs.enqueue}).
+     * Owes the ensure-run backstop of `conversationId` in the caller's transaction
+     * ({@link oweEnsureRun}): `seq` is the event the transaction wrote, `orgId` the conversation's.
      */
-    enqueue(data: RunJobData, jobId: string, kind?: ConversationKind): Promise<void>;
+    oweEnsureRun(conversationId: string, seq: number | string, orgId: string): Promise<boolean>;
     /**
-     * 中止一个孤儿运行（`POST .../abort` 用）：运行仍是 queued / running、会话锁无人持有、BullMQ 任务已不存在
-     * （queued 的还要超过宽限期，判定同 {@link ensureRun} 的孤儿清理）时，没有任何 worker 会收到中止广播——
-     * 直接写 `run_aborted`、status 置为 aborted、consumed_seq 至少推进到触发消息，返回 true。
-     * 不是孤儿（有人在执行或即将执行）或运行已结束 → 返回 false，由调用方走广播中止。
+     * Stops an orphaned run directly (`POST .../abort`, a tree stop): a `running` run whose
+     * conversation lock is free and whose BullMQ job is gone hears no stop broadcast, so its
+     * `run_aborted` is written here (status aborted, `consumed_seq` at least to its trigger): true. Not
+     * an orphan (someone runs it; a queued run's job is owed, or lost and re-added by the
+     * `run-orphan-check`, whose worker then sees the stop marker) or already finished: false, and the
+     * caller broadcasts the stop.
      */
     abortIfOrphaned(conversationId: string, runId: string, byAccountId: string | null, detail?: RunAbortDetail): Promise<boolean>;
     private abortOrphanIn;
     /**
-     * Repairs an orphaned run of a conversation nobody else repairs on demand (a sub-agent's: no
-     * `ensureRun` ever runs for it; sub-agents spec §7.5), with the same orphan test as
-     * {@link ensureRun}: the conversation lock is free, the run is still `queued` / `running`, its
-     * job is gone, and a `queued` one is past the grace period.
-     * - `queued` (the dispatch or resume committed, its enqueue was lost) → enqueued again under its
-     *   own job id (`runs.job_id`, else the trigger's), so concurrent repairs on several replicas
-     *   still leave one job: `"requeued"`;
-     * - `running` (its worker died without a terminal state and BullMQ dropped the job) → marked
-     *   failed `run.orphaned`; the caller reports it to the parent: `"failed"`.
-     * Not an orphan, or already finished → null.
+     * Ends a `queued` run in PG before any worker started it (an offboarded member's run, ruling
+     * R27): with the conversation lock free, under the conversation row lock, a run still `queued`
+     * is finished `aborted` with `run_aborted` (`consumed_seq` at least to its trigger), so a job
+     * that runs later finds it finished and starts nothing. True when it did; false when a worker
+     * holds the conversation or is in the middle of the run's first start (`run_started` written,
+     * `started_at` not yet set; it hears the stop), or the run is no longer queued. A run resumed
+     * from `waiting` to `queued` is ended like a new one. The caller writes the stop marker first, so a worker that takes the
+     * conversation meanwhile still stops; a worker that read the run before this commit re-checks it
+     * under the row locks before its `run_started` and never moves it back to `running`.
      */
-    repairOrphan(conversationId: string, runId: string): Promise<"requeued" | "failed" | null>;
-    private repairOrphanIn;
+    abortQueued(conversationId: string, runId: string, byAccountId: string | null): Promise<boolean>;
+    private abortQueuedIn;
     /** 会话当前未结束的运行（调用方已持有会话行锁）。 */
     private activeRun;
     /**
-     * 未结束的运行是否已是孤儿（调用方已确认会话锁无人持有）：waiting → 否（等人处理，本来就没有任务，
-     * 由人工决定或超时任务续跑）；queued 且仍在宽限期内 → 否（任务可能还没投递）；BullMQ 任务（续跑任务按
-     * `runs.job_id`）仍存在 → 否；否则是。
+     * Whether an active run is an orphan for {@link abortIfOrphaned} (the caller saw the conversation
+     * lock free): a `running` one whose BullMQ job (a resume's under `runs.job_id`) is gone. A
+     * `waiting` one never is (no job by design), nor a `queued` one (its job is owed in the
+     * transaction that queued it).
      */
     private isOrphan;
     /** 删除已结束运行的 checkpoint thread（失败只记日志）。 */
     private dropCheckpoints;
-    private deleteQueued;
 }