@capybara-hq/kernel 0.0.0-stage → 0.1.0-rc.0

package/dist/runtime/waits.d.ts

@@ -0,0 +1,233 @@
+import type { Logger } from "@capybara-hq/plugin-sdk";
+import { type Transaction } from "kysely";
+import type { EventLog } from "../conversations/event-log.js";
+import type { KernelDatabase, KernelDb } from "../db/kernel-schema.js";
+import type { ApprovalsService } from "../tools/approvals.service.js";
+import type { QuestionsService } from "../tools/questions.service.js";
+import type { RunAbortBy, RunAbortDetail } from "./abort.js";
+import type { KyselyCheckpointSaver } from "./checkpointer.js";
+import { type WaitingFor } from "./loop.js";
+import { MAINTENANCE_QUEUE_NAME, type Maintenance } from "./maintenance.js";
+import { type RunQueue } from "./run-queue.js";
+type KernelTrx = Transaction<KernelDatabase>;
+/** 等待超时任务所在的内核维护队列（见 maintenance.ts）。 */
+export { MAINTENANCE_QUEUE_NAME };
+/** 等待超时任务的名字与 jobId（每个审批 / 输入请求一个，重复投递被 BullMQ 忽略）。 */
+export declare const WAIT_TIMEOUT_JOB = "wait-timeout";
+export declare const waitTimeoutJobId: (waitId: string) => string;
+/** 续跑任务的 jobId：与按消息生成的 `run-<conversationId>-<seq>` 不冲突，同一等待只会续跑一次。 */
+export declare const resumeJobId: (conversationId: string, waitId: string) => string;
+/**
+ * 等待兜底扫描（Task 22）：任务名、job scheduler id（不含 `:`）、周期、宽限期与每类每次最多处理的条数。
+ * 宽限期：waiting 超过它、等待却已全部处理的运行才视为「续跑丢失」，避免与正常续跑（暂停后的复查等）竞争。
+ */
+export declare const WAIT_SWEEP_JOB = "wait-sweep";
+export declare const WAIT_SWEEP_SCHEDULER_ID = "wait-sweep-every-10m";
+export declare const WAIT_SWEEP_EVERY_MS = 600000;
+export declare const WAIT_SWEEP_GRACE_MS = 60000;
+export declare const WAIT_SWEEP_BATCH = 500;
+/** 一次扫描（本副本）实际生效的修复数。 */
+export interface WaitSweepResult {
+    /** 过期仍 pending、被本次扫描按超时处理的审批 / 输入请求。 */
+    expired: number;
+    /** 等待已全部处理、被本次扫描续跑的 waiting 运行。 */
+    resumed: number;
+    /** 运行已终态、被本次扫描取消的审批 / 输入请求。 */
+    cancelled: number;
+    /**
+     * Finished parent runs whose open `agent` calls had no `tool_result`, repaired (final review
+     * I3: the next run would send an unpaired tool call, which providers refuse).
+     */
+    repaired: number;
+    /** Queued sub-agent runs, and chat runs queued by a resume, whose job was lost, enqueued again. */
+    requeued: number;
+    /** Running sub-agent runs without job or lock, failed `run.orphaned` and reported to the parent. */
+    orphaned: number;
+    /** Active sub-agent runs of a finished parent run, asked to stop. */
+    stopped: number;
+}
+/** The data of a wait timeout job; `subagents` waits also carry their run and step. */
+export interface WaitTimeoutData {
+    kind: WaitingFor["kind"];
+    id: string;
+    runId?: string;
+    step?: number;
+}
+export interface WaitsDeps {
+    db: KernelDb;
+    eventLog: EventLog;
+    approvals: Pick<ApprovalsService, "expire" | "cancelPendingIn">;
+    questions: Pick<QuestionsService, "expire" | "cancelPendingIn">;
+    runQueue: Pick<RunQueue, "enqueue" | "repairOrphan">;
+    /**
+     * Sub-agents (spec §7.5, §12): `stop` stops a parent run's active children (only those of `step`
+     * when given, with `detail` on their `run_aborted`; `stopChildren`), `finished` reports a finished child to its parent
+     * (`onChildFinished`). Wired by the kernel; without it the sweep skips sub-agent repairs.
+     */
+    subagents?: {
+        stop: (parentRunId: string, byAccountId: string | null, opts?: {
+            step?: number;
+            detail?: RunAbortDetail;
+        }) => Promise<number>;
+        finished: (childRunId: string) => Promise<void>;
+        /**
+         * Writes, in `t`, the results of a stopped run's open `agent` calls (`ToolPipeline.
+         * stopOpenAgentCalls`); `fire` runs its observe hooks after the commit.
+         */
+        stopOpenCalls?: (t: KernelTrx, runId: string) => Promise<{
+            steps: number[];
+            callIds: Set<string>;
+            toSeq?: number | null;
+            fire: () => void;
+        }>;
+    };
+    /** 等待中被中止的运行删除其 checkpoint thread。 */
+    checkpoints?: Pick<KyselyCheckpointSaver, "deleteThread">;
+    /**
+     * Reads the run's stop marker (`RunAbort.requested`): between suspending and becoming `waiting`
+     * the run has no subscriber, so a stop arriving then only sits in the marker; checked once after
+     * `waiting` is set.
+     */
+    abortRequested?: (runId: string) => Promise<RunAbortBy | null>;
+    /** 内核维护队列：超时任务投递到这里，处理函数在构造时登记（Worker 由它在 worker 角色上启动）。 */
+    maintenance: Pick<Maintenance, "add" | "handle" | "schedule">;
+    log: Pick<Logger, "error" | "warn">;
+}
+/**
+ * 运行等待人处理（spec 4.4 / Task 9）：暂停、续跑、超时与中止。状态全部在 PG（`runs.status = 'waiting'`、
+ * `approvals` / `questions` 行），延迟与重试走 BullMQ（维护队列的 `wait-timeout` 任务），没有进程内状态。
+ *
+ * 只续跑一次：`resume` 是一条条件更新 `waiting → queued`（且该运行已没有 pending 的等待），并发调用只有一个成功，
+ * 成功者投递 jobId 为 `run-<conversationId>-resume-<waitId>` 的运行任务。人工决定、超时任务、
+ * 暂停后的复查都可能调用它，谁先谁后都只有一次续跑。
+ */
+export declare class Waits {
+    private readonly d;
+    constructor(d: WaitsDeps);
+    /**
+     * 兜底扫描（维护队列每 10 分钟一次，防延迟任务 / 续跑丢失）。每项修复都复用常规路径，与并发的超时任务、
+     * 人工决定、其他副本的扫描竞争时由行锁与条件更新保证只生效一次；每项生效的修复记一行 warn（只含 id）。
+     * 顺序：先取消终态运行的等待（这些不该再按超时处理），再处理过期的等待，最后续跑「续跑丢失」的运行。
+     * 单项失败只记日志，不影响其他项。
+     */
+    sweep(): Promise<WaitSweepResult>;
+    /**
+     * loop 返回 `suspended` 之后（worker 已释放会话锁与配额）：条件更新 `running → waiting`（记 `waiting_since`），
+     * 投递延迟到 `expiresAt` 的超时任务，再复查一次等待是否已被处理——处理者的 `resume` 可能发生在本次置 waiting
+     * 之前（那时条件更新不成立），复查保证这种情况下仍会续跑。
+     * 运行已不是 running（例如已被其他路径结束）→ 什么都不做。
+     * 暂停时已收到用户中止（`abort`），或置 waiting 之后补查到中止标记 → 直接按 {@link abortWaiting} 取消等待并结束
+     * 运行（置 waiting 之后到达的中止由 `POST .../abort` 看到 waiting 直接处理，两边不会都漏掉）。
+     */
+    suspend(runId: string, conversationId: string, waitingFor: WaitingFor, abort?: RunAbortBy): Promise<void>;
+    /**
+     * A sub-agent's run waits for an approval or an input (sub-agents spec §9): appends
+     * `subagent_waiting` to the parent conversation (idem `subagent_waiting:<childRunId>:<waitId>`),
+     * so the web and the channel outbound show the card there. The approval or question itself stays
+     * the child run's, with the parent run's starter as the person asked. Not a run of a sub-agent,
+     * or a `subagents` wait: nothing. Display only: a failure is logged, the wait goes on.
+     */
+    private surfaceInParent;
+    /**
+     * 续跑：一条条件更新 `waiting → queued`（且该运行没有 pending 的审批 / 输入请求），把本次等待时长累加进
+     * `waited_ms`，记下续跑任务的 jobId；成功（只有一个调用方）则投递运行任务并返回 true。
+     * 投递失败 → 撤回为 waiting（等待时长已计入，重新从现在起计），之后超时任务到点会再次续跑，并抛出。
+     */
+    resume(runId: string, waitId: string): Promise<boolean>;
+    /**
+     * 超时（`wait-timeout` 任务）：仍 pending 且已到期 → 置 expired 并写决定 / 回答事件；随后（无论本次是否生效）
+     * 尝试续跑——若人工决定之后的续跑没能发生（进程崩溃、投递失败），超时任务兜底。
+     * 尚未到期（任务被提前执行）→ 返回到期时间，由任务重新延迟。
+     */
+    expire(kind: WaitingFor["kind"], id: string, at?: {
+        runId: string;
+        step: number;
+    }): Promise<{
+        notDueUntil?: number;
+        acted?: boolean;
+    }>;
+    /**
+     * 中止等待中的运行（`POST .../abort`）：同一事务里锁会话行与运行行，运行仍是 waiting 时——取消它的 pending 审批 /
+     * 输入请求（写 `approval_decided{cancelled}` / `question_answered{cancelled}`），给等待中的调用写取消说明的
+     * `tool_result`（同一步里尚未执行的调用写「未执行」），再写 `run_aborted` 终态；提交后删除 checkpoint。
+     * 运行已不是 waiting（例如刚被续跑）→ 返回 false，由调用方走常规中止。
+     */
+    abortWaiting(conversationId: string, runId: string, byAccountId: string | null, detail?: RunAbortDetail): Promise<boolean>;
+    /** 扫描 3：运行已终态（如暂停途中失败）仍 pending 的审批 / 输入请求 → 取消（写决定 / 回答事件）。 */
+    private sweepFinishedRuns;
+    /**
+     * 同一事务里锁会话行与运行行（与 {@link abortWaiting} 同序），运行仍是终态时取消其 pending 等待；
+     * 提交后发布事件。返回取消的条数（并发的另一副本拿到锁时已无 pending → 0）。
+     */
+    private cancelWaitsOfFinished;
+    /** 扫描 1：`expires_at < now()` 仍 pending 的审批 / 输入请求 → 与超时任务同一路径（置 expired、写事件、续跑）。 */
+    private sweepOverdue;
+    /**
+     * Sub-agents waits past their `runs.wait_expires_at` (their timeout job was lost): expired the
+     * way the timeout job would ({@link expire}: the step's active children stop as `timed out`, the
+     * parent resumes once they are done). The step is the run's latest dispatch step: a waiting run
+     * has active children only from the step it waits in. A run stays here until its children have
+     * stopped; a pass that neither stopped a child nor resumed the run is not counted.
+     */
+    private sweepOverdueSubagents;
+    /**
+     * Sub-agent runs nobody will run (spec §7.5; `RunQueue.repairOrphan`, the same orphan test as
+     * `ensureRun`, which never runs for a child): `queued` past the grace period without a job → enqueued
+     * again (the dispatch committed, its enqueue was lost); `running` without job and lock → failed
+     * `run.orphaned` and reported to the parent, which then resumes. Also a chat run left `queued`
+     * by a resume whose enqueue was lost (final review m8): enqueued again under its resume job id
+     * (`ensureRun` would only repair it at the conversation's next message).
+     */
+    private sweepSubagentOrphans;
+    /**
+     * Finished parent runs (past the grace period, within the last day) with an `agent` call that has
+     * no `tool_result` (final review I3): a terminal path that skipped `stopOpenAgentCalls`, such as
+     * an old replica's abort, would otherwise leave an unpaired tool call that every later request of
+     * the conversation sends and providers refuse. Each is repaired under the conversation row lock
+     * with the normal `stopOpenCalls` (a child that ended gives its result, an active one the
+     * stopped text; idempotency keys make it safe against the regular paths); its active children
+     * are then stopped by {@link sweepChildrenOfFinished}.
+     */
+    private sweepOpenAgentCalls;
+    /**
+     * Active sub-agent runs of a parent run that finished more than the grace period ago (spec §7.5,
+     * §12: a parent's terminal state stops its children; this is the backstop) → stopped.
+     */
+    private sweepChildrenOfFinished;
+    /**
+     * 扫描 2：waiting 超过宽限期、却已没有 pending 等待的运行（决定提交后进程崩溃、续跑投递失败回滚等）→
+     * {@link resume}（条件更新 `waiting → queued` + 投递）。续跑任务的 jobId 用该运行最近一次等待的 id
+     * （与人工决定 / 超时任务一致，已投递过的同一续跑被 BullMQ 去重）。
+     */
+    private sweepStuckRuns;
+    /**
+     * The id of the run's latest wait (by creation time): an approval, an input request or a
+     * sub-agent dispatch, whose wait id is its child run id (the id the child's finisher resumes
+     * with, so a sweep resume and a late finisher resume share one job id).
+     */
+    private lastWaitId;
+    private process;
+    /** 投递超时任务（延迟到 expiresAt）。失败只记日志：人工处理仍可续跑，只是不会自动超时。 */
+    private scheduleTimeout;
+    /**
+     * Whether the wait is over: an approval or input request is no longer pending (decided, expired
+     * or cancelled); `subagents`: no child run of the run is active.
+     */
+    private isSettled;
+    /** 审批所在的步（`approval_requested` 事件里记的）。 */
+    private approvalStep;
+    /**
+     * 等待中被中止的运行：这些步里每个调用都补一条 `tool_result`（已有结果的按幂等键命中、不重复），
+     * 等待中的调用给取消说明，其余（排在等待调用之后、尚未执行的）给「未执行」说明。
+     */
+    private stoppedResults;
+}
+/**
+ * Phase D: finished chat runs (past the grace period, within the last day) triggered by a fork
+ * invocation that has no report (dispatched or not); the wait sweep writes it
+ * (`pairStoppedForks`).
+ */
+export declare function unreportedForkRuns(db: KernelDb): Promise<{
+    parent_run_id: string;
+    parent_conversation_id: string;
+}[]>;