@capybara-hq/kernel 0.1.0-rc.23 → 0.1.0-rc.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. package/bootstrap.json +25 -25
  2. package/dist/admin/admin.module.d.ts +2 -1
  3. package/dist/admin/admin.module.js +7 -3
  4. package/dist/admin/outbox-dead.controller.d.ts +23 -0
  5. package/dist/admin/outbox-dead.controller.js +62 -0
  6. package/dist/admin/outbox-dead.schemas.d.ts +8 -0
  7. package/dist/admin/outbox-dead.schemas.js +18 -0
  8. package/dist/admin/outbox-dead.service.d.ts +66 -0
  9. package/dist/admin/outbox-dead.service.js +179 -0
  10. package/dist/admin/overview.service.d.ts +40 -5
  11. package/dist/admin/overview.service.js +41 -7
  12. package/dist/agents/from-template.js +1 -1
  13. package/dist/background/finish.d.ts +5 -2
  14. package/dist/background/finish.js +14 -3
  15. package/dist/background/launch.d.ts +2 -1
  16. package/dist/background/launch.js +5 -2
  17. package/dist/background/store.d.ts +8 -0
  18. package/dist/background/store.js +24 -0
  19. package/dist/catalog/admin-catalog.controller.d.ts +21 -2
  20. package/dist/catalog/admin-catalog.controller.js +45 -6
  21. package/dist/catalog/admin-catalog.schemas.d.ts +5 -0
  22. package/dist/catalog/admin-catalog.schemas.js +8 -0
  23. package/dist/catalog/catalog.module.d.ts +2 -1
  24. package/dist/catalog/catalog.module.js +5 -1
  25. package/dist/catalog/marketplaces.service.d.ts +1 -1
  26. package/dist/catalog/platform-import-update.d.ts +50 -0
  27. package/dist/catalog/platform-import-update.js +116 -0
  28. package/dist/catalog/plugin-import.d.ts +36 -1
  29. package/dist/catalog/plugin-import.js +23 -1
  30. package/dist/catalog/plugin-upgrade.d.ts +18 -0
  31. package/dist/catalog/plugin-upgrade.js +19 -0
  32. package/dist/channels/channels.module.d.ts +5 -2
  33. package/dist/channels/channels.module.js +9 -2
  34. package/dist/channels/deliver-job.d.ts +25 -0
  35. package/dist/channels/deliver-job.js +25 -0
  36. package/dist/channels/inbound.d.ts +1 -1
  37. package/dist/channels/inbound.js +2 -15
  38. package/dist/channels/jobs.d.ts +28 -18
  39. package/dist/channels/jobs.js +63 -26
  40. package/dist/config/schema.d.ts +0 -3
  41. package/dist/config/schema.js +11 -20
  42. package/dist/conversations/agent-tree.d.ts +16 -0
  43. package/dist/conversations/agent-tree.js +36 -0
  44. package/dist/conversations/conversations.service.d.ts +44 -14
  45. package/dist/conversations/conversations.service.js +52 -38
  46. package/dist/conversations/event-log.d.ts +7 -10
  47. package/dist/conversations/event-log.js +24 -15
  48. package/dist/conversations/events.js +2 -1
  49. package/dist/conversations/queue.service.d.ts +8 -3
  50. package/dist/conversations/queue.service.js +37 -13
  51. package/dist/conversations/titles.d.ts +21 -8
  52. package/dist/conversations/titles.js +42 -15
  53. package/dist/db/kernel-schema.d.ts +26 -8
  54. package/dist/devices/devices.service.d.ts +12 -3
  55. package/dist/devices/devices.service.js +18 -3
  56. package/dist/errors.d.ts +11 -0
  57. package/dist/errors.js +11 -0
  58. package/dist/evals/run-replay.js +14 -8
  59. package/dist/extensions/extensions.module.js +3 -0
  60. package/dist/extensions/lifecycle.d.ts +6 -1
  61. package/dist/extensions/lifecycle.js +9 -2
  62. package/dist/extensions/usage-subscribers.d.ts +43 -0
  63. package/dist/extensions/usage-subscribers.js +108 -0
  64. package/dist/http/api.module.js +2 -0
  65. package/dist/index.d.ts +1 -0
  66. package/dist/index.js +1 -0
  67. package/dist/infra/cli/kernel-cli.module.js +5 -0
  68. package/dist/infra/events/kernel-events.d.ts +9 -35
  69. package/dist/infra/outbox/outbox-relay.module.d.ts +6 -0
  70. package/dist/infra/outbox/outbox-relay.module.js +18 -0
  71. package/dist/infra/outbox/outbox.dispatcher.d.ts +103 -0
  72. package/dist/infra/outbox/outbox.dispatcher.js +310 -0
  73. package/dist/infra/outbox/outbox.module.d.ts +24 -0
  74. package/dist/infra/outbox/outbox.module.js +50 -0
  75. package/dist/infra/outbox/outbox.relay.d.ts +31 -0
  76. package/dist/infra/outbox/outbox.relay.js +76 -0
  77. package/dist/infra/outbox/outbox.service.d.ts +51 -0
  78. package/dist/infra/outbox/outbox.service.js +131 -0
  79. package/dist/infra/outbox/registry.d.ts +120 -0
  80. package/dist/infra/outbox/registry.js +108 -0
  81. package/dist/infra/queue/queue-names.d.ts +8 -0
  82. package/dist/infra/queue/queue-names.js +16 -0
  83. package/dist/kernel.module.js +3 -0
  84. package/dist/local-devices/device-policy.service.d.ts +4 -0
  85. package/dist/local-devices/device-policy.service.js +61 -3
  86. package/dist/local-devices/machines.d.ts +13 -5
  87. package/dist/local-devices/machines.js +45 -7
  88. package/dist/local-devices/policy-summary.d.ts +44 -1
  89. package/dist/local-devices/policy-summary.js +69 -0
  90. package/dist/models/embeddings.d.ts +9 -9
  91. package/dist/models/embeddings.js +11 -18
  92. package/dist/models/models.module.js +3 -0
  93. package/dist/orgs/orgs.service.js +7 -0
  94. package/dist/overrides.d.ts +20 -3
  95. package/dist/overrides.js +10 -2
  96. package/dist/platform/exclusions.js +5 -0
  97. package/dist/platform/ops/admin.ops.js +18 -2
  98. package/dist/platform/ops/catalog-admin.ops.d.ts +3 -1
  99. package/dist/platform/ops/catalog-admin.ops.js +32 -4
  100. package/dist/platform/ops/outbox.ops.d.ts +21 -0
  101. package/dist/platform/ops/outbox.ops.js +125 -0
  102. package/dist/platform/ops/schedules.ops.js +40 -1
  103. package/dist/platform/platform-ops.module.d.ts +2 -1
  104. package/dist/platform/platform-ops.module.js +3 -0
  105. package/dist/platform/route-schemas.js +3 -0
  106. package/dist/roles/worker.module.js +9 -4
  107. package/dist/runtime/langgraph-loop.js +13 -5
  108. package/dist/runtime/limits.d.ts +15 -0
  109. package/dist/runtime/limits.js +25 -0
  110. package/dist/runtime/loop.d.ts +8 -3
  111. package/dist/runtime/loop.js +2 -1
  112. package/dist/runtime/maintenance.d.ts +4 -2
  113. package/dist/runtime/maintenance.js +17 -2
  114. package/dist/runtime/model-call.d.ts +14 -16
  115. package/dist/runtime/model-call.js +14 -24
  116. package/dist/runtime/queue-lost-jobs.d.ts +24 -0
  117. package/dist/runtime/queue-lost-jobs.js +51 -0
  118. package/dist/runtime/run-control.d.ts +42 -5
  119. package/dist/runtime/run-control.js +66 -12
  120. package/dist/runtime/run-finish.d.ts +39 -12
  121. package/dist/runtime/run-finish.js +97 -14
  122. package/dist/runtime/run-jobs.d.ts +95 -13
  123. package/dist/runtime/run-jobs.js +199 -28
  124. package/dist/runtime/run-orphan-check.job.d.ts +142 -0
  125. package/dist/runtime/run-orphan-check.job.js +339 -0
  126. package/dist/runtime/run-processors.d.ts +16 -9
  127. package/dist/runtime/run-processors.js +1 -1
  128. package/dist/runtime/run-queue.d.ts +64 -55
  129. package/dist/runtime/run-queue.js +147 -284
  130. package/dist/runtime/run-steps.d.ts +1 -2
  131. package/dist/runtime/run-steps.js +8 -11
  132. package/dist/runtime/run-worker.d.ts +6 -17
  133. package/dist/runtime/run-worker.js +32 -34
  134. package/dist/runtime/runtime.module.d.ts +2 -3
  135. package/dist/runtime/runtime.module.js +5 -6
  136. package/dist/runtime/step-log.d.ts +16 -4
  137. package/dist/runtime/step-log.js +26 -6
  138. package/dist/runtime/stream.controller.d.ts +2 -4
  139. package/dist/runtime/stream.controller.js +5 -11
  140. package/dist/runtime/subagents/child.js +1 -1
  141. package/dist/runtime/subagents/definition.d.ts +2 -1
  142. package/dist/runtime/subagents/definition.js +4 -2
  143. package/dist/runtime/subagents/deliveries.d.ts +6 -3
  144. package/dist/runtime/subagents/deliveries.js +16 -7
  145. package/dist/runtime/subagents/dispatch.d.ts +12 -12
  146. package/dist/runtime/subagents/dispatch.js +13 -38
  147. package/dist/runtime/subagents/messages.d.ts +5 -1
  148. package/dist/runtime/subagents/messages.js +11 -3
  149. package/dist/runtime/subagents/skill-fork.d.ts +3 -3
  150. package/dist/runtime/subagents/skill-fork.js +5 -5
  151. package/dist/runtime/subagents/subagents.service.js +1 -1
  152. package/dist/runtime/subagents/tree-jobs.d.ts +61 -0
  153. package/dist/runtime/subagents/tree-jobs.js +86 -0
  154. package/dist/runtime/usage-report-jobs.d.ts +23 -48
  155. package/dist/runtime/usage-report-jobs.js +67 -119
  156. package/dist/runtime/usage-report.d.ts +28 -77
  157. package/dist/runtime/usage-report.js +35 -213
  158. package/dist/runtime/waits.d.ts +48 -198
  159. package/dist/runtime/waits.js +103 -805
  160. package/dist/sandbox/sandbox.service.d.ts +17 -4
  161. package/dist/sandbox/sandbox.service.js +55 -14
  162. package/dist/schedules/calendar.d.ts +134 -0
  163. package/dist/schedules/calendar.js +410 -0
  164. package/dist/schedules/fire.d.ts +1 -3
  165. package/dist/schedules/fire.js +5 -10
  166. package/dist/schedules/run-tracker.d.ts +0 -5
  167. package/dist/schedules/run-tracker.js +0 -13
  168. package/dist/schedules/scan.job.js +15 -20
  169. package/dist/schedules/schedule-math.d.ts +42 -0
  170. package/dist/schedules/schedule-math.js +52 -0
  171. package/dist/schedules/scheduled-tasks.service.d.ts +16 -0
  172. package/dist/schedules/scheduled-tasks.service.js +53 -0
  173. package/dist/schedules/schedules.controller.d.ts +5 -1
  174. package/dist/schedules/schedules.controller.js +15 -1
  175. package/dist/schedules/schedules.schemas.d.ts +35 -0
  176. package/dist/schedules/schedules.schemas.js +27 -0
  177. package/dist/schedules/store.d.ts +5 -0
  178. package/dist/schedules/store.js +20 -0
  179. package/dist/tools/approvals.service.d.ts +26 -7
  180. package/dist/tools/approvals.service.js +88 -10
  181. package/dist/tools/builtin-names.d.ts +5 -3
  182. package/dist/tools/builtin-names.js +12 -4
  183. package/dist/tools/builtins/builtin-tools.d.ts +1 -2
  184. package/dist/tools/builtins/builtin-tools.js +21 -17
  185. package/dist/tools/builtins/sandbox/bash.js +9 -3
  186. package/dist/tools/builtins/sandbox/common.d.ts +18 -1
  187. package/dist/tools/builtins/sandbox/common.js +14 -1
  188. package/dist/tools/builtins/sandbox/index.d.ts +2 -1
  189. package/dist/tools/builtins/sandbox/index.js +4 -1
  190. package/dist/tools/builtins/sandbox/task-list.d.ts +12 -0
  191. package/dist/tools/builtins/sandbox/task-list.js +130 -0
  192. package/dist/tools/builtins/sandbox/task-output.d.ts +8 -1
  193. package/dist/tools/builtins/sandbox/task-output.js +21 -3
  194. package/dist/tools/builtins/web/extract.d.ts +2 -4
  195. package/dist/tools/builtins/web/extract.js +2 -5
  196. package/dist/tools/pipeline.d.ts +1 -1
  197. package/dist/tools/pipeline.js +1 -1
  198. package/dist/tools/questions.service.d.ts +15 -6
  199. package/dist/tools/questions.service.js +20 -8
  200. package/dist/tools/registry.d.ts +4 -2
  201. package/dist/tools/registry.js +4 -3
  202. package/dist/tools/tools.module.js +4 -3
  203. package/package.json +3 -3
  204. package/src/migrations/0118_device_policy_ops.sql +6 -0
  205. package/src/migrations/0119_outbox.sql +22 -0
  206. package/src/migrations/0120_drop_usage_report_sweep.sql +7 -0
@@ -7,7 +7,6 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
7
7
  var __param = (this && this.__param) || function (paramIndex, decorator) {
8
8
  return function (target, key) { decorator(target, key, paramIndex); }
9
9
  };
10
- import { getQueueToken } from "@nestjs/bullmq";
11
10
  import { Inject, Injectable } from "@nestjs/common";
12
11
  import { DelayedError } from "bullmq";
13
12
  import { sql } from "kysely";
@@ -18,18 +17,18 @@ import { KernelDb } from "../infra/database/kernel-db.js";
18
17
  import { TransactionHost } from "../infra/database/transaction-host.js";
19
18
  import { Transactional } from "../infra/database/transactional.decorator.js";
20
19
  import { KernelEvents } from "../infra/events/kernel-events.js";
21
- import { LockService } from "../infra/lock/lock.service.js";
22
- import { LockedCron } from "../infra/schedule/locked-cron.decorator.js";
20
+ import { OutboxService } from "../infra/outbox/outbox.service.js";
23
21
  import { KERNEL_LOGGER } from "../infra/tokens.js";
24
22
  import { ApprovalsService, NETWORK_ASK_CANCELLED_NOTE, } from "../tools/approvals.service.js";
25
23
  import { APPROVAL_CANCELLED_TEXT, INPUT_CANCELLED_TEXT, RUN_STOPPED_TEXT, ToolPipeline, } from "../tools/pipeline.js";
26
24
  import { QuestionsService } from "../tools/questions.service.js";
27
25
  import { RunAbort } from "./abort.js";
28
26
  import { KyselyCheckpointSaver } from "./checkpointer.js";
29
- import { subagentsWaitId, subagentWaitingIdemKey } from "./loop.js";
27
+ import { subagentWaitingIdemKey } from "./loop.js";
30
28
  import { MAINTENANCE_QUEUE_NAME } from "./maintenance-queue.js";
31
29
  import { assistantIdemKey } from "./model-call.js";
32
- import { notifyFinished, RunFinish, TERMINAL_STATUSES } from "./run-finish.js";
30
+ import { notifyFinished, RunFinish } from "./run-finish.js";
31
+ import { lockWaitRowsIn, RunResume, resumeJobId, TERMINAL_STATUSES } from "./run-jobs.js";
33
32
  import { ACTIVE_RUN_STATUSES, RunQueue } from "./run-queue.js";
34
33
  import { childResult, deliveryRowOfChildRun, finishedEvent, rootSurfaceOf, } from "./subagents/child.js";
35
34
  import { SubagentDeliveries } from "./subagents/deliveries.js";
@@ -38,28 +37,18 @@ export { MAINTENANCE_QUEUE_NAME };
38
37
  /** 等待超时任务的名字与 jobId(每个审批 / 输入请求一个,重复投递被 BullMQ 忽略)。 */
39
38
  export const WAIT_TIMEOUT_JOB = "wait-timeout";
40
39
  export const waitTimeoutJobId = (waitId) => `wait-timeout-${waitId}`;
41
- /** 续跑任务的 jobId:与按消息生成的 `run-<conversationId>-<seq>` 不冲突,同一等待只会续跑一次。 */
42
- export const resumeJobId = (conversationId, waitId) => `run-${conversationId}-resume-${waitId}`;
40
+ /** A resumed run's job id (`run-<conversationId>-resume-<waitId>`; `RunResume` owes it). */
41
+ export { resumeJobId };
43
42
  /**
44
- * 等待兜底扫描(Task 22):任务名、job scheduler id(不含 `:`)、周期、宽限期与每类每次最多处理的条数。
45
- * 宽限期:waiting 超过它、等待却已全部处理的运行才视为「续跑丢失」,避免与正常续跑(暂停后的复查等)竞争。
46
- */
47
- export const WAIT_SWEEP_JOB = "wait-sweep";
48
- /** The maintenance scheduler the sweep ran on before it became a `@LockedCron` (retired). */
49
- export { WAIT_SWEEP_SCHEDULER_ID } from "./job-ids.js";
50
- export const WAIT_SWEEP_GRACE_MS = 60_000;
51
- export const WAIT_SWEEP_BATCH = 500;
52
- const TERMINAL_RUN_STATUSES = ["completed", "failed", "aborted"];
53
- /** 超时任务处理出错(如 PG 暂不可用)时的重试。 */
54
- const TIMEOUT_JOB_ATTEMPTS = 5;
55
- const TIMEOUT_JOB_BACKOFF_MS = 5_000;
56
- /**
57
- * 运行等待人处理(spec 4.4 / Task 9):暂停、续跑、超时与中止。状态全部在 PG(`runs.status = 'waiting'`、
58
- * `approvals` / `questions` 行),延迟与重试走 BullMQ(维护队列的 `wait-timeout` 任务),没有进程内状态。
43
+ * Runs waiting for a person (spec 4.4 / Task 9): suspend, resume, timeout and stop. All state is in
44
+ * PG (`runs.status = 'waiting'`, the `approvals` / `questions` rows); delays and retries are BullMQ
45
+ * jobs (the maintenance queue's `wait-timeout`) owed through the outbox; no in-process state.
59
46
  *
60
- * 只续跑一次:`resume` 是一条条件更新 `waiting → queued`(且该运行已没有 pending 的等待),并发调用只有一个成功,
61
- * 成功者投递 jobId 为 `run-<conversationId>-resume-<waitId>` 的运行任务。人工决定、超时任务、
62
- * 暂停后的复查都可能调用它,谁先谁后都只有一次续跑。
47
+ * One resume only: a resume locks the run row and moves it `waiting → queued` only when it waits on
48
+ * nothing else, owing the run job `run-<conversationId>-resume-<waitId>` in the same transaction
49
+ * (transactional outbox spec §6). A decision does it in its own transaction
50
+ * (`RunResume.resumeIn`); the timeout job, the suspend's re-check and a fork child's finish
51
+ * through {@link resume}. Whichever comes first, the run resumes once.
63
52
  */
64
53
  let Waits = class Waits {
65
54
  kdb;
@@ -71,15 +60,17 @@ let Waits = class Waits {
71
60
  finisher;
72
61
  checkpoints;
73
62
  abort;
74
- timeouts;
75
- events;
63
+ outbox;
64
+ resumes;
76
65
  log;
77
66
  background;
78
67
  deliveries;
79
68
  txHost;
80
69
  /** Drops the `run.orphaned` subscription (the events may outlive this instance). */
81
70
  unsubscribe;
82
- constructor(kdb, eventLog, approvals, questions, runQueue,
71
+ constructor(kdb, eventLog, approvals, questions,
72
+ /** Stops a queued or running sub-agent no worker holds directly ({@link stopRun}). */
73
+ runQueue,
83
74
  /**
84
75
  * Writes, in the caller's transaction, the results of a stopped run's open `Agent` calls;
85
76
  * `fire` runs their observe hooks after the commit.
@@ -93,8 +84,10 @@ let Waits = class Waits {
93
84
  * `waiting` is set. Children are stopped through `request`.
94
85
  */
95
86
  abort,
96
- /** The maintenance queue: wait timeout jobs (`MaintenanceProcessor` consumes them). */
97
- timeouts, events, log,
87
+ /** Owes the wait timeout jobs (`MaintenanceProcessor` consumes them) with the suspend. */
88
+ outbox,
89
+ /** The resume of a waiting run, in the caller's transaction ({@link resume}). */
90
+ resumes, events, log,
98
91
  /** Stops a finished sub-agent run's background tasks ({@link childFinished}). */
99
92
  background,
100
93
  /** Delivers a background child's finish to its parent ({@link childFinished}). */
@@ -108,15 +101,15 @@ let Waits = class Waits {
108
101
  this.finisher = finisher;
109
102
  this.checkpoints = checkpoints;
110
103
  this.abort = abort;
111
- this.timeouts = timeouts;
112
- this.events = events;
104
+ this.outbox = outbox;
105
+ this.resumes = resumes;
113
106
  this.log = log;
114
107
  this.background = background;
115
108
  this.deliveries = deliveries;
116
109
  this.txHost = txHost;
117
- // an orphaned parent run (marked failed by `ensureRun`) stops its fork child at once
118
- // (background children keep running, agent map spec §5.4); the sweep's
119
- // `sweepForkChildrenOfFinished` is the backstop
110
+ // an orphaned parent run (failed by the `run-orphan-check`) stops its fork child at once
111
+ // (background children keep running, agent map spec §5.4); the `stop-tree` job its terminal
112
+ // transaction owed is the backstop
120
113
  this.unsubscribe = events.on("run.orphaned", ({ runId }) => this.stopChildren(runId, null)
121
114
  .then(() => { })
122
115
  .catch((err) => this.log.error("cannot stop the sub-agents of an orphaned run", { runId, err })));
@@ -130,55 +123,18 @@ let Waits = class Waits {
130
123
  return this.kdb.db;
131
124
  }
132
125
  /**
133
- * 兜底扫描(维护队列每 10 分钟一次,防延迟任务 / 续跑丢失)。每项修复都复用常规路径,与并发的超时任务、
134
- * 人工决定、其他副本的扫描竞争时由行锁与条件更新保证只生效一次;每项生效的修复记一行 warn(只含 id)。
135
- * 顺序:先取消终态运行的等待(这些不该再按超时处理),再处理过期的等待,最后续跑「续跑丢失」的运行。
136
- * 单项失败只记日志,不影响其他项。
137
- */
138
- async sweep() {
139
- const out = {
140
- expired: 0,
141
- resumed: 0,
142
- cancelled: 0,
143
- repaired: 0,
144
- requeued: 0,
145
- orphaned: 0,
146
- stopped: 0,
147
- delivered: 0,
148
- restarted: 0,
149
- };
150
- out.cancelled = await this.sweepFinishedRuns();
151
- out.expired = await this.sweepOverdue();
152
- Object.assign(out, await this.sweepSubagentOrphans());
153
- out.repaired = await this.sweepOpenAgentCalls();
154
- out.stopped =
155
- (await this.sweepForkChildrenOfFinished()) + (await this.sweepTreesOfDeletedRoots());
156
- out.delivered = await this.sweepUndelivered();
157
- out.restarted = await this.sweepParkedMessages();
158
- out.resumed = await this.sweepStuckRuns();
159
- return out;
160
- }
161
- /**
162
- * loop 返回 `suspended` 之后(worker 已释放会话锁与配额):条件更新 `running → waiting`(记 `waiting_since`),
163
- * 投递延迟到 `expiresAt` 的超时任务,再复查一次等待是否已被处理——处理者的 `resume` 可能发生在本次置 waiting
164
- * 之前(那时条件更新不成立),复查保证这种情况下仍会续跑。
165
- * 运行已不是 running(例如已被其他路径结束)→ 什么都不做。
166
- * 暂停时已收到用户中止(`abort`),或置 waiting 之后补查到中止标记 → 直接按 {@link abortWaiting} 取消等待并结束
167
- * 运行(置 waiting 之后到达的中止由 `POST .../abort` 看到 waiting 直接处理,两边不会都漏掉)。
126
+ * After the loop returned `suspended` (the worker released the conversation lock and the quota):
127
+ * in one transaction the conditional `running → waiting` (recording `waiting_since`) and the
128
+ * wait's timeout job (`runAt` = `expiresAt`); then a re-check whether the wait is already
129
+ * settled: a decision that came while the run was still `running` did not resume it, so the
130
+ * re-check does.
131
+ * A run no longer running (ended by another path) → nothing.
132
+ * A stop received while suspending (`abort`), or found in the stop marker after `waiting` is set
133
+ * → the wait is cancelled and the run ended as {@link abortWaiting} does (a stop arriving after
134
+ * `waiting` is set is handled by `POST .../abort`, which sees `waiting`; neither side misses it).
168
135
  */
169
136
  async suspend(runId, conversationId, waitingFor, abort) {
170
- const r = await this.db
171
- .updateTable("runs")
172
- .set({
173
- status: "waiting",
174
- waiting_since: sql `now()`,
175
- // a sub-agents wait has no row of its own: its expiry lives on the run, for the sweep
176
- wait_expires_at: waitingFor.kind === "subagents" ? new Date(waitingFor.expiresAt) : null,
177
- })
178
- .where("id", "=", runId)
179
- .where("status", "=", "running")
180
- .executeTakeFirst();
181
- if (Number(r.numUpdatedRows) === 0) {
137
+ if (!(await this.suspendIn(runId, waitingFor))) {
182
138
  this.log.warn("run is no longer running; not suspending", { runId });
183
139
  return;
184
140
  }
@@ -187,10 +143,37 @@ let Waits = class Waits {
187
143
  return;
188
144
  }
189
145
  await this.surfaceInRoot(runId, conversationId, waitingFor);
190
- await this.scheduleTimeout(waitingFor, runId);
146
+ // A decision that locked the run while it was still `running` wrote nothing to it
147
+ // (`RunResume.resumeIn`: "still-running"); its row lock held this suspend's update until it
148
+ // committed, so this read, a new statement after that commit, sees the decided wait.
191
149
  if (await this.isSettled(waitingFor, runId))
192
150
  await this.resume(runId, waitingFor.id);
193
151
  }
152
+ /**
153
+ * {@link suspend}'s transaction: the conditional `running → waiting` and the wait's timeout job
154
+ * owed with it ({@link oweTimeout}). False when the run was no longer running (nothing written).
155
+ */
156
+ async suspendIn(runId, waitingFor) {
157
+ const row = await this.db
158
+ .updateTable("runs")
159
+ .set({
160
+ status: "waiting",
161
+ waiting_since: sql `now()`,
162
+ // a sub-agents wait has no row of its own: its expiry is kept on the run (its timeout job
163
+ // carries the run and step)
164
+ wait_expires_at: waitingFor.kind === "subagents" ? new Date(waitingFor.expiresAt) : null,
165
+ })
166
+ .from("conversations")
167
+ .whereRef("conversations.id", "=", "runs.conversation_id")
168
+ .where("runs.id", "=", runId)
169
+ .where("runs.status", "=", "running")
170
+ .returning("conversations.org_id")
171
+ .executeTakeFirst();
172
+ if (!row)
173
+ return false;
174
+ await this.oweTimeout(waitingFor, runId, row.org_id);
175
+ return true;
176
+ }
194
177
  /**
195
178
  * A sub-agent's run waits for an approval or an input (sub-agents spec §9): appends
196
179
  * `subagent_waiting` to the **root** conversation of its tree, whatever its depth (agent map spec
@@ -255,59 +238,12 @@ let Waits = class Waits {
255
238
  }
256
239
  }
257
240
  /**
258
- * 续跑:一条条件更新 `waiting → queued`(且该运行没有 pending 的审批 / 输入请求),把本次等待时长累加进
259
- * `waited_ms`,记下续跑任务的 jobId;成功(只有一个调用方)则投递运行任务并返回 true。
260
- * 投递失败 → 撤回为 waiting(等待时长已计入,重新从现在起计),之后超时任务到点会再次续跑,并抛出。
241
+ * Resumes the run in its own transaction (`RunResume.resumeIn`: locks the run row and, when it
242
+ * is `waiting` on nothing else, sets it `queued` and owes its resume job); true when this call
243
+ * queued it. The wait timeout, the suspend's re-check and a fork child's finish use it; the decisions call `RunResume.resumeIn` in their own transactions.
261
244
  */
262
245
  async resume(runId, waitId) {
263
- const row = await this.db
264
- .updateTable("runs")
265
- .set({
266
- status: "queued",
267
- queued_at: sql `now()`,
268
- job_id: sql `'run-' || conversation_id || '-resume-' || ${waitId}::text`,
269
- waited_ms: sql `waited_ms + COALESCE(floor(extract(epoch FROM now() - waiting_since) * 1000), 0)::bigint`,
270
- waiting_since: null,
271
- wait_expires_at: null,
272
- })
273
- .where("id", "=", runId)
274
- .where("status", "=", "waiting")
275
- .where(({ not, exists, selectFrom }) => not(exists(selectFrom("approvals")
276
- .select(sql `1`.as("one"))
277
- .where("approvals.run_id", "=", runId)
278
- .where("approvals.status", "=", "pending"))))
279
- .where(({ not, exists, selectFrom }) => not(exists(selectFrom("questions")
280
- .select(sql `1`.as("one"))
281
- .where("questions.run_id", "=", runId)
282
- .where("questions.status", "=", "pending"))))
283
- // a fork invocation's sub-agent (phase D; agent map spec §5.4: background `Agent`
284
- // children never hold their parent run) is no longer active
285
- .where(({ not, exists, selectFrom }) => not(exists(selectFrom("subagent_dispatches as sd")
286
- .innerJoin("runs as cr", "cr.id", "sd.child_run_id")
287
- .select(sql `1`.as("one"))
288
- .where("sd.parent_run_id", "=", runId)
289
- .where("sd.for_skill", "=", true)
290
- .where("cr.status", "in", ACTIVE_RUN_STATUSES))))
291
- .returning(["conversation_id"])
292
- .executeTakeFirst();
293
- if (!row)
294
- return false;
295
- const jobId = resumeJobId(row.conversation_id, waitId);
296
- try {
297
- await this.runQueue.enqueue({ runId, conversationId: row.conversation_id }, jobId);
298
- }
299
- catch (err) {
300
- this.log.error("cannot enqueue resumed run; back to waiting", { runId, jobId, err });
301
- await this.db
302
- .updateTable("runs")
303
- .set({ status: "waiting", waiting_since: sql `now()`, job_id: null, queued_at: null })
304
- .where("id", "=", runId)
305
- .where("status", "=", "queued")
306
- .where("job_id", "=", jobId)
307
- .execute();
308
- throw err;
309
- }
310
- return true;
246
+ return (await this.resumes.resumeIn(runId, waitId)) === "queued";
311
247
  }
312
248
  /**
313
249
  * 超时(`wait-timeout` 任务):仍 pending 且已到期 → 置 expired 并写决定 / 回答事件;随后(无论本次是否生效)
@@ -360,10 +296,12 @@ let Waits = class Waits {
360
296
  .deleteThread(runId)
361
297
  .catch((err) => this.log.warn("checkpoint delete failed", { runId, err }));
362
298
  stopped?.fire();
363
- // a stopped sub-agent reports to its parent now, not at the next sweep (a no-op for other runs)
299
+ // a stopped sub-agent reports to its parent now, not when the owed `deliver-to-parent` job
300
+ // runs (a no-op for other runs)
364
301
  await this.childFinished(runId).catch((err) => this.log.error("cannot report stopped sub-agent run", { runId, err }));
365
302
  // a stopped run stops its fork child (spec §12), on its behalf (no account: it reports
366
- // "stopped with this run"); background children keep running; the wait sweep is the backstop
303
+ // "stopped with this run"); background children keep running; the `stop-tree` job the
304
+ // terminal transaction owed is the backstop
367
305
  await this.stopChildren(runId, null).catch((err) => this.log.error("cannot stop the sub-agents of a run", { runId, err }));
368
306
  return true;
369
307
  }
@@ -376,6 +314,7 @@ let Waits = class Waits {
376
314
  .where("id", "=", conversationId)
377
315
  .forUpdate()
378
316
  .executeTakeFirst();
317
+ await lockWaitRowsIn(t, runId);
379
318
  const run = await t
380
319
  .selectFrom("runs")
381
320
  .select("status")
@@ -444,561 +383,6 @@ let Waits = class Waits {
444
383
  },
445
384
  });
446
385
  }
447
- /** 扫描 3:运行已终态(如暂停途中失败)仍 pending 的审批 / 输入请求 → 取消(写决定 / 回答事件)。 */
448
- async sweepFinishedRuns() {
449
- const db = this.db;
450
- const runs = await db
451
- .selectFrom("runs")
452
- .select(["id", "conversation_id"])
453
- .where("status", "in", TERMINAL_RUN_STATUSES)
454
- .where(({ or, exists, selectFrom }) => or([
455
- exists(selectFrom("approvals")
456
- .select(sql `1`.as("one"))
457
- .whereRef("approvals.run_id", "=", "runs.id")
458
- .where("approvals.status", "=", "pending")),
459
- exists(selectFrom("questions")
460
- .select(sql `1`.as("one"))
461
- .whereRef("questions.run_id", "=", "runs.id")
462
- .where("questions.status", "=", "pending")),
463
- ]))
464
- .limit(WAIT_SWEEP_BATCH)
465
- .execute();
466
- let total = 0;
467
- for (const run of runs) {
468
- try {
469
- const n = await this.cancelWaitsOfFinished(run.conversation_id, run.id);
470
- if (n > 0) {
471
- total += n;
472
- this.log.warn("wait sweep: cancelled waits of finished run", {
473
- runId: run.id,
474
- conversationId: run.conversation_id,
475
- count: n,
476
- });
477
- }
478
- }
479
- catch (err) {
480
- this.log.error("wait sweep: cannot cancel waits of finished run", { runId: run.id, err });
481
- }
482
- }
483
- return total;
484
- }
485
- /**
486
- * 同一事务里锁会话行与运行行(与 {@link abortWaiting} 同序),运行仍是终态时取消其 pending 等待;
487
- * 提交后发布事件。返回取消的条数(并发的另一副本拿到锁时已无 pending → 0)。
488
- */
489
- async cancelWaitsOfFinished(conversationId, runId) {
490
- const r = await this.cancelWaitsOfFinishedIn(conversationId, runId);
491
- if (r > 0) {
492
- const last = await this.db
493
- .selectFrom("conversation_events")
494
- .select(sql `max(seq)`.as("seq"))
495
- .where("conversation_id", "=", conversationId)
496
- .executeTakeFirst();
497
- if (last?.seq != null)
498
- await this.eventLog.notify(conversationId, Number(last.seq));
499
- }
500
- return r;
501
- }
502
- async cancelWaitsOfFinishedIn(conversationId, runId) {
503
- const t = this.db;
504
- await t
505
- .selectFrom("conversations")
506
- .select("id")
507
- .where("id", "=", conversationId)
508
- .forUpdate()
509
- .executeTakeFirst();
510
- const run = await t
511
- .selectFrom("runs")
512
- .select("status")
513
- .where("id", "=", runId)
514
- .forUpdate()
515
- .executeTakeFirst();
516
- if (!run || !TERMINAL_RUN_STATUSES.includes(run.status))
517
- return 0;
518
- const a = await this.approvals.cancelPendingIn(runId);
519
- const q = await this.questions.cancelPendingIn(runId);
520
- return a.length + q.length;
521
- }
522
- /** 扫描 1:`expires_at < now()` 仍 pending 的审批 / 输入请求 → 与超时任务同一路径(置 expired、写事件、续跑)。 */
523
- async sweepOverdue() {
524
- const db = this.db;
525
- const approvals = await db
526
- .selectFrom("approvals")
527
- .select(["id", "run_id"])
528
- .where("status", "=", "pending")
529
- .where("expires_at", "<", sql `now()`)
530
- .orderBy("expires_at")
531
- .limit(WAIT_SWEEP_BATCH)
532
- .execute();
533
- const questions = await db
534
- .selectFrom("questions")
535
- .select(["id", "run_id"])
536
- .where("status", "=", "pending")
537
- .where("expires_at", "<", sql `now()`)
538
- .orderBy("expires_at")
539
- .limit(WAIT_SWEEP_BATCH)
540
- .execute();
541
- const due = [
542
- ...approvals.map((a) => ({ kind: "approval", id: a.id, runId: a.run_id })),
543
- ...questions.map((q) => ({ kind: "input", id: q.id, runId: q.run_id })),
544
- ];
545
- let total = 0;
546
- for (const w of due) {
547
- try {
548
- const r = w.kind === "approval"
549
- ? await this.approvals.expire(w.id)
550
- : await this.questions.expire(w.id);
551
- if (r.outcome !== "expired")
552
- continue;
553
- total++;
554
- this.log.warn("wait sweep: expired overdue wait", {
555
- kind: w.kind,
556
- id: w.id,
557
- runId: w.runId,
558
- });
559
- if (r.record)
560
- await this.resume(r.record.runId, w.id);
561
- }
562
- catch (err) {
563
- this.log.error("wait sweep: cannot expire overdue wait", { kind: w.kind, id: w.id, err });
564
- }
565
- }
566
- return total + (await this.sweepOverdueSubagents());
567
- }
568
- /**
569
- * Sub-agents waits past their `runs.wait_expires_at` (their timeout job was lost): expired the
570
- * way the timeout job would ({@link expire}: the fork invocation's active child stops as
571
- * `timed out`, the parent resumes once it is done). Only fork invocations wait (agent map spec
572
- * §5.4); their step is the run's fork dispatch step. A run stays here until its children have
573
- * stopped; a pass that neither stopped a child nor resumed the run is not counted.
574
- */
575
- async sweepOverdueSubagents() {
576
- const runs = await this.db
577
- .selectFrom("runs")
578
- .select(["id"])
579
- .where("status", "=", "waiting")
580
- .where("wait_expires_at", "<", sql `now()`)
581
- .orderBy("wait_expires_at")
582
- .limit(WAIT_SWEEP_BATCH)
583
- .execute();
584
- let total = 0;
585
- for (const r of runs) {
586
- try {
587
- const last = await this.db
588
- .selectFrom("subagent_dispatches")
589
- .select(sql `max(step)`.as("step"))
590
- .where("parent_run_id", "=", r.id)
591
- .where("for_skill", "=", true)
592
- .executeTakeFirst();
593
- if (last?.step == null)
594
- continue;
595
- const step = Number(last.step);
596
- const id = subagentsWaitId(r.id, step);
597
- // children asked to stop may take a while to end, so the same wait can come back on the
598
- // next sweep: only a pass that stopped a child or resumed the run counts and logs (M3)
599
- const { acted } = await this.expire("subagents", id, { runId: r.id, step });
600
- if (!acted)
601
- continue;
602
- this.log.warn("wait sweep: expired overdue wait", { kind: "subagents", id, runId: r.id });
603
- total++;
604
- }
605
- catch (err) {
606
- this.log.error("wait sweep: cannot expire overdue wait", {
607
- kind: "subagents",
608
- runId: r.id,
609
- err,
610
- });
611
- }
612
- }
613
- return total;
614
- }
615
- /**
616
- * Sub-agent runs nobody will run (spec §7.5; `RunQueue.repairOrphan`, the same orphan test as
617
- * `ensureRun`, which never runs for a child): `queued` past the grace period without a job → enqueued
618
- * again (the dispatch committed, its enqueue was lost); `running` without job and lock → failed
619
- * `run.orphaned` and reported to the parent, which then resumes. Also a chat run left `queued`
620
- * by a resume whose enqueue was lost (final review m8): enqueued again under its resume job id
621
- * (`ensureRun` would only repair it at the conversation's next message).
622
- */
623
- async sweepSubagentOrphans() {
624
- const out = { requeued: 0, orphaned: 0 };
625
- const runs = await this.db
626
- .selectFrom("runs")
627
- .innerJoin("conversations as c", "c.id", "runs.conversation_id")
628
- .select(["runs.id", "runs.conversation_id"])
629
- .where((eb) => eb.or([
630
- eb.and([eb("c.kind", "=", "subagent"), eb("runs.status", "in", ["queued", "running"])]),
631
- // a parent (or any chat run) set `queued` by Waits.resume whose replica died before its
632
- // queue.add: child completions make that a frequent cross-replica resume (m8); its job
633
- // id is the resume job id recorded with the update
634
- eb.and([
635
- eb("c.kind", "=", "chat"),
636
- eb("runs.status", "=", "queued"),
637
- eb("runs.job_id", "like", "run-%-resume-%"),
638
- ]),
639
- ]))
640
- .where(sql `COALESCE(runs.queued_at, runs.created_at)`, "<", sql `now() - ${WAIT_SWEEP_GRACE_MS}::integer * interval '1 millisecond'`)
641
- .limit(WAIT_SWEEP_BATCH)
642
- .execute();
643
- for (const run of runs) {
644
- try {
645
- const r = await this.runQueue.repairOrphan(run.conversation_id, run.id);
646
- if (r === null)
647
- continue;
648
- if (r === "failed") {
649
- out.orphaned++;
650
- await this.childFinished(run.id);
651
- }
652
- else
653
- out.requeued++;
654
- this.log.warn("wait sweep: repaired orphaned sub-agent run", {
655
- runId: run.id,
656
- conversationId: run.conversation_id,
657
- outcome: r,
658
- });
659
- }
660
- catch (err) {
661
- this.log.error("wait sweep: cannot repair orphaned sub-agent run", {
662
- runId: run.id,
663
- err,
664
- });
665
- }
666
- }
667
- return out;
668
- }
669
- /**
670
- * Idle sub-agent conversations (no active run, not deleted) with a message newer than what they
671
- * consumed, written past the grace period and within the last day: a `SendMessage` that reached a
672
- * run after its last model call is started by that run's post-job `ensureRun`
673
- * (`RunQueue.childWakeIn`), which a replica dying after the terminal commit skips. Re-derived with
674
- * the same `ensureRun` (its rules decide: a paused root starts nothing). A sub-agent whose last
675
- * run was stopped (`aborted`) can never start from a message, so it is left out here, and the
676
- * newest conversations come first (active within the last day), so parked messages that cannot
677
- * start never fill the batch. Returns how many runs started.
678
- */
679
- async sweepParkedMessages() {
680
- const rows = await this.db
681
- .selectFrom("conversations as c")
682
- .select("c.id")
683
- .where("c.kind", "=", "subagent")
684
- .where("c.deleted_at", "is", null)
685
- // every append bumps `updated_at`: a message of the last day implies it (an index-friendly
686
- // bound before the event lookups)
687
- .where("c.updated_at", ">", sql `now() - interval '1 day'`)
688
- .where((eb) => sql `(
689
- SELECT r.status FROM capybara.runs r WHERE r.conversation_id = ${eb.ref("c.id")}
690
- ORDER BY r.created_at DESC, r.id DESC LIMIT 1
691
- ) IS DISTINCT FROM 'aborted'`)
692
- .where(({ exists, selectFrom }) => exists(selectFrom("conversation_events as e")
693
- .select(sql `1`.as("one"))
694
- .whereRef("e.conversation_id", "=", "c.id")
695
- .where("e.type", "=", "user_message")
696
- .whereRef("e.seq", ">", "c.consumed_seq")
697
- .where("e.created_at", "<", sql `now() - ${WAIT_SWEEP_GRACE_MS}::integer * interval '1 millisecond'`)
698
- .where("e.created_at", ">", sql `now() - interval '1 day'`)))
699
- .where(({ not, exists, selectFrom }) => not(exists(selectFrom("runs as r")
700
- .select(sql `1`.as("one"))
701
- .whereRef("r.conversation_id", "=", "c.id")
702
- .where("r.status", "in", ACTIVE_RUN_STATUSES))))
703
- .orderBy("c.updated_at", "desc")
704
- .orderBy("c.id")
705
- .limit(WAIT_SWEEP_BATCH)
706
- .execute();
707
- let n = 0;
708
- for (const c of rows) {
709
- try {
710
- if ((await this.runQueue.ensureRun(c.id)) === null)
711
- continue;
712
- n++;
713
- this.log.warn("wait sweep: started a sub-agent's parked message", { conversationId: c.id });
714
- }
715
- catch (err) {
716
- this.log.error("wait sweep: cannot start a sub-agent's parked message", {
717
- conversationId: c.id,
718
- err,
719
- });
720
- }
721
- }
722
- return n;
723
- }
724
- /**
725
- * Finished runs (past the grace period, within the last day) triggered by a fork invocation that
726
- * has no report ({@link unreportedForkRuns}): a terminal path that skipped
727
- * `stopOpenAgentCalls`, such as an old replica's abort, would otherwise leave a fork lead without
728
- * its report. Each is repaired under the conversation row lock with the normal
729
- * `stopOpenAgentCalls` (an ended child gives its result, an active one the stopped text;
730
- * idempotency keys make it safe against the regular paths); an active fork child is then stopped
731
- * by {@link sweepForkChildrenOfFinished}. An `Agent` call has its result from its dispatch (agent
732
- * map spec §4.1).
733
- */
734
- async sweepOpenAgentCalls() {
735
- const runs = await unreportedForkRuns(this.db);
736
- let total = 0;
737
- for (const r of runs) {
738
- try {
739
- const st = {};
740
- const toSeq = await this.repairOpenAgentCallsIn(r.parent_conversation_id, r.parent_run_id, st);
741
- const fire = st.fire;
742
- if (fire === undefined)
743
- continue;
744
- if (toSeq !== null)
745
- await this.eventLog.notify(r.parent_conversation_id, toSeq);
746
- fire();
747
- total++;
748
- this.log.warn("wait sweep: wrote the missing fork report of a finished run", {
749
- runId: r.parent_run_id,
750
- conversationId: r.parent_conversation_id,
751
- });
752
- }
753
- catch (err) {
754
- this.log.error("wait sweep: cannot write the missing fork report of a finished run", {
755
- runId: r.parent_run_id,
756
- err,
757
- });
758
- }
759
- }
760
- return total;
761
- }
762
- /** One run's repair of {@link sweepOpenAgentCalls}; its observe hooks go back through `st`. */
763
- async repairOpenAgentCallsIn(conversationId, runId, st) {
764
- const t = this.db;
765
- await t
766
- .selectFrom("conversations")
767
- .select("id")
768
- .where("id", "=", conversationId)
769
- .forUpdate()
770
- .executeTakeFirst();
771
- const run = await t
772
- .selectFrom("runs")
773
- .select("status")
774
- .where("id", "=", runId)
775
- .executeTakeFirst();
776
- if (run === undefined || !TERMINAL_RUN_STATUSES.includes(run.status)) {
777
- return null;
778
- }
779
- const stopped = await this.pipeline.stopOpenAgentCalls(runId);
780
- if (stopped.callIds.size === 0)
781
- return null;
782
- st.fire = stopped.fire;
783
- return stopped.toSeq ?? null;
784
- }
785
- /**
786
- * Active fork sub-agent runs (phase D, `for_skill`) of a parent run that finished more than the
787
- * grace period ago (§12: the run's terminal state stops its fork child and reports it stopped;
788
- * this is the backstop) → stopped. Background `Agent` children of a finished run are the normal
789
- * case and keep running (agent map spec §5.4).
790
- */
791
- async sweepForkChildrenOfFinished() {
792
- const parents = await this.db
793
- .selectFrom("subagent_dispatches as sd")
794
- .innerJoin("runs as cr", "cr.id", "sd.child_run_id")
795
- .innerJoin("runs as pr", "pr.id", "sd.parent_run_id")
796
- .select("sd.parent_run_id")
797
- .distinct()
798
- .where("sd.for_skill", "=", true)
799
- .where("cr.status", "in", ACTIVE_RUN_STATUSES)
800
- .where("pr.status", "in", TERMINAL_RUN_STATUSES)
801
- .where("pr.finished_at", "<", sql `now() - ${WAIT_SWEEP_GRACE_MS}::integer * interval '1 millisecond'`)
802
- .limit(WAIT_SWEEP_BATCH)
803
- .execute();
804
- let total = 0;
805
- for (const p of parents) {
806
- try {
807
- const n = await this.stopChildren(p.parent_run_id, null);
808
- if (n === 0)
809
- continue;
810
- total += n;
811
- this.log.warn("wait sweep: stopped the fork sub-agent of a finished run", {
812
- runId: p.parent_run_id,
813
- count: n,
814
- });
815
- }
816
- catch (err) {
817
- this.log.error("wait sweep: cannot stop the fork sub-agent of a finished run", {
818
- runId: p.parent_run_id,
819
- err,
820
- });
821
- }
822
- }
823
- return total;
824
- }
825
- /**
826
- * Active sub-agent runs with a deleted ancestor (agent map spec §10: deleting the root stops the
827
- * whole tree; the delete's own {@link stopTree} is the normal path, this the backstop for a crash
828
- * between marking the root deleted and stopping its tree, or a dispatch that raced it) → stopped
829
- * on the deleter's behalf (no account). No grace period: nothing in a deleted tree should run.
830
- */
831
- async sweepTreesOfDeletedRoots() {
832
- const r = await sql `
833
- WITH RECURSIVE up(run_id, conversation_id, status, ancestor, n) AS (
834
- SELECT r.id, r.conversation_id, r.status, c.parent_conversation_id, 1
835
- FROM capybara.runs r JOIN capybara.conversations c ON c.id = r.conversation_id
836
- WHERE r.status IN ('queued', 'running', 'waiting') AND c.kind = 'subagent'
837
- AND c.parent_conversation_id IS NOT NULL
838
- UNION ALL
839
- SELECT up.run_id, up.conversation_id, up.status, c.parent_conversation_id, up.n + 1
840
- FROM up JOIN capybara.conversations c ON c.id = up.ancestor
841
- WHERE c.deleted_at IS NULL AND c.parent_conversation_id IS NOT NULL AND up.n < 16
842
- )
843
- SELECT DISTINCT up.run_id, up.conversation_id, up.status
844
- FROM up JOIN capybara.conversations a ON a.id = up.ancestor
845
- WHERE a.deleted_at IS NOT NULL
846
- LIMIT ${WAIT_SWEEP_BATCH}`.execute(this.db);
847
- let total = 0;
848
- for (const run of r.rows) {
849
- try {
850
- // already asked to stop (a worker holds it and stops it at its next check): not again
851
- if (run.status === "running" && (await this.abort.requested(run.run_id)) !== null)
852
- continue;
853
- await this.stopRun(run.conversation_id, run.run_id, run.status, null);
854
- total++;
855
- this.log.warn("wait sweep: stopped a sub-agent run of a deleted conversation's tree", {
856
- runId: run.run_id,
857
- });
858
- }
859
- catch (err) {
860
- this.log.error("wait sweep: cannot stop a sub-agent run of a deleted tree", {
861
- runId: run.run_id,
862
- err,
863
- });
864
- }
865
- }
866
- return total;
867
- }
868
- /**
869
- * Background sub-agent runs that finished more than the grace period ago (within the last day)
870
- * whose parent has no `subagent_finished` for them (agent map spec §10: the finisher died after
871
- * the child's terminal commit; the finish path writes `subagent_finished` and the delivery in one
872
- * transaction) → {@link childFinished}, idempotent against a late finisher. Keyed on the child
873
- * run, so a run nobody dispatched (a wake run, §5.3) is repaired too, and on
874
- * `subagent_finished`, not on the delivery, so a child that finished before background
875
- * dispatch (its result was the `Agent` call's, its `subagent_finished` written then) is never
876
- * delivered a second time. Fork invocations' runs, a run without a terminal event in its log
877
- * (nothing to deliver), a conversation never dispatched and a child whose parent conversation
878
- * was deleted are skipped; only a written delivery counts.
879
- */
880
- async sweepUndelivered() {
881
- const runs = await this.db
882
- .selectFrom("runs as cr")
883
- .innerJoin("conversations as cc", "cc.id", "cr.conversation_id")
884
- .innerJoin("conversations as pc", "pc.id", "cc.parent_conversation_id")
885
- .select("cr.id as child_run_id")
886
- .where("cc.kind", "=", "subagent")
887
- .where("pc.deleted_at", "is", null)
888
- .where("cr.status", "in", TERMINAL_RUN_STATUSES)
889
- .where("cr.finished_at", "<", sql `now() - ${WAIT_SWEEP_GRACE_MS}::integer * interval '1 millisecond'`)
890
- .where("cr.finished_at", ">", sql `now() - interval '1 day'`)
891
- .where(({ exists, selectFrom }) => exists(selectFrom("subagent_dispatches as sd")
892
- .select(sql `1`.as("one"))
893
- .whereRef("sd.child_conversation_id", "=", "cc.id")))
894
- .where(({ not, exists, selectFrom }) => not(exists(selectFrom("subagent_dispatches as fd")
895
- .select(sql `1`.as("one"))
896
- .whereRef("fd.child_run_id", "=", "cr.id")
897
- .where("fd.for_skill", "=", true))))
898
- .where(({ not, exists, selectFrom }) => not(exists(selectFrom("conversation_events as e")
899
- .select(sql `1`.as("one"))
900
- .whereRef("e.conversation_id", "=", "pc.id")
901
- .where("e.idem_key", "=", sql `'subagent_finished:' || cr.id`))))
902
- .where(({ exists, selectFrom }) => exists(selectFrom("conversation_events as t")
903
- .select(sql `1`.as("one"))
904
- .whereRef("t.conversation_id", "=", "cc.id")
905
- .whereRef("t.run_id", "=", "cr.id")
906
- .where("t.type", "in", ["run_finished", "run_failed", "run_aborted"])))
907
- .limit(WAIT_SWEEP_BATCH)
908
- .execute();
909
- let total = 0;
910
- for (const r of runs) {
911
- try {
912
- if (!(await this.childFinished(r.child_run_id)))
913
- continue;
914
- total++;
915
- this.log.warn("wait sweep: delivered the finish of a sub-agent run", {
916
- childRunId: r.child_run_id,
917
- });
918
- }
919
- catch (err) {
920
- this.log.error("wait sweep: cannot deliver the finish of a sub-agent run", {
921
- childRunId: r.child_run_id,
922
- err,
923
- });
924
- }
925
- }
926
- return total;
927
- }
928
- /**
929
- * 扫描 2:waiting 超过宽限期、却已没有 pending 等待的运行(决定提交后进程崩溃、续跑投递失败回滚等)→
930
- * {@link resume}(条件更新 `waiting → queued` + 投递)。续跑任务的 jobId 用该运行最近一次等待的 id
931
- * (与人工决定 / 超时任务一致,已投递过的同一续跑被 BullMQ 去重)。
932
- */
933
- async sweepStuckRuns() {
934
- const runs = await this.db
935
- .selectFrom("runs")
936
- .select(["id", "conversation_id"])
937
- .where("status", "=", "waiting")
938
- .where("waiting_since", "<", sql `now() - ${WAIT_SWEEP_GRACE_MS}::integer * interval '1 millisecond'`)
939
- .where(({ not, exists, selectFrom }) => not(exists(selectFrom("approvals")
940
- .select(sql `1`.as("one"))
941
- .whereRef("approvals.run_id", "=", "runs.id")
942
- .where("approvals.status", "=", "pending"))))
943
- .where(({ not, exists, selectFrom }) => not(exists(selectFrom("questions")
944
- .select(sql `1`.as("one"))
945
- .whereRef("questions.run_id", "=", "runs.id")
946
- .where("questions.status", "=", "pending"))))
947
- // the resume condition's third clause: a run with an active fork child is not stuck
948
- .where(({ not, exists, selectFrom }) => not(exists(selectFrom("subagent_dispatches as sd")
949
- .innerJoin("runs as cr", "cr.id", "sd.child_run_id")
950
- .select(sql `1`.as("one"))
951
- .whereRef("sd.parent_run_id", "=", "runs.id")
952
- .where("sd.for_skill", "=", true)
953
- .where("cr.status", "in", ACTIVE_RUN_STATUSES))))
954
- .limit(WAIT_SWEEP_BATCH)
955
- .execute();
956
- let total = 0;
957
- for (const run of runs) {
958
- try {
959
- const waitId = (await this.lastWaitId(run.id)) ?? run.id;
960
- if (!(await this.resume(run.id, waitId)))
961
- continue;
962
- total++;
963
- this.log.warn("wait sweep: resumed stuck waiting run", {
964
- runId: run.id,
965
- conversationId: run.conversation_id,
966
- waitId,
967
- });
968
- }
969
- catch (err) {
970
- this.log.error("wait sweep: cannot resume stuck waiting run", { runId: run.id, err });
971
- }
972
- }
973
- return total;
974
- }
975
- /**
976
- * The id of the run's latest wait (by creation time): an approval, an input request or a fork
977
- * invocation's dispatch, whose wait id is its child run id (the id the child's finisher resumes
978
- * with, so a sweep resume and a late finisher resume share one job id).
979
- */
980
- async lastWaitId(runId) {
981
- const row = await this.db
982
- .selectFrom(this.db
983
- .selectFrom("approvals")
984
- .select(["id", "created_at"])
985
- .where("run_id", "=", runId)
986
- .unionAll(this.db
987
- .selectFrom("questions")
988
- .select(["id", "created_at"])
989
- .where("run_id", "=", runId))
990
- .unionAll(this.db
991
- .selectFrom("subagent_dispatches")
992
- .select(["child_run_id as id", "created_at"])
993
- .where("parent_run_id", "=", runId)
994
- .where("for_skill", "=", true))
995
- .as("w"))
996
- .select("w.id")
997
- .orderBy("w.created_at", "desc")
998
- .limit(1)
999
- .executeTakeFirst();
1000
- return row?.id ?? null;
1001
- }
1002
386
  /**
1003
387
  * A `wait-timeout` job (dispatched by `MaintenanceProcessor`): expires the wait,
1004
388
  * or delays the job again until the wait's expiry when it ran early.
@@ -1012,24 +396,22 @@ let Waits = class Waits {
1012
396
  throw new DelayedError();
1013
397
  }
1014
398
  }
1015
- /** 投递超时任务(延迟到 expiresAt)。失败只记日志:人工处理仍可续跑,只是不会自动超时。 */
1016
- async scheduleTimeout(w, runId) {
399
+ /**
400
+ * Owes the wait's timeout job (`wait-timeout-<waitId>`, `runAt` = the wait's deadline) in the
401
+ * suspend's transaction (transactional outbox spec §6 "Resumes and timeouts").
402
+ */
403
+ async oweTimeout(w, runId, orgId) {
1017
404
  const data = w.kind === "subagents" && w.step !== undefined
1018
405
  ? { kind: w.kind, id: w.id, runId, step: w.step }
1019
406
  : { kind: w.kind, id: w.id };
1020
- try {
1021
- await this.timeouts.add(WAIT_TIMEOUT_JOB, data, {
1022
- jobId: waitTimeoutJobId(w.id),
1023
- delay: Math.max(0, new Date(w.expiresAt).getTime() - Date.now()),
1024
- attempts: TIMEOUT_JOB_ATTEMPTS,
1025
- backoff: { type: "exponential", delay: TIMEOUT_JOB_BACKOFF_MS },
1026
- removeOnComplete: true,
1027
- removeOnFail: true,
1028
- });
1029
- }
1030
- catch (err) {
1031
- this.log.error("cannot schedule wait timeout", { waitId: w.id, err });
1032
- }
407
+ await this.outbox.add({
408
+ queue: MAINTENANCE_QUEUE_NAME,
409
+ name: WAIT_TIMEOUT_JOB,
410
+ jobId: waitTimeoutJobId(w.id),
411
+ data,
412
+ runAt: new Date(w.expiresAt),
413
+ orgId,
414
+ });
1033
415
  }
1034
416
  /**
1035
417
  * Whether the wait is over: an approval or input request is no longer pending (decided, expired
@@ -1116,7 +498,8 @@ let Waits = class Waits {
1116
498
  * `subagent_finished` (idempotent; not model-visible) and tries
1117
499
  * {@link resume}`(parentRunId, childRunId)`, which succeeds once its parent run waits on nothing
1118
500
  * else. A run of a conversation never dispatched, or not terminal yet, is left alone. A crash in
1119
- * between is repaired by the wait sweep (the orphan repair calls this again). A fork child's
501
+ * between is repaired by the `deliver-to-parent` job its terminal transaction owed (the
502
+ * `run-orphan-check` calls this too). A fork child's
1120
503
  * running background tasks the terminal run started are then stopped (not those of a later run);
1121
504
  * a stopped background child's are all stopped, whichever of its runs started them; a background
1122
505
  * child that ended on its own keeps them, and their finish wakes it (agent map spec §5.3). True
@@ -1244,13 +627,8 @@ let Waits = class Waits {
1244
627
  .execute()).sort((a, b) => (level.get(a.conversation_id) ?? 0) - (level.get(b.conversation_id) ?? 0));
1245
628
  for (const r of runs) {
1246
629
  try {
1247
- if (await this.stopRun(r.conversation_id, r.id, r.status, byAccountId)) {
1248
- // its approval / question cards changed (RunControl.abortWaiting's event)
1249
- await this.events.emit("run.wait-stopped", {
1250
- conversationId: r.conversation_id,
1251
- runId: r.id,
1252
- });
1253
- }
630
+ // a stopped wait's card updates are owed by its transaction's appends (outbox spec §8.1)
631
+ await this.stopRun(r.conversation_id, r.id, r.status, byAccountId);
1254
632
  }
1255
633
  catch (err) {
1256
634
  this.log.error("cannot stop a sub-agent run of the tree", { runId: r.id, err });
@@ -1321,13 +699,13 @@ let Waits = class Waits {
1321
699
  };
1322
700
  __decorate([
1323
701
  Transactional({ propagation: "requires_new" })
1324
- ], Waits.prototype, "abortWaitingIn", null);
702
+ ], Waits.prototype, "suspendIn", null);
1325
703
  __decorate([
1326
- Transactional({ propagation: "requires_new" })
1327
- ], Waits.prototype, "cancelWaitsOfFinishedIn", null);
704
+ Transactional()
705
+ ], Waits.prototype, "resume", null);
1328
706
  __decorate([
1329
707
  Transactional({ propagation: "requires_new" })
1330
- ], Waits.prototype, "repairOpenAgentCallsIn", null);
708
+ ], Waits.prototype, "abortWaitingIn", null);
1331
709
  Waits = __decorate([
1332
710
  Injectable(),
1333
711
  __param(0, Inject(KernelDb)),
@@ -1339,92 +717,12 @@ Waits = __decorate([
1339
717
  __param(6, Inject(RunFinish)),
1340
718
  __param(7, Inject(KyselyCheckpointSaver)),
1341
719
  __param(8, Inject(RunAbort)),
1342
- __param(9, Inject(getQueueToken(MAINTENANCE_QUEUE_NAME))),
1343
- __param(10, Inject(KernelEvents)),
1344
- __param(11, Inject(KERNEL_LOGGER)),
1345
- __param(12, Inject(BackgroundTaskStopper)),
1346
- __param(13, Inject(SubagentDeliveries)),
1347
- __param(14, Inject(TransactionHost))
720
+ __param(9, Inject(OutboxService)),
721
+ __param(10, Inject(RunResume)),
722
+ __param(11, Inject(KernelEvents)),
723
+ __param(12, Inject(KERNEL_LOGGER)),
724
+ __param(13, Inject(BackgroundTaskStopper)),
725
+ __param(14, Inject(SubagentDeliveries)),
726
+ __param(15, Inject(TransactionHost))
1348
727
  ], Waits);
1349
728
  export { Waits };
1350
- /**
1351
- * Phase D: finished chat runs (past the grace period, within the last day) triggered by a fork
1352
- * invocation that has no report (dispatched or not); the wait sweep writes it
1353
- * (`pairStoppedForks`).
1354
- */
1355
- export function unreportedForkRuns(db) {
1356
- return (db
1357
- .selectFrom("runs as r")
1358
- .innerJoin("conversation_events as m", (j) => j.onRef("m.conversation_id", "=", "r.conversation_id").onRef("m.seq", "=", "r.trigger_seq"))
1359
- // a sub-agent's task message carries the skill record too: chat conversations only (I1)
1360
- .innerJoin("conversations as c", "c.id", "r.conversation_id")
1361
- .select(["r.id as parent_run_id", "r.conversation_id as parent_conversation_id"])
1362
- .where("c.kind", "=", "chat")
1363
- .where("r.status", "in", TERMINAL_RUN_STATUSES)
1364
- .where("r.finished_at", "<", sql `now() - ${WAIT_SWEEP_GRACE_MS}::integer * interval '1 millisecond'`)
1365
- .where("r.finished_at", ">", sql `now() - interval '1 day'`)
1366
- .where("m.type", "=", "user_message")
1367
- .where(sql `m.payload -> 'skill' ->> 'mode' = 'fork'`)
1368
- .where(({ not, exists, selectFrom }) => not(exists(selectFrom("conversation_events as e")
1369
- .select(sql `1`.as("one"))
1370
- .whereRef("e.conversation_id", "=", "r.conversation_id")
1371
- .where("e.idem_key", "=", sql `'skill_fork_result:' || r.trigger_seq`))))
1372
- .limit(WAIT_SWEEP_BATCH)
1373
- .execute());
1374
- }
1375
- /**
1376
- * A decided approval / answered question resumes the waiting run on this replica (domain events,
1377
- * spec §3b; the wait sweep resumes a run whose resume was lost). Its IM delivery follows the same
1378
- * events (`ChannelJobs`).
1379
- */
1380
- let DecisionResumer = class DecisionResumer {
1381
- unsubscribe;
1382
- constructor(waits, events) {
1383
- this.unsubscribe = [
1384
- events.on("approval.decided", async (a) => {
1385
- await waits.resume(a.runId, a.id);
1386
- }),
1387
- events.on("question.answered", async (q) => {
1388
- await waits.resume(q.runId, q.id);
1389
- }),
1390
- ];
1391
- }
1392
- /** Unsubscribes after the stop phases: events emitted while they drain are still handled. */
1393
- onApplicationShutdown() {
1394
- for (const off of this.unsubscribe)
1395
- off();
1396
- }
1397
- };
1398
- DecisionResumer = __decorate([
1399
- Injectable(),
1400
- __param(0, Inject(Waits)),
1401
- __param(1, Inject(KernelEvents))
1402
- ], DecisionResumer);
1403
- export { DecisionResumer };
1404
- /**
1405
- * The wait sweep, every 10 minutes, once across replicas: {@link Waits.sweep}. Replaces
1406
- * the maintenance scheduler `wait-sweep-every-10m` (retired at worker start).
1407
- */
1408
- let WaitSweepJob = class WaitSweepJob {
1409
- waits;
1410
- lockService;
1411
- log;
1412
- constructor(waits, lockService, log) {
1413
- this.waits = waits;
1414
- this.lockService = lockService;
1415
- this.log = log;
1416
- }
1417
- async run() {
1418
- await this.waits.sweep();
1419
- }
1420
- };
1421
- __decorate([
1422
- LockedCron("*/10 * * * *", { name: WAIT_SWEEP_JOB })
1423
- ], WaitSweepJob.prototype, "run", null);
1424
- WaitSweepJob = __decorate([
1425
- Injectable(),
1426
- __param(0, Inject(Waits)),
1427
- __param(1, Inject(LockService)),
1428
- __param(2, Inject(KERNEL_LOGGER))
1429
- ], WaitSweepJob);
1430
- export { WaitSweepJob };