@capybara-hq/kernel 0.1.0-rc.23 → 0.1.0-rc.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. package/bootstrap.json +25 -25
  2. package/dist/admin/admin.module.d.ts +2 -1
  3. package/dist/admin/admin.module.js +7 -3
  4. package/dist/admin/outbox-dead.controller.d.ts +23 -0
  5. package/dist/admin/outbox-dead.controller.js +62 -0
  6. package/dist/admin/outbox-dead.schemas.d.ts +8 -0
  7. package/dist/admin/outbox-dead.schemas.js +18 -0
  8. package/dist/admin/outbox-dead.service.d.ts +66 -0
  9. package/dist/admin/outbox-dead.service.js +179 -0
  10. package/dist/admin/overview.service.d.ts +40 -5
  11. package/dist/admin/overview.service.js +41 -7
  12. package/dist/agents/from-template.js +1 -1
  13. package/dist/background/finish.d.ts +5 -2
  14. package/dist/background/finish.js +14 -3
  15. package/dist/background/launch.d.ts +2 -1
  16. package/dist/background/launch.js +5 -2
  17. package/dist/background/store.d.ts +8 -0
  18. package/dist/background/store.js +24 -0
  19. package/dist/catalog/admin-catalog.controller.d.ts +21 -2
  20. package/dist/catalog/admin-catalog.controller.js +45 -6
  21. package/dist/catalog/admin-catalog.schemas.d.ts +5 -0
  22. package/dist/catalog/admin-catalog.schemas.js +8 -0
  23. package/dist/catalog/catalog.module.d.ts +2 -1
  24. package/dist/catalog/catalog.module.js +5 -1
  25. package/dist/catalog/marketplaces.service.d.ts +1 -1
  26. package/dist/catalog/platform-import-update.d.ts +50 -0
  27. package/dist/catalog/platform-import-update.js +116 -0
  28. package/dist/catalog/plugin-import.d.ts +36 -1
  29. package/dist/catalog/plugin-import.js +23 -1
  30. package/dist/catalog/plugin-upgrade.d.ts +18 -0
  31. package/dist/catalog/plugin-upgrade.js +19 -0
  32. package/dist/channels/channels.module.d.ts +5 -2
  33. package/dist/channels/channels.module.js +9 -2
  34. package/dist/channels/deliver-job.d.ts +25 -0
  35. package/dist/channels/deliver-job.js +25 -0
  36. package/dist/channels/inbound.d.ts +1 -1
  37. package/dist/channels/inbound.js +2 -15
  38. package/dist/channels/jobs.d.ts +28 -18
  39. package/dist/channels/jobs.js +63 -26
  40. package/dist/config/schema.d.ts +0 -3
  41. package/dist/config/schema.js +11 -20
  42. package/dist/conversations/agent-tree.d.ts +16 -0
  43. package/dist/conversations/agent-tree.js +36 -0
  44. package/dist/conversations/conversations.service.d.ts +44 -14
  45. package/dist/conversations/conversations.service.js +52 -38
  46. package/dist/conversations/event-log.d.ts +7 -10
  47. package/dist/conversations/event-log.js +24 -15
  48. package/dist/conversations/events.js +2 -1
  49. package/dist/conversations/queue.service.d.ts +8 -3
  50. package/dist/conversations/queue.service.js +37 -13
  51. package/dist/conversations/titles.d.ts +21 -8
  52. package/dist/conversations/titles.js +42 -15
  53. package/dist/db/kernel-schema.d.ts +26 -8
  54. package/dist/devices/devices.service.d.ts +12 -3
  55. package/dist/devices/devices.service.js +18 -3
  56. package/dist/errors.d.ts +11 -0
  57. package/dist/errors.js +11 -0
  58. package/dist/evals/run-replay.js +14 -8
  59. package/dist/extensions/extensions.module.js +3 -0
  60. package/dist/extensions/lifecycle.d.ts +6 -1
  61. package/dist/extensions/lifecycle.js +9 -2
  62. package/dist/extensions/usage-subscribers.d.ts +43 -0
  63. package/dist/extensions/usage-subscribers.js +108 -0
  64. package/dist/http/api.module.js +2 -0
  65. package/dist/index.d.ts +1 -0
  66. package/dist/index.js +1 -0
  67. package/dist/infra/cli/kernel-cli.module.js +5 -0
  68. package/dist/infra/events/kernel-events.d.ts +9 -35
  69. package/dist/infra/outbox/outbox-relay.module.d.ts +6 -0
  70. package/dist/infra/outbox/outbox-relay.module.js +18 -0
  71. package/dist/infra/outbox/outbox.dispatcher.d.ts +103 -0
  72. package/dist/infra/outbox/outbox.dispatcher.js +310 -0
  73. package/dist/infra/outbox/outbox.module.d.ts +24 -0
  74. package/dist/infra/outbox/outbox.module.js +50 -0
  75. package/dist/infra/outbox/outbox.relay.d.ts +31 -0
  76. package/dist/infra/outbox/outbox.relay.js +76 -0
  77. package/dist/infra/outbox/outbox.service.d.ts +51 -0
  78. package/dist/infra/outbox/outbox.service.js +131 -0
  79. package/dist/infra/outbox/registry.d.ts +120 -0
  80. package/dist/infra/outbox/registry.js +108 -0
  81. package/dist/infra/queue/queue-names.d.ts +8 -0
  82. package/dist/infra/queue/queue-names.js +16 -0
  83. package/dist/kernel.module.js +3 -0
  84. package/dist/local-devices/device-policy.service.d.ts +4 -0
  85. package/dist/local-devices/device-policy.service.js +61 -3
  86. package/dist/local-devices/machines.d.ts +13 -5
  87. package/dist/local-devices/machines.js +45 -7
  88. package/dist/local-devices/policy-summary.d.ts +44 -1
  89. package/dist/local-devices/policy-summary.js +69 -0
  90. package/dist/models/embeddings.d.ts +9 -9
  91. package/dist/models/embeddings.js +11 -18
  92. package/dist/models/models.module.js +3 -0
  93. package/dist/orgs/orgs.service.js +7 -0
  94. package/dist/overrides.d.ts +20 -3
  95. package/dist/overrides.js +10 -2
  96. package/dist/platform/exclusions.js +5 -0
  97. package/dist/platform/ops/admin.ops.js +18 -2
  98. package/dist/platform/ops/catalog-admin.ops.d.ts +3 -1
  99. package/dist/platform/ops/catalog-admin.ops.js +32 -4
  100. package/dist/platform/ops/outbox.ops.d.ts +21 -0
  101. package/dist/platform/ops/outbox.ops.js +125 -0
  102. package/dist/platform/ops/schedules.ops.js +40 -1
  103. package/dist/platform/platform-ops.module.d.ts +2 -1
  104. package/dist/platform/platform-ops.module.js +3 -0
  105. package/dist/platform/route-schemas.js +3 -0
  106. package/dist/roles/worker.module.js +9 -4
  107. package/dist/runtime/langgraph-loop.js +13 -5
  108. package/dist/runtime/limits.d.ts +15 -0
  109. package/dist/runtime/limits.js +25 -0
  110. package/dist/runtime/loop.d.ts +8 -3
  111. package/dist/runtime/loop.js +2 -1
  112. package/dist/runtime/maintenance.d.ts +4 -2
  113. package/dist/runtime/maintenance.js +17 -2
  114. package/dist/runtime/model-call.d.ts +14 -16
  115. package/dist/runtime/model-call.js +14 -24
  116. package/dist/runtime/queue-lost-jobs.d.ts +24 -0
  117. package/dist/runtime/queue-lost-jobs.js +51 -0
  118. package/dist/runtime/run-control.d.ts +42 -5
  119. package/dist/runtime/run-control.js +66 -12
  120. package/dist/runtime/run-finish.d.ts +39 -12
  121. package/dist/runtime/run-finish.js +97 -14
  122. package/dist/runtime/run-jobs.d.ts +95 -13
  123. package/dist/runtime/run-jobs.js +199 -28
  124. package/dist/runtime/run-orphan-check.job.d.ts +142 -0
  125. package/dist/runtime/run-orphan-check.job.js +339 -0
  126. package/dist/runtime/run-processors.d.ts +16 -9
  127. package/dist/runtime/run-processors.js +1 -1
  128. package/dist/runtime/run-queue.d.ts +64 -55
  129. package/dist/runtime/run-queue.js +147 -284
  130. package/dist/runtime/run-steps.d.ts +1 -2
  131. package/dist/runtime/run-steps.js +8 -11
  132. package/dist/runtime/run-worker.d.ts +6 -17
  133. package/dist/runtime/run-worker.js +32 -34
  134. package/dist/runtime/runtime.module.d.ts +2 -3
  135. package/dist/runtime/runtime.module.js +5 -6
  136. package/dist/runtime/step-log.d.ts +16 -4
  137. package/dist/runtime/step-log.js +26 -6
  138. package/dist/runtime/stream.controller.d.ts +2 -4
  139. package/dist/runtime/stream.controller.js +5 -11
  140. package/dist/runtime/subagents/child.js +1 -1
  141. package/dist/runtime/subagents/definition.d.ts +2 -1
  142. package/dist/runtime/subagents/definition.js +4 -2
  143. package/dist/runtime/subagents/deliveries.d.ts +6 -3
  144. package/dist/runtime/subagents/deliveries.js +16 -7
  145. package/dist/runtime/subagents/dispatch.d.ts +12 -12
  146. package/dist/runtime/subagents/dispatch.js +13 -38
  147. package/dist/runtime/subagents/messages.d.ts +5 -1
  148. package/dist/runtime/subagents/messages.js +11 -3
  149. package/dist/runtime/subagents/skill-fork.d.ts +3 -3
  150. package/dist/runtime/subagents/skill-fork.js +5 -5
  151. package/dist/runtime/subagents/subagents.service.js +1 -1
  152. package/dist/runtime/subagents/tree-jobs.d.ts +61 -0
  153. package/dist/runtime/subagents/tree-jobs.js +86 -0
  154. package/dist/runtime/usage-report-jobs.d.ts +23 -48
  155. package/dist/runtime/usage-report-jobs.js +67 -119
  156. package/dist/runtime/usage-report.d.ts +28 -77
  157. package/dist/runtime/usage-report.js +35 -213
  158. package/dist/runtime/waits.d.ts +48 -198
  159. package/dist/runtime/waits.js +103 -805
  160. package/dist/sandbox/sandbox.service.d.ts +17 -4
  161. package/dist/sandbox/sandbox.service.js +55 -14
  162. package/dist/schedules/calendar.d.ts +134 -0
  163. package/dist/schedules/calendar.js +410 -0
  164. package/dist/schedules/fire.d.ts +1 -3
  165. package/dist/schedules/fire.js +5 -10
  166. package/dist/schedules/run-tracker.d.ts +0 -5
  167. package/dist/schedules/run-tracker.js +0 -13
  168. package/dist/schedules/scan.job.js +15 -20
  169. package/dist/schedules/schedule-math.d.ts +42 -0
  170. package/dist/schedules/schedule-math.js +52 -0
  171. package/dist/schedules/scheduled-tasks.service.d.ts +16 -0
  172. package/dist/schedules/scheduled-tasks.service.js +53 -0
  173. package/dist/schedules/schedules.controller.d.ts +5 -1
  174. package/dist/schedules/schedules.controller.js +15 -1
  175. package/dist/schedules/schedules.schemas.d.ts +35 -0
  176. package/dist/schedules/schedules.schemas.js +27 -0
  177. package/dist/schedules/store.d.ts +5 -0
  178. package/dist/schedules/store.js +20 -0
  179. package/dist/tools/approvals.service.d.ts +26 -7
  180. package/dist/tools/approvals.service.js +88 -10
  181. package/dist/tools/builtin-names.d.ts +5 -3
  182. package/dist/tools/builtin-names.js +12 -4
  183. package/dist/tools/builtins/builtin-tools.d.ts +1 -2
  184. package/dist/tools/builtins/builtin-tools.js +21 -17
  185. package/dist/tools/builtins/sandbox/bash.js +9 -3
  186. package/dist/tools/builtins/sandbox/common.d.ts +18 -1
  187. package/dist/tools/builtins/sandbox/common.js +14 -1
  188. package/dist/tools/builtins/sandbox/index.d.ts +2 -1
  189. package/dist/tools/builtins/sandbox/index.js +4 -1
  190. package/dist/tools/builtins/sandbox/task-list.d.ts +12 -0
  191. package/dist/tools/builtins/sandbox/task-list.js +130 -0
  192. package/dist/tools/builtins/sandbox/task-output.d.ts +8 -1
  193. package/dist/tools/builtins/sandbox/task-output.js +21 -3
  194. package/dist/tools/builtins/web/extract.d.ts +2 -4
  195. package/dist/tools/builtins/web/extract.js +2 -5
  196. package/dist/tools/pipeline.d.ts +1 -1
  197. package/dist/tools/pipeline.js +1 -1
  198. package/dist/tools/questions.service.d.ts +15 -6
  199. package/dist/tools/questions.service.js +20 -8
  200. package/dist/tools/registry.d.ts +4 -2
  201. package/dist/tools/registry.js +4 -3
  202. package/dist/tools/tools.module.js +4 -3
  203. package/package.json +3 -3
  204. package/src/migrations/0118_device_policy_ops.sql +6 -0
  205. package/src/migrations/0119_outbox.sql +22 -0
  206. package/src/migrations/0120_drop_usage_report_sweep.sql +7 -0
@@ -14,47 +14,78 @@ import { pauseQueueIn } from "../conversations/queue.js";
14
14
  import { KernelDb } from "../infra/database/kernel-db.js";
15
15
  import { TransactionHost } from "../infra/database/transaction-host.js";
16
16
  import { Transactional } from "../infra/database/transactional.decorator.js";
17
+ import { OutboxService } from "../infra/outbox/outbox.service.js";
17
18
  import { ScheduledTaskRunTracker } from "../schedules/run-tracker.js";
18
- export const TERMINAL_STATUSES = new Set([
19
- "completed",
20
- "failed",
21
- "aborted",
22
- ]);
19
+ import { ApprovalsService } from "../tools/approvals.service.js";
20
+ import { QuestionsService } from "../tools/questions.service.js";
21
+ import { ACTIVE_RUN_STATUSES, lockWaitRowsIn, oweEnsureRun, TERMINAL_STATUSES, } from "./run-jobs.js";
22
+ import { pairStoppedForks } from "./subagents/skill-fork.js";
23
+ import { oweDeliverToParent, oweStopChildren } from "./subagents/tree-jobs.js";
23
24
  /**
24
- * Writes runs' terminal states (one instance; the run worker, the run queue's orphan repairs, the
25
- * waits and the eval replays finish runs through it).
25
+ * Writes runs' terminal states (one instance; the run worker, the run queue's orphan abort, the
26
+ * `run-orphan-check`, the waits and the eval replays finish runs through it).
26
27
  */
27
28
  let RunFinish = class RunFinish {
28
29
  kdb;
29
30
  eventLog;
30
31
  txHost;
31
32
  taskRuns;
33
+ outbox;
34
+ approvals;
35
+ questions;
32
36
  constructor(kdb, eventLog, txHost,
33
37
  /** A scheduled task's run follows its kernel run's end in the same transaction (spec §6). */
34
- taskRuns) {
38
+ taskRuns,
39
+ /** The run's end owes the conversation's ensure-run backstop (transactional outbox spec §6). */
40
+ outbox,
41
+ /** A finished run's pending approvals are cancelled in its terminal transaction. */
42
+ approvals,
43
+ /** ... and its pending input requests. */
44
+ questions) {
35
45
  this.kdb = kdb;
36
46
  this.eventLog = eventLog;
37
47
  this.txHost = txHost;
38
48
  this.taskRuns = taskRuns;
49
+ this.outbox = outbox;
50
+ this.approvals = approvals;
51
+ this.questions = questions;
39
52
  }
40
53
  /**
41
54
  * 在调用方的事务内写运行的终态:会话行锁 → 运行仍未结束才写终态事件、推进 `consumed_seq`、更新 `runs.status`。
55
+ * It also owes the conversation's ensure-run backstop under the terminal event's seq.
56
+ *
57
+ * A finished run waits on nothing: its pending approvals and input requests (one asked just
58
+ * before the run failed, while it was still `running`) are cancelled before its terminal event
59
+ * (`approval_decided` / `question_answered` with `cancelled`). Lock order, as every path that
60
+ * writes them: the conversation row, the run's approval and input request rows, the run row.
61
+ *
62
+ * Sub-agents (transactional outbox spec §6): before the terminal event the run's open fork
63
+ * invocation gets its report (`pairStoppedForks`; an `Agent` call has its result from its
64
+ * dispatch), so no terminal path leaves an open call. After it, the stops and the delivery the
65
+ * run's end causes are owed here (runtime/subagents/tree-jobs.ts), the caller still doing them
66
+ * right after its commit: `stop-tree` when an active fork child remains or a sub-agent run a
67
+ * person stopped has sub-agents below it, `deliver-to-parent` for a sub-agent's run. The
68
+ * delivery is a job, not written here: this transaction holds the child's conversation and run
69
+ * rows, and the parent's conversation row comes first in the dispatch's lock order (parent
70
+ * conversation, parent run, budget run, child conversation).
42
71
  * 返回终态事件的 seq;运行已是终态(例如已被其他副本标记失败)→ 返回 null,什么都不写。
43
72
  *
44
73
  * `consumed_seq = GREATEST(consumed_seq, consumed, runs.trigger_seq)`:不回退,且失败 / 中止 / 完成都至少
45
74
  * 消费到触发该运行的消息——否则结束时的 ensureRun 会对同一条消息无限重试(失败)或卡住会话。
46
75
  * 超时(`floorToTrigger: false`)只取 `GREATEST(consumed_seq, consumed)`:没吸收的消息留给下一轮(R34)。
47
- * 加锁顺序与 `RunQueue.ensureRun` 一致(先会话行、后 runs 行),不会互相死锁。
76
+ * Lock order as `RunQueue.ensureRun` and the decisions take it: the conversation row, the run's
77
+ * approval and input request rows, the run row; they never deadlock with each other.
48
78
  * 调用方在提交后对非 null 的返回值调用 {@link notifyFinished}。
49
79
  */
50
80
  async finishIn(a) {
51
81
  const t = this.kdb.db;
52
- await t
82
+ const conv = await t
53
83
  .selectFrom("conversations")
54
- .select("id")
84
+ .select(["kind", "org_id"])
55
85
  .where("id", "=", a.conversationId)
56
86
  .forUpdate()
57
87
  .executeTakeFirst();
88
+ const pending = await lockWaitRowsIn(t, a.runId);
58
89
  const run = await t
59
90
  .selectFrom("runs")
60
91
  .select("status")
@@ -63,7 +94,11 @@ let RunFinish = class RunFinish {
63
94
  .executeTakeFirst();
64
95
  if (!run || TERMINAL_STATUSES.has(run.status))
65
96
  return null;
66
- await a.beforeTerminal?.();
97
+ if (pending.approvals)
98
+ await this.approvals.cancelPendingIn(a.runId);
99
+ if (pending.questions)
100
+ await this.questions.cancelPendingIn(a.runId);
101
+ await pairStoppedForks(t, this.eventLog, a.runId);
67
102
  const [ev] = await this.eventLog.appendIn(a.conversationId, [
68
103
  { ...a.event, runId: a.runId, idemKey: `${a.event.type}:${a.runId}` },
69
104
  ]);
@@ -85,7 +120,42 @@ let RunFinish = class RunFinish {
85
120
  // a failed or stopped run pauses the queued messages until the person resumes (spec §5.7)
86
121
  if (a.status !== "completed")
87
122
  await pauseQueueIn(t, a.conversationId);
88
- return ev.seq;
123
+ const seq = ev.seq;
124
+ // what arrived during the run starts the next one after the commit (the run worker's post-job
125
+ // ensureRun); owed here too, for when that never happens (an eval's hidden conversation never
126
+ // starts runs)
127
+ if (conv && (conv.kind === "chat" || conv.kind === "subagent")) {
128
+ await oweEnsureRun(this.outbox, a.conversationId, seq, conv.org_id);
129
+ }
130
+ if (conv)
131
+ await this.oweTreeWork(a, conv.kind, conv.org_id);
132
+ return seq;
133
+ }
134
+ /** {@link finishIn}'s owed stops and delivery of a run's sub-agent tree. */
135
+ async oweTreeWork(a, kind, orgId) {
136
+ const t = this.kdb.db;
137
+ if (kind === "subagent")
138
+ await oweDeliverToParent(this.outbox, a.runId, orgId);
139
+ const forkChild = await t
140
+ .selectFrom("subagent_dispatches as sd")
141
+ .innerJoin("runs as cr", "cr.id", "sd.child_run_id")
142
+ .select("sd.child_run_id")
143
+ .where("sd.parent_run_id", "=", a.runId)
144
+ .where("sd.for_skill", "=", true)
145
+ .where("cr.status", "in", ACTIVE_RUN_STATUSES)
146
+ .limit(1)
147
+ .executeTakeFirst();
148
+ const stoppedWithChildren = kind === "subagent" &&
149
+ personStopped(a.event) &&
150
+ (await t
151
+ .selectFrom("conversations")
152
+ .select("id")
153
+ .where("parent_conversation_id", "=", a.conversationId)
154
+ .where("kind", "=", "subagent")
155
+ .limit(1)
156
+ .executeTakeFirst()) !== undefined;
157
+ if (forkChild || stoppedWithChildren)
158
+ await oweStopChildren(this.outbox, a.runId, orgId);
89
159
  }
90
160
  /** 自开事务写终态并在提交后发布会话通知;返回值同 {@link finishIn}。 */
91
161
  async finish(a) {
@@ -109,9 +179,22 @@ RunFinish = __decorate([
109
179
  __param(0, Inject(KernelDb)),
110
180
  __param(1, Inject(EventLog)),
111
181
  __param(2, Inject(TransactionHost)),
112
- __param(3, Inject(ScheduledTaskRunTracker))
182
+ __param(3, Inject(ScheduledTaskRunTracker)),
183
+ __param(4, Inject(OutboxService)),
184
+ __param(5, Inject(ApprovalsService)),
185
+ __param(6, Inject(QuestionsService))
113
186
  ], RunFinish);
114
187
  export { RunFinish };
188
+ /**
189
+ * A `run_aborted` a person caused (an account and no `detail`): a sub-agent run stopped that way
190
+ * stops its descendants (`Waits.stopDescendantsOfStopped`, agent map spec §6).
191
+ */
192
+ function personStopped(event) {
193
+ if (event.type !== "run_aborted")
194
+ return false;
195
+ const p = event.payload;
196
+ return typeof p?.byAccountId === "string" && p.detail === undefined;
197
+ }
115
198
  /** The message of a `run_failed` event's error, for the task run (undefined otherwise). */
116
199
  function failureMessage(event) {
117
200
  if (event.type !== "run_failed")
@@ -1,6 +1,8 @@
1
1
  import type { Job, Queue } from "bullmq";
2
- import type { ConversationKind } from "../db/kernel-schema.js";
2
+ import type { ConversationKind, KernelSchemaDb } from "../db/kernel-schema.js";
3
3
  import { KernelDb } from "../infra/database/kernel-db.js";
4
+ import { TransactionHost } from "../infra/database/transaction-host.js";
5
+ import { OutboxService } from "../infra/outbox/outbox.service.js";
4
6
  import type { KernelLogger } from "../log.js";
5
7
  /** 内核运行队列的 BullMQ 队列名(不含 `:`;与插件队列 `cap.<插件>.<名>` 不会重名)。 */
6
8
  export declare const RUN_QUEUE_NAME = "cap-kernel-runs";
@@ -10,9 +12,23 @@ export declare const RUN_QUEUE_NAME = "cap-kernel-runs";
10
12
  */
11
13
  export declare const SUBAGENT_RUN_QUEUE_NAME = "cap-kernel-subagent-runs";
12
14
  /** The run queue of a conversation kind: sub-agent conversations have their own. */
13
- export declare const runQueueNameFor: (kind: ConversationKind) => string;
15
+ export declare const runQueueNameFor: (kind: ConversationKind) => typeof RUN_QUEUE_NAME | typeof SUBAGENT_RUN_QUEUE_NAME;
14
16
  /** 未结束的运行状态(waiting:等人处理审批 / 输入请求,没有任务、不占会话锁与配额)。 */
15
17
  export declare const ACTIVE_RUN_STATUSES: readonly ["queued", "running", "waiting"];
18
+ /** 运行的终态。 */
19
+ export type TerminalStatus = "completed" | "failed" | "aborted";
20
+ /** The terminal run statuses (here, not in `run-finish.ts`, which the sub-agent modules import). */
21
+ export declare const TERMINAL_STATUSES: ReadonlySet<string>;
22
+ /**
23
+ * Locks a run's approval and input request rows (`FOR UPDATE`, in id order) in the caller's
24
+ * transaction, after its conversation row and before its run row: the one lock order of every path
25
+ * that writes them (conversation, decided rows, run; ruling R19), so a stop, a run's end, a decision
26
+ * and a timeout never deadlock. Returns whether any of them is still pending.
27
+ */
28
+ export declare function lockWaitRowsIn(t: KernelSchemaDb, runId: string): Promise<{
29
+ approvals: boolean;
30
+ questions: boolean;
31
+ }>;
16
32
  /** 运行任务的数据。 */
17
33
  export interface RunJobData {
18
34
  runId: string;
@@ -21,26 +37,92 @@ export interface RunJobData {
21
37
  /** 同一触发点(会话 + 最新用户消息 seq)只投递一次。 */
22
38
  export declare const runJobId: (conversationId: string, maxUserSeq: number) => string;
23
39
  /**
24
- * The producer side of the two run queues (`cap-kernel-runs`, `cap-kernel-subagent-runs`): adds
25
- * run jobs under the caller's job id and looks them up. The run queue (`ensureRun`, orphan
26
- * repairs), the waits (resumes) and the sub-agent dispatch enqueue through it; the run worker
27
- * consumes both queues.
40
+ * A resumed run's job id: apart from the per-message `run-<conversationId>-<seq>`, and one per
41
+ * wait, so the same wait resumes its run once.
42
+ */
43
+ export declare const resumeJobId: (conversationId: string, waitId: string) => string;
44
+ /** The BullMQ job name of every run job, on both run queues (an outbox registry name). */
45
+ export declare const RUN_JOB = "run";
46
+ /**
47
+ * The ensure-run backstop (transactional outbox spec §6 "Sending a message"): a maintenance job
48
+ * that calls `RunQueue.ensureRun` for a conversation. Every transaction that may start a run
49
+ * after its commit (a message, a queue change, a background task's finish, a sub-agent's delivery,
50
+ * a run's end) owes one, {@link ENSURE_RUN_BACKSTOP_MS} later, and its caller still calls
51
+ * `ensureRun` right after the commit: the job only matters when that call never happened (the
52
+ * process died) or failed (Redis was down).
53
+ */
54
+ export declare const ENSURE_RUN_JOB = "ensure-run";
55
+ /** How long after its transaction the ensure-run backstop fires (a backstop, not a second call). */
56
+ export declare const ENSURE_RUN_BACKSTOP_MS = 30000;
57
+ /** The ensure-run job's data. */
58
+ export interface EnsureRunJobData {
59
+ conversationId: string;
60
+ }
61
+ /**
62
+ * The ensure-run job id: one per conversation and the seq of the event the transaction wrote (a
63
+ * write without an event of its own uses the conversation's newest seq; no `:`).
64
+ */
65
+ export declare const ensureRunJobId: (conversationId: string, seq: number | string) => string;
66
+ /**
67
+ * Owes the ensure-run backstop of `conversationId` in the caller's transaction (throws outside
68
+ * one). False when the same job id is already owed (the owed job does the same).
69
+ */
70
+ export declare function oweEnsureRun(outbox: Pick<OutboxService, "add">, conversationId: string, seq: number | string, orgId: string): Promise<boolean>;
71
+ /**
72
+ * The producer side of the two run queues (`cap-kernel-runs`, `cap-kernel-subagent-runs`): owes
73
+ * run jobs in the caller's transaction ({@link owe}: `ensureRun`, the sub-agent dispatch, the
74
+ * `run-orphan-check`) and looks them up; the run worker consumes both queues.
28
75
  */
29
76
  export declare class RunJobs {
30
- private readonly kdb;
77
+ private readonly outbox;
31
78
  private readonly queues;
32
- constructor(kdb: KernelDb, runs: Queue<RunJobData>, subagentRuns: Queue<RunJobData>, log: KernelLogger);
79
+ constructor(runs: Queue<RunJobData>, subagentRuns: Queue<RunJobData>, log: KernelLogger, outbox: OutboxService);
33
80
  /** The BullMQ queue `name` (default: conversation runs). */
34
81
  queue(name?: string): Queue<RunJobData>;
35
82
  /** A run job by id on the queue `name`, or undefined. */
36
83
  getJob(jobId: string, name?: string): Promise<Job<RunJobData> | undefined>;
37
84
  /**
38
- * Enqueues a run job under the caller's job id (`ensureRun`, resumes, dispatches, repairs). The
39
- * queue follows the conversation's kind (a sub-agent's runs go to
40
- * {@link SUBAGENT_RUN_QUEUE_NAME}); pass `kind` when the caller knows it, else it is read.
85
+ * Owes the run job in the caller's transaction (transactional outbox spec §6): the outbox row
86
+ * commits or rolls back with the run row, on the queue of the conversation's `kind`. Returns
87
+ * false when the same job id is already owed (nothing written; the caller creates no run).
41
88
  */
42
- enqueue(data: RunJobData, jobId: string, kind?: ConversationKind): Promise<void>;
89
+ owe(data: RunJobData, jobId: string, kind: ConversationKind, orgId: string): Promise<boolean>;
90
+ /** {@link oweEnsureRun} in the caller's transaction. */
91
+ oweEnsureRun(conversationId: string, seq: number | string, orgId: string): Promise<boolean>;
43
92
  }
44
- /** The run queues' producer (the sub-agent dispatch, the run queue and the waits use it). */
93
+ /**
94
+ * What {@link RunResume.resumeIn} found: `queued`, the run was waiting on nothing else and now is
95
+ * queued with its resume job owed; `still-running`, the run has not reached `waiting` yet (its
96
+ * suspend's re-check resumes it); `other-wait-pending`, the run still waits on another approval,
97
+ * input request or fork sub-agent; `not-waiting`, the run is queued or finished (or gone).
98
+ */
99
+ export type ResumeOutcome = "queued" | "still-running" | "other-wait-pending" | "not-waiting";
100
+ /**
101
+ * The resume of a waiting run (transactional outbox spec §6 "Resumes and timeouts", "Decisions
102
+ * resume in their own transaction"), in the caller's transaction. A decision (approvals, questions,
103
+ * a reply's skips) calls it after writing the decided row, so the lock order is the decided row,
104
+ * then the `runs` row. It lives beside the run jobs, not in `Waits`, so the tools domain (which the
105
+ * runtime imports) can call it; `Waits.resume` wraps it in a transaction of its own.
106
+ */
107
+ export declare class RunResume {
108
+ private readonly kdb;
109
+ private readonly outbox;
110
+ readonly txHost: TransactionHost;
111
+ constructor(kdb: KernelDb, outbox: Pick<OutboxService, "add">, txHost: TransactionHost);
112
+ /**
113
+ * Locks the run row (`FOR UPDATE`); when it is `waiting` and waits on nothing else (no pending
114
+ * approval or input request, no active fork sub-agent), sets it `queued` (adding this wait's
115
+ * time to `waited_ms`, recording the resume job id) and owes the resume job
116
+ * {@link resumeJobId}`(conversation, waitId)` on its conversation kind's run queue. A `running`
117
+ * run is left alone: its suspend has not set `waiting` yet and re-checks the wait after.
118
+ */
119
+ resumeIn(runId: string, waitId: string): Promise<ResumeOutcome>;
120
+ }
121
+ /**
122
+ * The run queues' producer (the sub-agent dispatch, the run queue and the waits use it), the
123
+ * resume of a waiting run (the waits and the decisions of the tools domain use it) and the
124
+ * lost-jobs canary of the run queues (the `run-orphan-check` reports it, the admin overview reads
125
+ * it).
126
+ */
45
127
  export declare class RunJobsModule {
46
128
  }
@@ -9,9 +9,15 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
9
9
  };
10
10
  import { getQueueToken } from "@nestjs/bullmq";
11
11
  import { Inject, Injectable, Module } from "@nestjs/common";
12
+ import { sql } from "kysely";
12
13
  import { KernelDb } from "../infra/database/kernel-db.js";
14
+ import { TransactionHost } from "../infra/database/transaction-host.js";
15
+ import { Transactional } from "../infra/database/transactional.decorator.js";
16
+ import { OutboxService } from "../infra/outbox/outbox.service.js";
13
17
  import { QueueModule } from "../infra/queue/queue.module.js";
14
18
  import { KERNEL_LOGGER } from "../infra/tokens.js";
19
+ import { MAINTENANCE_QUEUE_NAME } from "./maintenance-queue.js";
20
+ import { QueueLostJobs } from "./queue-lost-jobs.js";
15
21
  /** 内核运行队列的 BullMQ 队列名(不含 `:`;与插件队列 `cap.<插件>.<名>` 不会重名)。 */
16
22
  export const RUN_QUEUE_NAME = "cap-kernel-runs";
17
23
  /**
@@ -23,21 +29,87 @@ export const SUBAGENT_RUN_QUEUE_NAME = "cap-kernel-subagent-runs";
23
29
  export const runQueueNameFor = (kind) => kind === "subagent" ? SUBAGENT_RUN_QUEUE_NAME : RUN_QUEUE_NAME;
24
30
  /** 未结束的运行状态(waiting:等人处理审批 / 输入请求,没有任务、不占会话锁与配额)。 */
25
31
  export const ACTIVE_RUN_STATUSES = ["queued", "running", "waiting"];
32
+ /** The terminal run statuses (here, not in `run-finish.ts`, which the sub-agent modules import). */
33
+ export const TERMINAL_STATUSES = new Set([
34
+ "completed",
35
+ "failed",
36
+ "aborted",
37
+ ]);
38
+ /**
39
+ * Locks a run's approval and input request rows (`FOR UPDATE`, in id order) in the caller's
40
+ * transaction, after its conversation row and before its run row: the one lock order of every path
41
+ * that writes them (conversation, decided rows, run; ruling R19), so a stop, a run's end, a decision
42
+ * and a timeout never deadlock. Returns whether any of them is still pending.
43
+ */
44
+ export async function lockWaitRowsIn(t, runId) {
45
+ const approvals = await t
46
+ .selectFrom("approvals")
47
+ .select("status")
48
+ .where("run_id", "=", runId)
49
+ .orderBy("id")
50
+ .forUpdate()
51
+ .execute();
52
+ const questions = await t
53
+ .selectFrom("questions")
54
+ .select("status")
55
+ .where("run_id", "=", runId)
56
+ .orderBy("id")
57
+ .forUpdate()
58
+ .execute();
59
+ return {
60
+ approvals: approvals.some((r) => r.status === "pending"),
61
+ questions: questions.some((r) => r.status === "pending"),
62
+ };
63
+ }
26
64
  /** 同一触发点(会话 + 最新用户消息 seq)只投递一次。 */
27
65
  export const runJobId = (conversationId, maxUserSeq) => `run-${conversationId}-${maxUserSeq}`;
28
- /** The options of every run job: one attempt (the run's own paths handle failures), not kept. */
29
- const RUN_JOB_OPTIONS = { attempts: 1, removeOnComplete: true, removeOnFail: true };
30
66
  /**
31
- * The producer side of the two run queues (`cap-kernel-runs`, `cap-kernel-subagent-runs`): adds
32
- * run jobs under the caller's job id and looks them up. The run queue (`ensureRun`, orphan
33
- * repairs), the waits (resumes) and the sub-agent dispatch enqueue through it; the run worker
34
- * consumes both queues.
67
+ * A resumed run's job id: apart from the per-message `run-<conversationId>-<seq>`, and one per
68
+ * wait, so the same wait resumes its run once.
69
+ */
70
+ export const resumeJobId = (conversationId, waitId) => `run-${conversationId}-resume-${waitId}`;
71
+ /** The BullMQ job name of every run job, on both run queues (an outbox registry name). */
72
+ export const RUN_JOB = "run";
73
+ /**
74
+ * The ensure-run backstop (transactional outbox spec §6 "Sending a message"): a maintenance job
75
+ * that calls `RunQueue.ensureRun` for a conversation. Every transaction that may start a run
76
+ * after its commit (a message, a queue change, a background task's finish, a sub-agent's delivery,
77
+ * a run's end) owes one, {@link ENSURE_RUN_BACKSTOP_MS} later, and its caller still calls
78
+ * `ensureRun` right after the commit: the job only matters when that call never happened (the
79
+ * process died) or failed (Redis was down).
80
+ */
81
+ export const ENSURE_RUN_JOB = "ensure-run";
82
+ /** How long after its transaction the ensure-run backstop fires (a backstop, not a second call). */
83
+ export const ENSURE_RUN_BACKSTOP_MS = 30_000;
84
+ /**
85
+ * The ensure-run job id: one per conversation and the seq of the event the transaction wrote (a
86
+ * write without an event of its own uses the conversation's newest seq; no `:`).
87
+ */
88
+ export const ensureRunJobId = (conversationId, seq) => `ensure-run-${conversationId}-${seq}`;
89
+ /**
90
+ * Owes the ensure-run backstop of `conversationId` in the caller's transaction (throws outside
91
+ * one). False when the same job id is already owed (the owed job does the same).
92
+ */
93
+ export function oweEnsureRun(outbox, conversationId, seq, orgId) {
94
+ return outbox.add({
95
+ queue: MAINTENANCE_QUEUE_NAME,
96
+ name: ENSURE_RUN_JOB,
97
+ jobId: ensureRunJobId(conversationId, seq),
98
+ data: { conversationId },
99
+ runAt: new Date(Date.now() + ENSURE_RUN_BACKSTOP_MS),
100
+ orgId,
101
+ });
102
+ }
103
+ /**
104
+ * The producer side of the two run queues (`cap-kernel-runs`, `cap-kernel-subagent-runs`): owes
105
+ * run jobs in the caller's transaction ({@link owe}: `ensureRun`, the sub-agent dispatch, the
106
+ * `run-orphan-check`) and looks them up; the run worker consumes both queues.
35
107
  */
36
108
  let RunJobs = class RunJobs {
37
- kdb;
109
+ outbox;
38
110
  queues;
39
- constructor(kdb, runs, subagentRuns, log) {
40
- this.kdb = kdb;
111
+ constructor(runs, subagentRuns, log, outbox) {
112
+ this.outbox = outbox;
41
113
  this.queues = { [RUN_QUEUE_NAME]: runs, [SUBAGENT_RUN_QUEUE_NAME]: subagentRuns };
42
114
  for (const [name, q] of Object.entries(this.queues)) {
43
115
  q.on("error", (err) => log.error("run queue error", { err, queue: name }));
@@ -55,39 +127,138 @@ let RunJobs = class RunJobs {
55
127
  return this.queue(name).getJob(jobId);
56
128
  }
57
129
  /**
58
- * Enqueues a run job under the caller's job id (`ensureRun`, resumes, dispatches, repairs). The
59
- * queue follows the conversation's kind (a sub-agent's runs go to
60
- * {@link SUBAGENT_RUN_QUEUE_NAME}); pass `kind` when the caller knows it, else it is read.
130
+ * Owes the run job in the caller's transaction (transactional outbox spec §6): the outbox row
131
+ * commits or rolls back with the run row, on the queue of the conversation's `kind`. Returns
132
+ * false when the same job id is already owed (nothing written; the caller creates no run).
61
133
  */
62
- async enqueue(data, jobId, kind) {
63
- const k = kind ??
64
- (await this.kdb.db
65
- .selectFrom("conversations")
66
- .select("kind")
67
- .where("id", "=", data.conversationId)
68
- .executeTakeFirst())?.kind ??
69
- "chat";
70
- await this.queue(runQueueNameFor(k)).add("run", data, { jobId, ...RUN_JOB_OPTIONS });
134
+ owe(data, jobId, kind, orgId) {
135
+ return this.outbox.add({
136
+ queue: runQueueNameFor(kind),
137
+ name: RUN_JOB,
138
+ jobId,
139
+ data,
140
+ orgId,
141
+ });
142
+ }
143
+ /** {@link oweEnsureRun} in the caller's transaction. */
144
+ oweEnsureRun(conversationId, seq, orgId) {
145
+ return oweEnsureRun(this.outbox, conversationId, seq, orgId);
71
146
  }
72
147
  };
73
148
  RunJobs = __decorate([
74
149
  Injectable(),
75
- __param(0, Inject(KernelDb)),
76
- __param(1, Inject(getQueueToken(RUN_QUEUE_NAME))),
77
- __param(2, Inject(getQueueToken(SUBAGENT_RUN_QUEUE_NAME))),
78
- __param(3, Inject(KERNEL_LOGGER))
150
+ __param(0, Inject(getQueueToken(RUN_QUEUE_NAME))),
151
+ __param(1, Inject(getQueueToken(SUBAGENT_RUN_QUEUE_NAME))),
152
+ __param(2, Inject(KERNEL_LOGGER)),
153
+ __param(3, Inject(OutboxService))
79
154
  ], RunJobs);
80
155
  export { RunJobs };
156
+ /**
157
+ * The resume of a waiting run (transactional outbox spec §6 "Resumes and timeouts", "Decisions
158
+ * resume in their own transaction"), in the caller's transaction. A decision (approvals, questions,
159
+ * a reply's skips) calls it after writing the decided row, so the lock order is the decided row,
160
+ * then the `runs` row. It lives beside the run jobs, not in `Waits`, so the tools domain (which the
161
+ * runtime imports) can call it; `Waits.resume` wraps it in a transaction of its own.
162
+ */
163
+ let RunResume = class RunResume {
164
+ kdb;
165
+ outbox;
166
+ txHost;
167
+ constructor(kdb, outbox, txHost) {
168
+ this.kdb = kdb;
169
+ this.outbox = outbox;
170
+ this.txHost = txHost;
171
+ }
172
+ /**
173
+ * Locks the run row (`FOR UPDATE`); when it is `waiting` and waits on nothing else (no pending
174
+ * approval or input request, no active fork sub-agent), sets it `queued` (adding this wait's
175
+ * time to `waited_ms`, recording the resume job id) and owes the resume job
176
+ * {@link resumeJobId}`(conversation, waitId)` on its conversation kind's run queue. A `running`
177
+ * run is left alone: its suspend has not set `waiting` yet and re-checks the wait after.
178
+ */
179
+ async resumeIn(runId, waitId) {
180
+ const t = this.kdb.db;
181
+ const run = await t
182
+ .selectFrom("runs")
183
+ .select("status")
184
+ .where("id", "=", runId)
185
+ .forUpdate()
186
+ .executeTakeFirst();
187
+ if (run?.status === "running")
188
+ return "still-running";
189
+ if (run?.status !== "waiting")
190
+ return "not-waiting";
191
+ const row = await t
192
+ .updateTable("runs")
193
+ .set({
194
+ status: "queued",
195
+ queued_at: sql `now()`,
196
+ job_id: sql `'run-' || conversation_id || '-resume-' || ${waitId}::text`,
197
+ waited_ms: sql `waited_ms + COALESCE(floor(extract(epoch FROM now() - waiting_since) * 1000), 0)::bigint`,
198
+ waiting_since: null,
199
+ wait_expires_at: null,
200
+ })
201
+ .where("id", "=", runId)
202
+ .where(({ not, exists, selectFrom }) => not(exists(selectFrom("approvals")
203
+ .select(sql `1`.as("one"))
204
+ .where("approvals.run_id", "=", runId)
205
+ .where("approvals.status", "=", "pending"))))
206
+ .where(({ not, exists, selectFrom }) => not(exists(selectFrom("questions")
207
+ .select(sql `1`.as("one"))
208
+ .where("questions.run_id", "=", runId)
209
+ .where("questions.status", "=", "pending"))))
210
+ // a fork invocation's sub-agent (agent map spec §5.4: background `Agent` children never
211
+ // hold their parent run) is no longer active
212
+ .where(({ not, exists, selectFrom }) => not(exists(selectFrom("subagent_dispatches as sd")
213
+ .innerJoin("runs as cr", "cr.id", "sd.child_run_id")
214
+ .select(sql `1`.as("one"))
215
+ .where("sd.parent_run_id", "=", runId)
216
+ .where("sd.for_skill", "=", true)
217
+ .where("cr.status", "in", ACTIVE_RUN_STATUSES))))
218
+ .returning("conversation_id")
219
+ .executeTakeFirst();
220
+ if (!row)
221
+ return "other-wait-pending";
222
+ const conv = await t
223
+ .selectFrom("conversations")
224
+ .select(["kind", "org_id"])
225
+ .where("id", "=", row.conversation_id)
226
+ .executeTakeFirstOrThrow();
227
+ await this.outbox.add({
228
+ queue: runQueueNameFor(conv.kind),
229
+ name: RUN_JOB,
230
+ jobId: resumeJobId(row.conversation_id, waitId),
231
+ data: { runId, conversationId: row.conversation_id },
232
+ orgId: conv.org_id,
233
+ });
234
+ return "queued";
235
+ }
236
+ };
237
+ __decorate([
238
+ Transactional({ propagation: "mandatory" })
239
+ ], RunResume.prototype, "resumeIn", null);
240
+ RunResume = __decorate([
241
+ Injectable(),
242
+ __param(0, Inject(KernelDb)),
243
+ __param(1, Inject(OutboxService)),
244
+ __param(2, Inject(TransactionHost))
245
+ ], RunResume);
246
+ export { RunResume };
81
247
  /** The run queues' registration (re-exported: the admin health reads their counts). */
82
248
  const RUN_QUEUES_MODULE = QueueModule.registerQueue(RUN_QUEUE_NAME, SUBAGENT_RUN_QUEUE_NAME);
83
- /** The run queues' producer (the sub-agent dispatch, the run queue and the waits use it). */
249
+ /**
250
+ * The run queues' producer (the sub-agent dispatch, the run queue and the waits use it), the
251
+ * resume of a waiting run (the waits and the decisions of the tools domain use it) and the
252
+ * lost-jobs canary of the run queues (the `run-orphan-check` reports it, the admin overview reads
253
+ * it).
254
+ */
84
255
  let RunJobsModule = class RunJobsModule {
85
256
  };
86
257
  RunJobsModule = __decorate([
87
258
  Module({
88
259
  imports: [RUN_QUEUES_MODULE],
89
- providers: [RunJobs],
90
- exports: [RunJobs, RUN_QUEUES_MODULE],
260
+ providers: [RunJobs, RunResume, QueueLostJobs],
261
+ exports: [RunJobs, RunResume, QueueLostJobs, RUN_QUEUES_MODULE],
91
262
  })
92
263
  ], RunJobsModule);
93
264
  export { RunJobsModule };