@capybara-hq/kernel 0.1.0-rc.24 → 0.1.0-rc.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. package/dist/admin/admin.module.js +2 -0
  2. package/dist/admin/health.service.d.ts +2 -2
  3. package/dist/admin/health.service.js +6 -2
  4. package/dist/agents/agents.module.js +3 -0
  5. package/dist/agents/agents.service.d.ts +6 -1
  6. package/dist/agents/agents.service.js +11 -2
  7. package/dist/catalog/catalog-import.module.js +0 -4
  8. package/dist/catalog/catalog-scope.service.d.ts +63 -26
  9. package/dist/catalog/catalog-scope.service.js +146 -71
  10. package/dist/catalog/catalog.module.js +4 -3
  11. package/dist/catalog/catalog.service.d.ts +11 -4
  12. package/dist/catalog/catalog.service.js +21 -5
  13. package/dist/catalog/cli.d.ts +4 -3
  14. package/dist/catalog/cli.js +8 -13
  15. package/dist/catalog/job-ids.d.ts +8 -20
  16. package/dist/catalog/job-ids.js +11 -16
  17. package/dist/catalog/marketplaces.service.d.ts +1 -1
  18. package/dist/catalog/org-plugin-uploads.d.ts +11 -4
  19. package/dist/catalog/org-plugin-uploads.js +21 -6
  20. package/dist/catalog/owned-items.d.ts +6 -13
  21. package/dist/catalog/owned-items.js +11 -29
  22. package/dist/catalog/owned-plugin-cleanup.d.ts +8 -17
  23. package/dist/catalog/owned-plugin-cleanup.js +12 -66
  24. package/dist/catalog/platform-components.d.ts +56 -34
  25. package/dist/catalog/platform-components.js +68 -78
  26. package/dist/catalog/plugin-import.d.ts +28 -25
  27. package/dist/catalog/plugin-import.js +90 -76
  28. package/dist/catalog/plugin-upgrade.d.ts +4 -4
  29. package/dist/catalog/plugin-upgrade.js +7 -19
  30. package/dist/catalog/scope-jobs.d.ts +59 -0
  31. package/dist/catalog/scope-jobs.js +65 -0
  32. package/dist/catalog/scope-sync-workers.d.ts +22 -0
  33. package/dist/catalog/scope-sync-workers.js +58 -2
  34. package/dist/catalog/scope-sync.d.ts +40 -65
  35. package/dist/catalog/scope-sync.js +114 -122
  36. package/dist/channels/inbound.js +2 -2
  37. package/dist/channels/threads.d.ts +4 -3
  38. package/dist/channels/threads.js +4 -3
  39. package/dist/conversations/conversations.controller.js +4 -2
  40. package/dist/conversations/conversations.service.d.ts +17 -5
  41. package/dist/conversations/conversations.service.js +17 -5
  42. package/dist/conversations/events.d.ts +21 -0
  43. package/dist/conversations/events.js +24 -1
  44. package/dist/conversations/source.d.ts +10 -0
  45. package/dist/conversations/source.js +17 -0
  46. package/dist/conversations/tree.d.ts +10 -0
  47. package/dist/conversations/tree.js +35 -0
  48. package/dist/db/kernel-schema.d.ts +9 -14
  49. package/dist/devices/devices.service.d.ts +11 -8
  50. package/dist/devices/devices.service.js +16 -23
  51. package/dist/evals/run-replay.d.ts +13 -2
  52. package/dist/evals/run-replay.js +28 -1
  53. package/dist/evals/snapshot.d.ts +7 -1
  54. package/dist/evals/snapshot.js +2 -1
  55. package/dist/extensions/hooks.d.ts +22 -3
  56. package/dist/extensions/hooks.js +15 -1
  57. package/dist/extensions/lifecycle-hooks.d.ts +99 -0
  58. package/dist/extensions/lifecycle-hooks.js +273 -0
  59. package/dist/extensions/org-config.d.ts +13 -1
  60. package/dist/extensions/org-config.js +19 -2
  61. package/dist/extensions/permission-view.d.ts +9 -5
  62. package/dist/extensions/permission-view.js +9 -4
  63. package/dist/extensions/point-defaults.d.ts +28 -1
  64. package/dist/extensions/point-defaults.js +44 -13
  65. package/dist/inbox/inbox.service.d.ts +6 -20
  66. package/dist/inbox/inbox.service.js +0 -3
  67. package/dist/inbox/org-notices.d.ts +1 -1
  68. package/dist/inbox/platform-inbox.service.d.ts +3 -32
  69. package/dist/inbox/platform-inbox.service.js +5 -54
  70. package/dist/infra/outbox/registry.d.ts +124 -0
  71. package/dist/infra/outbox/registry.js +126 -0
  72. package/dist/infra/queue/queue-names.d.ts +1 -1
  73. package/dist/infra/queue/queue-names.js +1 -0
  74. package/dist/orgs/directory-sync.js +1 -1
  75. package/dist/orgs/members.service.js +1 -1
  76. package/dist/orgs/offboarding.d.ts +50 -57
  77. package/dist/orgs/offboarding.js +117 -162
  78. package/dist/orgs/orgs.module.js +3 -0
  79. package/dist/orgs/orgs.service.d.ts +11 -5
  80. package/dist/orgs/orgs.service.js +15 -6
  81. package/dist/overrides.d.ts +3 -0
  82. package/dist/overrides.js +2 -0
  83. package/dist/platform/ops/inbox.ops.js +4 -20
  84. package/dist/platform/ops/platform.ops.d.ts +1 -0
  85. package/dist/platform/ops/platform.ops.js +6 -3
  86. package/dist/roles/worker-role.d.ts +10 -5
  87. package/dist/roles/worker-role.js +19 -9
  88. package/dist/roles/worker.module.js +2 -4
  89. package/dist/runtime/context/build.d.ts +3 -0
  90. package/dist/runtime/context/build.js +15 -0
  91. package/dist/runtime/context/plugin-context.d.ts +20 -0
  92. package/dist/runtime/context/plugin-context.js +30 -0
  93. package/dist/runtime/langgraph-loop.js +4 -0
  94. package/dist/runtime/loop.d.ts +8 -0
  95. package/dist/runtime/maintenance.d.ts +5 -3
  96. package/dist/runtime/maintenance.js +36 -11
  97. package/dist/runtime/model-call.d.ts +6 -1
  98. package/dist/runtime/model-call.js +1 -0
  99. package/dist/runtime/plugin-context.service.d.ts +69 -0
  100. package/dist/runtime/plugin-context.service.js +208 -0
  101. package/dist/runtime/run-control.d.ts +6 -1
  102. package/dist/runtime/run-control.js +13 -0
  103. package/dist/runtime/run-queue.d.ts +13 -0
  104. package/dist/runtime/run-queue.js +63 -0
  105. package/dist/runtime/run-worker.d.ts +22 -1
  106. package/dist/runtime/run-worker.js +107 -10
  107. package/dist/runtime/runtime.module.js +2 -0
  108. package/dist/runtime/subagents/dispatch.js +1 -0
  109. package/dist/runtime/subagents/skill-fork.js +3 -0
  110. package/dist/sandbox/jobs.d.ts +20 -10
  111. package/dist/sandbox/jobs.js +23 -17
  112. package/dist/schedules/fire.d.ts +25 -4
  113. package/dist/schedules/fire.js +87 -13
  114. package/dist/schedules/job-ids.d.ts +66 -17
  115. package/dist/schedules/job-ids.js +81 -13
  116. package/dist/schedules/run-tracker.js +2 -1
  117. package/dist/schedules/scan.job.d.ts +13 -32
  118. package/dist/schedules/scan.job.js +35 -144
  119. package/dist/schedules/scheduled-tasks.service.d.ts +6 -6
  120. package/dist/schedules/scheduled-tasks.service.js +17 -14
  121. package/dist/schedules/schedules.module.js +0 -4
  122. package/dist/skills/changes.controller.d.ts +1 -1
  123. package/dist/skills/changes.controller.js +1 -1
  124. package/dist/tools/approvals.service.d.ts +15 -4
  125. package/dist/tools/approvals.service.js +26 -10
  126. package/dist/tools/pipeline.d.ts +11 -2
  127. package/dist/tools/pipeline.js +5 -0
  128. package/dist/tools/questions.service.d.ts +15 -4
  129. package/dist/tools/questions.service.js +21 -19
  130. package/dist/tools/registry.d.ts +8 -1
  131. package/dist/tools/registry.js +9 -0
  132. package/package.json +3 -3
  133. package/src/migrations/0121_drop_platform_migration_runs.sql +12 -0
  134. package/src/migrations/0122_drop_scheduled_task_runs_rechecked_at.sql +5 -0
  135. package/src/migrations/0123_pgvector.sql +27 -0
  136. package/src/migrations/0124_conversation_source.sql +36 -0
  137. package/dist/catalog/platform-migration.d.ts +0 -115
  138. package/dist/catalog/platform-migration.js +0 -583
  139. package/dist/infra/queue/add-job-once.d.ts +0 -14
  140. package/dist/infra/queue/add-job-once.js +0 -16
@@ -12,38 +12,41 @@ import { Inject, Injectable } from "@nestjs/common";
12
12
  import { sql } from "kysely";
13
13
  import { retargetSetupRequests } from "../agents/pending-connector-requests.js";
14
14
  import { EventLog } from "../conversations/event-log.js";
15
+ import { lockConversationTrees } from "../conversations/tree.js";
15
16
  import { ApiError } from "../errors.js";
16
17
  import { enabledExtensionsQuery } from "../extensions/enabled.js";
18
+ import { LifecycleHooks } from "../extensions/lifecycle-hooks.js";
17
19
  import { ExtensionRegistry } from "../extensions/registry.js";
18
20
  import { audit } from "../identity/audit.js";
19
21
  import { SessionService } from "../identity/sessions.js";
20
22
  import { KernelDb } from "../infra/database/kernel-db.js";
21
23
  import { TransactionHost } from "../infra/database/transaction-host.js";
22
24
  import { Transactional } from "../infra/database/transactional.decorator.js";
23
- import { LockService } from "../infra/lock/lock.service.js";
24
- import { LockedCron } from "../infra/schedule/locked-cron.decorator.js";
25
- import { KERNEL_LOGGER, REDIS } from "../infra/tokens.js";
25
+ import { OutboxService } from "../infra/outbox/outbox.service.js";
26
+ import { KERNEL_LOGGER } from "../infra/tokens.js";
27
+ import { MAINTENANCE_QUEUE_NAME } from "../runtime/maintenance-queue.js";
26
28
  import { RunControl } from "../runtime/run-control.js";
27
- import { SandboxJobs } from "../sandbox/jobs.js";
29
+ import { oweDeviceDestroy } from "../sandbox/jobs.js";
28
30
  import { ApprovalsService } from "../tools/approvals.service.js";
29
31
  import { Credentials } from "../tools/credentials.js";
30
32
  import { QuestionsService } from "../tools/questions.service.js";
31
- /** The safety-net sweep: cron job name (no `:`), every 10 minutes, and the rows per page. */
32
- export const OFFBOARD_SWEEP_JOB = "offboard-sweep";
33
- export const OFFBOARD_SWEEP_BATCH = 500;
34
- /** 一次兜底扫描最多翻的页数(每页 {@link OFFBOARD_SWEEP_BATCH} 条),防止单次任务无限长。 */
35
- export const OFFBOARD_SWEEP_MAX_PAGES = 20;
36
- /** 已经广播中止的运行的标记保留时长(秒),长于扫描周期:期间扫描不再重复广播。 */
37
- export const OFFBOARD_ABORT_MARK_TTL_SECONDS = 1800;
38
- /** 「已请求中止」标记(Redis 键)。 */
39
- export const offboardAbortKey = (runId) => `capk:offboard-abort:${runId}`;
33
+ /**
34
+ * The stop of a departed member's run (transactional outbox spec §8.1), owed by the offboard
35
+ * transaction on the maintenance queue under {@link runStopJobId}; `MaintenanceProcessor` runs
36
+ * {@link Offboarding.stopRun}.
37
+ */
38
+ export const RUN_STOP_JOB = "run-stop";
39
+ /** The `run-stop` job id: a run is stopped once (no `:`). */
40
+ export const runStopJobId = (runId) => `run-stop-${runId}`;
40
41
  const ACTIVE = ["queued", "running", "waiting"];
41
42
  /** 通讯录同步加入的成员关系的 `source` 前缀(`directory:<插件名>`)。 */
42
43
  const DIRECTORY_SOURCE_PREFIX = "directory:";
43
44
  /**
44
45
  * 成员离职(spec 3.3):管理员移出、本人离开与通讯录同步共用这一条路径,全部写入在调用方的同一个事务内,
45
- * 要么都生效要么都不生效。Redis 吊销标记、事件发布与中止进行中的运行是提交后的尽力而为({@link afterOffboard}),
46
- * 漏掉的运行由兜底扫描({@link sweep}:发起人已不是该组织成员的进行中运行)中止。框架无关。
46
+ * 要么都生效要么都不生效。The stops of the person's runs and the destruction of their devices are owed
47
+ * in that transaction (outbox jobs, transactional outbox spec §8.1); the Redis revocation markers
48
+ * and the event publishes after the commit are hints ({@link afterOffboard}: `revoked_at` in PG is
49
+ * the authority).
47
50
  *
48
51
  * 已接受的窄窗口:离职提交时正在进行的 OAuth token 刷新(`Credentials` 刷新锁内)可能在提交之后把此人的
49
52
  * 个人凭据重新写回(upsert)。凭据只在此人以成员身份调用工具时使用,而其运行已被中止、成员关系已删除,
@@ -53,31 +56,33 @@ let Offboarding = class Offboarding {
53
56
  db;
54
57
  txHost;
55
58
  sessions;
56
- redis;
57
59
  registry;
58
60
  eventLog;
59
- sandboxJobs;
61
+ outbox;
60
62
  runs;
61
63
  credentials;
62
64
  approvals;
63
65
  questions;
66
+ lifecycle;
64
67
  log;
65
- constructor(db, txHost, sessions, redis, registry, eventLog,
66
- /** The departed member's devices: one-off maintenance jobs, destroyed by a worker. */
67
- sandboxJobs,
68
- /** Stops the departed member's runs (`by: null`, a system action). */
69
- runs, credentials, approvals, questions, log) {
68
+ constructor(db, txHost, sessions, registry, eventLog,
69
+ /** Owes the run stops and device destroys in the offboard transaction. */
70
+ outbox,
71
+ /** Stops the departed member's runs (`by: null`, a system action) in the `run-stop` job. */
72
+ runs, credentials, approvals, questions,
73
+ /** Owes `org.member_removed` to the subscribing plugins, in the offboard transaction. */
74
+ lifecycle, log) {
70
75
  this.db = db;
71
76
  this.txHost = txHost;
72
77
  this.sessions = sessions;
73
- this.redis = redis;
74
78
  this.registry = registry;
75
79
  this.eventLog = eventLog;
76
- this.sandboxJobs = sandboxJobs;
80
+ this.outbox = outbox;
77
81
  this.runs = runs;
78
82
  this.credentials = credentials;
79
83
  this.approvals = approvals;
80
84
  this.questions = questions;
85
+ this.lifecycle = lifecycle;
81
86
  this.log = log;
82
87
  }
83
88
  /**
@@ -88,7 +93,8 @@ let Offboarding = class Offboarding {
88
93
  * account (both are account-wide, so conservatively all); a person who leaves (`left`) keeps
89
94
  * them, and the session's current org is cleared (the membership check ends the access);
90
95
  * 3. 删除其在该组织所有连接器上的个人凭据;
91
- * 4. 取消其作为处理人的 pending 审批与提问(写 cancelled 事件),记下此人在该组织进行中的运行(提交后中止);
96
+ * 4. 取消其作为处理人的 pending 审批与提问(写 cancelled 事件); owes a `run-stop` job for each of the
97
+ * person's active runs in the org;
92
98
  * 5. 其负责的该组织 agent 转给组织所有者(最早加入的 owner)并置 `needs_reassignment = true`;
93
99
  * 6. 归档其在该组织各渠道连接上的 IM 话题行(重新加入后在 IM 里开始新的对话,不接着旧会话),
94
100
  * 删除其在该组织的「以后都允许」工具授权(`tool_grants`);
@@ -96,9 +102,10 @@ let Offboarding = class Offboarding {
96
102
  * 以及成员关系自身的 `directory:<插件>` 来源,不论此人当初是不是由通讯录加入(手动重新加入后再被移出也不会被同步加回);
97
103
  * 同步不再把此人加回,直到有人手动重新加入({@link MembersService.add} 删除排除);
98
104
  * 8. 审计 `org.member.offboarded`;
99
- * 9. Soft-deletes the person's devices in the org and clears their default row (the
100
- * instances and files are destroyed after the commit, best effort; the hourly sweep finishes
101
- * any left).
105
+ * 9. Soft-deletes the person's devices in the org and clears their default row, and owes the
106
+ * destruction of each (instance and files, `device-destroy-<deviceId>`);
107
+ * 10. Owes the `org.member_removed` lifecycle hook deliveries (outbox jobs, one per subscribing
108
+ * plugin).
102
109
  */
103
110
  async offboard(orgId, accountId, reason, actor = {}) {
104
111
  const deleted = await this.db
@@ -126,17 +133,26 @@ let Offboarding = class Offboarding {
126
133
  .orderBy("id")
127
134
  .forNoKeyUpdate()
128
135
  .execute();
129
- // then the member's conversations in id order, before any of their queue rows (final review
130
- // m2): queue dispatch, send-now and remove lock a conversation, then its queue rows; deleting
131
- // the rows first and updating the conversations after could deadlock with them (40P01)
132
- await this.db
136
+ // then, in one pass, the member's conversations and those holding a pending approval or
137
+ // question this offboarding cancels, before any of their queue, approval or question rows
138
+ // (final review m2: queue dispatch, send-now and remove lock a conversation, then its queue
139
+ // rows). Locked tree by tree, root first (`lockConversationTrees`), not in id order: a reply
140
+ // locks its root and then the sub-agents' conversations whose network asks it skips, and a
141
+ // child's id may sort before its root's (40P01)
142
+ const own = await this.db
133
143
  .selectFrom("conversations")
134
144
  .select("id")
135
145
  .where("org_id", "=", orgId)
136
146
  .where("account_id", "=", accountId)
137
- .orderBy("id")
138
- .forNoKeyUpdate()
139
147
  .execute();
148
+ const approvalConvs = await this.approvals.pendingConversationsFor(orgId, accountId);
149
+ const questionConvs = await this.questions.pendingConversationsFor(orgId, accountId);
150
+ const locked = await lockConversationTrees(this.db.db, [
151
+ ...own.map((c) => c.id),
152
+ ...approvalConvs,
153
+ ...questionConvs,
154
+ ]);
155
+ const isLocked = new Set(locked);
140
156
  const owner = await this.db
141
157
  .selectFrom("memberships")
142
158
  .select("account_id")
@@ -180,8 +196,9 @@ let Offboarding = class Offboarding {
180
196
  : [];
181
197
  // in this transaction (the approvals and questions services join it: `mandatory`)
182
198
  const credentials = await this.credentials.clearAccount(orgId, accountId);
183
- const approvals = await this.approvals.cancelPendingFor(orgId, accountId);
184
- const questions = await this.questions.cancelPendingFor(orgId, accountId);
199
+ // only in the conversations locked above (one asked meanwhile elsewhere waits for its run's stop)
200
+ const approvals = await this.approvals.cancelPendingFor(orgId, accountId, approvalConvs.filter((c) => isLocked.has(c)));
201
+ const questions = await this.questions.cancelPendingFor(orgId, accountId, questionConvs.filter((c) => isLocked.has(c)));
185
202
  const notify = new Map(approvals.seqs);
186
203
  for (const [c, seq] of questions.seqs)
187
204
  notify.set(c, Math.max(notify.get(c) ?? 0, seq));
@@ -201,6 +218,16 @@ let Offboarding = class Offboarding {
201
218
  eb("conversations.account_id", "=", accountId),
202
219
  ]))
203
220
  .execute();
221
+ // their stops are owed with the offboarding (the job reads the run's state when it runs)
222
+ for (const r of runs) {
223
+ await this.outbox.add({
224
+ queue: MAINTENANCE_QUEUE_NAME,
225
+ name: RUN_STOP_JOB,
226
+ jobId: runStopJobId(r.id),
227
+ data: { conversationId: r.conversation_id, runId: r.id },
228
+ orgId,
229
+ });
230
+ }
204
231
  const agents = await this.db
205
232
  .updateTable("agents")
206
233
  .set({
@@ -268,6 +295,8 @@ let Offboarding = class Offboarding {
268
295
  .where("deleted_at", "is", null)
269
296
  .returning("id")
270
297
  .execute();
298
+ for (const d of devices)
299
+ await oweDeviceDestroy(this.outbox, d.id, orgId);
271
300
  await this.db
272
301
  .deleteFrom("device_defaults")
273
302
  .where("org_id", "=", orgId)
@@ -297,6 +326,10 @@ let Offboarding = class Offboarding {
297
326
  ...(queued.length > 0 && { queuedMessagesDeleted: queued.length }),
298
327
  },
299
328
  });
329
+ await this.lifecycle.recordIn("org.member_removed", {
330
+ orgId,
331
+ accountId,
332
+ });
300
333
  return {
301
334
  role: deleted.role,
302
335
  revokedSessionIds: sessions.map((s) => s.id),
@@ -335,9 +368,9 @@ let Offboarding = class Offboarding {
335
368
  }
336
369
  /**
337
370
  * After the commit (callers register it with the infra `afterCommit` inside their transaction):
338
- * writes the session revocation markers, publishes the cancelled events, stops the person's
339
- * runs in the org and schedules destroying their devices. A failing step only logs a
340
- * warning.
371
+ * writes the session revocation markers and publishes the cancelled events and queue changes,
372
+ * all hints (the run stops and device destroys were owed in the transaction). A failing step
373
+ * only logs a warning.
341
374
  */
342
375
  async afterOffboard(r) {
343
376
  const { sessions, log, eventLog } = this;
@@ -363,102 +396,51 @@ let Offboarding = class Offboarding {
363
396
  log.warn("offboarding: queue change notify failed", { conversationId, err });
364
397
  }
365
398
  }
366
- for (const run of r.runs)
367
- await this.stopRun(run);
368
- if (r.devices.length > 0) {
369
- try {
370
- await this.sandboxJobs.destroy(r.devices);
371
- }
372
- catch (err) {
373
- log.warn("offboarding: cannot schedule device destruction", {
374
- count: r.devices.length,
375
- err,
376
- });
377
- }
378
- }
379
399
  }
380
400
  /**
381
- * 中止一个运行,顺序同会话中止接口:waiting → 直接取消等待并写终态;孤儿 → 直接写终态;
382
- * 否则经 Redis 广播中止(运行所在副本收到后结束;尚未开始的运行开始时补查中止标记)。失败只记 warn。
383
- * 不像会话中止那样随后起新一轮运行:离职者的待处理消息不再处理。
401
+ * The `run-stop` job (`MaintenanceProcessor`): stops a departed member's run the way the
402
+ * conversation's stop does (`POST .../abort`), by no account: a waiting run is aborted directly,
403
+ * a queued run is ended in PG (ruling R27: it never starts later, however long it was held), an
404
+ * orphan is finished directly, a running run is asked to stop through Redis (its replica ends it). A finished run is left alone. No
405
+ * new run follows, as after a person's stop: the departed member's queued messages were deleted.
406
+ * A failure throws (the job is retried).
384
407
  */
385
- async stopRun(run) {
408
+ async stopRun(data) {
409
+ const run = await this.db
410
+ .selectFrom("runs")
411
+ .select("status")
412
+ .where("id", "=", data.runId)
413
+ .executeTakeFirst();
414
+ if (run === undefined || !ACTIVE.includes(run.status))
415
+ return;
386
416
  const ctl = this.runs;
387
- try {
388
- if (run.status === "waiting" &&
389
- (await ctl.abortWaiting(run.conversationId, run.runId, null))) {
390
- return;
391
- }
392
- if (await ctl.abortIfOrphaned(run.conversationId, run.runId, null))
393
- return;
394
- await ctl.requestAbort(run.runId, null);
395
- // 广播中止的运行要等所在副本到达步骤边界才真正结束:广播成功后记标记,扫描在其有效期内不再重复广播
396
- // (广播失败不记,下一轮扫描重试)
397
- await this.redis
398
- .set(offboardAbortKey(run.runId), "1", "EX", OFFBOARD_ABORT_MARK_TTL_SECONDS)
399
- .catch((err) => this.log.warn("offboarding: cannot mark abort", { err }));
400
- }
401
- catch (err) {
402
- this.log.warn("offboarding: cannot stop run", { runId: run.runId, err });
417
+ if (run.status === "waiting" &&
418
+ (await ctl.abortWaiting(data.conversationId, data.runId, null))) {
419
+ return;
403
420
  }
404
- }
405
- /**
406
- * 兜底扫描(维护队列每 10 分钟):进行中、且发起人已不是该会话所属组织成员的运行 → 中止
407
- * (提交后的中止没做成时,如进程崩溃)。按 `(runs.created_at, runs.id)` 键集分页翻完(每页 500 条,
408
- * 至多 20 页):经广播中止、尚未真正结束的运行仍是进行中,若只取前 500 条,超过 500 条卡住的运行会
409
- * 一直占满这一页,后面的永远轮不到。已广播中止、仍带 `capk:offboard-abort:<runId>` 标记(30 分钟)的运行跳过,
410
- * 不每轮重复广播;标记过期仍未结束的再处理一次。返回本次处理(未跳过)的运行数。
411
- * 现任成员(含匿名访客这样的 guest 成员)发起的运行不受影响。
412
- */
413
- async sweep(batch = OFFBOARD_SWEEP_BATCH) {
414
- const db = this.db;
415
- // 游标的时间用文本保存完整的微秒精度(JS Date 只有毫秒,会让同一行再被取到)
416
- let after;
417
- let total = 0;
418
- for (let page = 0; page < OFFBOARD_SWEEP_MAX_PAGES; page++) {
419
- const cursor = after;
420
- const rows = await db
421
+ if (run.status === "queued") {
422
+ // a queued run may wait for hours: it is ended in PG, never left to the stop marker's TTL
423
+ // (ruling R27); the marker first, so a worker that takes it meanwhile stops too
424
+ await ctl.requestAbort(data.runId, null);
425
+ if (await ctl.abortQueued(data.conversationId, data.runId, null))
426
+ return;
427
+ // a worker took it (now running) or it moved on: the marker stops it; re-read and go on
428
+ const now = await this.db
421
429
  .selectFrom("runs")
422
- .innerJoin("conversations", "conversations.id", "runs.conversation_id")
423
- .select(["runs.id", "runs.conversation_id", "runs.status"])
424
- .select(sql `runs.created_at::text`.as("created_at_text"))
425
- .where("runs.status", "in", ACTIVE)
426
- // a removed member's sub-agents run with their identity and credentials: stopped too
427
- .where("conversations.kind", "in", ["chat", "subagent"])
428
- .where(({ not, exists, selectFrom }) => not(exists(selectFrom("memberships")
429
- .select(sql `1`.as("one"))
430
- .whereRef("memberships.org_id", "=", "conversations.org_id")
431
- .whereRef("memberships.account_id", "=", "runs.trigger_account_id"))))
432
- .$if(cursor !== undefined, (q) => q.where(sql `(runs.created_at, runs.id) > (${cursor?.createdAt}::timestamptz, ${cursor?.id})`))
433
- .orderBy("runs.created_at")
434
- .orderBy("runs.id")
435
- .limit(batch)
436
- .execute();
437
- for (const r of rows) {
438
- if (await this.recentlyAborted(r.id))
439
- continue;
440
- total++;
441
- this.log.warn("offboard sweep: stopping run of a former member", { runId: r.id });
442
- await this.stopRun({
443
- conversationId: r.conversation_id,
444
- runId: r.id,
445
- status: r.status,
446
- });
430
+ .select("status")
431
+ .where("id", "=", data.runId)
432
+ .executeTakeFirst();
433
+ if (now?.status === "queued") {
434
+ throw new Error(`run ${data.runId} is still queued; the run-stop job retries`);
447
435
  }
448
- const last = rows[rows.length - 1];
449
- if (rows.length < batch || !last)
450
- break;
451
- after = { createdAt: last.created_at_text, id: last.id };
452
- }
453
- return total;
454
- }
455
- async recentlyAborted(runId) {
456
- try {
457
- return (await this.redis.exists(offboardAbortKey(runId))) === 1;
458
- }
459
- catch {
460
- return false;
436
+ if (now?.status !== "waiting")
437
+ return;
438
+ await ctl.abortWaiting(data.conversationId, data.runId, null);
439
+ return;
461
440
  }
441
+ if (await ctl.abortIfOrphaned(data.conversationId, data.runId, null))
442
+ return;
443
+ await ctl.requestAbort(data.runId, null);
462
444
  }
463
445
  };
464
446
  __decorate([
@@ -469,41 +451,14 @@ Offboarding = __decorate([
469
451
  __param(0, Inject(KernelDb)),
470
452
  __param(1, Inject(TransactionHost)),
471
453
  __param(2, Inject(SessionService)),
472
- __param(3, Inject(REDIS)),
473
- __param(4, Inject(ExtensionRegistry)),
474
- __param(5, Inject(EventLog)),
475
- __param(6, Inject(SandboxJobs)),
476
- __param(7, Inject(RunControl)),
477
- __param(8, Inject(Credentials)),
478
- __param(9, Inject(ApprovalsService)),
479
- __param(10, Inject(QuestionsService)),
454
+ __param(3, Inject(ExtensionRegistry)),
455
+ __param(4, Inject(EventLog)),
456
+ __param(5, Inject(OutboxService)),
457
+ __param(6, Inject(RunControl)),
458
+ __param(7, Inject(Credentials)),
459
+ __param(8, Inject(ApprovalsService)),
460
+ __param(9, Inject(QuestionsService)),
461
+ __param(10, Inject(LifecycleHooks)),
480
462
  __param(11, Inject(KERNEL_LOGGER))
481
463
  ], Offboarding);
482
464
  export { Offboarding };
483
- /**
484
- * The offboarding safety-net sweep ({@link Offboarding.sweep}) every 10 minutes, once per tick
485
- * across replicas; a worker-role job (`WorkerModule`).
486
- */
487
- let OffboardSweepJob = class OffboardSweepJob {
488
- offboarding;
489
- lockService;
490
- log;
491
- constructor(offboarding, lockService, log) {
492
- this.offboarding = offboarding;
493
- this.lockService = lockService;
494
- this.log = log;
495
- }
496
- async run() {
497
- await this.offboarding.sweep();
498
- }
499
- };
500
- __decorate([
501
- LockedCron("*/10 * * * *", { name: OFFBOARD_SWEEP_JOB })
502
- ], OffboardSweepJob.prototype, "run", null);
503
- OffboardSweepJob = __decorate([
504
- Injectable(),
505
- __param(0, Inject(Offboarding)),
506
- __param(1, Inject(LockService)),
507
- __param(2, Inject(KERNEL_LOGGER))
508
- ], OffboardSweepJob);
509
- export { OffboardSweepJob };
@@ -9,6 +9,7 @@ import { CatalogModule } from "../catalog/catalog.module.js";
9
9
  import { EventLogModule } from "../conversations/event-log.js";
10
10
  import { EvalBudgetModule } from "../evals/budget.js";
11
11
  import { ExtensionsModule } from "../extensions/extensions.module.js";
12
+ import { LifecycleHooksModule } from "../extensions/lifecycle-hooks.js";
12
13
  import { IdentityModule } from "../identity/identity.module.js";
13
14
  import { QueueModule } from "../infra/queue/queue.module.js";
14
15
  import { MAINTENANCE_QUEUE_NAME } from "../runtime/maintenance-queue.js";
@@ -37,6 +38,8 @@ OrgsModule = __decorate([
37
38
  OrgLockModule,
38
39
  OrgSettingsModule,
39
40
  ExtensionsModule,
41
+ // offboarding and the org delete record lifecycle hook deliveries
42
+ LifecycleHooksModule,
40
43
  // offboarding clears credentials and cancels approvals and questions
41
44
  ToolsModule,
42
45
  // offboarding stops the member's runs
@@ -1,6 +1,7 @@
1
1
  import { OwnedItems } from "../catalog/owned-items.js";
2
2
  import { PluginScopeSync } from "../catalog/scope-sync.js";
3
3
  import { EvalBudget } from "../evals/budget.js";
4
+ import { LifecycleHooks } from "../extensions/lifecycle-hooks.js";
4
5
  import type { Account } from "../identity/accounts.js";
5
6
  import { ApiTokenService } from "../identity/api-tokens.js";
6
7
  import { SettingsService } from "../identity/settings.js";
@@ -32,17 +33,21 @@ export declare class OrgsService {
32
33
  private readonly events;
33
34
  /** Cancels the deleted org's running eval runs (in the delete's transaction). */
34
35
  private readonly evalBudget;
35
- /** A new org gets the plugins its scope offers it (the scope sync, after the commit). */
36
+ /** A new org gets the plugins its scope offers it (the scope sync, owed with the org). */
36
37
  private readonly scopeSync;
37
38
  /** What the deleted org owns is transferred or removed in the delete's transaction (P3-20). */
38
39
  private readonly ownedItems;
40
+ /** Records `org.deleted` for the subscribing plugins in the delete's transaction. */
41
+ private readonly lifecycle;
39
42
  constructor(db: KernelDb, txHost: TransactionHost, settings: SettingsService, apiTokens: ApiTokenService, orgLock: OrgLock, events: KernelEvents,
40
43
  /** Cancels the deleted org's running eval runs (in the delete's transaction). */
41
44
  evalBudget: Pick<EvalBudget, "cancelRuns">,
42
- /** A new org gets the plugins its scope offers it (the scope sync, after the commit). */
45
+ /** A new org gets the plugins its scope offers it (the scope sync, owed with the org). */
43
46
  scopeSync: Pick<PluginScopeSync, "enqueueOrg">,
44
47
  /** What the deleted org owns is transferred or removed in the delete's transaction (P3-20). */
45
- ownedItems: Pick<OwnedItems, "onOrgDeleted">);
48
+ ownedItems: Pick<OwnedItems, "onOrgDeleted">,
49
+ /** Records `org.deleted` for the subscribing plugins in the delete's transaction. */
50
+ lifecycle: Pick<LifecycleHooks, "recordIn">);
46
51
  /**
47
52
  * Creates an org with the actor as owner, in the caller's transaction when there is one (the
48
53
  * setup wizard creates it with the account); a slug conflict is `org.slug_taken` either way, but
@@ -79,8 +84,9 @@ export declare class OrgsService {
79
84
  * sessions' current org pointing at it, and transfers or removes the plugins and promoted
80
85
  * components it owns ({@link OwnedItems.onOrgDeleted}, P3-20: shared ones become the platform's,
81
86
  * the others are removed, a removed plugin's extension taken out by its cleanup job after the
82
- * commit); after the commit emits `org.deleted` (channel connections go with the org: the
83
- * runtime's connection reads filter deleted orgs).
87
+ * commit) and owes the `org.deleted` lifecycle hook deliveries (outbox jobs); after the commit
88
+ * emits `org.deleted` (channel connections go with the org: the runtime's connection reads filter
89
+ * deleted orgs).
84
90
  */
85
91
  softDelete(orgId: string, actor: Account, actorTokenId?: string): Promise<void>;
86
92
  private softDeleteIn;
@@ -15,6 +15,7 @@ import { cancelPendingRequests } from "../catalog/request-rules.js";
15
15
  import { PluginScopeSync } from "../catalog/scope-sync.js";
16
16
  import { ApiError } from "../errors.js";
17
17
  import { EvalBudget } from "../evals/budget.js";
18
+ import { LifecycleHooks } from "../extensions/lifecycle-hooks.js";
18
19
  import { ApiTokenService } from "../identity/api-tokens.js";
19
20
  import { audit } from "../identity/audit.js";
20
21
  import { SettingsService } from "../identity/settings.js";
@@ -85,13 +86,16 @@ let OrgsService = class OrgsService {
85
86
  evalBudget;
86
87
  scopeSync;
87
88
  ownedItems;
89
+ lifecycle;
88
90
  constructor(db, txHost, settings, apiTokens, orgLock, events,
89
91
  /** Cancels the deleted org's running eval runs (in the delete's transaction). */
90
92
  evalBudget,
91
- /** A new org gets the plugins its scope offers it (the scope sync, after the commit). */
93
+ /** A new org gets the plugins its scope offers it (the scope sync, owed with the org). */
92
94
  scopeSync,
93
95
  /** What the deleted org owns is transferred or removed in the delete's transaction (P3-20). */
94
- ownedItems) {
96
+ ownedItems,
97
+ /** Records `org.deleted` for the subscribing plugins in the delete's transaction. */
98
+ lifecycle) {
95
99
  this.db = db;
96
100
  this.txHost = txHost;
97
101
  this.settings = settings;
@@ -101,6 +105,7 @@ let OrgsService = class OrgsService {
101
105
  this.evalBudget = evalBudget;
102
106
  this.scopeSync = scopeSync;
103
107
  this.ownedItems = ownedItems;
108
+ this.lifecycle = lifecycle;
104
109
  }
105
110
  /**
106
111
  * Creates an org with the actor as owner, in the caller's transaction when there is one (the
@@ -137,7 +142,7 @@ let OrgsService = class OrgsService {
137
142
  target: row.id,
138
143
  detail: { slug, name },
139
144
  });
140
- afterCommit(() => this.scopeSync.enqueueOrg(row.id));
145
+ await this.scopeSync.enqueueOrg(row.id);
141
146
  return toOrg(row);
142
147
  }
143
148
  /** 账号所属的(未软删除的)组织及其角色,按创建时间升序。 */
@@ -222,8 +227,9 @@ let OrgsService = class OrgsService {
222
227
  * sessions' current org pointing at it, and transfers or removes the plugins and promoted
223
228
  * components it owns ({@link OwnedItems.onOrgDeleted}, P3-20: shared ones become the platform's,
224
229
  * the others are removed, a removed plugin's extension taken out by its cleanup job after the
225
- * commit); after the commit emits `org.deleted` (channel connections go with the org: the
226
- * runtime's connection reads filter deleted orgs).
230
+ * commit) and owes the `org.deleted` lifecycle hook deliveries (outbox jobs); after the commit
231
+ * emits `org.deleted` (channel connections go with the org: the runtime's connection reads filter
232
+ * deleted orgs).
227
233
  */
228
234
  async softDelete(orgId, actor, actorTokenId) {
229
235
  await this.softDeleteIn(orgId, actor, actorTokenId);
@@ -263,6 +269,8 @@ let OrgsService = class OrgsService {
263
269
  action: "org.deleted",
264
270
  target: orgId,
265
271
  });
272
+ // owed with the delete, carrying no org id, so the org's own outbox cleanup keeps them
273
+ await this.lifecycle.recordIn("org.deleted", { orgId });
266
274
  // channel connections follow the org after the root commit (none after a rollback)
267
275
  afterCommit(() => this.events.emit("org.deleted", { orgId }));
268
276
  }
@@ -286,6 +294,7 @@ OrgsService = __decorate([
286
294
  __param(5, Inject(KernelEvents)),
287
295
  __param(6, Inject(EvalBudget)),
288
296
  __param(7, Inject(PluginScopeSync)),
289
- __param(8, Inject(OwnedItems))
297
+ __param(8, Inject(OwnedItems)),
298
+ __param(9, Inject(LifecycleHooks))
290
299
  ], OrgsService);
291
300
  export { OrgsService };
@@ -1,6 +1,7 @@
1
1
  import type { NestInterceptor } from "@nestjs/common";
2
2
  import { APP_INTERCEPTOR } from "@nestjs/core";
3
3
  import { ADMIN_USAGE_TIMINGS, type AdminUsageTimings } from "./admin/usage.service.js";
4
+ import { SCOPE_BACKSTOP_DELAY } from "./catalog/scope-jobs.js";
4
5
  import { CHANNEL_WEBHOOK_TIMINGS, type ChannelWebhookTimings } from "./channels/webhook.js";
5
6
  import type { EvalReplayHooks } from "./evals/run-replay.js";
6
7
  import { EVAL_REPLAY_HOOKS } from "./evals/worker.js";
@@ -46,6 +47,7 @@ import { EMBED_TOKEN_CLOCK } from "./ui/embed-token.js";
46
47
  * - `OUTBOX_FAST_PATH_SETTLED`: receives each detached outbox fast-path pass (a test awaits them);
47
48
  * - `ENSURE_RUN_FAILS`: returns true to make `RunQueue.ensureRun` throw (Redis unreachable, or the
48
49
  * direct call after a commit skipped by a crash);
50
+ * - `SCOPE_BACKSTOP_DELAY`: the delay of a confirmation's scope sync backstop (ruling R24);
49
51
  * - `APP_INTERCEPTOR`: one more global interceptor (a probe of the matched routes).
50
52
  */
51
53
  interface KernelSeams {
@@ -69,6 +71,7 @@ interface KernelSeams {
69
71
  [USAGE_SUBSCRIBERS_CLOCK]: () => number;
70
72
  [OUTBOX_FAST_PATH_SETTLED]: (pass: Promise<void>) => void;
71
73
  [ENSURE_RUN_FAILS]: () => boolean;
74
+ [SCOPE_BACKSTOP_DELAY]: number;
72
75
  [APP_INTERCEPTOR]: NestInterceptor;
73
76
  }
74
77
  /**
package/dist/overrides.js CHANGED
@@ -1,5 +1,6 @@
1
1
  import { APP_INTERCEPTOR } from "@nestjs/core";
2
2
  import { ADMIN_USAGE_TIMINGS } from "./admin/usage.service.js";
3
+ import { SCOPE_BACKSTOP_DELAY } from "./catalog/scope-jobs.js";
3
4
  import { CHANNEL_WEBHOOK_TIMINGS } from "./channels/webhook.js";
4
5
  import { EVAL_REPLAY_HOOKS } from "./evals/worker.js";
5
6
  import { BOOTSTRAP_LOCK_TTL } from "./extensions/bootstrap.js";
@@ -40,6 +41,7 @@ const SEAM_TOKENS = [
40
41
  OUTBOX_DISPATCH_FAILS,
41
42
  OUTBOX_FAST_PATH_SETTLED,
42
43
  ENSURE_RUN_FAILS,
44
+ SCOPE_BACKSTOP_DELAY,
43
45
  APP_INTERCEPTOR,
44
46
  ];
45
47
  const allSeamsListed = true;