@capybara-hq/kernel 0.1.0-rc.24 → 0.1.0-rc.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/admin/admin.module.js +2 -0
- package/dist/admin/health.service.d.ts +2 -2
- package/dist/admin/health.service.js +6 -2
- package/dist/agents/agents.module.js +3 -0
- package/dist/agents/agents.service.d.ts +6 -1
- package/dist/agents/agents.service.js +11 -2
- package/dist/catalog/catalog-import.module.js +0 -4
- package/dist/catalog/catalog-scope.service.d.ts +63 -26
- package/dist/catalog/catalog-scope.service.js +146 -71
- package/dist/catalog/catalog.module.js +4 -3
- package/dist/catalog/catalog.service.d.ts +11 -4
- package/dist/catalog/catalog.service.js +21 -5
- package/dist/catalog/cli.d.ts +4 -3
- package/dist/catalog/cli.js +8 -13
- package/dist/catalog/job-ids.d.ts +8 -20
- package/dist/catalog/job-ids.js +11 -16
- package/dist/catalog/marketplaces.service.d.ts +1 -1
- package/dist/catalog/org-plugin-uploads.d.ts +11 -4
- package/dist/catalog/org-plugin-uploads.js +21 -6
- package/dist/catalog/owned-items.d.ts +6 -13
- package/dist/catalog/owned-items.js +11 -29
- package/dist/catalog/owned-plugin-cleanup.d.ts +8 -17
- package/dist/catalog/owned-plugin-cleanup.js +12 -66
- package/dist/catalog/platform-components.d.ts +56 -34
- package/dist/catalog/platform-components.js +68 -78
- package/dist/catalog/plugin-import.d.ts +28 -25
- package/dist/catalog/plugin-import.js +90 -76
- package/dist/catalog/plugin-upgrade.d.ts +4 -4
- package/dist/catalog/plugin-upgrade.js +7 -19
- package/dist/catalog/scope-jobs.d.ts +59 -0
- package/dist/catalog/scope-jobs.js +65 -0
- package/dist/catalog/scope-sync-workers.d.ts +22 -0
- package/dist/catalog/scope-sync-workers.js +58 -2
- package/dist/catalog/scope-sync.d.ts +40 -65
- package/dist/catalog/scope-sync.js +114 -122
- package/dist/channels/inbound.js +2 -2
- package/dist/channels/threads.d.ts +4 -3
- package/dist/channels/threads.js +4 -3
- package/dist/conversations/conversations.controller.js +4 -2
- package/dist/conversations/conversations.service.d.ts +17 -5
- package/dist/conversations/conversations.service.js +17 -5
- package/dist/conversations/events.d.ts +21 -0
- package/dist/conversations/events.js +24 -1
- package/dist/conversations/source.d.ts +10 -0
- package/dist/conversations/source.js +17 -0
- package/dist/conversations/tree.d.ts +10 -0
- package/dist/conversations/tree.js +35 -0
- package/dist/db/kernel-schema.d.ts +9 -14
- package/dist/devices/devices.service.d.ts +11 -8
- package/dist/devices/devices.service.js +16 -23
- package/dist/evals/run-replay.d.ts +13 -2
- package/dist/evals/run-replay.js +28 -1
- package/dist/evals/snapshot.d.ts +7 -1
- package/dist/evals/snapshot.js +2 -1
- package/dist/extensions/hooks.d.ts +22 -3
- package/dist/extensions/hooks.js +15 -1
- package/dist/extensions/lifecycle-hooks.d.ts +99 -0
- package/dist/extensions/lifecycle-hooks.js +273 -0
- package/dist/extensions/org-config.d.ts +13 -1
- package/dist/extensions/org-config.js +19 -2
- package/dist/extensions/permission-view.d.ts +9 -5
- package/dist/extensions/permission-view.js +9 -4
- package/dist/extensions/point-defaults.d.ts +28 -1
- package/dist/extensions/point-defaults.js +44 -13
- package/dist/inbox/inbox.service.d.ts +6 -20
- package/dist/inbox/inbox.service.js +0 -3
- package/dist/inbox/org-notices.d.ts +1 -1
- package/dist/inbox/platform-inbox.service.d.ts +3 -32
- package/dist/inbox/platform-inbox.service.js +5 -54
- package/dist/infra/outbox/registry.d.ts +124 -0
- package/dist/infra/outbox/registry.js +126 -0
- package/dist/infra/queue/queue-names.d.ts +1 -1
- package/dist/infra/queue/queue-names.js +1 -0
- package/dist/orgs/directory-sync.js +1 -1
- package/dist/orgs/members.service.js +1 -1
- package/dist/orgs/offboarding.d.ts +50 -57
- package/dist/orgs/offboarding.js +117 -162
- package/dist/orgs/orgs.module.js +3 -0
- package/dist/orgs/orgs.service.d.ts +11 -5
- package/dist/orgs/orgs.service.js +15 -6
- package/dist/overrides.d.ts +3 -0
- package/dist/overrides.js +2 -0
- package/dist/platform/ops/inbox.ops.js +4 -20
- package/dist/platform/ops/platform.ops.d.ts +1 -0
- package/dist/platform/ops/platform.ops.js +6 -3
- package/dist/roles/worker-role.d.ts +10 -5
- package/dist/roles/worker-role.js +19 -9
- package/dist/roles/worker.module.js +2 -4
- package/dist/runtime/context/build.d.ts +3 -0
- package/dist/runtime/context/build.js +15 -0
- package/dist/runtime/context/plugin-context.d.ts +20 -0
- package/dist/runtime/context/plugin-context.js +30 -0
- package/dist/runtime/langgraph-loop.js +4 -0
- package/dist/runtime/loop.d.ts +8 -0
- package/dist/runtime/maintenance.d.ts +5 -3
- package/dist/runtime/maintenance.js +36 -11
- package/dist/runtime/model-call.d.ts +6 -1
- package/dist/runtime/model-call.js +1 -0
- package/dist/runtime/plugin-context.service.d.ts +69 -0
- package/dist/runtime/plugin-context.service.js +208 -0
- package/dist/runtime/run-control.d.ts +6 -1
- package/dist/runtime/run-control.js +13 -0
- package/dist/runtime/run-queue.d.ts +13 -0
- package/dist/runtime/run-queue.js +63 -0
- package/dist/runtime/run-worker.d.ts +22 -1
- package/dist/runtime/run-worker.js +107 -10
- package/dist/runtime/runtime.module.js +2 -0
- package/dist/runtime/subagents/dispatch.js +1 -0
- package/dist/runtime/subagents/skill-fork.js +3 -0
- package/dist/sandbox/jobs.d.ts +20 -10
- package/dist/sandbox/jobs.js +23 -17
- package/dist/schedules/fire.d.ts +25 -4
- package/dist/schedules/fire.js +87 -13
- package/dist/schedules/job-ids.d.ts +66 -17
- package/dist/schedules/job-ids.js +81 -13
- package/dist/schedules/run-tracker.js +2 -1
- package/dist/schedules/scan.job.d.ts +13 -32
- package/dist/schedules/scan.job.js +35 -144
- package/dist/schedules/scheduled-tasks.service.d.ts +6 -6
- package/dist/schedules/scheduled-tasks.service.js +17 -14
- package/dist/schedules/schedules.module.js +0 -4
- package/dist/skills/changes.controller.d.ts +1 -1
- package/dist/skills/changes.controller.js +1 -1
- package/dist/tools/approvals.service.d.ts +15 -4
- package/dist/tools/approvals.service.js +26 -10
- package/dist/tools/pipeline.d.ts +11 -2
- package/dist/tools/pipeline.js +5 -0
- package/dist/tools/questions.service.d.ts +15 -4
- package/dist/tools/questions.service.js +21 -19
- package/dist/tools/registry.d.ts +8 -1
- package/dist/tools/registry.js +9 -0
- package/package.json +3 -3
- package/src/migrations/0121_drop_platform_migration_runs.sql +12 -0
- package/src/migrations/0122_drop_scheduled_task_runs_rechecked_at.sql +5 -0
- package/src/migrations/0123_pgvector.sql +27 -0
- package/src/migrations/0124_conversation_source.sql +36 -0
- package/dist/catalog/platform-migration.d.ts +0 -115
- package/dist/catalog/platform-migration.js +0 -583
- package/dist/infra/queue/add-job-once.d.ts +0 -14
- package/dist/infra/queue/add-job-once.js +0 -16
package/dist/orgs/offboarding.js
CHANGED
|
@@ -12,38 +12,41 @@ import { Inject, Injectable } from "@nestjs/common";
|
|
|
12
12
|
import { sql } from "kysely";
|
|
13
13
|
import { retargetSetupRequests } from "../agents/pending-connector-requests.js";
|
|
14
14
|
import { EventLog } from "../conversations/event-log.js";
|
|
15
|
+
import { lockConversationTrees } from "../conversations/tree.js";
|
|
15
16
|
import { ApiError } from "../errors.js";
|
|
16
17
|
import { enabledExtensionsQuery } from "../extensions/enabled.js";
|
|
18
|
+
import { LifecycleHooks } from "../extensions/lifecycle-hooks.js";
|
|
17
19
|
import { ExtensionRegistry } from "../extensions/registry.js";
|
|
18
20
|
import { audit } from "../identity/audit.js";
|
|
19
21
|
import { SessionService } from "../identity/sessions.js";
|
|
20
22
|
import { KernelDb } from "../infra/database/kernel-db.js";
|
|
21
23
|
import { TransactionHost } from "../infra/database/transaction-host.js";
|
|
22
24
|
import { Transactional } from "../infra/database/transactional.decorator.js";
|
|
23
|
-
import {
|
|
24
|
-
import {
|
|
25
|
-
import {
|
|
25
|
+
import { OutboxService } from "../infra/outbox/outbox.service.js";
|
|
26
|
+
import { KERNEL_LOGGER } from "../infra/tokens.js";
|
|
27
|
+
import { MAINTENANCE_QUEUE_NAME } from "../runtime/maintenance-queue.js";
|
|
26
28
|
import { RunControl } from "../runtime/run-control.js";
|
|
27
|
-
import {
|
|
29
|
+
import { oweDeviceDestroy } from "../sandbox/jobs.js";
|
|
28
30
|
import { ApprovalsService } from "../tools/approvals.service.js";
|
|
29
31
|
import { Credentials } from "../tools/credentials.js";
|
|
30
32
|
import { QuestionsService } from "../tools/questions.service.js";
|
|
31
|
-
/**
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
export const offboardAbortKey = (runId) => `capk:offboard-abort:${runId}`;
|
|
33
|
+
/**
|
|
34
|
+
* The stop of a departed member's run (transactional outbox spec §8.1), owed by the offboard
|
|
35
|
+
* transaction on the maintenance queue under {@link runStopJobId}; `MaintenanceProcessor` runs
|
|
36
|
+
* {@link Offboarding.stopRun}.
|
|
37
|
+
*/
|
|
38
|
+
export const RUN_STOP_JOB = "run-stop";
|
|
39
|
+
/** The `run-stop` job id: a run is stopped once (no `:`). */
|
|
40
|
+
export const runStopJobId = (runId) => `run-stop-${runId}`;
|
|
40
41
|
const ACTIVE = ["queued", "running", "waiting"];
|
|
41
42
|
/** 通讯录同步加入的成员关系的 `source` 前缀(`directory:<插件名>`)。 */
|
|
42
43
|
const DIRECTORY_SOURCE_PREFIX = "directory:";
|
|
43
44
|
/**
|
|
44
45
|
* 成员离职(spec 3.3):管理员移出、本人离开与通讯录同步共用这一条路径,全部写入在调用方的同一个事务内,
|
|
45
|
-
* 要么都生效要么都不生效。
|
|
46
|
-
*
|
|
46
|
+
* 要么都生效要么都不生效。The stops of the person's runs and the destruction of their devices are owed
|
|
47
|
+
* in that transaction (outbox jobs, transactional outbox spec §8.1); the Redis revocation markers
|
|
48
|
+
* and the event publishes after the commit are hints ({@link afterOffboard}: `revoked_at` in PG is
|
|
49
|
+
* the authority).
|
|
47
50
|
*
|
|
48
51
|
* 已接受的窄窗口:离职提交时正在进行的 OAuth token 刷新(`Credentials` 刷新锁内)可能在提交之后把此人的
|
|
49
52
|
* 个人凭据重新写回(upsert)。凭据只在此人以成员身份调用工具时使用,而其运行已被中止、成员关系已删除,
|
|
@@ -53,31 +56,33 @@ let Offboarding = class Offboarding {
|
|
|
53
56
|
db;
|
|
54
57
|
txHost;
|
|
55
58
|
sessions;
|
|
56
|
-
redis;
|
|
57
59
|
registry;
|
|
58
60
|
eventLog;
|
|
59
|
-
|
|
61
|
+
outbox;
|
|
60
62
|
runs;
|
|
61
63
|
credentials;
|
|
62
64
|
approvals;
|
|
63
65
|
questions;
|
|
66
|
+
lifecycle;
|
|
64
67
|
log;
|
|
65
|
-
constructor(db, txHost, sessions,
|
|
66
|
-
/**
|
|
67
|
-
|
|
68
|
-
/** Stops the departed member's runs (`by: null`, a system action). */
|
|
69
|
-
runs, credentials, approvals, questions,
|
|
68
|
+
constructor(db, txHost, sessions, registry, eventLog,
|
|
69
|
+
/** Owes the run stops and device destroys in the offboard transaction. */
|
|
70
|
+
outbox,
|
|
71
|
+
/** Stops the departed member's runs (`by: null`, a system action) in the `run-stop` job. */
|
|
72
|
+
runs, credentials, approvals, questions,
|
|
73
|
+
/** Owes `org.member_removed` to the subscribing plugins, in the offboard transaction. */
|
|
74
|
+
lifecycle, log) {
|
|
70
75
|
this.db = db;
|
|
71
76
|
this.txHost = txHost;
|
|
72
77
|
this.sessions = sessions;
|
|
73
|
-
this.redis = redis;
|
|
74
78
|
this.registry = registry;
|
|
75
79
|
this.eventLog = eventLog;
|
|
76
|
-
this.
|
|
80
|
+
this.outbox = outbox;
|
|
77
81
|
this.runs = runs;
|
|
78
82
|
this.credentials = credentials;
|
|
79
83
|
this.approvals = approvals;
|
|
80
84
|
this.questions = questions;
|
|
85
|
+
this.lifecycle = lifecycle;
|
|
81
86
|
this.log = log;
|
|
82
87
|
}
|
|
83
88
|
/**
|
|
@@ -88,7 +93,8 @@ let Offboarding = class Offboarding {
|
|
|
88
93
|
* account (both are account-wide, so conservatively all); a person who leaves (`left`) keeps
|
|
89
94
|
* them, and the session's current org is cleared (the membership check ends the access);
|
|
90
95
|
* 3. 删除其在该组织所有连接器上的个人凭据;
|
|
91
|
-
* 4. 取消其作为处理人的 pending 审批与提问(写 cancelled
|
|
96
|
+
* 4. 取消其作为处理人的 pending 审批与提问(写 cancelled 事件); owes a `run-stop` job for each of the
|
|
97
|
+
* person's active runs in the org;
|
|
92
98
|
* 5. 其负责的该组织 agent 转给组织所有者(最早加入的 owner)并置 `needs_reassignment = true`;
|
|
93
99
|
* 6. 归档其在该组织各渠道连接上的 IM 话题行(重新加入后在 IM 里开始新的对话,不接着旧会话),
|
|
94
100
|
* 删除其在该组织的「以后都允许」工具授权(`tool_grants`);
|
|
@@ -96,9 +102,10 @@ let Offboarding = class Offboarding {
|
|
|
96
102
|
* 以及成员关系自身的 `directory:<插件>` 来源,不论此人当初是不是由通讯录加入(手动重新加入后再被移出也不会被同步加回);
|
|
97
103
|
* 同步不再把此人加回,直到有人手动重新加入({@link MembersService.add} 删除排除);
|
|
98
104
|
* 8. 审计 `org.member.offboarded`;
|
|
99
|
-
* 9. Soft-deletes the person's devices in the org and clears their default row
|
|
100
|
-
*
|
|
101
|
-
*
|
|
105
|
+
* 9. Soft-deletes the person's devices in the org and clears their default row, and owes the
|
|
106
|
+
* destruction of each (instance and files, `device-destroy-<deviceId>`);
|
|
107
|
+
* 10. Owes the `org.member_removed` lifecycle hook deliveries (outbox jobs, one per subscribing
|
|
108
|
+
* plugin).
|
|
102
109
|
*/
|
|
103
110
|
async offboard(orgId, accountId, reason, actor = {}) {
|
|
104
111
|
const deleted = await this.db
|
|
@@ -126,17 +133,26 @@ let Offboarding = class Offboarding {
|
|
|
126
133
|
.orderBy("id")
|
|
127
134
|
.forNoKeyUpdate()
|
|
128
135
|
.execute();
|
|
129
|
-
// then the member's conversations
|
|
130
|
-
//
|
|
131
|
-
//
|
|
132
|
-
|
|
136
|
+
// then, in one pass, the member's conversations and those holding a pending approval or
|
|
137
|
+
// question this offboarding cancels, before any of their queue, approval or question rows
|
|
138
|
+
// (final review m2: queue dispatch, send-now and remove lock a conversation, then its queue
|
|
139
|
+
// rows). Locked tree by tree, root first (`lockConversationTrees`), not in id order: a reply
|
|
140
|
+
// locks its root and then the sub-agents' conversations whose network asks it skips, and a
|
|
141
|
+
// child's id may sort before its root's (40P01)
|
|
142
|
+
const own = await this.db
|
|
133
143
|
.selectFrom("conversations")
|
|
134
144
|
.select("id")
|
|
135
145
|
.where("org_id", "=", orgId)
|
|
136
146
|
.where("account_id", "=", accountId)
|
|
137
|
-
.orderBy("id")
|
|
138
|
-
.forNoKeyUpdate()
|
|
139
147
|
.execute();
|
|
148
|
+
const approvalConvs = await this.approvals.pendingConversationsFor(orgId, accountId);
|
|
149
|
+
const questionConvs = await this.questions.pendingConversationsFor(orgId, accountId);
|
|
150
|
+
const locked = await lockConversationTrees(this.db.db, [
|
|
151
|
+
...own.map((c) => c.id),
|
|
152
|
+
...approvalConvs,
|
|
153
|
+
...questionConvs,
|
|
154
|
+
]);
|
|
155
|
+
const isLocked = new Set(locked);
|
|
140
156
|
const owner = await this.db
|
|
141
157
|
.selectFrom("memberships")
|
|
142
158
|
.select("account_id")
|
|
@@ -180,8 +196,9 @@ let Offboarding = class Offboarding {
|
|
|
180
196
|
: [];
|
|
181
197
|
// in this transaction (the approvals and questions services join it: `mandatory`)
|
|
182
198
|
const credentials = await this.credentials.clearAccount(orgId, accountId);
|
|
183
|
-
|
|
184
|
-
const
|
|
199
|
+
// only in the conversations locked above (one asked meanwhile elsewhere waits for its run's stop)
|
|
200
|
+
const approvals = await this.approvals.cancelPendingFor(orgId, accountId, approvalConvs.filter((c) => isLocked.has(c)));
|
|
201
|
+
const questions = await this.questions.cancelPendingFor(orgId, accountId, questionConvs.filter((c) => isLocked.has(c)));
|
|
185
202
|
const notify = new Map(approvals.seqs);
|
|
186
203
|
for (const [c, seq] of questions.seqs)
|
|
187
204
|
notify.set(c, Math.max(notify.get(c) ?? 0, seq));
|
|
@@ -201,6 +218,16 @@ let Offboarding = class Offboarding {
|
|
|
201
218
|
eb("conversations.account_id", "=", accountId),
|
|
202
219
|
]))
|
|
203
220
|
.execute();
|
|
221
|
+
// their stops are owed with the offboarding (the job reads the run's state when it runs)
|
|
222
|
+
for (const r of runs) {
|
|
223
|
+
await this.outbox.add({
|
|
224
|
+
queue: MAINTENANCE_QUEUE_NAME,
|
|
225
|
+
name: RUN_STOP_JOB,
|
|
226
|
+
jobId: runStopJobId(r.id),
|
|
227
|
+
data: { conversationId: r.conversation_id, runId: r.id },
|
|
228
|
+
orgId,
|
|
229
|
+
});
|
|
230
|
+
}
|
|
204
231
|
const agents = await this.db
|
|
205
232
|
.updateTable("agents")
|
|
206
233
|
.set({
|
|
@@ -268,6 +295,8 @@ let Offboarding = class Offboarding {
|
|
|
268
295
|
.where("deleted_at", "is", null)
|
|
269
296
|
.returning("id")
|
|
270
297
|
.execute();
|
|
298
|
+
for (const d of devices)
|
|
299
|
+
await oweDeviceDestroy(this.outbox, d.id, orgId);
|
|
271
300
|
await this.db
|
|
272
301
|
.deleteFrom("device_defaults")
|
|
273
302
|
.where("org_id", "=", orgId)
|
|
@@ -297,6 +326,10 @@ let Offboarding = class Offboarding {
|
|
|
297
326
|
...(queued.length > 0 && { queuedMessagesDeleted: queued.length }),
|
|
298
327
|
},
|
|
299
328
|
});
|
|
329
|
+
await this.lifecycle.recordIn("org.member_removed", {
|
|
330
|
+
orgId,
|
|
331
|
+
accountId,
|
|
332
|
+
});
|
|
300
333
|
return {
|
|
301
334
|
role: deleted.role,
|
|
302
335
|
revokedSessionIds: sessions.map((s) => s.id),
|
|
@@ -335,9 +368,9 @@ let Offboarding = class Offboarding {
|
|
|
335
368
|
}
|
|
336
369
|
/**
|
|
337
370
|
* After the commit (callers register it with the infra `afterCommit` inside their transaction):
|
|
338
|
-
* writes the session revocation markers
|
|
339
|
-
*
|
|
340
|
-
* warning.
|
|
371
|
+
* writes the session revocation markers and publishes the cancelled events and queue changes,
|
|
372
|
+
* all hints (the run stops and device destroys were owed in the transaction). A failing step
|
|
373
|
+
* only logs a warning.
|
|
341
374
|
*/
|
|
342
375
|
async afterOffboard(r) {
|
|
343
376
|
const { sessions, log, eventLog } = this;
|
|
@@ -363,102 +396,51 @@ let Offboarding = class Offboarding {
|
|
|
363
396
|
log.warn("offboarding: queue change notify failed", { conversationId, err });
|
|
364
397
|
}
|
|
365
398
|
}
|
|
366
|
-
for (const run of r.runs)
|
|
367
|
-
await this.stopRun(run);
|
|
368
|
-
if (r.devices.length > 0) {
|
|
369
|
-
try {
|
|
370
|
-
await this.sandboxJobs.destroy(r.devices);
|
|
371
|
-
}
|
|
372
|
-
catch (err) {
|
|
373
|
-
log.warn("offboarding: cannot schedule device destruction", {
|
|
374
|
-
count: r.devices.length,
|
|
375
|
-
err,
|
|
376
|
-
});
|
|
377
|
-
}
|
|
378
|
-
}
|
|
379
399
|
}
|
|
380
400
|
/**
|
|
381
|
-
*
|
|
382
|
-
*
|
|
383
|
-
*
|
|
401
|
+
* The `run-stop` job (`MaintenanceProcessor`): stops a departed member's run the way the
|
|
402
|
+
* conversation's stop does (`POST .../abort`), by no account: a waiting run is aborted directly,
|
|
403
|
+
* a queued run is ended in PG (ruling R27: it never starts later, however long it was held), an
|
|
404
|
+
* orphan is finished directly, a running run is asked to stop through Redis (its replica ends it). A finished run is left alone. No
|
|
405
|
+
* new run follows, as after a person's stop: the departed member's queued messages were deleted.
|
|
406
|
+
* A failure throws (the job is retried).
|
|
384
407
|
*/
|
|
385
|
-
async stopRun(
|
|
408
|
+
async stopRun(data) {
|
|
409
|
+
const run = await this.db
|
|
410
|
+
.selectFrom("runs")
|
|
411
|
+
.select("status")
|
|
412
|
+
.where("id", "=", data.runId)
|
|
413
|
+
.executeTakeFirst();
|
|
414
|
+
if (run === undefined || !ACTIVE.includes(run.status))
|
|
415
|
+
return;
|
|
386
416
|
const ctl = this.runs;
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
return;
|
|
391
|
-
}
|
|
392
|
-
if (await ctl.abortIfOrphaned(run.conversationId, run.runId, null))
|
|
393
|
-
return;
|
|
394
|
-
await ctl.requestAbort(run.runId, null);
|
|
395
|
-
// 广播中止的运行要等所在副本到达步骤边界才真正结束:广播成功后记标记,扫描在其有效期内不再重复广播
|
|
396
|
-
// (广播失败不记,下一轮扫描重试)
|
|
397
|
-
await this.redis
|
|
398
|
-
.set(offboardAbortKey(run.runId), "1", "EX", OFFBOARD_ABORT_MARK_TTL_SECONDS)
|
|
399
|
-
.catch((err) => this.log.warn("offboarding: cannot mark abort", { err }));
|
|
400
|
-
}
|
|
401
|
-
catch (err) {
|
|
402
|
-
this.log.warn("offboarding: cannot stop run", { runId: run.runId, err });
|
|
417
|
+
if (run.status === "waiting" &&
|
|
418
|
+
(await ctl.abortWaiting(data.conversationId, data.runId, null))) {
|
|
419
|
+
return;
|
|
403
420
|
}
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
*/
|
|
413
|
-
async sweep(batch = OFFBOARD_SWEEP_BATCH) {
|
|
414
|
-
const db = this.db;
|
|
415
|
-
// 游标的时间用文本保存完整的微秒精度(JS Date 只有毫秒,会让同一行再被取到)
|
|
416
|
-
let after;
|
|
417
|
-
let total = 0;
|
|
418
|
-
for (let page = 0; page < OFFBOARD_SWEEP_MAX_PAGES; page++) {
|
|
419
|
-
const cursor = after;
|
|
420
|
-
const rows = await db
|
|
421
|
+
if (run.status === "queued") {
|
|
422
|
+
// a queued run may wait for hours: it is ended in PG, never left to the stop marker's TTL
|
|
423
|
+
// (ruling R27); the marker first, so a worker that takes it meanwhile stops too
|
|
424
|
+
await ctl.requestAbort(data.runId, null);
|
|
425
|
+
if (await ctl.abortQueued(data.conversationId, data.runId, null))
|
|
426
|
+
return;
|
|
427
|
+
// a worker took it (now running) or it moved on: the marker stops it; re-read and go on
|
|
428
|
+
const now = await this.db
|
|
421
429
|
.selectFrom("runs")
|
|
422
|
-
.
|
|
423
|
-
.
|
|
424
|
-
.
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
.where("conversations.kind", "in", ["chat", "subagent"])
|
|
428
|
-
.where(({ not, exists, selectFrom }) => not(exists(selectFrom("memberships")
|
|
429
|
-
.select(sql `1`.as("one"))
|
|
430
|
-
.whereRef("memberships.org_id", "=", "conversations.org_id")
|
|
431
|
-
.whereRef("memberships.account_id", "=", "runs.trigger_account_id"))))
|
|
432
|
-
.$if(cursor !== undefined, (q) => q.where(sql `(runs.created_at, runs.id) > (${cursor?.createdAt}::timestamptz, ${cursor?.id})`))
|
|
433
|
-
.orderBy("runs.created_at")
|
|
434
|
-
.orderBy("runs.id")
|
|
435
|
-
.limit(batch)
|
|
436
|
-
.execute();
|
|
437
|
-
for (const r of rows) {
|
|
438
|
-
if (await this.recentlyAborted(r.id))
|
|
439
|
-
continue;
|
|
440
|
-
total++;
|
|
441
|
-
this.log.warn("offboard sweep: stopping run of a former member", { runId: r.id });
|
|
442
|
-
await this.stopRun({
|
|
443
|
-
conversationId: r.conversation_id,
|
|
444
|
-
runId: r.id,
|
|
445
|
-
status: r.status,
|
|
446
|
-
});
|
|
430
|
+
.select("status")
|
|
431
|
+
.where("id", "=", data.runId)
|
|
432
|
+
.executeTakeFirst();
|
|
433
|
+
if (now?.status === "queued") {
|
|
434
|
+
throw new Error(`run ${data.runId} is still queued; the run-stop job retries`);
|
|
447
435
|
}
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
}
|
|
453
|
-
return total;
|
|
454
|
-
}
|
|
455
|
-
async recentlyAborted(runId) {
|
|
456
|
-
try {
|
|
457
|
-
return (await this.redis.exists(offboardAbortKey(runId))) === 1;
|
|
458
|
-
}
|
|
459
|
-
catch {
|
|
460
|
-
return false;
|
|
436
|
+
if (now?.status !== "waiting")
|
|
437
|
+
return;
|
|
438
|
+
await ctl.abortWaiting(data.conversationId, data.runId, null);
|
|
439
|
+
return;
|
|
461
440
|
}
|
|
441
|
+
if (await ctl.abortIfOrphaned(data.conversationId, data.runId, null))
|
|
442
|
+
return;
|
|
443
|
+
await ctl.requestAbort(data.runId, null);
|
|
462
444
|
}
|
|
463
445
|
};
|
|
464
446
|
__decorate([
|
|
@@ -469,41 +451,14 @@ Offboarding = __decorate([
|
|
|
469
451
|
__param(0, Inject(KernelDb)),
|
|
470
452
|
__param(1, Inject(TransactionHost)),
|
|
471
453
|
__param(2, Inject(SessionService)),
|
|
472
|
-
__param(3, Inject(
|
|
473
|
-
__param(4, Inject(
|
|
474
|
-
__param(5, Inject(
|
|
475
|
-
__param(6, Inject(
|
|
476
|
-
__param(7, Inject(
|
|
477
|
-
__param(8, Inject(
|
|
478
|
-
__param(9, Inject(
|
|
479
|
-
__param(10, Inject(
|
|
454
|
+
__param(3, Inject(ExtensionRegistry)),
|
|
455
|
+
__param(4, Inject(EventLog)),
|
|
456
|
+
__param(5, Inject(OutboxService)),
|
|
457
|
+
__param(6, Inject(RunControl)),
|
|
458
|
+
__param(7, Inject(Credentials)),
|
|
459
|
+
__param(8, Inject(ApprovalsService)),
|
|
460
|
+
__param(9, Inject(QuestionsService)),
|
|
461
|
+
__param(10, Inject(LifecycleHooks)),
|
|
480
462
|
__param(11, Inject(KERNEL_LOGGER))
|
|
481
463
|
], Offboarding);
|
|
482
464
|
export { Offboarding };
|
|
483
|
-
/**
|
|
484
|
-
* The offboarding safety-net sweep ({@link Offboarding.sweep}) every 10 minutes, once per tick
|
|
485
|
-
* across replicas; a worker-role job (`WorkerModule`).
|
|
486
|
-
*/
|
|
487
|
-
let OffboardSweepJob = class OffboardSweepJob {
|
|
488
|
-
offboarding;
|
|
489
|
-
lockService;
|
|
490
|
-
log;
|
|
491
|
-
constructor(offboarding, lockService, log) {
|
|
492
|
-
this.offboarding = offboarding;
|
|
493
|
-
this.lockService = lockService;
|
|
494
|
-
this.log = log;
|
|
495
|
-
}
|
|
496
|
-
async run() {
|
|
497
|
-
await this.offboarding.sweep();
|
|
498
|
-
}
|
|
499
|
-
};
|
|
500
|
-
__decorate([
|
|
501
|
-
LockedCron("*/10 * * * *", { name: OFFBOARD_SWEEP_JOB })
|
|
502
|
-
], OffboardSweepJob.prototype, "run", null);
|
|
503
|
-
OffboardSweepJob = __decorate([
|
|
504
|
-
Injectable(),
|
|
505
|
-
__param(0, Inject(Offboarding)),
|
|
506
|
-
__param(1, Inject(LockService)),
|
|
507
|
-
__param(2, Inject(KERNEL_LOGGER))
|
|
508
|
-
], OffboardSweepJob);
|
|
509
|
-
export { OffboardSweepJob };
|
package/dist/orgs/orgs.module.js
CHANGED
|
@@ -9,6 +9,7 @@ import { CatalogModule } from "../catalog/catalog.module.js";
|
|
|
9
9
|
import { EventLogModule } from "../conversations/event-log.js";
|
|
10
10
|
import { EvalBudgetModule } from "../evals/budget.js";
|
|
11
11
|
import { ExtensionsModule } from "../extensions/extensions.module.js";
|
|
12
|
+
import { LifecycleHooksModule } from "../extensions/lifecycle-hooks.js";
|
|
12
13
|
import { IdentityModule } from "../identity/identity.module.js";
|
|
13
14
|
import { QueueModule } from "../infra/queue/queue.module.js";
|
|
14
15
|
import { MAINTENANCE_QUEUE_NAME } from "../runtime/maintenance-queue.js";
|
|
@@ -37,6 +38,8 @@ OrgsModule = __decorate([
|
|
|
37
38
|
OrgLockModule,
|
|
38
39
|
OrgSettingsModule,
|
|
39
40
|
ExtensionsModule,
|
|
41
|
+
// offboarding and the org delete record lifecycle hook deliveries
|
|
42
|
+
LifecycleHooksModule,
|
|
40
43
|
// offboarding clears credentials and cancels approvals and questions
|
|
41
44
|
ToolsModule,
|
|
42
45
|
// offboarding stops the member's runs
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { OwnedItems } from "../catalog/owned-items.js";
|
|
2
2
|
import { PluginScopeSync } from "../catalog/scope-sync.js";
|
|
3
3
|
import { EvalBudget } from "../evals/budget.js";
|
|
4
|
+
import { LifecycleHooks } from "../extensions/lifecycle-hooks.js";
|
|
4
5
|
import type { Account } from "../identity/accounts.js";
|
|
5
6
|
import { ApiTokenService } from "../identity/api-tokens.js";
|
|
6
7
|
import { SettingsService } from "../identity/settings.js";
|
|
@@ -32,17 +33,21 @@ export declare class OrgsService {
|
|
|
32
33
|
private readonly events;
|
|
33
34
|
/** Cancels the deleted org's running eval runs (in the delete's transaction). */
|
|
34
35
|
private readonly evalBudget;
|
|
35
|
-
/** A new org gets the plugins its scope offers it (the scope sync,
|
|
36
|
+
/** A new org gets the plugins its scope offers it (the scope sync, owed with the org). */
|
|
36
37
|
private readonly scopeSync;
|
|
37
38
|
/** What the deleted org owns is transferred or removed in the delete's transaction (P3-20). */
|
|
38
39
|
private readonly ownedItems;
|
|
40
|
+
/** Records `org.deleted` for the subscribing plugins in the delete's transaction. */
|
|
41
|
+
private readonly lifecycle;
|
|
39
42
|
constructor(db: KernelDb, txHost: TransactionHost, settings: SettingsService, apiTokens: ApiTokenService, orgLock: OrgLock, events: KernelEvents,
|
|
40
43
|
/** Cancels the deleted org's running eval runs (in the delete's transaction). */
|
|
41
44
|
evalBudget: Pick<EvalBudget, "cancelRuns">,
|
|
42
|
-
/** A new org gets the plugins its scope offers it (the scope sync,
|
|
45
|
+
/** A new org gets the plugins its scope offers it (the scope sync, owed with the org). */
|
|
43
46
|
scopeSync: Pick<PluginScopeSync, "enqueueOrg">,
|
|
44
47
|
/** What the deleted org owns is transferred or removed in the delete's transaction (P3-20). */
|
|
45
|
-
ownedItems: Pick<OwnedItems, "onOrgDeleted"
|
|
48
|
+
ownedItems: Pick<OwnedItems, "onOrgDeleted">,
|
|
49
|
+
/** Records `org.deleted` for the subscribing plugins in the delete's transaction. */
|
|
50
|
+
lifecycle: Pick<LifecycleHooks, "recordIn">);
|
|
46
51
|
/**
|
|
47
52
|
* Creates an org with the actor as owner, in the caller's transaction when there is one (the
|
|
48
53
|
* setup wizard creates it with the account); a slug conflict is `org.slug_taken` either way, but
|
|
@@ -79,8 +84,9 @@ export declare class OrgsService {
|
|
|
79
84
|
* sessions' current org pointing at it, and transfers or removes the plugins and promoted
|
|
80
85
|
* components it owns ({@link OwnedItems.onOrgDeleted}, P3-20: shared ones become the platform's,
|
|
81
86
|
* the others are removed, a removed plugin's extension taken out by its cleanup job after the
|
|
82
|
-
* commit)
|
|
83
|
-
* runtime's connection reads filter
|
|
87
|
+
* commit) and owes the `org.deleted` lifecycle hook deliveries (outbox jobs); after the commit
|
|
88
|
+
* emits `org.deleted` (channel connections go with the org: the runtime's connection reads filter
|
|
89
|
+
* deleted orgs).
|
|
84
90
|
*/
|
|
85
91
|
softDelete(orgId: string, actor: Account, actorTokenId?: string): Promise<void>;
|
|
86
92
|
private softDeleteIn;
|
|
@@ -15,6 +15,7 @@ import { cancelPendingRequests } from "../catalog/request-rules.js";
|
|
|
15
15
|
import { PluginScopeSync } from "../catalog/scope-sync.js";
|
|
16
16
|
import { ApiError } from "../errors.js";
|
|
17
17
|
import { EvalBudget } from "../evals/budget.js";
|
|
18
|
+
import { LifecycleHooks } from "../extensions/lifecycle-hooks.js";
|
|
18
19
|
import { ApiTokenService } from "../identity/api-tokens.js";
|
|
19
20
|
import { audit } from "../identity/audit.js";
|
|
20
21
|
import { SettingsService } from "../identity/settings.js";
|
|
@@ -85,13 +86,16 @@ let OrgsService = class OrgsService {
|
|
|
85
86
|
evalBudget;
|
|
86
87
|
scopeSync;
|
|
87
88
|
ownedItems;
|
|
89
|
+
lifecycle;
|
|
88
90
|
constructor(db, txHost, settings, apiTokens, orgLock, events,
|
|
89
91
|
/** Cancels the deleted org's running eval runs (in the delete's transaction). */
|
|
90
92
|
evalBudget,
|
|
91
|
-
/** A new org gets the plugins its scope offers it (the scope sync,
|
|
93
|
+
/** A new org gets the plugins its scope offers it (the scope sync, owed with the org). */
|
|
92
94
|
scopeSync,
|
|
93
95
|
/** What the deleted org owns is transferred or removed in the delete's transaction (P3-20). */
|
|
94
|
-
ownedItems
|
|
96
|
+
ownedItems,
|
|
97
|
+
/** Records `org.deleted` for the subscribing plugins in the delete's transaction. */
|
|
98
|
+
lifecycle) {
|
|
95
99
|
this.db = db;
|
|
96
100
|
this.txHost = txHost;
|
|
97
101
|
this.settings = settings;
|
|
@@ -101,6 +105,7 @@ let OrgsService = class OrgsService {
|
|
|
101
105
|
this.evalBudget = evalBudget;
|
|
102
106
|
this.scopeSync = scopeSync;
|
|
103
107
|
this.ownedItems = ownedItems;
|
|
108
|
+
this.lifecycle = lifecycle;
|
|
104
109
|
}
|
|
105
110
|
/**
|
|
106
111
|
* Creates an org with the actor as owner, in the caller's transaction when there is one (the
|
|
@@ -137,7 +142,7 @@ let OrgsService = class OrgsService {
|
|
|
137
142
|
target: row.id,
|
|
138
143
|
detail: { slug, name },
|
|
139
144
|
});
|
|
140
|
-
|
|
145
|
+
await this.scopeSync.enqueueOrg(row.id);
|
|
141
146
|
return toOrg(row);
|
|
142
147
|
}
|
|
143
148
|
/** 账号所属的(未软删除的)组织及其角色,按创建时间升序。 */
|
|
@@ -222,8 +227,9 @@ let OrgsService = class OrgsService {
|
|
|
222
227
|
* sessions' current org pointing at it, and transfers or removes the plugins and promoted
|
|
223
228
|
* components it owns ({@link OwnedItems.onOrgDeleted}, P3-20: shared ones become the platform's,
|
|
224
229
|
* the others are removed, a removed plugin's extension taken out by its cleanup job after the
|
|
225
|
-
* commit)
|
|
226
|
-
* runtime's connection reads filter
|
|
230
|
+
* commit) and owes the `org.deleted` lifecycle hook deliveries (outbox jobs); after the commit
|
|
231
|
+
* emits `org.deleted` (channel connections go with the org: the runtime's connection reads filter
|
|
232
|
+
* deleted orgs).
|
|
227
233
|
*/
|
|
228
234
|
async softDelete(orgId, actor, actorTokenId) {
|
|
229
235
|
await this.softDeleteIn(orgId, actor, actorTokenId);
|
|
@@ -263,6 +269,8 @@ let OrgsService = class OrgsService {
|
|
|
263
269
|
action: "org.deleted",
|
|
264
270
|
target: orgId,
|
|
265
271
|
});
|
|
272
|
+
// owed with the delete, carrying no org id, so the org's own outbox cleanup keeps them
|
|
273
|
+
await this.lifecycle.recordIn("org.deleted", { orgId });
|
|
266
274
|
// channel connections follow the org after the root commit (none after a rollback)
|
|
267
275
|
afterCommit(() => this.events.emit("org.deleted", { orgId }));
|
|
268
276
|
}
|
|
@@ -286,6 +294,7 @@ OrgsService = __decorate([
|
|
|
286
294
|
__param(5, Inject(KernelEvents)),
|
|
287
295
|
__param(6, Inject(EvalBudget)),
|
|
288
296
|
__param(7, Inject(PluginScopeSync)),
|
|
289
|
-
__param(8, Inject(OwnedItems))
|
|
297
|
+
__param(8, Inject(OwnedItems)),
|
|
298
|
+
__param(9, Inject(LifecycleHooks))
|
|
290
299
|
], OrgsService);
|
|
291
300
|
export { OrgsService };
|
package/dist/overrides.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { NestInterceptor } from "@nestjs/common";
|
|
2
2
|
import { APP_INTERCEPTOR } from "@nestjs/core";
|
|
3
3
|
import { ADMIN_USAGE_TIMINGS, type AdminUsageTimings } from "./admin/usage.service.js";
|
|
4
|
+
import { SCOPE_BACKSTOP_DELAY } from "./catalog/scope-jobs.js";
|
|
4
5
|
import { CHANNEL_WEBHOOK_TIMINGS, type ChannelWebhookTimings } from "./channels/webhook.js";
|
|
5
6
|
import type { EvalReplayHooks } from "./evals/run-replay.js";
|
|
6
7
|
import { EVAL_REPLAY_HOOKS } from "./evals/worker.js";
|
|
@@ -46,6 +47,7 @@ import { EMBED_TOKEN_CLOCK } from "./ui/embed-token.js";
|
|
|
46
47
|
* - `OUTBOX_FAST_PATH_SETTLED`: receives each detached outbox fast-path pass (a test awaits them);
|
|
47
48
|
* - `ENSURE_RUN_FAILS`: returns true to make `RunQueue.ensureRun` throw (Redis unreachable, or the
|
|
48
49
|
* direct call after a commit skipped by a crash);
|
|
50
|
+
* - `SCOPE_BACKSTOP_DELAY`: the delay of a confirmation's scope sync backstop (ruling R24);
|
|
49
51
|
* - `APP_INTERCEPTOR`: one more global interceptor (a probe of the matched routes).
|
|
50
52
|
*/
|
|
51
53
|
interface KernelSeams {
|
|
@@ -69,6 +71,7 @@ interface KernelSeams {
|
|
|
69
71
|
[USAGE_SUBSCRIBERS_CLOCK]: () => number;
|
|
70
72
|
[OUTBOX_FAST_PATH_SETTLED]: (pass: Promise<void>) => void;
|
|
71
73
|
[ENSURE_RUN_FAILS]: () => boolean;
|
|
74
|
+
[SCOPE_BACKSTOP_DELAY]: number;
|
|
72
75
|
[APP_INTERCEPTOR]: NestInterceptor;
|
|
73
76
|
}
|
|
74
77
|
/**
|
package/dist/overrides.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { APP_INTERCEPTOR } from "@nestjs/core";
|
|
2
2
|
import { ADMIN_USAGE_TIMINGS } from "./admin/usage.service.js";
|
|
3
|
+
import { SCOPE_BACKSTOP_DELAY } from "./catalog/scope-jobs.js";
|
|
3
4
|
import { CHANNEL_WEBHOOK_TIMINGS } from "./channels/webhook.js";
|
|
4
5
|
import { EVAL_REPLAY_HOOKS } from "./evals/worker.js";
|
|
5
6
|
import { BOOTSTRAP_LOCK_TTL } from "./extensions/bootstrap.js";
|
|
@@ -40,6 +41,7 @@ const SEAM_TOKENS = [
|
|
|
40
41
|
OUTBOX_DISPATCH_FAILS,
|
|
41
42
|
OUTBOX_FAST_PATH_SETTLED,
|
|
42
43
|
ENSURE_RUN_FAILS,
|
|
44
|
+
SCOPE_BACKSTOP_DELAY,
|
|
43
45
|
APP_INTERCEPTOR,
|
|
44
46
|
];
|
|
45
47
|
const allSeamsListed = true;
|