@capybara-hq/kernel 0.1.0-rc.25 → 0.1.0-rc.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/admin/outbox-dead.service.d.ts +7 -3
- package/dist/admin/outbox-dead.service.js +17 -6
- package/dist/admin/overview.service.d.ts +6 -6
- package/dist/admin/overview.service.js +7 -7
- package/dist/background/wake-rules.d.ts +5 -0
- package/dist/background/wake-rules.js +16 -5
- package/dist/catalog/marketplaces.service.d.ts +1 -1
- package/dist/channels/channels.module.d.ts +9 -8
- package/dist/channels/channels.module.js +11 -10
- package/dist/channels/connections.service.d.ts +3 -3
- package/dist/channels/connections.service.js +6 -6
- package/dist/channels/deliver-job.d.ts +10 -3
- package/dist/channels/deliver-job.js +7 -3
- package/dist/channels/jobs.d.ts +11 -4
- package/dist/channels/jobs.js +18 -3
- package/dist/channels/outbound.d.ts +5 -2
- package/dist/channels/outbound.js +9 -2
- package/dist/conversations/conversations.controller.d.ts +4 -2
- package/dist/conversations/conversations.controller.js +4 -2
- package/dist/conversations/conversations.service.d.ts +37 -4
- package/dist/conversations/conversations.service.js +77 -20
- package/dist/conversations/events.d.ts +6 -0
- package/dist/conversations/events.js +6 -0
- package/dist/db/kernel-schema.d.ts +10 -1
- package/dist/devices/devices.service.d.ts +14 -6
- package/dist/devices/devices.service.js +20 -7
- package/dist/evals/evals.module.js +5 -1
- package/dist/evals/jobs.d.ts +14 -6
- package/dist/evals/jobs.js +21 -19
- package/dist/evals/runs.service.d.ts +5 -6
- package/dist/evals/runs.service.js +11 -33
- package/dist/infra/outbox/outbox.service.d.ts +6 -0
- package/dist/infra/outbox/outbox.service.js +51 -17
- package/dist/infra/outbox/registry.d.ts +52 -0
- package/dist/infra/outbox/registry.js +57 -0
- package/dist/local-devices/device-policy.service.d.ts +131 -26
- package/dist/local-devices/device-policy.service.js +341 -145
- package/dist/local-devices/machines.d.ts +17 -2
- package/dist/local-devices/machines.js +66 -5
- package/dist/local-devices/policy-op-jobs.d.ts +15 -0
- package/dist/local-devices/policy-op-jobs.js +12 -0
- package/dist/local-devices/policy-summary.d.ts +36 -3
- package/dist/local-devices/policy-summary.js +47 -1
- package/dist/orgs/orgs.service.js +4 -1
- package/dist/overrides.d.ts +4 -0
- package/dist/overrides.js +2 -0
- package/dist/platform/ops/conversations.ops.js +1 -1
- package/dist/runtime/maintenance.d.ts +6 -3
- package/dist/runtime/maintenance.js +21 -7
- package/dist/runtime/plugin-context.service.d.ts +9 -3
- package/dist/runtime/plugin-context.service.js +17 -5
- package/dist/runtime/run-context.service.d.ts +7 -3
- package/dist/runtime/run-context.service.js +12 -2
- package/dist/runtime/run-finish.d.ts +2 -2
- package/dist/runtime/run-finish.js +11 -6
- package/dist/runtime/run-jobs.d.ts +21 -4
- package/dist/runtime/run-jobs.js +29 -6
- package/dist/runtime/run-orphan-check.job.d.ts +14 -11
- package/dist/runtime/run-orphan-check.job.js +16 -11
- package/dist/runtime/run-queue.d.ts +23 -31
- package/dist/runtime/run-queue.js +62 -64
- package/dist/runtime/run-status.d.ts +12 -0
- package/dist/runtime/run-status.js +21 -0
- package/dist/runtime/run-worker.d.ts +10 -2
- package/dist/runtime/run-worker.js +44 -19
- package/dist/runtime/step-log.d.ts +6 -5
- package/dist/runtime/step-log.js +6 -5
- package/dist/runtime/usage-report.d.ts +3 -1
- package/dist/runtime/usage-report.js +4 -2
- package/dist/runtime/waits.d.ts +16 -6
- package/dist/runtime/waits.js +37 -15
- package/dist/schedules/scan.job.d.ts +6 -6
- package/dist/schedules/scan.job.js +6 -6
- package/dist/skills/catalog.d.ts +7 -3
- package/dist/skills/catalog.js +12 -2
- package/dist/tools/approvals.service.d.ts +96 -19
- package/dist/tools/approvals.service.js +300 -67
- package/dist/tools/builtins/sandbox/task-output.js +21 -6
- package/dist/tools/device-op-resolver.d.ts +40 -0
- package/dist/tools/device-op-resolver.js +121 -0
- package/dist/tools/network-asks.d.ts +1 -1
- package/dist/tools/network-asks.js +4 -3
- package/dist/tools/tools.module.js +4 -0
- package/package.json +2 -2
- package/src/migrations/0125_conversation_tree_stopped_at.sql +5 -0
|
@@ -8,11 +8,10 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
|
8
8
|
return function (target, key) { decorator(target, key, paramIndex); }
|
|
9
9
|
};
|
|
10
10
|
import { randomUUID } from "node:crypto";
|
|
11
|
-
import { getQueueToken } from "@nestjs/bullmq";
|
|
12
11
|
import { Inject, Injectable, Optional } from "@nestjs/common";
|
|
13
12
|
import { sql } from "kysely";
|
|
14
13
|
import { visibleTo } from "../agents/agent-lookup.js";
|
|
15
|
-
import {
|
|
14
|
+
import { wakeDecisionIn } from "../background/wake-rules.js";
|
|
16
15
|
import { EventLog } from "../conversations/event-log.js";
|
|
17
16
|
import { dispatchQueuedIn, pauseQueueWith } from "../conversations/queue.js";
|
|
18
17
|
import { conversationTree } from "../conversations/tree.js";
|
|
@@ -25,7 +24,6 @@ import { linkStartedRunIn } from "../schedules/run-tracker.js";
|
|
|
25
24
|
import { scheduledTaskRunAt } from "../schedules/unattended.js";
|
|
26
25
|
import { KyselyCheckpointSaver } from "./checkpointer.js";
|
|
27
26
|
import { ConversationLock } from "./conversation-lock.js";
|
|
28
|
-
import { MAINTENANCE_QUEUE_NAME } from "./maintenance-queue.js";
|
|
29
27
|
import { notifyFinished, RunFinish } from "./run-finish.js";
|
|
30
28
|
import { ACTIVE_RUN_STATUSES, RunJobs, runJobId, runQueueNameFor } from "./run-jobs.js";
|
|
31
29
|
import { rootConversationOf } from "./subagents/child.js";
|
|
@@ -37,12 +35,6 @@ export { ACTIVE_RUN_STATUSES, ENSURE_RUN_BACKSTOP_MS, ENSURE_RUN_JOB, ensureRunJ
|
|
|
37
35
|
* kernel never provides it.
|
|
38
36
|
*/
|
|
39
37
|
export const ENSURE_RUN_FAILS = Symbol("ENSURE_RUN_FAILS");
|
|
40
|
-
/**
|
|
41
|
-
* Margin added to the delayed wake job: its window is read on the database's clock, the delay runs
|
|
42
|
-
* on this process's and Redis's, so the job does not usually fire just before the window ends (one
|
|
43
|
-
* that does reschedules itself under its next retry id, `bgWakeJobId`).
|
|
44
|
-
*/
|
|
45
|
-
const WAKE_JOB_MARGIN_MS = 1000;
|
|
46
38
|
/**
|
|
47
39
|
* 运行队列(生产方):`ensureRun` 在会话有未消费的用户消息、且没有进行中的运行时插入 `runs` 行并投递任务。
|
|
48
40
|
* 「是否已有运行」以 PG 为准(会话行锁内检查 `runs` 中的 queued / running),并参考会话锁的持有者;
|
|
@@ -58,13 +50,10 @@ let RunQueue = class RunQueue {
|
|
|
58
50
|
log;
|
|
59
51
|
txHost;
|
|
60
52
|
config;
|
|
61
|
-
maintenance;
|
|
62
53
|
ensureFails;
|
|
63
54
|
constructor(kdb, eventLog, lock, jobs, finisher,
|
|
64
55
|
/** An orphan aborted directly loses its checkpoint thread (thread_id = runId). */
|
|
65
|
-
checkpoints, log, txHost, config,
|
|
66
|
-
/** The producer side of the delayed `bg-wake` job (`MaintenanceProcessor` consumes it). */
|
|
67
|
-
maintenance, ensureFails = () => false) {
|
|
56
|
+
checkpoints, log, txHost, config, ensureFails = () => false) {
|
|
68
57
|
this.kdb = kdb;
|
|
69
58
|
this.eventLog = eventLog;
|
|
70
59
|
this.lock = lock;
|
|
@@ -74,7 +63,6 @@ let RunQueue = class RunQueue {
|
|
|
74
63
|
this.log = log;
|
|
75
64
|
this.txHost = txHost;
|
|
76
65
|
this.config = config;
|
|
77
|
-
this.maintenance = maintenance;
|
|
78
66
|
this.ensureFails = ensureFails;
|
|
79
67
|
}
|
|
80
68
|
get db() {
|
|
@@ -90,7 +78,7 @@ let RunQueue = class RunQueue {
|
|
|
90
78
|
* once it is sent; with no head either, a wake-eligible finished background task or sub-agent
|
|
91
79
|
* delivery (background Bash spec §3, agent map spec §5.3, {@link wakeDecisionIn}): the run is
|
|
92
80
|
* then attributed to the event's trigger account and joins the tree's budget window, and an
|
|
93
|
-
* event inside the wake window
|
|
81
|
+
* event inside the wake window owes the delayed `bg-wake` job instead. A sub-agent's
|
|
94
82
|
* conversation is only woken this way ({@link childWakeIn}).
|
|
95
83
|
*
|
|
96
84
|
* An active run (queued, running or waiting) is left alone: a run nobody runs any more is the
|
|
@@ -111,12 +99,9 @@ let RunQueue = class RunQueue {
|
|
|
111
99
|
dispatchedSwitch: false,
|
|
112
100
|
dispatchedUnarchived: false,
|
|
113
101
|
pausedNow: false,
|
|
114
|
-
|
|
102
|
+
wakeRetry: opts.wakeRetry ?? 0,
|
|
115
103
|
};
|
|
116
104
|
const created = await this.ensureRunIn(conversationId, st);
|
|
117
|
-
if (st.wakeLater !== null) {
|
|
118
|
-
await this.scheduleWake(conversationId, st.wakeLater, opts.wakeRetry ?? 0);
|
|
119
|
-
}
|
|
120
105
|
const { dispatchedSeq, dispatchedSwitch, dispatchedUnarchived, pausedNow } = st;
|
|
121
106
|
// a pause is published too (batch 5 review I1): the page shows the reason without a reload
|
|
122
107
|
if (pausedNow)
|
|
@@ -144,6 +129,7 @@ let RunQueue = class RunQueue {
|
|
|
144
129
|
"queue_paused",
|
|
145
130
|
"account_id",
|
|
146
131
|
"deleted_at",
|
|
132
|
+
"tree_stopped_at",
|
|
147
133
|
])
|
|
148
134
|
.where("id", "=", conversationId)
|
|
149
135
|
.forUpdate()
|
|
@@ -227,8 +213,11 @@ let RunQueue = class RunQueue {
|
|
|
227
213
|
// Nothing queued either: a finished background task may wake the idle conversation
|
|
228
214
|
// (background Bash spec §3; a person's message or the queue always goes first)
|
|
229
215
|
const wake = await wakeDecisionIn(t, { id: conversationId, orgId: conv.org_id, rootId: conversationId }, Number(conv.consumed_seq), this.config.get().runtime.wakeOnBackgroundTask, (accountId) => authorCanSend(t, conversationId, accountId));
|
|
230
|
-
|
|
231
|
-
|
|
216
|
+
// inside the wake window: the delayed `bg-wake` job is owed here, so the wake survives a
|
|
217
|
+
// process that dies after the commit (transactional outbox spec §3.1)
|
|
218
|
+
if (wake?.kind === "later") {
|
|
219
|
+
await this.jobs.oweBgWake(conversationId, wake, st.wakeRetry, conv.org_id);
|
|
220
|
+
}
|
|
232
221
|
if (wake?.kind !== "wake")
|
|
233
222
|
return null;
|
|
234
223
|
last = { seq: String(wake.seq), accountId: wake.triggerAccountId };
|
|
@@ -284,6 +273,15 @@ let RunQueue = class RunQueue {
|
|
|
284
273
|
// the task run its message started follows this run from now on (spec §6)
|
|
285
274
|
if (taskRunId !== null)
|
|
286
275
|
await linkStartedRunIn(t, taskRunId, runId, conversationId);
|
|
276
|
+
// the root runs again: a person's stop of its tree no longer holds the tree's wakes
|
|
277
|
+
// (agent map spec §6); read with the row lock above, so the hot path issues no statement
|
|
278
|
+
if (conv.tree_stopped_at !== null) {
|
|
279
|
+
await t
|
|
280
|
+
.updateTable("conversations")
|
|
281
|
+
.set({ tree_stopped_at: null })
|
|
282
|
+
.where("id", "=", conversationId)
|
|
283
|
+
.execute();
|
|
284
|
+
}
|
|
287
285
|
return runId;
|
|
288
286
|
}
|
|
289
287
|
/**
|
|
@@ -308,8 +306,9 @@ let RunQueue = class RunQueue {
|
|
|
308
306
|
}
|
|
309
307
|
/**
|
|
310
308
|
* Owes the new run's job in {@link ensureRunIn}'s transaction, before its run row is written.
|
|
311
|
-
* False (logged): the same job id is still owed, so the caller writes no run
|
|
312
|
-
*
|
|
309
|
+
* False (logged): the same job id is still owed, so the caller writes no run. A later run of the
|
|
310
|
+
* same trigger has its own `<runJobId>-<n>` ({@link jobIdFor}), so this is a should-not-happen
|
|
311
|
+
* (the conversation has one active run at a time).
|
|
313
312
|
*/
|
|
314
313
|
async oweJob(conversationId, runId, jobId, kind, orgId) {
|
|
315
314
|
if (await this.jobs.owe({ runId, conversationId }, jobId, kind, orgId))
|
|
@@ -328,10 +327,12 @@ let RunQueue = class RunQueue {
|
|
|
328
327
|
* racing the run's end starts a new run, never lost); and it is woken by its own background
|
|
329
328
|
* task's finish or a delivery from its own sub-agent, under the same rules as a chat
|
|
330
329
|
* conversation. Neither happens once its last run was stopped (`aborted`): a stopped sub-agent is
|
|
331
|
-
* started again only by its parent
|
|
332
|
-
*
|
|
333
|
-
*
|
|
334
|
-
*
|
|
330
|
+
* started again only by its parent; nor while a person's stop of the root holds the tree
|
|
331
|
+
* (`conversations.tree_stopped_at`, cleared by the root's next run). Both read the queue and the
|
|
332
|
+
* trigger account's access on the root of its tree; a message needs no wake setting and has no
|
|
333
|
+
* wake window. The run counts against the root's budget window and runs on the sub-agent queue
|
|
334
|
+
* under `runJobId(child, boundary seq)`. An active run is never started past (one nobody runs is
|
|
335
|
+
* the `run-orphan-check`'s).
|
|
335
336
|
*/
|
|
336
337
|
async childWakeIn(conversationId, conv, st) {
|
|
337
338
|
const t = this.db;
|
|
@@ -353,10 +354,16 @@ let RunQueue = class RunQueue {
|
|
|
353
354
|
if (last?.status === "aborted")
|
|
354
355
|
return null;
|
|
355
356
|
const rootId = await rootConversationOf(t, conversationId);
|
|
357
|
+
// a tree a person stopped at its root (agent map spec §6, changed in acceptance 2026-11-13)
|
|
358
|
+
// starts no sub-agent run here until the root runs again: an idle sub-agent the stop found
|
|
359
|
+
// with nothing running (or one whose report raced the stop) is not woken by a late finish
|
|
360
|
+
if (await treeStopped(t, rootId))
|
|
361
|
+
return null;
|
|
356
362
|
const wake = (await this.pendingMessageIn(conversationId, Number(conv.consumed_seq), rootId)) ??
|
|
357
363
|
(await wakeDecisionIn(t, { id: conversationId, orgId: conv.org_id, rootId }, Number(conv.consumed_seq), this.config.get().runtime.wakeOnBackgroundTask, (accountId) => authorCanSend(t, rootId, accountId)));
|
|
358
|
-
if (wake?.kind === "later")
|
|
359
|
-
st.
|
|
364
|
+
if (wake?.kind === "later") {
|
|
365
|
+
await this.jobs.oweBgWake(conversationId, wake, st.wakeRetry, conv.org_id);
|
|
366
|
+
}
|
|
360
367
|
if (wake?.kind !== "wake")
|
|
361
368
|
return null;
|
|
362
369
|
const jobId = await this.jobIdFor(conversationId, wake.seq);
|
|
@@ -397,8 +404,8 @@ let RunQueue = class RunQueue {
|
|
|
397
404
|
return runId;
|
|
398
405
|
}
|
|
399
406
|
/**
|
|
400
|
-
* The newest unconsumed message of an idle sub-agent (a `SendMessage` its last run did not
|
|
401
|
-
* as a run boundary, or null: none; the root's queue is paused or the root deleted (the
|
|
407
|
+
* The newest unconsumed message of an idle sub-agent (a `SendMessage` its last run did not
|
|
408
|
+
* absorb) as a run boundary, or null: none; the root's queue is paused or the root deleted (the
|
|
402
409
|
* message waits for the unpause, {@link wakeHeldTree}); or its author can no longer send to the
|
|
403
410
|
* root (it waits for a later run). The caller already refused a stopped sub-agent (last run
|
|
404
411
|
* `aborted`): a message that reached it while running and then stopped is parked, and the next
|
|
@@ -496,27 +503,6 @@ let RunQueue = class RunQueue {
|
|
|
496
503
|
.where("r.status", "in", ACTIVE_RUN_STATUSES))))
|
|
497
504
|
.execute();
|
|
498
505
|
}
|
|
499
|
-
/**
|
|
500
|
-
* Schedules the delayed `bg-wake` job at the end of the conversation's wake window (after the
|
|
501
|
-
* commit): one job per conversation and window end, so finishes inside one window share it. It
|
|
502
|
-
* calls {@link ensureRun}, which wakes the conversation if a finish is still unconsumed. A failed
|
|
503
|
-
* enqueue is logged: the next message, run end or page opening re-derives the wake.
|
|
504
|
-
*/
|
|
505
|
-
async scheduleWake(conversationId, w, retry) {
|
|
506
|
-
const data = { conversationId, ...(retry > 0 && { retry }) };
|
|
507
|
-
try {
|
|
508
|
-
await this.maintenance.add(BG_WAKE_JOB, data, {
|
|
509
|
-
// a job that fired early reschedules under a new id (its own is still taken)
|
|
510
|
-
jobId: bgWakeJobId(conversationId, w.windowEnd, retry),
|
|
511
|
-
delay: w.delayMs + WAKE_JOB_MARGIN_MS,
|
|
512
|
-
removeOnComplete: true,
|
|
513
|
-
removeOnFail: true,
|
|
514
|
-
});
|
|
515
|
-
}
|
|
516
|
-
catch (err) {
|
|
517
|
-
this.log.error("cannot schedule the background task wake", { conversationId, err });
|
|
518
|
-
}
|
|
519
|
-
}
|
|
520
506
|
/**
|
|
521
507
|
* Owes the ensure-run backstop of `conversationId` in the caller's transaction
|
|
522
508
|
* ({@link oweEnsureRun}): `seq` is the event the transaction wrote, `orgId` the conversation's.
|
|
@@ -527,8 +513,8 @@ let RunQueue = class RunQueue {
|
|
|
527
513
|
/**
|
|
528
514
|
* Stops an orphaned run directly (`POST .../abort`, a tree stop): a `running` run whose
|
|
529
515
|
* conversation lock is free and whose BullMQ job is gone hears no stop broadcast, so its
|
|
530
|
-
* `run_aborted` is written here (status aborted, `consumed_seq` at least to its trigger): true.
|
|
531
|
-
* an orphan (someone runs it; a queued run's job is owed, or lost and re-added by the
|
|
516
|
+
* `run_aborted` is written here (status aborted, `consumed_seq` at least to its trigger): true.
|
|
517
|
+
* Not an orphan (someone runs it; a queued run's job is owed, or lost and re-added by the
|
|
532
518
|
* `run-orphan-check`, whose worker then sees the stop marker) or already finished: false, and the
|
|
533
519
|
* caller broadcasts the stop.
|
|
534
520
|
*/
|
|
@@ -569,14 +555,15 @@ let RunQueue = class RunQueue {
|
|
|
569
555
|
}
|
|
570
556
|
/**
|
|
571
557
|
* Ends a `queued` run in PG before any worker started it (an offboarded member's run, ruling
|
|
572
|
-
* R27): with the conversation lock free, under the conversation row lock, a run still `queued`
|
|
573
|
-
*
|
|
574
|
-
*
|
|
575
|
-
*
|
|
558
|
+
* R27): with the conversation lock free, under the conversation row lock, a run still `queued` is
|
|
559
|
+
* finished `aborted` with `run_aborted` (`consumed_seq` at least to its trigger), so a job that
|
|
560
|
+
* runs later finds it finished and starts nothing. True when it did; false when a worker holds
|
|
561
|
+
* the conversation or is in the middle of the run's first start (`run_started` written,
|
|
576
562
|
* `started_at` not yet set; it hears the stop), or the run is no longer queued. A run resumed
|
|
577
|
-
* from `waiting` to `queued` is ended like a new one. The caller writes the stop marker first, so
|
|
578
|
-
* conversation meanwhile still stops; a worker that read the run before
|
|
579
|
-
* under the row locks before its `run_started` and never moves it back
|
|
563
|
+
* from `waiting` to `queued` is ended like a new one. The caller writes the stop marker first, so
|
|
564
|
+
* a worker that takes the conversation meanwhile still stops; a worker that read the run before
|
|
565
|
+
* this commit re-checks it under the row locks before its `run_started` and never moves it back
|
|
566
|
+
* to `running`.
|
|
580
567
|
*/
|
|
581
568
|
async abortQueued(conversationId, runId, byAccountId) {
|
|
582
569
|
if ((await this.lock.holder(conversationId)) !== null)
|
|
@@ -678,9 +665,8 @@ RunQueue = __decorate([
|
|
|
678
665
|
__param(6, Inject(KERNEL_LOGGER)),
|
|
679
666
|
__param(7, Inject(TransactionHost)),
|
|
680
667
|
__param(8, Inject(KernelConfigService)),
|
|
681
|
-
__param(9,
|
|
682
|
-
__param(
|
|
683
|
-
__param(10, Inject(ENSURE_RUN_FAILS))
|
|
668
|
+
__param(9, Optional()),
|
|
669
|
+
__param(9, Inject(ENSURE_RUN_FAILS))
|
|
684
670
|
], RunQueue);
|
|
685
671
|
export { RunQueue };
|
|
686
672
|
/** The conversation's owner is active and can still see its agent (`isAccessible`'s rule). */
|
|
@@ -705,3 +691,15 @@ async function authorCanSend(t, conversationId, accountId) {
|
|
|
705
691
|
.executeTakeFirst();
|
|
706
692
|
return row !== undefined;
|
|
707
693
|
}
|
|
694
|
+
/**
|
|
695
|
+
* Whether a person's stop of the root conversation holds its tree (`tree_stopped_at`, agent map
|
|
696
|
+
* spec §6): set by the stop, cleared when the root next starts a run.
|
|
697
|
+
*/
|
|
698
|
+
async function treeStopped(t, rootId) {
|
|
699
|
+
const root = await t
|
|
700
|
+
.selectFrom("conversations")
|
|
701
|
+
.select("tree_stopped_at")
|
|
702
|
+
.where("id", "=", rootId)
|
|
703
|
+
.executeTakeFirst();
|
|
704
|
+
return root?.tree_stopped_at != null;
|
|
705
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { KernelSchemaDb } from "../db/kernel-schema.js";
|
|
2
|
+
/** 运行的终态。 */
|
|
3
|
+
export type TerminalStatus = "completed" | "failed" | "aborted";
|
|
4
|
+
/** The terminal run statuses (a leaf module: the event writers of a run start import it too). */
|
|
5
|
+
export declare const TERMINAL_STATUSES: ReadonlySet<string>;
|
|
6
|
+
/**
|
|
7
|
+
* Whether run `runId` has ended, read in the caller's transaction with its row locked: call it
|
|
8
|
+
* after the conversation row lock (lock order conversation, then run, as `RunFinish.finishIn`), so
|
|
9
|
+
* no stop can end the run between this read and the transaction's appends. A missing run is not
|
|
10
|
+
* ended (a caller may record for a run id without a row).
|
|
11
|
+
*/
|
|
12
|
+
export declare function runEndedIn(db: KernelSchemaDb, runId: string): Promise<boolean>;
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/** The terminal run statuses (a leaf module: the event writers of a run start import it too). */
|
|
2
|
+
export const TERMINAL_STATUSES = new Set([
|
|
3
|
+
"completed",
|
|
4
|
+
"failed",
|
|
5
|
+
"aborted",
|
|
6
|
+
]);
|
|
7
|
+
/**
|
|
8
|
+
* Whether run `runId` has ended, read in the caller's transaction with its row locked: call it
|
|
9
|
+
* after the conversation row lock (lock order conversation, then run, as `RunFinish.finishIn`), so
|
|
10
|
+
* no stop can end the run between this read and the transaction's appends. A missing run is not
|
|
11
|
+
* ended (a caller may record for a run id without a row).
|
|
12
|
+
*/
|
|
13
|
+
export async function runEndedIn(db, runId) {
|
|
14
|
+
const row = await db
|
|
15
|
+
.selectFrom("runs")
|
|
16
|
+
.select("status")
|
|
17
|
+
.where("id", "=", runId)
|
|
18
|
+
.forUpdate()
|
|
19
|
+
.executeTakeFirst();
|
|
20
|
+
return row !== undefined && TERMINAL_STATUSES.has(row.status);
|
|
21
|
+
}
|
|
@@ -139,8 +139,10 @@ export declare class RunWorker {
|
|
|
139
139
|
/**
|
|
140
140
|
* After any terminal commit of a run that is not a sub-agent's (sub-agents spec §12): stops its
|
|
141
141
|
* active fork invocation child on its behalf (no account); background `Agent` children keep
|
|
142
|
-
* running (agent map spec §5.4)
|
|
143
|
-
*
|
|
142
|
+
* running (agent map spec §5.4), unless a person stopped the run: then the sub-agents of its
|
|
143
|
+
* windows stop too, also one it dispatched after the stop read the tree (agent map spec §6,
|
|
144
|
+
* changed in acceptance 2026-11-13). The terminal transaction owed the same stops
|
|
145
|
+
* (`stop-tree`), the backstop when this call never happens.
|
|
144
146
|
*/
|
|
145
147
|
private stopChildren;
|
|
146
148
|
/**
|
|
@@ -188,6 +190,12 @@ export declare class RunWorker {
|
|
|
188
190
|
* handles it). Resolved for the conversation's org (its effective provider config).
|
|
189
191
|
*/
|
|
190
192
|
private fallBackIfUnavailable;
|
|
193
|
+
/**
|
|
194
|
+
* Whether the run is gone or terminal, read in the caller's transaction under the locks
|
|
195
|
+
* (conversation, then run, as `RunFinish.finishIn`): the job read its status before taking the
|
|
196
|
+
* conversation lock, and a stop may have ended it since.
|
|
197
|
+
*/
|
|
198
|
+
private endedUnderLocks;
|
|
191
199
|
/**
|
|
192
200
|
* `run_started` under the conversation row lock, with the model / effort switch in effect then
|
|
193
201
|
* (a chat run's `saved` version under the conversation's selection): what the run uses.
|
|
@@ -209,8 +209,10 @@ let RunWorker = class RunWorker {
|
|
|
209
209
|
/**
|
|
210
210
|
* After any terminal commit of a run that is not a sub-agent's (sub-agents spec §12): stops its
|
|
211
211
|
* active fork invocation child on its behalf (no account); background `Agent` children keep
|
|
212
|
-
* running (agent map spec §5.4)
|
|
213
|
-
*
|
|
212
|
+
* running (agent map spec §5.4), unless a person stopped the run: then the sub-agents of its
|
|
213
|
+
* windows stop too, also one it dispatched after the stop read the tree (agent map spec §6,
|
|
214
|
+
* changed in acceptance 2026-11-13). The terminal transaction owed the same stops
|
|
215
|
+
* (`stop-tree`), the backstop when this call never happens.
|
|
214
216
|
*/
|
|
215
217
|
async stopChildren(runId) {
|
|
216
218
|
try {
|
|
@@ -219,6 +221,12 @@ let RunWorker = class RunWorker {
|
|
|
219
221
|
catch (err) {
|
|
220
222
|
this.log.error("cannot stop the sub-agents of a finished run", { runId, err });
|
|
221
223
|
}
|
|
224
|
+
try {
|
|
225
|
+
await this.waits.stopDescendantsOfStopped(runId);
|
|
226
|
+
}
|
|
227
|
+
catch (err) {
|
|
228
|
+
this.log.error("cannot stop the sub-agents of a stopped run", { runId, err });
|
|
229
|
+
}
|
|
222
230
|
}
|
|
223
231
|
/**
|
|
224
232
|
* After a sub-agent run's terminal state committed (sub-agents spec §7.3): reports it to the
|
|
@@ -624,17 +632,23 @@ let RunWorker = class RunWorker {
|
|
|
624
632
|
if (toSeq !== undefined)
|
|
625
633
|
await this.eventLog.notify(conversationId, toSeq);
|
|
626
634
|
}
|
|
627
|
-
//
|
|
628
|
-
|
|
635
|
+
// Skills catalog snapshot (ruling M17): written before the run is running, and again on a
|
|
636
|
+
// crash retry (deduped by its idem key). Both records re-read the run under the conversation
|
|
637
|
+
// and run row locks in their append transactions: a stop that ended it meanwhile (a retried
|
|
638
|
+
// start reads `queued` before any lock) wins, nothing is appended after its end, and the run
|
|
639
|
+
// never starts.
|
|
640
|
+
const catalogged = await this.skillsCatalog.record({
|
|
629
641
|
orgId: conv.org_id,
|
|
630
642
|
agentId: conv.agent_id,
|
|
631
643
|
conversationId,
|
|
632
644
|
runId,
|
|
633
645
|
agent,
|
|
634
646
|
}, { log: this.log });
|
|
647
|
+
if (!catalogged)
|
|
648
|
+
return "finished";
|
|
635
649
|
// run context (spec §5): after the catalog, before the run is running, so before any
|
|
636
650
|
// compaction and model call; a retried run start finds its event or no change
|
|
637
|
-
await this.runContext.record({
|
|
651
|
+
const contexted = await this.runContext.record({
|
|
638
652
|
orgId: conv.org_id,
|
|
639
653
|
conversationId,
|
|
640
654
|
runId,
|
|
@@ -648,9 +662,12 @@ let RunWorker = class RunWorker {
|
|
|
648
662
|
},
|
|
649
663
|
}),
|
|
650
664
|
});
|
|
665
|
+
if (!contexted)
|
|
666
|
+
return "finished";
|
|
651
667
|
// plugin context (memory plugin spec §6.1): after the run context, still before the run is
|
|
652
|
-
// running; the providers are asked outside any transaction, a chat run only
|
|
653
|
-
|
|
668
|
+
// running; the providers are asked outside any transaction, a chat run only, so a stop may
|
|
669
|
+
// end the run meanwhile: its append transaction re-reads the run under the locks
|
|
670
|
+
const pluginContexted = await this.pluginContext.record({
|
|
654
671
|
orgId: conv.org_id,
|
|
655
672
|
conversationId,
|
|
656
673
|
runId,
|
|
@@ -667,6 +684,8 @@ let RunWorker = class RunWorker {
|
|
|
667
684
|
// the run's own abort watch starts with its execution below: until then, the shutdown
|
|
668
685
|
signal: state.shutdown.signal,
|
|
669
686
|
});
|
|
687
|
+
if (!pluginContexted)
|
|
688
|
+
return "finished";
|
|
670
689
|
await this.hooks?.beforeRunning?.(runId, conversationId);
|
|
671
690
|
// never over a terminal status: a run finished meanwhile stays finished
|
|
672
691
|
const updated = await this.db
|
|
@@ -905,28 +924,34 @@ let RunWorker = class RunWorker {
|
|
|
905
924
|
await this.eventLog.publishChanged(conversationId, "selection");
|
|
906
925
|
}
|
|
907
926
|
/**
|
|
908
|
-
*
|
|
909
|
-
* (
|
|
927
|
+
* Whether the run is gone or terminal, read in the caller's transaction under the locks
|
|
928
|
+
* (conversation, then run, as `RunFinish.finishIn`): the job read its status before taking the
|
|
929
|
+
* conversation lock, and a stop may have ended it since.
|
|
910
930
|
*/
|
|
911
|
-
async
|
|
912
|
-
|
|
913
|
-
current, saved) {
|
|
914
|
-
const t = this.db;
|
|
915
|
-
await t
|
|
931
|
+
async endedUnderLocks(conversationId, runId) {
|
|
932
|
+
await this.db
|
|
916
933
|
.selectFrom("conversations")
|
|
917
934
|
.select("id")
|
|
918
935
|
.where("id", "=", conversationId)
|
|
919
936
|
.forUpdate()
|
|
920
937
|
.execute();
|
|
921
|
-
|
|
922
|
-
// the job read it before taking the conversation lock, and a stop may have ended it since
|
|
923
|
-
const row = await t
|
|
938
|
+
const row = await this.db
|
|
924
939
|
.selectFrom("runs")
|
|
925
940
|
.select("status")
|
|
926
|
-
.where("id", "=",
|
|
941
|
+
.where("id", "=", runId)
|
|
927
942
|
.forUpdate()
|
|
928
943
|
.executeTakeFirst();
|
|
929
|
-
|
|
944
|
+
return !row || TERMINAL_STATUSES.has(row.status);
|
|
945
|
+
}
|
|
946
|
+
/**
|
|
947
|
+
* `run_started` under the conversation row lock, with the model / effort switch in effect then
|
|
948
|
+
* (a chat run's `saved` version under the conversation's selection): what the run uses.
|
|
949
|
+
*/
|
|
950
|
+
async appendRunStarted(conversationId, run,
|
|
951
|
+
/** The definition so far (a sub-agent's frozen one is final). */
|
|
952
|
+
current, saved) {
|
|
953
|
+
const t = this.db;
|
|
954
|
+
if (await this.endedUnderLocks(conversationId, run.id)) {
|
|
930
955
|
return { toSeq: undefined, agent: current, terminal: true };
|
|
931
956
|
}
|
|
932
957
|
const agent = saved !== undefined
|
|
@@ -9,10 +9,10 @@ import { UsageReportJobs } from "./usage-report-jobs.js";
|
|
|
9
9
|
export type StepAuditEntry = Parameters<typeof audit>[1];
|
|
10
10
|
/**
|
|
11
11
|
* The writes of a run step that go to the event log together with another row, each in a
|
|
12
|
-
* transaction of its own (the caller notifies the log after the commit): a model call with the
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
12
|
+
* transaction of its own (the caller notifies the log after the commit): a model call with the tool
|
|
13
|
+
* definitions it sent, a denial with its audit row, usage events with their owed `usage.report`
|
|
14
|
+
* deliveries (transactional outbox spec §8.1), and a sub-agent's usage counted on its parent run.
|
|
15
|
+
* The step functions (`model-call.ts`, `hook-calls.ts`) reach it through their deps.
|
|
16
16
|
*/
|
|
17
17
|
export declare class StepLog {
|
|
18
18
|
private readonly kdb;
|
|
@@ -35,7 +35,8 @@ export declare class StepLog {
|
|
|
35
35
|
* the whole tree counts on one run): `counter.budgetRunId`, the run's own window
|
|
36
36
|
* (`SubagentParent.budgetRunId`), else the parent run's ({@link budgetRunOf}). A usage event
|
|
37
37
|
* already in the log (an idempotent hit after a crash) adds nothing, so each call counts exactly
|
|
38
|
-
* once; the new usage events' `usage.report` deliveries are owed in the same transaction. Returns
|
|
38
|
+
* once; the new usage events' `usage.report` deliveries are owed in the same transaction. Returns
|
|
39
|
+
* what was written (hits included) and the highest new seq (null: nothing new).
|
|
39
40
|
*/
|
|
40
41
|
appendCounted(conversationId: string, counter: {
|
|
41
42
|
parentRunId: string;
|
package/dist/runtime/step-log.js
CHANGED
|
@@ -20,10 +20,10 @@ import { budgetRunOf } from "./subagents/limits.js";
|
|
|
20
20
|
import { UsageReportJobs, UsageReportJobsModule } from "./usage-report-jobs.js";
|
|
21
21
|
/**
|
|
22
22
|
* The writes of a run step that go to the event log together with another row, each in a
|
|
23
|
-
* transaction of its own (the caller notifies the log after the commit): a model call with the
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
23
|
+
* transaction of its own (the caller notifies the log after the commit): a model call with the tool
|
|
24
|
+
* definitions it sent, a denial with its audit row, usage events with their owed `usage.report`
|
|
25
|
+
* deliveries (transactional outbox spec §8.1), and a sub-agent's usage counted on its parent run.
|
|
26
|
+
* The step functions (`model-call.ts`, `hook-calls.ts`) reach it through their deps.
|
|
27
27
|
*/
|
|
28
28
|
let StepLog = class StepLog {
|
|
29
29
|
kdb;
|
|
@@ -53,7 +53,8 @@ let StepLog = class StepLog {
|
|
|
53
53
|
* the whole tree counts on one run): `counter.budgetRunId`, the run's own window
|
|
54
54
|
* (`SubagentParent.budgetRunId`), else the parent run's ({@link budgetRunOf}). A usage event
|
|
55
55
|
* already in the log (an idempotent hit after a crash) adds nothing, so each call counts exactly
|
|
56
|
-
* once; the new usage events' `usage.report` deliveries are owed in the same transaction. Returns
|
|
56
|
+
* once; the new usage events' `usage.report` deliveries are owed in the same transaction. Returns
|
|
57
|
+
* what was written (hits included) and the highest new seq (null: nothing new).
|
|
57
58
|
*/
|
|
58
59
|
async appendCounted(conversationId, counter, events) {
|
|
59
60
|
const t = this.kdb.db;
|
|
@@ -58,7 +58,9 @@ export declare function narrowPending(stored: readonly string[] | undefined, fai
|
|
|
58
58
|
* `model_usage` row and owes job `usage-model-<rowId>` with it; the payload
|
|
59
59
|
* (`purpose: "embedding"`, `eventId` = the row id, the calling `extension`, no conversation fields)
|
|
60
60
|
* is read from the row when the job runs, and the row's `reported_at` is set once nothing is left
|
|
61
|
-
* to deliver; a job that gives up after its last attempt sets `report_failed_at`.
|
|
61
|
+
* to deliver; a job that gives up after its last attempt sets `report_failed_at`. A row with
|
|
62
|
+
* `reported_at IS NULL` is not necessarily pending: no job is owed when no enabled extension
|
|
63
|
+
* subscribed to `usage.report` at the time of the call, and such a row is never reported.
|
|
62
64
|
*/
|
|
63
65
|
export declare class UsageReporter {
|
|
64
66
|
private readonly kdb;
|
|
@@ -113,7 +113,9 @@ export function narrowPending(stored, failed) {
|
|
|
113
113
|
* `model_usage` row and owes job `usage-model-<rowId>` with it; the payload
|
|
114
114
|
* (`purpose: "embedding"`, `eventId` = the row id, the calling `extension`, no conversation fields)
|
|
115
115
|
* is read from the row when the job runs, and the row's `reported_at` is set once nothing is left
|
|
116
|
-
* to deliver; a job that gives up after its last attempt sets `report_failed_at`.
|
|
116
|
+
* to deliver; a job that gives up after its last attempt sets `report_failed_at`. A row with
|
|
117
|
+
* `reported_at IS NULL` is not necessarily pending: no job is owed when no enabled extension
|
|
118
|
+
* subscribed to `usage.report` at the time of the call, and such a row is never reported.
|
|
117
119
|
*/
|
|
118
120
|
let UsageReporter = class UsageReporter {
|
|
119
121
|
kdb;
|
|
@@ -359,7 +361,7 @@ let UsageReporter = class UsageReporter {
|
|
|
359
361
|
/** One delivery job (the `@Processor` of {@link USAGE_REPORT_QUEUE} calls it). */
|
|
360
362
|
async process(j, token) {
|
|
361
363
|
const payload = await this.payloadOfJob(j.data);
|
|
362
|
-
// conversation
|
|
364
|
+
// conversation deleted meanwhile: nothing to report
|
|
363
365
|
if (payload === null)
|
|
364
366
|
return;
|
|
365
367
|
const subscribers = await this.subscribers(payload.orgId);
|
package/dist/runtime/waits.d.ts
CHANGED
|
@@ -230,6 +230,13 @@ export declare class Waits implements OnApplicationShutdown {
|
|
|
230
230
|
* background tasks of every covered sub-agent conversation stop, whether it was running or idle.
|
|
231
231
|
* Returns the number of runs asked to stop.
|
|
232
232
|
*
|
|
233
|
+
* `windowsUpTo` (a root run's id; only after that root run a person stopped ended,
|
|
234
|
+
* {@link stopDescendantsOfStopped}) narrows the stop to the runs of the budget windows opened up
|
|
235
|
+
* to that run (`runs.budget_run_id`, agent map spec §6): a run of a window a later root run
|
|
236
|
+
* opened (the person's next message, started meanwhile) is never stopped by the late call, and
|
|
237
|
+
* the background task sweep is left out (a stopped child's own end stops its tasks,
|
|
238
|
+
* {@link childFinished}).
|
|
239
|
+
*
|
|
233
240
|
* Lock order: nothing is held across runs. Each run's stop is its own transaction (or none), and
|
|
234
241
|
* one that writes the terminal state locks that run's conversation row, then its run row, the
|
|
235
242
|
* order of `StepLog.appendCounted` (conversation row, then the root's budget run row) and of
|
|
@@ -238,14 +245,17 @@ export declare class Waits implements OnApplicationShutdown {
|
|
|
238
245
|
*/
|
|
239
246
|
stopTree(conversationId: string, byAccountId: string | null, opts?: {
|
|
240
247
|
includeSelf?: boolean;
|
|
248
|
+
windowsUpTo?: string;
|
|
241
249
|
}): Promise<number>;
|
|
242
250
|
/**
|
|
243
|
-
* After a
|
|
244
|
-
*
|
|
245
|
-
*
|
|
246
|
-
*
|
|
247
|
-
*
|
|
248
|
-
*
|
|
251
|
+
* After a run's terminal commit (its worker): a run a person stopped (`run_aborted` with an
|
|
252
|
+
* account and no `detail`; {@link stopTree} or a person's stop of the root) stops the active runs
|
|
253
|
+
* below its conversation as well, so a descendant it dispatched after the tree stop read the tree
|
|
254
|
+
* still stops (dispatch needs its run `running`, so after its terminal commit the set is final).
|
|
255
|
+
* A root's run stops only the runs of the budget windows up to it (`windowsUpTo`): the person's
|
|
256
|
+
* next message may already have started a new run with sub-agents of its own (agent map spec §6,
|
|
257
|
+
* changed in acceptance 2026-11-13). Any other end leaves its children running (§5.4). Returns
|
|
258
|
+
* the number of runs asked to stop.
|
|
249
259
|
*/
|
|
250
260
|
stopDescendantsOfStopped(runId: string): Promise<number>;
|
|
251
261
|
/**
|