@capybara-hq/kernel 0.1.0-rc.25 → 0.1.0-rc.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/admin/outbox-dead.service.d.ts +7 -3
- package/dist/admin/outbox-dead.service.js +17 -6
- package/dist/admin/overview.service.d.ts +6 -6
- package/dist/admin/overview.service.js +7 -7
- package/dist/background/wake-rules.d.ts +5 -0
- package/dist/background/wake-rules.js +16 -5
- package/dist/catalog/catalog-content.d.ts +6 -1
- package/dist/catalog/catalog-content.js +2 -1
- package/dist/catalog/marketplaces.service.d.ts +1 -1
- package/dist/catalog/org-catalog.service.js +5 -2
- package/dist/catalog/plugin-install.d.ts +5 -0
- package/dist/catalog/plugin-install.js +37 -1
- package/dist/channels/channels.module.d.ts +9 -8
- package/dist/channels/channels.module.js +11 -10
- package/dist/channels/connections.service.d.ts +3 -3
- package/dist/channels/connections.service.js +6 -6
- package/dist/channels/deliver-job.d.ts +10 -3
- package/dist/channels/deliver-job.js +7 -3
- package/dist/channels/jobs.d.ts +11 -4
- package/dist/channels/jobs.js +18 -3
- package/dist/channels/outbound.d.ts +5 -2
- package/dist/channels/outbound.js +9 -2
- package/dist/conversations/conversations.controller.d.ts +4 -2
- package/dist/conversations/conversations.controller.js +4 -2
- package/dist/conversations/conversations.service.d.ts +37 -4
- package/dist/conversations/conversations.service.js +77 -20
- package/dist/conversations/events.d.ts +6 -0
- package/dist/conversations/events.js +6 -0
- package/dist/db/kernel-schema.d.ts +10 -1
- package/dist/devices/devices.service.d.ts +14 -6
- package/dist/devices/devices.service.js +20 -7
- package/dist/evals/evals.module.js +5 -1
- package/dist/evals/jobs.d.ts +14 -6
- package/dist/evals/jobs.js +21 -19
- package/dist/evals/runs.service.d.ts +5 -6
- package/dist/evals/runs.service.js +11 -33
- package/dist/infra/outbox/outbox.service.d.ts +6 -0
- package/dist/infra/outbox/outbox.service.js +51 -17
- package/dist/infra/outbox/registry.d.ts +52 -0
- package/dist/infra/outbox/registry.js +57 -0
- package/dist/local-devices/device-policy.service.d.ts +131 -26
- package/dist/local-devices/device-policy.service.js +341 -145
- package/dist/local-devices/machines.d.ts +17 -2
- package/dist/local-devices/machines.js +66 -5
- package/dist/local-devices/policy-op-jobs.d.ts +15 -0
- package/dist/local-devices/policy-op-jobs.js +12 -0
- package/dist/local-devices/policy-summary.d.ts +36 -3
- package/dist/local-devices/policy-summary.js +47 -1
- package/dist/orgs/orgs.service.js +4 -1
- package/dist/overrides.d.ts +4 -0
- package/dist/overrides.js +2 -0
- package/dist/platform/ops/conversations.ops.js +1 -1
- package/dist/runtime/maintenance.d.ts +6 -3
- package/dist/runtime/maintenance.js +21 -7
- package/dist/runtime/plugin-context.service.d.ts +9 -3
- package/dist/runtime/plugin-context.service.js +17 -5
- package/dist/runtime/run-context.service.d.ts +7 -3
- package/dist/runtime/run-context.service.js +12 -2
- package/dist/runtime/run-finish.d.ts +2 -2
- package/dist/runtime/run-finish.js +11 -6
- package/dist/runtime/run-jobs.d.ts +21 -4
- package/dist/runtime/run-jobs.js +29 -6
- package/dist/runtime/run-orphan-check.job.d.ts +14 -11
- package/dist/runtime/run-orphan-check.job.js +16 -11
- package/dist/runtime/run-queue.d.ts +23 -31
- package/dist/runtime/run-queue.js +62 -64
- package/dist/runtime/run-status.d.ts +12 -0
- package/dist/runtime/run-status.js +21 -0
- package/dist/runtime/run-worker.d.ts +10 -2
- package/dist/runtime/run-worker.js +44 -19
- package/dist/runtime/step-log.d.ts +6 -5
- package/dist/runtime/step-log.js +6 -5
- package/dist/runtime/usage-report.d.ts +3 -1
- package/dist/runtime/usage-report.js +4 -2
- package/dist/runtime/waits.d.ts +16 -6
- package/dist/runtime/waits.js +37 -15
- package/dist/schedules/scan.job.d.ts +6 -6
- package/dist/schedules/scan.job.js +6 -6
- package/dist/skills/catalog.d.ts +7 -3
- package/dist/skills/catalog.js +12 -2
- package/dist/tools/approvals.service.d.ts +96 -19
- package/dist/tools/approvals.service.js +300 -67
- package/dist/tools/builtins/sandbox/task-output.js +21 -6
- package/dist/tools/connectors.service.js +0 -4
- package/dist/tools/device-op-resolver.d.ts +40 -0
- package/dist/tools/device-op-resolver.js +121 -0
- package/dist/tools/network-asks.d.ts +1 -1
- package/dist/tools/network-asks.js +4 -3
- package/dist/tools/tools.module.js +4 -0
- package/package.json +2 -2
- package/src/migrations/0125_conversation_tree_stopped_at.sql +5 -0
|
@@ -58,8 +58,8 @@ export declare class RunFinish {
|
|
|
58
58
|
* invocation gets its report (`pairStoppedForks`; an `Agent` call has its result from its
|
|
59
59
|
* dispatch), so no terminal path leaves an open call. After it, the stops and the delivery the
|
|
60
60
|
* run's end causes are owed here (runtime/subagents/tree-jobs.ts), the caller still doing them
|
|
61
|
-
* right after its commit: `stop-tree` when an active fork child remains or a
|
|
62
|
-
*
|
|
61
|
+
* right after its commit: `stop-tree` when an active fork child remains or a run a person
|
|
62
|
+
* stopped (a root's or a sub-agent's) has sub-agents below it, `deliver-to-parent` for a sub-agent's run. The
|
|
63
63
|
* delivery is a job, not written here: this transaction holds the child's conversation and run
|
|
64
64
|
* rows, and the parent's conversation row comes first in the dispatch's lock order (parent
|
|
65
65
|
* conversation, parent run, budget run, child conversation).
|
|
@@ -63,8 +63,8 @@ let RunFinish = class RunFinish {
|
|
|
63
63
|
* invocation gets its report (`pairStoppedForks`; an `Agent` call has its result from its
|
|
64
64
|
* dispatch), so no terminal path leaves an open call. After it, the stops and the delivery the
|
|
65
65
|
* run's end causes are owed here (runtime/subagents/tree-jobs.ts), the caller still doing them
|
|
66
|
-
* right after its commit: `stop-tree` when an active fork child remains or a
|
|
67
|
-
*
|
|
66
|
+
* right after its commit: `stop-tree` when an active fork child remains or a run a person
|
|
67
|
+
* stopped (a root's or a sub-agent's) has sub-agents below it, `deliver-to-parent` for a sub-agent's run. The
|
|
68
68
|
* delivery is a job, not written here: this transaction holds the child's conversation and run
|
|
69
69
|
* rows, and the parent's conversation row comes first in the dispatch's lock order (parent
|
|
70
70
|
* conversation, parent run, budget run, child conversation).
|
|
@@ -102,9 +102,14 @@ let RunFinish = class RunFinish {
|
|
|
102
102
|
const [ev] = await this.eventLog.appendIn(a.conversationId, [
|
|
103
103
|
{ ...a.event, runId: a.runId, idemKey: `${a.event.type}:${a.runId}` },
|
|
104
104
|
]);
|
|
105
|
+
// a root run a person stopped holds the root's tree (agent map spec §6, acceptance
|
|
106
|
+
// 2026-11-13): marked with its terminal state, under the same row lock, so no wake decided
|
|
107
|
+
// after this commit starts the root from a sub-agent's delivery
|
|
108
|
+
const holdTree = conv?.kind === "chat" && personStopped(a.event);
|
|
105
109
|
await t
|
|
106
110
|
.updateTable("conversations")
|
|
107
111
|
.set({
|
|
112
|
+
...(holdTree && { tree_stopped_at: sql `now()` }),
|
|
108
113
|
consumed_seq: a.floorToTrigger === false
|
|
109
114
|
? sql `GREATEST(consumed_seq, ${a.consumed ?? 0}::bigint)`
|
|
110
115
|
: sql `GREATEST(consumed_seq, ${a.consumed ?? 0}::bigint, (SELECT r.trigger_seq FROM capybara.runs r WHERE r.id = ${a.runId}))`,
|
|
@@ -145,8 +150,8 @@ let RunFinish = class RunFinish {
|
|
|
145
150
|
.where("cr.status", "in", ACTIVE_RUN_STATUSES)
|
|
146
151
|
.limit(1)
|
|
147
152
|
.executeTakeFirst();
|
|
148
|
-
|
|
149
|
-
|
|
153
|
+
// a root's too (agent map spec §6, changed in acceptance 2026-11-13)
|
|
154
|
+
const stoppedWithChildren = personStopped(a.event) &&
|
|
150
155
|
(await t
|
|
151
156
|
.selectFrom("conversations")
|
|
152
157
|
.select("id")
|
|
@@ -186,8 +191,8 @@ RunFinish = __decorate([
|
|
|
186
191
|
], RunFinish);
|
|
187
192
|
export { RunFinish };
|
|
188
193
|
/**
|
|
189
|
-
* A `run_aborted` a person caused (an account and no `detail`): a
|
|
190
|
-
*
|
|
194
|
+
* A `run_aborted` a person caused (an account and no `detail`): a run stopped that way stops its
|
|
195
|
+
* descendants (`Waits.stopDescendantsOfStopped`, agent map spec §6; a root's, those of its windows).
|
|
191
196
|
*/
|
|
192
197
|
function personStopped(event) {
|
|
193
198
|
if (event.type !== "run_aborted")
|
|
@@ -15,10 +15,7 @@ export declare const SUBAGENT_RUN_QUEUE_NAME = "cap-kernel-subagent-runs";
|
|
|
15
15
|
export declare const runQueueNameFor: (kind: ConversationKind) => typeof RUN_QUEUE_NAME | typeof SUBAGENT_RUN_QUEUE_NAME;
|
|
16
16
|
/** 未结束的运行状态(waiting:等人处理审批 / 输入请求,没有任务、不占会话锁与配额)。 */
|
|
17
17
|
export declare const ACTIVE_RUN_STATUSES: readonly ["queued", "running", "waiting"];
|
|
18
|
-
|
|
19
|
-
export type TerminalStatus = "completed" | "failed" | "aborted";
|
|
20
|
-
/** The terminal run statuses (here, not in `run-finish.ts`, which the sub-agent modules import). */
|
|
21
|
-
export declare const TERMINAL_STATUSES: ReadonlySet<string>;
|
|
18
|
+
export { TERMINAL_STATUSES, type TerminalStatus } from "./run-status.js";
|
|
22
19
|
/**
|
|
23
20
|
* Locks a run's approval and input request rows (`FOR UPDATE`, in id order) in the caller's
|
|
24
21
|
* transaction, after its conversation row and before its run row: the one lock order of every path
|
|
@@ -89,7 +86,27 @@ export declare class RunJobs {
|
|
|
89
86
|
owe(data: RunJobData, jobId: string, kind: ConversationKind, orgId: string): Promise<boolean>;
|
|
90
87
|
/** {@link oweEnsureRun} in the caller's transaction. */
|
|
91
88
|
oweEnsureRun(conversationId: string, seq: number | string, orgId: string): Promise<boolean>;
|
|
89
|
+
/**
|
|
90
|
+
* Owes the delayed `bg-wake` job at the end of the conversation's wake window in the caller's
|
|
91
|
+
* transaction (the `ensureRun` that decided `later`): one job per conversation and window end
|
|
92
|
+
* ({@link bgWakeJobId}), so finishes inside one window share it; `retry` > 0 is a job that fired
|
|
93
|
+
* before its window ended. The relay adds it with the remaining delay (a row due later skips the
|
|
94
|
+
* fast path); `runAt` carries {@link WAKE_JOB_MARGIN_MS}, since the window is read on the
|
|
95
|
+
* database's clock and the delay runs on this process's and Redis's. Every `ensureRun` inside the
|
|
96
|
+
* window (each append that calls it) owes it again: a no-op while the row is pending, and once it
|
|
97
|
+
* was delivered a new row whose add BullMQ dedupes by the job id while the delayed job exists.
|
|
98
|
+
*/
|
|
99
|
+
oweBgWake(conversationId: string, w: {
|
|
100
|
+
windowEnd: number;
|
|
101
|
+
delayMs: number;
|
|
102
|
+
}, retry: number, orgId: string): Promise<boolean>;
|
|
92
103
|
}
|
|
104
|
+
/**
|
|
105
|
+
* Margin added to the delayed wake job: its window is read on the database's clock, the delay runs
|
|
106
|
+
* on this process's and Redis's, so the job does not usually fire just before the window ends (one
|
|
107
|
+
* that does reschedules itself under its next retry id, `bgWakeJobId`).
|
|
108
|
+
*/
|
|
109
|
+
export declare const WAKE_JOB_MARGIN_MS = 1000;
|
|
93
110
|
/**
|
|
94
111
|
* What {@link RunResume.resumeIn} found: `queued`, the run was waiting on nothing else and now is
|
|
95
112
|
* queued with its resume job owed; `still-running`, the run has not reached `waiting` yet (its
|
package/dist/runtime/run-jobs.js
CHANGED
|
@@ -10,6 +10,7 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
|
10
10
|
import { getQueueToken } from "@nestjs/bullmq";
|
|
11
11
|
import { Inject, Injectable, Module } from "@nestjs/common";
|
|
12
12
|
import { sql } from "kysely";
|
|
13
|
+
import { BG_WAKE_JOB, bgWakeJobId } from "../background/wake-rules.js";
|
|
13
14
|
import { KernelDb } from "../infra/database/kernel-db.js";
|
|
14
15
|
import { TransactionHost } from "../infra/database/transaction-host.js";
|
|
15
16
|
import { Transactional } from "../infra/database/transactional.decorator.js";
|
|
@@ -29,12 +30,8 @@ export const SUBAGENT_RUN_QUEUE_NAME = "cap-kernel-subagent-runs";
|
|
|
29
30
|
export const runQueueNameFor = (kind) => kind === "subagent" ? SUBAGENT_RUN_QUEUE_NAME : RUN_QUEUE_NAME;
|
|
30
31
|
/** 未结束的运行状态(waiting:等人处理审批 / 输入请求,没有任务、不占会话锁与配额)。 */
|
|
31
32
|
export const ACTIVE_RUN_STATUSES = ["queued", "running", "waiting"];
|
|
32
|
-
|
|
33
|
-
export
|
|
34
|
-
"completed",
|
|
35
|
-
"failed",
|
|
36
|
-
"aborted",
|
|
37
|
-
]);
|
|
33
|
+
// The terminal run statuses (here, not in `run-finish.ts`, which the sub-agent modules import)
|
|
34
|
+
export { TERMINAL_STATUSES } from "./run-status.js";
|
|
38
35
|
/**
|
|
39
36
|
* Locks a run's approval and input request rows (`FOR UPDATE`, in id order) in the caller's
|
|
40
37
|
* transaction, after its conversation row and before its run row: the one lock order of every path
|
|
@@ -144,6 +141,26 @@ let RunJobs = class RunJobs {
|
|
|
144
141
|
oweEnsureRun(conversationId, seq, orgId) {
|
|
145
142
|
return oweEnsureRun(this.outbox, conversationId, seq, orgId);
|
|
146
143
|
}
|
|
144
|
+
/**
|
|
145
|
+
* Owes the delayed `bg-wake` job at the end of the conversation's wake window in the caller's
|
|
146
|
+
* transaction (the `ensureRun` that decided `later`): one job per conversation and window end
|
|
147
|
+
* ({@link bgWakeJobId}), so finishes inside one window share it; `retry` > 0 is a job that fired
|
|
148
|
+
* before its window ended. The relay adds it with the remaining delay (a row due later skips the
|
|
149
|
+
* fast path); `runAt` carries {@link WAKE_JOB_MARGIN_MS}, since the window is read on the
|
|
150
|
+
* database's clock and the delay runs on this process's and Redis's. Every `ensureRun` inside the
|
|
151
|
+
* window (each append that calls it) owes it again: a no-op while the row is pending, and once it
|
|
152
|
+
* was delivered a new row whose add BullMQ dedupes by the job id while the delayed job exists.
|
|
153
|
+
*/
|
|
154
|
+
oweBgWake(conversationId, w, retry, orgId) {
|
|
155
|
+
return this.outbox.add({
|
|
156
|
+
queue: MAINTENANCE_QUEUE_NAME,
|
|
157
|
+
name: BG_WAKE_JOB,
|
|
158
|
+
jobId: bgWakeJobId(conversationId, w.windowEnd, retry),
|
|
159
|
+
data: { conversationId, ...(retry > 0 && { retry }) },
|
|
160
|
+
runAt: new Date(Date.now() + w.delayMs + WAKE_JOB_MARGIN_MS),
|
|
161
|
+
orgId,
|
|
162
|
+
});
|
|
163
|
+
}
|
|
147
164
|
};
|
|
148
165
|
RunJobs = __decorate([
|
|
149
166
|
Injectable(),
|
|
@@ -153,6 +170,12 @@ RunJobs = __decorate([
|
|
|
153
170
|
__param(3, Inject(OutboxService))
|
|
154
171
|
], RunJobs);
|
|
155
172
|
export { RunJobs };
|
|
173
|
+
/**
|
|
174
|
+
* Margin added to the delayed wake job: its window is read on the database's clock, the delay runs
|
|
175
|
+
* on this process's and Redis's, so the job does not usually fire just before the window ends (one
|
|
176
|
+
* that does reschedules itself under its next retry id, `bgWakeJobId`).
|
|
177
|
+
*/
|
|
178
|
+
export const WAKE_JOB_MARGIN_MS = 1000;
|
|
156
179
|
/**
|
|
157
180
|
* The resume of a waiting run (transactional outbox spec §6 "Resumes and timeouts", "Decisions
|
|
158
181
|
* resume in their own transaction"), in the caller's transaction. A decision (approvals, questions,
|
|
@@ -13,7 +13,7 @@ import { RunFinish } from "./run-finish.js";
|
|
|
13
13
|
import { RunJobs } from "./run-jobs.js";
|
|
14
14
|
import { RunQueue } from "./run-queue.js";
|
|
15
15
|
import { Waits } from "./waits.js";
|
|
16
|
-
/** The check's `@LockedInterval` job name (its run lock key
|
|
16
|
+
/** The check's `@LockedInterval` job name (its run lock key: `cron:run-orphan-check`). */
|
|
17
17
|
export declare const RUN_ORPHAN_CHECK_JOB = "run-orphan-check";
|
|
18
18
|
/** The check's period: a fixed constant, not a config field. */
|
|
19
19
|
export declare const RUN_ORPHAN_CHECK_PERIOD_MS = 60000;
|
|
@@ -55,7 +55,10 @@ export interface OrphanCandidate {
|
|
|
55
55
|
/**
|
|
56
56
|
* One page of the active runs of chat and sub-agent conversations in `statuses` that entered their
|
|
57
57
|
* state more than `minAgeMs` ago, by (state start, id), after `after` (an eval's hidden
|
|
58
|
-
* conversation drives its runs itself).
|
|
58
|
+
* conversation drives its runs itself). A deleted org's runs are never candidates, running ones
|
|
59
|
+
* included: its deletion purged its owed jobs on purpose, so their absence is no lost job and
|
|
60
|
+
* nothing is re-added, and the org's data is invisible from then on, so a running orphan is left
|
|
61
|
+
* as it is rather than failed.
|
|
59
62
|
*/
|
|
60
63
|
export declare function orphanCandidates(db: KernelSchemaDb, statuses: readonly ("queued" | "running")[], minAgeMs: number, limit: number, after?: Pick<OrphanCandidate, "since" | "runId">): Promise<OrphanCandidate[]>;
|
|
61
64
|
/**
|
|
@@ -65,12 +68,12 @@ export declare function orphanCandidates(db: KernelSchemaDb, statuses: readonly
|
|
|
65
68
|
*/
|
|
66
69
|
export declare function jobInOutbox(db: KernelSchemaDb, queue: string, jobId: string): Promise<boolean>;
|
|
67
70
|
/**
|
|
68
|
-
* The `queued` runs (in their state for over `minAgeMs`) whose job is neither in the outbox
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
* run again after both (a job that finished took its run out of `queued` first), so a run
|
|
72
|
-
* flight is never reported. The `run-orphan-check` uses it; so does the dual-replica test
|
|
73
|
-
* which asserts at its teardown that the kernel left none (`minAgeMs` 0).
|
|
71
|
+
* The `queued` runs (in their state for over `minAgeMs`) whose job is neither in the outbox (owed,
|
|
72
|
+
* or dead) nor in BullMQ (`jobExists`), and that are still queued after both looks: the run queue
|
|
73
|
+
* lost their jobs. The outbox is read before BullMQ (a row is deleted only after its job was added)
|
|
74
|
+
* and the run again after both (a job that finished took its run out of `queued` first), so a run
|
|
75
|
+
* in flight is never reported. The `run-orphan-check` uses it; so does the dual-replica test
|
|
76
|
+
* harness, which asserts at its teardown that the kernel left none (`minAgeMs` 0).
|
|
74
77
|
*/
|
|
75
78
|
export declare function queuedRunsWithoutJob(db: KernelSchemaDb, jobExists: (queue: string, jobId: string) => Promise<boolean>, opts: {
|
|
76
79
|
minAgeMs: number;
|
|
@@ -85,9 +88,9 @@ export declare function queuedRunsWithoutJob(db: KernelSchemaDb, jobExists: (que
|
|
|
85
88
|
* `run.orphaned` (its terminal transaction cancels its waits and owes what every run end owes),
|
|
86
89
|
* then, after the commit, its checkpoints are dropped, its fork child stopped (`run.orphaned`), a
|
|
87
90
|
* sub-agent's failure delivered to its parent at once and the conversation's next run started.
|
|
88
|
-
* - `queued`, with no outbox row either (pending or dead): a canary. Every queued run has its job
|
|
89
|
-
* own transaction, so this means Redis lost data. It is logged as an error, counted
|
|
90
|
-
* overview ({@link QueueLostJobs}) and its job owed again so the run is not stuck.
|
|
91
|
+
* - `queued`, with no outbox row either (pending or dead): a canary. Every queued run has its job
|
|
92
|
+
* owed in its own transaction, so this means Redis lost data. It is logged as an error, counted
|
|
93
|
+
* for the admin overview ({@link QueueLostJobs}) and its job owed again so the run is not stuck.
|
|
91
94
|
* Redis is read before each repair's transaction, which then re-reads the run under its
|
|
92
95
|
* conversation row lock and acts only if it is still in the same state since the same time. Read
|
|
93
96
|
* paths never repair runs.
|
|
@@ -25,7 +25,7 @@ import { notifyFinished, RunFinish } from "./run-finish.js";
|
|
|
25
25
|
import { RunJobs, runJobId, runQueueNameFor } from "./run-jobs.js";
|
|
26
26
|
import { RunQueue } from "./run-queue.js";
|
|
27
27
|
import { Waits } from "./waits.js";
|
|
28
|
-
/** The check's `@LockedInterval` job name (its run lock key
|
|
28
|
+
/** The check's `@LockedInterval` job name (its run lock key: `cron:run-orphan-check`). */
|
|
29
29
|
export const RUN_ORPHAN_CHECK_JOB = "run-orphan-check";
|
|
30
30
|
/** The check's period: a fixed constant, not a config field. */
|
|
31
31
|
export const RUN_ORPHAN_CHECK_PERIOD_MS = 60_000;
|
|
@@ -49,12 +49,16 @@ const SINCE = sql `GREATEST(COALESCE(r.queued_at, r.created_at), COALESCE(r.star
|
|
|
49
49
|
/**
|
|
50
50
|
* One page of the active runs of chat and sub-agent conversations in `statuses` that entered their
|
|
51
51
|
* state more than `minAgeMs` ago, by (state start, id), after `after` (an eval's hidden
|
|
52
|
-
* conversation drives its runs itself).
|
|
52
|
+
* conversation drives its runs itself). A deleted org's runs are never candidates, running ones
|
|
53
|
+
* included: its deletion purged its owed jobs on purpose, so their absence is no lost job and
|
|
54
|
+
* nothing is re-added, and the org's data is invisible from then on, so a running orphan is left
|
|
55
|
+
* as it is rather than failed.
|
|
53
56
|
*/
|
|
54
57
|
export async function orphanCandidates(db, statuses, minAgeMs, limit, after) {
|
|
55
58
|
let q = db
|
|
56
59
|
.selectFrom("runs as r")
|
|
57
60
|
.innerJoin("conversations as c", "c.id", "r.conversation_id")
|
|
61
|
+
.innerJoin("orgs as o", "o.id", "c.org_id")
|
|
58
62
|
.select([
|
|
59
63
|
"r.id",
|
|
60
64
|
"r.conversation_id",
|
|
@@ -67,6 +71,7 @@ export async function orphanCandidates(db, statuses, minAgeMs, limit, after) {
|
|
|
67
71
|
])
|
|
68
72
|
.where("r.status", "in", statuses)
|
|
69
73
|
.where("c.kind", "in", ["chat", "subagent"])
|
|
74
|
+
.where("o.deleted_at", "is", null)
|
|
70
75
|
.where(SINCE, "<", sql `now() - ${minAgeMs}::integer * interval '1 millisecond'`);
|
|
71
76
|
if (after !== undefined) {
|
|
72
77
|
q = q.where(sql `(${SINCE}, r.id) > (${after.since}::timestamptz, ${after.runId})`);
|
|
@@ -110,12 +115,12 @@ async function* candidatePages(db, statuses, minAgeMs) {
|
|
|
110
115
|
}
|
|
111
116
|
}
|
|
112
117
|
/**
|
|
113
|
-
* The `queued` runs (in their state for over `minAgeMs`) whose job is neither in the outbox
|
|
114
|
-
*
|
|
115
|
-
*
|
|
116
|
-
* run again after both (a job that finished took its run out of `queued` first), so a run
|
|
117
|
-
* flight is never reported. The `run-orphan-check` uses it; so does the dual-replica test
|
|
118
|
-
* which asserts at its teardown that the kernel left none (`minAgeMs` 0).
|
|
118
|
+
* The `queued` runs (in their state for over `minAgeMs`) whose job is neither in the outbox (owed,
|
|
119
|
+
* or dead) nor in BullMQ (`jobExists`), and that are still queued after both looks: the run queue
|
|
120
|
+
* lost their jobs. The outbox is read before BullMQ (a row is deleted only after its job was added)
|
|
121
|
+
* and the run again after both (a job that finished took its run out of `queued` first), so a run
|
|
122
|
+
* in flight is never reported. The `run-orphan-check` uses it; so does the dual-replica test
|
|
123
|
+
* harness, which asserts at its teardown that the kernel left none (`minAgeMs` 0).
|
|
119
124
|
*/
|
|
120
125
|
export async function queuedRunsWithoutJob(db, jobExists, opts) {
|
|
121
126
|
const out = [];
|
|
@@ -146,9 +151,9 @@ export async function queuedRunsWithoutJob(db, jobExists, opts) {
|
|
|
146
151
|
* `run.orphaned` (its terminal transaction cancels its waits and owes what every run end owes),
|
|
147
152
|
* then, after the commit, its checkpoints are dropped, its fork child stopped (`run.orphaned`), a
|
|
148
153
|
* sub-agent's failure delivered to its parent at once and the conversation's next run started.
|
|
149
|
-
* - `queued`, with no outbox row either (pending or dead): a canary. Every queued run has its job
|
|
150
|
-
* own transaction, so this means Redis lost data. It is logged as an error, counted
|
|
151
|
-
* overview ({@link QueueLostJobs}) and its job owed again so the run is not stuck.
|
|
154
|
+
* - `queued`, with no outbox row either (pending or dead): a canary. Every queued run has its job
|
|
155
|
+
* owed in its own transaction, so this means Redis lost data. It is logged as an error, counted
|
|
156
|
+
* for the admin overview ({@link QueueLostJobs}) and its job owed again so the run is not stuck.
|
|
152
157
|
* Redis is read before each repair's transaction, which then re-reads the run under its
|
|
153
158
|
* conversation row lock and acts only if it is still in the same state since the same time. Read
|
|
154
159
|
* paths never repair runs.
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import type { Logger } from "@capybara-hq/plugin-sdk";
|
|
2
|
-
import type { Queue } from "bullmq";
|
|
3
2
|
import { EventLog } from "../conversations/event-log.js";
|
|
4
3
|
import { KernelConfigService } from "../infra/config/kernel-config.service.js";
|
|
5
4
|
import { KernelDb } from "../infra/database/kernel-db.js";
|
|
@@ -32,14 +31,10 @@ export declare class RunQueue {
|
|
|
32
31
|
private readonly log;
|
|
33
32
|
readonly txHost: TransactionHost;
|
|
34
33
|
private readonly config;
|
|
35
|
-
/** The producer side of the delayed `bg-wake` job (`MaintenanceProcessor` consumes it). */
|
|
36
|
-
private readonly maintenance;
|
|
37
34
|
private readonly ensureFails;
|
|
38
35
|
constructor(kdb: KernelDb, eventLog: EventLog, lock: Pick<ConversationLock, "holder">, jobs: RunJobs, finisher: Pick<RunFinish, "finishIn">,
|
|
39
36
|
/** An orphan aborted directly loses its checkpoint thread (thread_id = runId). */
|
|
40
|
-
checkpoints: Pick<KyselyCheckpointSaver, "deleteThread">, log: Pick<Logger, "error" | "warn">, txHost: TransactionHost, config: KernelConfigService,
|
|
41
|
-
/** The producer side of the delayed `bg-wake` job (`MaintenanceProcessor` consumes it). */
|
|
42
|
-
maintenance: Pick<Queue, "add">, ensureFails?: () => boolean);
|
|
37
|
+
checkpoints: Pick<KyselyCheckpointSaver, "deleteThread">, log: Pick<Logger, "error" | "warn">, txHost: TransactionHost, config: KernelConfigService, ensureFails?: () => boolean);
|
|
43
38
|
private get db();
|
|
44
39
|
/**
|
|
45
40
|
* 有未消费的用户消息(最新 user_message 的 seq > `consumed_seq`)、会话锁无人持有、
|
|
@@ -51,7 +46,7 @@ export declare class RunQueue {
|
|
|
51
46
|
* once it is sent; with no head either, a wake-eligible finished background task or sub-agent
|
|
52
47
|
* delivery (background Bash spec §3, agent map spec §5.3, {@link wakeDecisionIn}): the run is
|
|
53
48
|
* then attributed to the event's trigger account and joins the tree's budget window, and an
|
|
54
|
-
* event inside the wake window
|
|
49
|
+
* event inside the wake window owes the delayed `bg-wake` job instead. A sub-agent's
|
|
55
50
|
* conversation is only woken this way ({@link childWakeIn}).
|
|
56
51
|
*
|
|
57
52
|
* An active run (queued, running or waiting) is left alone: a run nobody runs any more is the
|
|
@@ -77,8 +72,9 @@ export declare class RunQueue {
|
|
|
77
72
|
private jobIdFor;
|
|
78
73
|
/**
|
|
79
74
|
* Owes the new run's job in {@link ensureRunIn}'s transaction, before its run row is written.
|
|
80
|
-
* False (logged): the same job id is still owed, so the caller writes no run
|
|
81
|
-
*
|
|
75
|
+
* False (logged): the same job id is still owed, so the caller writes no run. A later run of the
|
|
76
|
+
* same trigger has its own `<runJobId>-<n>` ({@link jobIdFor}), so this is a should-not-happen
|
|
77
|
+
* (the conversation has one active run at a time).
|
|
82
78
|
*/
|
|
83
79
|
private oweJob;
|
|
84
80
|
/**
|
|
@@ -89,15 +85,17 @@ export declare class RunQueue {
|
|
|
89
85
|
* racing the run's end starts a new run, never lost); and it is woken by its own background
|
|
90
86
|
* task's finish or a delivery from its own sub-agent, under the same rules as a chat
|
|
91
87
|
* conversation. Neither happens once its last run was stopped (`aborted`): a stopped sub-agent is
|
|
92
|
-
* started again only by its parent
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
*
|
|
88
|
+
* started again only by its parent; nor while a person's stop of the root holds the tree
|
|
89
|
+
* (`conversations.tree_stopped_at`, cleared by the root's next run). Both read the queue and the
|
|
90
|
+
* trigger account's access on the root of its tree; a message needs no wake setting and has no
|
|
91
|
+
* wake window. The run counts against the root's budget window and runs on the sub-agent queue
|
|
92
|
+
* under `runJobId(child, boundary seq)`. An active run is never started past (one nobody runs is
|
|
93
|
+
* the `run-orphan-check`'s).
|
|
96
94
|
*/
|
|
97
95
|
private childWakeIn;
|
|
98
96
|
/**
|
|
99
|
-
* The newest unconsumed message of an idle sub-agent (a `SendMessage` its last run did not
|
|
100
|
-
* as a run boundary, or null: none; the root's queue is paused or the root deleted (the
|
|
97
|
+
* The newest unconsumed message of an idle sub-agent (a `SendMessage` its last run did not
|
|
98
|
+
* absorb) as a run boundary, or null: none; the root's queue is paused or the root deleted (the
|
|
101
99
|
* message waits for the unpause, {@link wakeHeldTree}); or its author can no longer send to the
|
|
102
100
|
* root (it waits for a later run). The caller already refused a stopped sub-agent (last run
|
|
103
101
|
* `aborted`): a message that reached it while running and then stopped is parked, and the next
|
|
@@ -126,13 +124,6 @@ export declare class RunQueue {
|
|
|
126
124
|
* with their org and newest seq. Reads only (no row locks).
|
|
127
125
|
*/
|
|
128
126
|
private heldTree;
|
|
129
|
-
/**
|
|
130
|
-
* Schedules the delayed `bg-wake` job at the end of the conversation's wake window (after the
|
|
131
|
-
* commit): one job per conversation and window end, so finishes inside one window share it. It
|
|
132
|
-
* calls {@link ensureRun}, which wakes the conversation if a finish is still unconsumed. A failed
|
|
133
|
-
* enqueue is logged: the next message, run end or page opening re-derives the wake.
|
|
134
|
-
*/
|
|
135
|
-
private scheduleWake;
|
|
136
127
|
/**
|
|
137
128
|
* Owes the ensure-run backstop of `conversationId` in the caller's transaction
|
|
138
129
|
* ({@link oweEnsureRun}): `seq` is the event the transaction wrote, `orgId` the conversation's.
|
|
@@ -141,8 +132,8 @@ export declare class RunQueue {
|
|
|
141
132
|
/**
|
|
142
133
|
* Stops an orphaned run directly (`POST .../abort`, a tree stop): a `running` run whose
|
|
143
134
|
* conversation lock is free and whose BullMQ job is gone hears no stop broadcast, so its
|
|
144
|
-
* `run_aborted` is written here (status aborted, `consumed_seq` at least to its trigger): true.
|
|
145
|
-
* an orphan (someone runs it; a queued run's job is owed, or lost and re-added by the
|
|
135
|
+
* `run_aborted` is written here (status aborted, `consumed_seq` at least to its trigger): true.
|
|
136
|
+
* Not an orphan (someone runs it; a queued run's job is owed, or lost and re-added by the
|
|
146
137
|
* `run-orphan-check`, whose worker then sees the stop marker) or already finished: false, and the
|
|
147
138
|
* caller broadcasts the stop.
|
|
148
139
|
*/
|
|
@@ -150,14 +141,15 @@ export declare class RunQueue {
|
|
|
150
141
|
private abortOrphanIn;
|
|
151
142
|
/**
|
|
152
143
|
* Ends a `queued` run in PG before any worker started it (an offboarded member's run, ruling
|
|
153
|
-
* R27): with the conversation lock free, under the conversation row lock, a run still `queued`
|
|
154
|
-
*
|
|
155
|
-
*
|
|
156
|
-
*
|
|
144
|
+
* R27): with the conversation lock free, under the conversation row lock, a run still `queued` is
|
|
145
|
+
* finished `aborted` with `run_aborted` (`consumed_seq` at least to its trigger), so a job that
|
|
146
|
+
* runs later finds it finished and starts nothing. True when it did; false when a worker holds
|
|
147
|
+
* the conversation or is in the middle of the run's first start (`run_started` written,
|
|
157
148
|
* `started_at` not yet set; it hears the stop), or the run is no longer queued. A run resumed
|
|
158
|
-
* from `waiting` to `queued` is ended like a new one. The caller writes the stop marker first, so
|
|
159
|
-
* conversation meanwhile still stops; a worker that read the run before
|
|
160
|
-
* under the row locks before its `run_started` and never moves it back
|
|
149
|
+
* from `waiting` to `queued` is ended like a new one. The caller writes the stop marker first, so
|
|
150
|
+
* a worker that takes the conversation meanwhile still stops; a worker that read the run before
|
|
151
|
+
* this commit re-checks it under the row locks before its `run_started` and never moves it back
|
|
152
|
+
* to `running`.
|
|
161
153
|
*/
|
|
162
154
|
abortQueued(conversationId: string, runId: string, byAccountId: string | null): Promise<boolean>;
|
|
163
155
|
private abortQueuedIn;
|