@capybara-hq/kernel 0.1.0-rc.23 → 0.1.0-rc.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bootstrap.json +25 -25
- package/dist/admin/admin.module.d.ts +2 -1
- package/dist/admin/admin.module.js +7 -3
- package/dist/admin/outbox-dead.controller.d.ts +23 -0
- package/dist/admin/outbox-dead.controller.js +62 -0
- package/dist/admin/outbox-dead.schemas.d.ts +8 -0
- package/dist/admin/outbox-dead.schemas.js +18 -0
- package/dist/admin/outbox-dead.service.d.ts +66 -0
- package/dist/admin/outbox-dead.service.js +179 -0
- package/dist/admin/overview.service.d.ts +40 -5
- package/dist/admin/overview.service.js +41 -7
- package/dist/agents/from-template.js +1 -1
- package/dist/background/finish.d.ts +5 -2
- package/dist/background/finish.js +14 -3
- package/dist/background/launch.d.ts +2 -1
- package/dist/background/launch.js +5 -2
- package/dist/background/store.d.ts +8 -0
- package/dist/background/store.js +24 -0
- package/dist/catalog/admin-catalog.controller.d.ts +21 -2
- package/dist/catalog/admin-catalog.controller.js +45 -6
- package/dist/catalog/admin-catalog.schemas.d.ts +5 -0
- package/dist/catalog/admin-catalog.schemas.js +8 -0
- package/dist/catalog/catalog.module.d.ts +2 -1
- package/dist/catalog/catalog.module.js +5 -1
- package/dist/catalog/marketplaces.service.d.ts +1 -1
- package/dist/catalog/platform-import-update.d.ts +50 -0
- package/dist/catalog/platform-import-update.js +116 -0
- package/dist/catalog/plugin-import.d.ts +36 -1
- package/dist/catalog/plugin-import.js +23 -1
- package/dist/catalog/plugin-upgrade.d.ts +18 -0
- package/dist/catalog/plugin-upgrade.js +19 -0
- package/dist/channels/channels.module.d.ts +5 -2
- package/dist/channels/channels.module.js +9 -2
- package/dist/channels/deliver-job.d.ts +25 -0
- package/dist/channels/deliver-job.js +25 -0
- package/dist/channels/inbound.d.ts +1 -1
- package/dist/channels/inbound.js +2 -15
- package/dist/channels/jobs.d.ts +28 -18
- package/dist/channels/jobs.js +63 -26
- package/dist/config/schema.d.ts +0 -3
- package/dist/config/schema.js +11 -20
- package/dist/conversations/agent-tree.d.ts +16 -0
- package/dist/conversations/agent-tree.js +36 -0
- package/dist/conversations/conversations.service.d.ts +44 -14
- package/dist/conversations/conversations.service.js +52 -38
- package/dist/conversations/event-log.d.ts +7 -10
- package/dist/conversations/event-log.js +24 -15
- package/dist/conversations/events.js +2 -1
- package/dist/conversations/queue.service.d.ts +8 -3
- package/dist/conversations/queue.service.js +37 -13
- package/dist/conversations/titles.d.ts +21 -8
- package/dist/conversations/titles.js +42 -15
- package/dist/db/kernel-schema.d.ts +26 -8
- package/dist/devices/devices.service.d.ts +12 -3
- package/dist/devices/devices.service.js +18 -3
- package/dist/errors.d.ts +11 -0
- package/dist/errors.js +11 -0
- package/dist/evals/run-replay.js +14 -8
- package/dist/extensions/extensions.module.js +3 -0
- package/dist/extensions/lifecycle.d.ts +6 -1
- package/dist/extensions/lifecycle.js +9 -2
- package/dist/extensions/usage-subscribers.d.ts +43 -0
- package/dist/extensions/usage-subscribers.js +108 -0
- package/dist/http/api.module.js +2 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/infra/cli/kernel-cli.module.js +5 -0
- package/dist/infra/events/kernel-events.d.ts +9 -35
- package/dist/infra/outbox/outbox-relay.module.d.ts +6 -0
- package/dist/infra/outbox/outbox-relay.module.js +18 -0
- package/dist/infra/outbox/outbox.dispatcher.d.ts +103 -0
- package/dist/infra/outbox/outbox.dispatcher.js +310 -0
- package/dist/infra/outbox/outbox.module.d.ts +24 -0
- package/dist/infra/outbox/outbox.module.js +50 -0
- package/dist/infra/outbox/outbox.relay.d.ts +31 -0
- package/dist/infra/outbox/outbox.relay.js +76 -0
- package/dist/infra/outbox/outbox.service.d.ts +51 -0
- package/dist/infra/outbox/outbox.service.js +131 -0
- package/dist/infra/outbox/registry.d.ts +120 -0
- package/dist/infra/outbox/registry.js +108 -0
- package/dist/infra/queue/queue-names.d.ts +8 -0
- package/dist/infra/queue/queue-names.js +16 -0
- package/dist/kernel.module.js +3 -0
- package/dist/local-devices/device-policy.service.d.ts +4 -0
- package/dist/local-devices/device-policy.service.js +61 -3
- package/dist/local-devices/machines.d.ts +13 -5
- package/dist/local-devices/machines.js +45 -7
- package/dist/local-devices/policy-summary.d.ts +44 -1
- package/dist/local-devices/policy-summary.js +69 -0
- package/dist/models/embeddings.d.ts +9 -9
- package/dist/models/embeddings.js +11 -18
- package/dist/models/models.module.js +3 -0
- package/dist/orgs/orgs.service.js +7 -0
- package/dist/overrides.d.ts +20 -3
- package/dist/overrides.js +10 -2
- package/dist/platform/exclusions.js +5 -0
- package/dist/platform/ops/admin.ops.js +18 -2
- package/dist/platform/ops/catalog-admin.ops.d.ts +3 -1
- package/dist/platform/ops/catalog-admin.ops.js +32 -4
- package/dist/platform/ops/outbox.ops.d.ts +21 -0
- package/dist/platform/ops/outbox.ops.js +125 -0
- package/dist/platform/ops/schedules.ops.js +40 -1
- package/dist/platform/platform-ops.module.d.ts +2 -1
- package/dist/platform/platform-ops.module.js +3 -0
- package/dist/platform/route-schemas.js +3 -0
- package/dist/roles/worker.module.js +9 -4
- package/dist/runtime/langgraph-loop.js +13 -5
- package/dist/runtime/limits.d.ts +15 -0
- package/dist/runtime/limits.js +25 -0
- package/dist/runtime/loop.d.ts +8 -3
- package/dist/runtime/loop.js +2 -1
- package/dist/runtime/maintenance.d.ts +4 -2
- package/dist/runtime/maintenance.js +17 -2
- package/dist/runtime/model-call.d.ts +14 -16
- package/dist/runtime/model-call.js +14 -24
- package/dist/runtime/queue-lost-jobs.d.ts +24 -0
- package/dist/runtime/queue-lost-jobs.js +51 -0
- package/dist/runtime/run-control.d.ts +42 -5
- package/dist/runtime/run-control.js +66 -12
- package/dist/runtime/run-finish.d.ts +39 -12
- package/dist/runtime/run-finish.js +97 -14
- package/dist/runtime/run-jobs.d.ts +95 -13
- package/dist/runtime/run-jobs.js +199 -28
- package/dist/runtime/run-orphan-check.job.d.ts +142 -0
- package/dist/runtime/run-orphan-check.job.js +339 -0
- package/dist/runtime/run-processors.d.ts +16 -9
- package/dist/runtime/run-processors.js +1 -1
- package/dist/runtime/run-queue.d.ts +64 -55
- package/dist/runtime/run-queue.js +147 -284
- package/dist/runtime/run-steps.d.ts +1 -2
- package/dist/runtime/run-steps.js +8 -11
- package/dist/runtime/run-worker.d.ts +6 -17
- package/dist/runtime/run-worker.js +32 -34
- package/dist/runtime/runtime.module.d.ts +2 -3
- package/dist/runtime/runtime.module.js +5 -6
- package/dist/runtime/step-log.d.ts +16 -4
- package/dist/runtime/step-log.js +26 -6
- package/dist/runtime/stream.controller.d.ts +2 -4
- package/dist/runtime/stream.controller.js +5 -11
- package/dist/runtime/subagents/child.js +1 -1
- package/dist/runtime/subagents/definition.d.ts +2 -1
- package/dist/runtime/subagents/definition.js +4 -2
- package/dist/runtime/subagents/deliveries.d.ts +6 -3
- package/dist/runtime/subagents/deliveries.js +16 -7
- package/dist/runtime/subagents/dispatch.d.ts +12 -12
- package/dist/runtime/subagents/dispatch.js +13 -38
- package/dist/runtime/subagents/messages.d.ts +5 -1
- package/dist/runtime/subagents/messages.js +11 -3
- package/dist/runtime/subagents/skill-fork.d.ts +3 -3
- package/dist/runtime/subagents/skill-fork.js +5 -5
- package/dist/runtime/subagents/subagents.service.js +1 -1
- package/dist/runtime/subagents/tree-jobs.d.ts +61 -0
- package/dist/runtime/subagents/tree-jobs.js +86 -0
- package/dist/runtime/usage-report-jobs.d.ts +23 -48
- package/dist/runtime/usage-report-jobs.js +67 -119
- package/dist/runtime/usage-report.d.ts +28 -77
- package/dist/runtime/usage-report.js +35 -213
- package/dist/runtime/waits.d.ts +48 -198
- package/dist/runtime/waits.js +103 -805
- package/dist/sandbox/sandbox.service.d.ts +17 -4
- package/dist/sandbox/sandbox.service.js +55 -14
- package/dist/schedules/calendar.d.ts +134 -0
- package/dist/schedules/calendar.js +410 -0
- package/dist/schedules/fire.d.ts +1 -3
- package/dist/schedules/fire.js +5 -10
- package/dist/schedules/run-tracker.d.ts +0 -5
- package/dist/schedules/run-tracker.js +0 -13
- package/dist/schedules/scan.job.js +15 -20
- package/dist/schedules/schedule-math.d.ts +42 -0
- package/dist/schedules/schedule-math.js +52 -0
- package/dist/schedules/scheduled-tasks.service.d.ts +16 -0
- package/dist/schedules/scheduled-tasks.service.js +53 -0
- package/dist/schedules/schedules.controller.d.ts +5 -1
- package/dist/schedules/schedules.controller.js +15 -1
- package/dist/schedules/schedules.schemas.d.ts +35 -0
- package/dist/schedules/schedules.schemas.js +27 -0
- package/dist/schedules/store.d.ts +5 -0
- package/dist/schedules/store.js +20 -0
- package/dist/tools/approvals.service.d.ts +26 -7
- package/dist/tools/approvals.service.js +88 -10
- package/dist/tools/builtin-names.d.ts +5 -3
- package/dist/tools/builtin-names.js +12 -4
- package/dist/tools/builtins/builtin-tools.d.ts +1 -2
- package/dist/tools/builtins/builtin-tools.js +21 -17
- package/dist/tools/builtins/sandbox/bash.js +9 -3
- package/dist/tools/builtins/sandbox/common.d.ts +18 -1
- package/dist/tools/builtins/sandbox/common.js +14 -1
- package/dist/tools/builtins/sandbox/index.d.ts +2 -1
- package/dist/tools/builtins/sandbox/index.js +4 -1
- package/dist/tools/builtins/sandbox/task-list.d.ts +12 -0
- package/dist/tools/builtins/sandbox/task-list.js +130 -0
- package/dist/tools/builtins/sandbox/task-output.d.ts +8 -1
- package/dist/tools/builtins/sandbox/task-output.js +21 -3
- package/dist/tools/builtins/web/extract.d.ts +2 -4
- package/dist/tools/builtins/web/extract.js +2 -5
- package/dist/tools/pipeline.d.ts +1 -1
- package/dist/tools/pipeline.js +1 -1
- package/dist/tools/questions.service.d.ts +15 -6
- package/dist/tools/questions.service.js +20 -8
- package/dist/tools/registry.d.ts +4 -2
- package/dist/tools/registry.js +4 -3
- package/dist/tools/tools.module.js +4 -3
- package/package.json +3 -3
- package/src/migrations/0118_device_policy_ops.sql +6 -0
- package/src/migrations/0119_outbox.sql +22 -0
- package/src/migrations/0120_drop_usage_report_sweep.sql +7 -0
|
@@ -9,7 +9,7 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
|
9
9
|
};
|
|
10
10
|
import { randomUUID } from "node:crypto";
|
|
11
11
|
import { getQueueToken } from "@nestjs/bullmq";
|
|
12
|
-
import { Inject, Injectable } from "@nestjs/common";
|
|
12
|
+
import { Inject, Injectable, Optional } from "@nestjs/common";
|
|
13
13
|
import { sql } from "kysely";
|
|
14
14
|
import { visibleTo } from "../agents/agent-lookup.js";
|
|
15
15
|
import { BG_WAKE_JOB, bgWakeJobId, wakeDecisionIn, } from "../background/wake-rules.js";
|
|
@@ -20,24 +20,23 @@ import { KernelConfigService } from "../infra/config/kernel-config.service.js";
|
|
|
20
20
|
import { KernelDb } from "../infra/database/kernel-db.js";
|
|
21
21
|
import { TransactionHost } from "../infra/database/transaction-host.js";
|
|
22
22
|
import { Transactional } from "../infra/database/transactional.decorator.js";
|
|
23
|
-
import { KernelEvents } from "../infra/events/kernel-events.js";
|
|
24
23
|
import { KERNEL_LOGGER } from "../infra/tokens.js";
|
|
25
|
-
import { linkStartedRunIn
|
|
24
|
+
import { linkStartedRunIn } from "../schedules/run-tracker.js";
|
|
26
25
|
import { scheduledTaskRunAt } from "../schedules/unattended.js";
|
|
27
|
-
import { ToolPipeline } from "../tools/pipeline.js";
|
|
28
26
|
import { KyselyCheckpointSaver } from "./checkpointer.js";
|
|
29
27
|
import { ConversationLock } from "./conversation-lock.js";
|
|
30
28
|
import { MAINTENANCE_QUEUE_NAME } from "./maintenance-queue.js";
|
|
31
29
|
import { notifyFinished, RunFinish } from "./run-finish.js";
|
|
32
|
-
import { ACTIVE_RUN_STATUSES, RunJobs, runJobId, runQueueNameFor
|
|
30
|
+
import { ACTIVE_RUN_STATUSES, RunJobs, runJobId, runQueueNameFor } from "./run-jobs.js";
|
|
33
31
|
import { rootConversationOf } from "./subagents/child.js";
|
|
34
32
|
import { budgetWindowOf } from "./subagents/limits.js";
|
|
35
|
-
export { ACTIVE_RUN_STATUSES, RUN_QUEUE_NAME, runJobId, runQueueNameFor, SUBAGENT_RUN_QUEUE_NAME, } from "./run-jobs.js";
|
|
33
|
+
export { ACTIVE_RUN_STATUSES, ENSURE_RUN_BACKSTOP_MS, ENSURE_RUN_JOB, ensureRunJobId, RUN_QUEUE_NAME, runJobId, runQueueNameFor, SUBAGENT_RUN_QUEUE_NAME, } from "./run-jobs.js";
|
|
36
34
|
/**
|
|
37
|
-
*
|
|
38
|
-
*
|
|
35
|
+
* Test seam (`KernelOverride`): returns true to make {@link RunQueue.ensureRun} throw before it
|
|
36
|
+
* does anything (Redis unreachable, or a direct call after a commit that never happened). The
|
|
37
|
+
* kernel never provides it.
|
|
39
38
|
*/
|
|
40
|
-
const
|
|
39
|
+
export const ENSURE_RUN_FAILS = Symbol("ENSURE_RUN_FAILS");
|
|
41
40
|
/**
|
|
42
41
|
* Margin added to the delayed wake job: its window is read on the database's clock, the delay runs
|
|
43
42
|
* on this process's and Redis's, so the job does not usually fire just before the window ends (one
|
|
@@ -56,51 +55,36 @@ let RunQueue = class RunQueue {
|
|
|
56
55
|
jobs;
|
|
57
56
|
finisher;
|
|
58
57
|
checkpoints;
|
|
59
|
-
pipeline;
|
|
60
|
-
events;
|
|
61
58
|
log;
|
|
62
59
|
txHost;
|
|
63
60
|
config;
|
|
64
61
|
maintenance;
|
|
62
|
+
ensureFails;
|
|
65
63
|
constructor(kdb, eventLog, lock, jobs, finisher,
|
|
66
|
-
/**
|
|
67
|
-
checkpoints,
|
|
68
|
-
/** The run's open `Agent` call results, in the orphan repair's transaction (sub-agents spec §12). */
|
|
69
|
-
pipeline, events, log, txHost, config,
|
|
64
|
+
/** An orphan aborted directly loses its checkpoint thread (thread_id = runId). */
|
|
65
|
+
checkpoints, log, txHost, config,
|
|
70
66
|
/** The producer side of the delayed `bg-wake` job (`MaintenanceProcessor` consumes it). */
|
|
71
|
-
maintenance) {
|
|
67
|
+
maintenance, ensureFails = () => false) {
|
|
72
68
|
this.kdb = kdb;
|
|
73
69
|
this.eventLog = eventLog;
|
|
74
70
|
this.lock = lock;
|
|
75
71
|
this.jobs = jobs;
|
|
76
72
|
this.finisher = finisher;
|
|
77
73
|
this.checkpoints = checkpoints;
|
|
78
|
-
this.pipeline = pipeline;
|
|
79
|
-
this.events = events;
|
|
80
74
|
this.log = log;
|
|
81
75
|
this.txHost = txHost;
|
|
82
76
|
this.config = config;
|
|
83
77
|
this.maintenance = maintenance;
|
|
78
|
+
this.ensureFails = ensureFails;
|
|
84
79
|
}
|
|
85
80
|
get db() {
|
|
86
81
|
return this.kdb.db;
|
|
87
82
|
}
|
|
88
|
-
/**
|
|
89
|
-
* The `beforeTerminal` of an orphan repair: writes the run's open `Agent` call results in the
|
|
90
|
-
* repair's transaction; `fire()` runs their observe hooks after the commit (review m1).
|
|
91
|
-
*/
|
|
92
|
-
stopOpenCalls(runId) {
|
|
93
|
-
let fire = () => { };
|
|
94
|
-
return {
|
|
95
|
-
beforeTerminal: async () => {
|
|
96
|
-
fire = (await this.pipeline.stopOpenAgentCalls(runId)).fire;
|
|
97
|
-
},
|
|
98
|
-
fire: () => fire(),
|
|
99
|
-
};
|
|
100
|
-
}
|
|
101
83
|
/**
|
|
102
84
|
* 有未消费的用户消息(最新 user_message 的 seq > `consumed_seq`)、会话锁无人持有、
|
|
103
|
-
* 且没有 queued / running / waiting 的运行时:插入 `runs`(queued
|
|
85
|
+
* 且没有 queued / running / waiting 的运行时:插入 `runs`(queued),返回 runId;否则返回 null。
|
|
86
|
+
* The run's job is owed in the same transaction (transactional outbox spec §6): the outbox
|
|
87
|
+
* delivers it after the commit, so a queued run never lacks its job.
|
|
104
88
|
*
|
|
105
89
|
* The boundary (`trigger_seq`) is that newest message; with none unconsumed, the queue's head
|
|
106
90
|
* once it is sent; with no head either, a wake-eligible finished background task or sub-agent
|
|
@@ -109,45 +93,31 @@ let RunQueue = class RunQueue {
|
|
|
109
93
|
* event inside the wake window schedules the delayed `bg-wake` job instead. A sub-agent's
|
|
110
94
|
* conversation is only woken this way ({@link childWakeIn}).
|
|
111
95
|
*
|
|
112
|
-
*
|
|
113
|
-
*
|
|
114
|
-
*
|
|
115
|
-
* 再按常规判断是否起新一轮,并删除它的 checkpoint thread。会话因此不会永远卡在「有运行」。
|
|
116
|
-
* 除了写入消息后,打开 SSE、读事件、运行结束后也会调用它(幂等),孤儿因此在有人查看时就被清理。
|
|
96
|
+
* An active run (queued, running or waiting) is left alone: a run nobody runs any more is the
|
|
97
|
+
* `run-orphan-check`'s to repair (transactional outbox spec §6), never this call's, and reads
|
|
98
|
+
* never call it.
|
|
117
99
|
*
|
|
118
|
-
* agent
|
|
100
|
+
* A conversation whose agent is soft-deleted starts no run.
|
|
119
101
|
*/
|
|
120
102
|
async ensureRun(conversationId,
|
|
121
103
|
/** From the `bg-wake` job: it fired `wakeRetry - 1` times before its window ended. */
|
|
122
104
|
opts = {}) {
|
|
105
|
+
if (this.ensureFails())
|
|
106
|
+
throw new Error("ensureRun failure (test seam)");
|
|
123
107
|
if ((await this.lock.holder(conversationId)) !== null)
|
|
124
108
|
return null;
|
|
125
109
|
const st = {
|
|
126
|
-
orphanSeq: null,
|
|
127
|
-
orphanRunId: null,
|
|
128
110
|
dispatchedSeq: null,
|
|
129
111
|
dispatchedSwitch: false,
|
|
130
112
|
dispatchedUnarchived: false,
|
|
131
113
|
pausedNow: false,
|
|
132
|
-
orphanStop: undefined,
|
|
133
114
|
wakeLater: null,
|
|
134
|
-
repairRunId: null,
|
|
135
115
|
};
|
|
136
116
|
const created = await this.ensureRunIn(conversationId, st);
|
|
137
|
-
if (st.repairRunId !== null) {
|
|
138
|
-
// a sub-agent's run nobody runs any more: requeued when queued, failed when running (its
|
|
139
|
-
// finish then reaches the parent through the wait sweep's undelivered repair)
|
|
140
|
-
await this.repairOrphan(conversationId, st.repairRunId);
|
|
141
|
-
}
|
|
142
117
|
if (st.wakeLater !== null) {
|
|
143
118
|
await this.scheduleWake(conversationId, st.wakeLater, opts.wakeRetry ?? 0);
|
|
144
119
|
}
|
|
145
|
-
const {
|
|
146
|
-
const { dispatchedUnarchived } = st;
|
|
147
|
-
if (orphanSeq !== null) {
|
|
148
|
-
await notifyFinished(this.eventLog, conversationId, orphanSeq, "failed");
|
|
149
|
-
orphanStop?.fire();
|
|
150
|
-
}
|
|
120
|
+
const { dispatchedSeq, dispatchedSwitch, dispatchedUnarchived, pausedNow } = st;
|
|
151
121
|
// a pause is published too (batch 5 review I1): the page shows the reason without a reload
|
|
152
122
|
if (pausedNow)
|
|
153
123
|
await this.eventLog.publishChanged(conversationId, "queue");
|
|
@@ -159,38 +129,7 @@ let RunQueue = class RunQueue {
|
|
|
159
129
|
if (dispatchedUnarchived)
|
|
160
130
|
await this.eventLog.publishChanged(conversationId, "archived");
|
|
161
131
|
}
|
|
162
|
-
|
|
163
|
-
await this.dropCheckpoints(orphanRunId);
|
|
164
|
-
// a lost parent stops its children now, not at the next sweep (review m2)
|
|
165
|
-
if (orphanSeq !== null) {
|
|
166
|
-
await this.events.emit("run.orphaned", { runId: orphanRunId });
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
if (!created)
|
|
170
|
-
return null;
|
|
171
|
-
const { runId, jobId, kind } = created;
|
|
172
|
-
try {
|
|
173
|
-
const queue = this.jobs.queue(runQueueNameFor(kind));
|
|
174
|
-
await this.jobs.enqueue({ runId, conversationId }, jobId, kind);
|
|
175
|
-
// 从 Redis 读回核对:检查与 add 之间出现了同 jobId 的任务时 add 被静默忽略,撤销本次插入,
|
|
176
|
-
// 不留下没有任务的 queued 行。读不到说明任务已被领取并移除(只可能是本次投递的),不撤销。
|
|
177
|
-
const stored = await queue.getJob(jobId);
|
|
178
|
-
if (stored && stored.data.runId !== runId) {
|
|
179
|
-
this.log.warn("run job already queued for this trigger", {
|
|
180
|
-
conversationId,
|
|
181
|
-
runId,
|
|
182
|
-
existingRunId: stored.data.runId,
|
|
183
|
-
});
|
|
184
|
-
await this.deleteQueued(runId);
|
|
185
|
-
return null;
|
|
186
|
-
}
|
|
187
|
-
}
|
|
188
|
-
catch (err) {
|
|
189
|
-
// 投递失败:撤销 queued 行,避免会话卡在「有运行但没有任务」。
|
|
190
|
-
await this.deleteQueued(runId).catch(() => { });
|
|
191
|
-
throw err;
|
|
192
|
-
}
|
|
193
|
-
return runId;
|
|
132
|
+
return created;
|
|
194
133
|
}
|
|
195
134
|
/** {@link ensureRun}'s transaction: what it decided goes back through `st`. */
|
|
196
135
|
async ensureRunIn(conversationId, st) {
|
|
@@ -232,20 +171,14 @@ let RunQueue = class RunQueue {
|
|
|
232
171
|
seq: newest.seq,
|
|
233
172
|
accountId: newest.payload.authorAccountId,
|
|
234
173
|
};
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
//
|
|
241
|
-
//
|
|
242
|
-
|
|
243
|
-
}
|
|
244
|
-
else if (!last || Number(last.seq) <= Number(conv.consumed_seq)) {
|
|
245
|
-
// Nothing unconsumed: the queue's head goes out when the conversation is idle (no active
|
|
246
|
-
// run, which includes runs waiting for approval or an answer) and the queue is not paused.
|
|
247
|
-
// Interjections are always unconsumed messages, so the queue never jumps ahead of them.
|
|
248
|
-
if (conv.queue_paused || idleActive)
|
|
174
|
+
// an active run (one waiting for approval or an answer too) takes what arrives at its next
|
|
175
|
+
// step or is followed by the next run after its end
|
|
176
|
+
if (await this.activeRun(t, conversationId))
|
|
177
|
+
return null;
|
|
178
|
+
if (!last || Number(last.seq) <= Number(conv.consumed_seq)) {
|
|
179
|
+
// Nothing unconsumed: the queue's head goes out when the queue is not paused. Interjections
|
|
180
|
+
// are always unconsumed messages, so the queue never jumps ahead of them.
|
|
181
|
+
if (conv.queue_paused)
|
|
249
182
|
return null;
|
|
250
183
|
const owner = await t
|
|
251
184
|
.selectFrom("agents")
|
|
@@ -302,34 +235,6 @@ let RunQueue = class RunQueue {
|
|
|
302
235
|
isWake = true;
|
|
303
236
|
}
|
|
304
237
|
}
|
|
305
|
-
const active = await this.activeRun(t, conversationId);
|
|
306
|
-
if (active) {
|
|
307
|
-
if (!(await this.isOrphan(conversationId, active, conv.kind)))
|
|
308
|
-
return null;
|
|
309
|
-
this.log.warn("marking orphaned run as failed", { conversationId, runId: active.id });
|
|
310
|
-
st.orphanRunId = active.id;
|
|
311
|
-
const stop = this.stopOpenCalls(active.id);
|
|
312
|
-
st.orphanStop = stop;
|
|
313
|
-
st.orphanSeq = await this.finisher.finishIn({
|
|
314
|
-
conversationId,
|
|
315
|
-
runId: active.id,
|
|
316
|
-
status: "failed",
|
|
317
|
-
consumed: null,
|
|
318
|
-
...(stop.beforeTerminal !== undefined && { beforeTerminal: stop.beforeTerminal }),
|
|
319
|
-
event: {
|
|
320
|
-
type: "run_failed",
|
|
321
|
-
payload: {
|
|
322
|
-
error: { code: "run.orphaned", message: "The run was lost and will not continue" },
|
|
323
|
-
},
|
|
324
|
-
},
|
|
325
|
-
});
|
|
326
|
-
// 孤儿运行的触发消息已被消费:之后没有更新的消息就不起新一轮。
|
|
327
|
-
if (Number(last.seq) <= Math.max(Number(conv.consumed_seq), Number(active.trigger_seq))) {
|
|
328
|
-
return null;
|
|
329
|
-
}
|
|
330
|
-
}
|
|
331
|
-
// what is consumed now (an orphan's failure consumed up to its trigger)
|
|
332
|
-
const consumedNow = Math.max(Number(conv.consumed_seq), st.orphanRunId !== null && active ? Number(active.trigger_seq) : 0);
|
|
333
238
|
// phase D run boundary: the first unconsumed fork invocation is the trigger of the run that
|
|
334
239
|
// absorbs it (`forkBoundary`); later messages wait for the next run
|
|
335
240
|
const fork = await t
|
|
@@ -337,7 +242,7 @@ let RunQueue = class RunQueue {
|
|
|
337
242
|
.select(["seq", "payload"])
|
|
338
243
|
.where("conversation_id", "=", conversationId)
|
|
339
244
|
.where("type", "=", "user_message")
|
|
340
|
-
.where("seq", ">",
|
|
245
|
+
.where("seq", ">", conv.consumed_seq)
|
|
341
246
|
.where(sql `payload -> 'skill' ->> 'mode' = 'fork'`)
|
|
342
247
|
.orderBy("seq")
|
|
343
248
|
.limit(1)
|
|
@@ -345,18 +250,7 @@ let RunQueue = class RunQueue {
|
|
|
345
250
|
const trigger = fork
|
|
346
251
|
? { seq: fork.seq, accountId: fork.payload.authorAccountId }
|
|
347
252
|
: last;
|
|
348
|
-
|
|
349
|
-
// 先显式查同一触发点的任务;仍存在(例如属于刚结束、尚未从队列移除的运行)就不插入 runs 行。
|
|
350
|
-
const jobId = runJobId(conversationId, Number(trigger.seq));
|
|
351
|
-
const existing = await this.jobs.getJob(jobId);
|
|
352
|
-
if (existing) {
|
|
353
|
-
this.log.warn("run job for this trigger already exists; not enqueuing", {
|
|
354
|
-
conversationId,
|
|
355
|
-
jobId,
|
|
356
|
-
existingRunId: existing.data?.runId,
|
|
357
|
-
});
|
|
358
|
-
return null;
|
|
359
|
-
}
|
|
253
|
+
const jobId = await this.jobIdFor(conversationId, Number(trigger.seq));
|
|
360
254
|
const agent = await t
|
|
361
255
|
.selectFrom("agents")
|
|
362
256
|
.select(["current_version", "deleted_at"])
|
|
@@ -366,6 +260,8 @@ let RunQueue = class RunQueue {
|
|
|
366
260
|
if (agent.deleted_at !== null)
|
|
367
261
|
return null;
|
|
368
262
|
const runId = randomUUID();
|
|
263
|
+
if (!(await this.oweJob(conversationId, runId, jobId.id, "chat", conv.org_id)))
|
|
264
|
+
return null;
|
|
369
265
|
// scheduled tasks spec §4: a task's message makes the run unattended; a wake inherits it
|
|
370
266
|
// from the run that produced its event; a person's message never does
|
|
371
267
|
const taskRunId = await scheduledTaskRunAt(t, conversationId, trigger.seq);
|
|
@@ -378,6 +274,7 @@ let RunQueue = class RunQueue {
|
|
|
378
274
|
agent_version: agent.current_version,
|
|
379
275
|
trigger_account_id: trigger.accountId,
|
|
380
276
|
trigger_seq: Number(trigger.seq),
|
|
277
|
+
job_id: jobId.recorded,
|
|
381
278
|
// a chat conversation always has its person-triggered run before any wake (a wake event
|
|
382
279
|
// comes from a run of it or of its tree); null would only make the wake run its own window
|
|
383
280
|
budget_run_id: isWake ? await budgetWindowOf(t, conversationId) : null,
|
|
@@ -387,7 +284,41 @@ let RunQueue = class RunQueue {
|
|
|
387
284
|
// the task run its message started follows this run from now on (spec §6)
|
|
388
285
|
if (taskRunId !== null)
|
|
389
286
|
await linkStartedRunIn(t, taskRunId, runId, conversationId);
|
|
390
|
-
return
|
|
287
|
+
return runId;
|
|
288
|
+
}
|
|
289
|
+
/**
|
|
290
|
+
* The job id of a new run for `seq` (in {@link ensureRunIn}'s transaction, read from PG only):
|
|
291
|
+
* the trigger's `runJobId` for its first run; a later run of the same trigger (a run that hit its
|
|
292
|
+
* deadline leaves the messages it did not absorb unconsumed, R34) gets `<runJobId>-<n>`,
|
|
293
|
+
* recorded in `runs.job_id`, so its job is never taken for the earlier run's (BullMQ ignores an
|
|
294
|
+
* add under the id of a job it still holds).
|
|
295
|
+
*/
|
|
296
|
+
async jobIdFor(conversationId, seq) {
|
|
297
|
+
const prior = await this.db
|
|
298
|
+
.selectFrom("runs")
|
|
299
|
+
.select(sql `count(*)`.as("n"))
|
|
300
|
+
.where("conversation_id", "=", conversationId)
|
|
301
|
+
.where("trigger_seq", "=", String(seq))
|
|
302
|
+
.executeTakeFirst();
|
|
303
|
+
const n = Number(prior?.n ?? 0);
|
|
304
|
+
if (n === 0)
|
|
305
|
+
return { id: runJobId(conversationId, seq), recorded: null };
|
|
306
|
+
const id = `${runJobId(conversationId, seq)}-${n + 1}`;
|
|
307
|
+
return { id, recorded: id };
|
|
308
|
+
}
|
|
309
|
+
/**
|
|
310
|
+
* Owes the new run's job in {@link ensureRunIn}'s transaction, before its run row is written.
|
|
311
|
+
* False (logged): the same job id is still owed, so the caller writes no run (a second run of
|
|
312
|
+
* one trigger; the conversation's unique active run makes this a should-not-happen).
|
|
313
|
+
*/
|
|
314
|
+
async oweJob(conversationId, runId, jobId, kind, orgId) {
|
|
315
|
+
if (await this.jobs.owe({ runId, conversationId }, jobId, kind, orgId))
|
|
316
|
+
return true;
|
|
317
|
+
this.log.warn("run job for this trigger already owed; not starting a run", {
|
|
318
|
+
conversationId,
|
|
319
|
+
jobId,
|
|
320
|
+
});
|
|
321
|
+
return false;
|
|
391
322
|
}
|
|
392
323
|
/**
|
|
393
324
|
* {@link ensureRunIn} for a sub-agent's conversation (agent map spec §5.3), under its row lock.
|
|
@@ -400,17 +331,12 @@ let RunQueue = class RunQueue {
|
|
|
400
331
|
* started again only by its parent. Both read the queue and the trigger account's access on the root
|
|
401
332
|
* of its tree; a message needs no wake setting and has no wake window. The run counts against
|
|
402
333
|
* the root's budget window and runs on the sub-agent queue under `runJobId(child, boundary
|
|
403
|
-
* seq)`. An active run is
|
|
404
|
-
* started past.
|
|
334
|
+
* seq)`. An active run is never started past (one nobody runs is the `run-orphan-check`'s).
|
|
405
335
|
*/
|
|
406
336
|
async childWakeIn(conversationId, conv, st) {
|
|
407
337
|
const t = this.db;
|
|
408
|
-
|
|
409
|
-
if (active) {
|
|
410
|
-
if (active.status !== "waiting")
|
|
411
|
-
st.repairRunId = active.id;
|
|
338
|
+
if (await this.activeRun(t, conversationId))
|
|
412
339
|
return null;
|
|
413
|
-
}
|
|
414
340
|
// a stopped sub-agent (its last run `aborted`: a person's or a tree stop) stays stopped: no
|
|
415
341
|
// message, task finish or delivery that reached it before or during the stop wakes it (a
|
|
416
342
|
// descendant's report that landed just before the stop would otherwise restart the stopped
|
|
@@ -433,16 +359,7 @@ let RunQueue = class RunQueue {
|
|
|
433
359
|
st.wakeLater = wake;
|
|
434
360
|
if (wake?.kind !== "wake")
|
|
435
361
|
return null;
|
|
436
|
-
const jobId =
|
|
437
|
-
const existing = await this.jobs.getJob(jobId, runQueueNameFor("subagent"));
|
|
438
|
-
if (existing) {
|
|
439
|
-
this.log.warn("run job for this trigger already exists; not enqueuing", {
|
|
440
|
-
conversationId,
|
|
441
|
-
jobId,
|
|
442
|
-
existingRunId: existing.data?.runId,
|
|
443
|
-
});
|
|
444
|
-
return null;
|
|
445
|
-
}
|
|
362
|
+
const jobId = await this.jobIdFor(conversationId, wake.seq);
|
|
446
363
|
const agent = await t
|
|
447
364
|
.selectFrom("agents")
|
|
448
365
|
.select(["current_version", "deleted_at"])
|
|
@@ -459,6 +376,9 @@ let RunQueue = class RunQueue {
|
|
|
459
376
|
.limit(1)
|
|
460
377
|
.executeTakeFirst();
|
|
461
378
|
const runId = randomUUID();
|
|
379
|
+
if (!(await this.oweJob(conversationId, runId, jobId.id, "subagent", conv.org_id))) {
|
|
380
|
+
return null;
|
|
381
|
+
}
|
|
462
382
|
await t
|
|
463
383
|
.insertInto("runs")
|
|
464
384
|
.values({
|
|
@@ -468,12 +388,13 @@ let RunQueue = class RunQueue {
|
|
|
468
388
|
agent_version: previous?.agent_version ?? agent.current_version,
|
|
469
389
|
trigger_account_id: wake.triggerAccountId,
|
|
470
390
|
trigger_seq: wake.seq,
|
|
391
|
+
job_id: jobId.recorded,
|
|
471
392
|
budget_run_id: await budgetWindowOf(t, rootId),
|
|
472
393
|
// a parent's message or a wake: unattended when the run behind it was (scheduled tasks §4)
|
|
473
394
|
scheduled_task_run_id: await scheduledTaskRunAt(t, conversationId, wake.seq),
|
|
474
395
|
})
|
|
475
396
|
.execute();
|
|
476
|
-
return
|
|
397
|
+
return runId;
|
|
477
398
|
}
|
|
478
399
|
/**
|
|
479
400
|
* The newest unconsumed message of an idle sub-agent (a `SendMessage` its last run did not absorb)
|
|
@@ -512,18 +433,52 @@ let RunQueue = class RunQueue {
|
|
|
512
433
|
* Re-derives the wakes a paused root held for its tree (agent map spec §5.3: a paused root queue
|
|
513
434
|
* holds the wakes of the whole tree): after the root's queue is unpaused, {@link ensureRun} on
|
|
514
435
|
* every sub-agent conversation below it that has an unconsumed wake event or message and no
|
|
515
|
-
* active run (the root itself is the caller's own `ensureRun`). Errors are logged; the
|
|
516
|
-
*
|
|
436
|
+
* active run (the root itself is the caller's own `ensureRun`). Errors are logged; the
|
|
437
|
+
* ensure-run backstop the releasing transaction owed ({@link oweHeldTreeIn}) wakes it later.
|
|
517
438
|
*/
|
|
518
439
|
async wakeHeldTree(rootId) {
|
|
440
|
+
for (const c of await this.heldTree(rootId)) {
|
|
441
|
+
try {
|
|
442
|
+
await this.ensureRun(c.id);
|
|
443
|
+
}
|
|
444
|
+
catch (err) {
|
|
445
|
+
this.log.warn("cannot wake a held sub-agent", { conversationId: c.id, err });
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
/**
|
|
450
|
+
* Owes, in the caller's transaction (the one that unpauses the root's queue or sends its item
|
|
451
|
+
* now), the ensure-run backstop of every sub-agent {@link wakeHeldTree} would wake after the
|
|
452
|
+
* commit, under the sub-agent's newest seq (transactional outbox spec §6): a process that dies
|
|
453
|
+
* before that call leaves no held wake behind. A root whose queue is still paused (or that is
|
|
454
|
+
* deleted) releases nothing and owes nothing.
|
|
455
|
+
*/
|
|
456
|
+
async oweHeldTreeIn(rootId) {
|
|
457
|
+
const root = await this.db
|
|
458
|
+
.selectFrom("conversations")
|
|
459
|
+
.select(["queue_paused", "deleted_at"])
|
|
460
|
+
.where("id", "=", rootId)
|
|
461
|
+
.executeTakeFirst();
|
|
462
|
+
if (!root || root.queue_paused || root.deleted_at !== null)
|
|
463
|
+
return;
|
|
464
|
+
for (const c of await this.heldTree(rootId)) {
|
|
465
|
+
await this.jobs.oweEnsureRun(c.id, c.seq, c.org_id);
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
/**
|
|
469
|
+
* The sub-agent conversations below `rootId` a paused root holds: not deleted, with an
|
|
470
|
+
* unconsumed wake event (a finished background task, a delivery) or message, and no active run;
|
|
471
|
+
* with their org and newest seq. Reads only (no row locks).
|
|
472
|
+
*/
|
|
473
|
+
async heldTree(rootId) {
|
|
519
474
|
const ids = (await conversationTree(this.db, rootId))
|
|
520
475
|
.filter((c) => c.level > 0)
|
|
521
476
|
.map((c) => c.id);
|
|
522
477
|
if (ids.length === 0)
|
|
523
|
-
return;
|
|
524
|
-
|
|
478
|
+
return [];
|
|
479
|
+
return this.db
|
|
525
480
|
.selectFrom("conversations as c")
|
|
526
|
-
.select("c.id")
|
|
481
|
+
.select(["c.id", "c.org_id", "c.last_seq as seq"])
|
|
527
482
|
.where("c.id", "in", ids)
|
|
528
483
|
.where("c.deleted_at", "is", null)
|
|
529
484
|
.where(({ exists, selectFrom }) => exists(selectFrom("conversation_events as e")
|
|
@@ -540,14 +495,6 @@ let RunQueue = class RunQueue {
|
|
|
540
495
|
.whereRef("r.conversation_id", "=", "c.id")
|
|
541
496
|
.where("r.status", "in", ACTIVE_RUN_STATUSES))))
|
|
542
497
|
.execute();
|
|
543
|
-
for (const c of held) {
|
|
544
|
-
try {
|
|
545
|
-
await this.ensureRun(c.id);
|
|
546
|
-
}
|
|
547
|
-
catch (err) {
|
|
548
|
-
this.log.warn("cannot wake a held sub-agent", { conversationId: c.id, err });
|
|
549
|
-
}
|
|
550
|
-
}
|
|
551
498
|
}
|
|
552
499
|
/**
|
|
553
500
|
* Schedules the delayed `bg-wake` job at the end of the conversation's wake window (after the
|
|
@@ -571,31 +518,31 @@ let RunQueue = class RunQueue {
|
|
|
571
518
|
}
|
|
572
519
|
}
|
|
573
520
|
/**
|
|
574
|
-
*
|
|
575
|
-
*
|
|
521
|
+
* Owes the ensure-run backstop of `conversationId` in the caller's transaction
|
|
522
|
+
* ({@link oweEnsureRun}): `seq` is the event the transaction wrote, `orgId` the conversation's.
|
|
576
523
|
*/
|
|
577
|
-
|
|
578
|
-
return this.jobs.
|
|
524
|
+
oweEnsureRun(conversationId, seq, orgId) {
|
|
525
|
+
return this.jobs.oweEnsureRun(conversationId, seq, orgId);
|
|
579
526
|
}
|
|
580
527
|
/**
|
|
581
|
-
*
|
|
582
|
-
*
|
|
583
|
-
*
|
|
584
|
-
*
|
|
528
|
+
* Stops an orphaned run directly (`POST .../abort`, a tree stop): a `running` run whose
|
|
529
|
+
* conversation lock is free and whose BullMQ job is gone hears no stop broadcast, so its
|
|
530
|
+
* `run_aborted` is written here (status aborted, `consumed_seq` at least to its trigger): true. Not
|
|
531
|
+
* an orphan (someone runs it; a queued run's job is owed, or lost and re-added by the
|
|
532
|
+
* `run-orphan-check`, whose worker then sees the stop marker) or already finished: false, and the
|
|
533
|
+
* caller broadcasts the stop.
|
|
585
534
|
*/
|
|
586
535
|
async abortIfOrphaned(conversationId, runId, byAccountId, detail) {
|
|
587
536
|
if ((await this.lock.holder(conversationId)) !== null)
|
|
588
537
|
return false;
|
|
589
|
-
const
|
|
590
|
-
const toSeq = await this.abortOrphanIn(conversationId, runId, byAccountId, detail, stop);
|
|
538
|
+
const toSeq = await this.abortOrphanIn(conversationId, runId, byAccountId, detail);
|
|
591
539
|
if (toSeq === null)
|
|
592
540
|
return false;
|
|
593
541
|
await notifyFinished(this.eventLog, conversationId, toSeq, "aborted");
|
|
594
|
-
stop.fire();
|
|
595
542
|
await this.dropCheckpoints(runId);
|
|
596
543
|
return true;
|
|
597
544
|
}
|
|
598
|
-
async abortOrphanIn(conversationId, runId, byAccountId, detail
|
|
545
|
+
async abortOrphanIn(conversationId, runId, byAccountId, detail) {
|
|
599
546
|
const t = this.db;
|
|
600
547
|
const conv = await t
|
|
601
548
|
.selectFrom("conversations")
|
|
@@ -614,101 +561,32 @@ let RunQueue = class RunQueue {
|
|
|
614
561
|
runId,
|
|
615
562
|
status: "aborted",
|
|
616
563
|
consumed: null,
|
|
617
|
-
beforeTerminal: stop.beforeTerminal,
|
|
618
564
|
event: {
|
|
619
565
|
type: "run_aborted",
|
|
620
566
|
payload: { byAccountId, reason: "user", ...(detail !== undefined && { detail }) },
|
|
621
567
|
},
|
|
622
568
|
});
|
|
623
569
|
}
|
|
624
|
-
/**
|
|
625
|
-
* Repairs an orphaned run of a conversation nobody else repairs on demand (a sub-agent's: no
|
|
626
|
-
* `ensureRun` ever runs for it; sub-agents spec §7.5), with the same orphan test as
|
|
627
|
-
* {@link ensureRun}: the conversation lock is free, the run is still `queued` / `running`, its
|
|
628
|
-
* job is gone, and a `queued` one is past the grace period.
|
|
629
|
-
* - `queued` (the dispatch or resume committed, its enqueue was lost) → enqueued again under its
|
|
630
|
-
* own job id (`runs.job_id`, else the trigger's), so concurrent repairs on several replicas
|
|
631
|
-
* still leave one job: `"requeued"`;
|
|
632
|
-
* - `running` (its worker died without a terminal state and BullMQ dropped the job) → marked
|
|
633
|
-
* failed `run.orphaned`; the caller reports it to the parent: `"failed"`.
|
|
634
|
-
* Not an orphan, or already finished → null.
|
|
635
|
-
*/
|
|
636
|
-
async repairOrphan(conversationId, runId) {
|
|
637
|
-
if ((await this.lock.holder(conversationId)) !== null)
|
|
638
|
-
return null;
|
|
639
|
-
const r = await this.repairOrphanIn(conversationId, runId);
|
|
640
|
-
if (!r)
|
|
641
|
-
return null;
|
|
642
|
-
if (r.kind === "requeued") {
|
|
643
|
-
await this.enqueue({ runId, conversationId }, r.jobId, r.convKind);
|
|
644
|
-
return "requeued";
|
|
645
|
-
}
|
|
646
|
-
if (r.seq !== null)
|
|
647
|
-
await notifyFinished(this.eventLog, conversationId, r.seq, "failed");
|
|
648
|
-
await this.dropCheckpoints(runId);
|
|
649
|
-
return "failed";
|
|
650
|
-
}
|
|
651
|
-
async repairOrphanIn(conversationId, runId) {
|
|
652
|
-
const t = this.db;
|
|
653
|
-
const conv = await t
|
|
654
|
-
.selectFrom("conversations")
|
|
655
|
-
.select("kind")
|
|
656
|
-
.where("id", "=", conversationId)
|
|
657
|
-
.forUpdate()
|
|
658
|
-
.executeTakeFirst();
|
|
659
|
-
const active = await this.activeRun(t, conversationId);
|
|
660
|
-
if (!conv || !active || active.id !== runId)
|
|
661
|
-
return null;
|
|
662
|
-
if (!(await this.isOrphan(conversationId, active, conv.kind)))
|
|
663
|
-
return null;
|
|
664
|
-
if (active.status === "queued") {
|
|
665
|
-
const jobId = active.job_id ?? runJobId(conversationId, Number(active.trigger_seq));
|
|
666
|
-
return { kind: "requeued", jobId, convKind: conv.kind };
|
|
667
|
-
}
|
|
668
|
-
this.log.warn("marking orphaned run as failed", { conversationId, runId });
|
|
669
|
-
const seq = await this.finisher.finishIn({
|
|
670
|
-
conversationId,
|
|
671
|
-
runId,
|
|
672
|
-
status: "failed",
|
|
673
|
-
consumed: null,
|
|
674
|
-
event: {
|
|
675
|
-
type: "run_failed",
|
|
676
|
-
payload: {
|
|
677
|
-
error: { code: "run.orphaned", message: "The run was lost and will not continue" },
|
|
678
|
-
},
|
|
679
|
-
},
|
|
680
|
-
});
|
|
681
|
-
return { kind: "failed", seq };
|
|
682
|
-
}
|
|
683
570
|
/** 会话当前未结束的运行(调用方已持有会话行锁)。 */
|
|
684
571
|
activeRun(t, conversationId) {
|
|
685
572
|
return t
|
|
686
573
|
.selectFrom("runs")
|
|
687
|
-
.select([
|
|
688
|
-
"id",
|
|
689
|
-
"status",
|
|
690
|
-
"trigger_seq",
|
|
691
|
-
"trigger_account_id",
|
|
692
|
-
"job_id",
|
|
693
|
-
sql `COALESCE(queued_at, created_at) < now() - make_interval(secs => ${ORPHAN_QUEUED_GRACE_SECONDS})`.as("past_grace"),
|
|
694
|
-
])
|
|
574
|
+
.select(["id", "status", "trigger_seq", "job_id"])
|
|
695
575
|
.where("conversation_id", "=", conversationId)
|
|
696
576
|
.where("status", "in", ACTIVE_RUN_STATUSES)
|
|
697
577
|
.executeTakeFirst();
|
|
698
578
|
}
|
|
699
579
|
/**
|
|
700
|
-
*
|
|
701
|
-
*
|
|
702
|
-
* `
|
|
580
|
+
* Whether an active run is an orphan for {@link abortIfOrphaned} (the caller saw the conversation
|
|
581
|
+
* lock free): a `running` one whose BullMQ job (a resume's under `runs.job_id`) is gone. A
|
|
582
|
+
* `waiting` one never is (no job by design), nor a `queued` one (its job is owed in the
|
|
583
|
+
* transaction that queued it).
|
|
703
584
|
*/
|
|
704
585
|
async isOrphan(conversationId, active, kind) {
|
|
705
|
-
if (active.status
|
|
706
|
-
return false;
|
|
707
|
-
if (active.status === "queued" && !active.past_grace)
|
|
586
|
+
if (active.status !== "running")
|
|
708
587
|
return false;
|
|
709
588
|
const jobId = active.job_id ?? runJobId(conversationId, Number(active.trigger_seq));
|
|
710
|
-
|
|
711
|
-
return !job;
|
|
589
|
+
return (await this.jobs.getJob(jobId, runQueueNameFor(kind))) === undefined;
|
|
712
590
|
}
|
|
713
591
|
/** 删除已结束运行的 checkpoint thread(失败只记日志)。 */
|
|
714
592
|
async dropCheckpoints(runId) {
|
|
@@ -716,31 +594,16 @@ let RunQueue = class RunQueue {
|
|
|
716
594
|
.deleteThread(runId)
|
|
717
595
|
.catch((err) => this.log.warn("checkpoint delete failed", { runId, err }));
|
|
718
596
|
}
|
|
719
|
-
async deleteQueued(runId) {
|
|
720
|
-
const gone = await this.db
|
|
721
|
-
.deleteFrom("runs")
|
|
722
|
-
.where("id", "=", runId)
|
|
723
|
-
.where("status", "=", "queued")
|
|
724
|
-
.returning("scheduled_task_run_id")
|
|
725
|
-
.executeTakeFirst();
|
|
726
|
-
// its scheduled task run waits for the message's next run again
|
|
727
|
-
if (gone?.scheduled_task_run_id != null) {
|
|
728
|
-
await unlinkRunIn(this.db, gone.scheduled_task_run_id, runId);
|
|
729
|
-
}
|
|
730
|
-
}
|
|
731
597
|
};
|
|
732
598
|
__decorate([
|
|
733
599
|
Transactional({ propagation: "requires_new" })
|
|
734
600
|
], RunQueue.prototype, "ensureRunIn", null);
|
|
735
601
|
__decorate([
|
|
736
|
-
Transactional({ propagation: "
|
|
737
|
-
], RunQueue.prototype, "
|
|
602
|
+
Transactional({ propagation: "mandatory" })
|
|
603
|
+
], RunQueue.prototype, "oweHeldTreeIn", null);
|
|
738
604
|
__decorate([
|
|
739
605
|
Transactional({ propagation: "requires_new" })
|
|
740
|
-
], RunQueue.prototype, "
|
|
741
|
-
__decorate([
|
|
742
|
-
Transactional()
|
|
743
|
-
], RunQueue.prototype, "deleteQueued", null);
|
|
606
|
+
], RunQueue.prototype, "abortOrphanIn", null);
|
|
744
607
|
RunQueue = __decorate([
|
|
745
608
|
Injectable(),
|
|
746
609
|
__param(0, Inject(KernelDb)),
|
|
@@ -749,12 +612,12 @@ RunQueue = __decorate([
|
|
|
749
612
|
__param(3, Inject(RunJobs)),
|
|
750
613
|
__param(4, Inject(RunFinish)),
|
|
751
614
|
__param(5, Inject(KyselyCheckpointSaver)),
|
|
752
|
-
__param(6, Inject(
|
|
753
|
-
__param(7, Inject(
|
|
754
|
-
__param(8, Inject(
|
|
755
|
-
__param(9, Inject(
|
|
756
|
-
__param(10,
|
|
757
|
-
__param(
|
|
615
|
+
__param(6, Inject(KERNEL_LOGGER)),
|
|
616
|
+
__param(7, Inject(TransactionHost)),
|
|
617
|
+
__param(8, Inject(KernelConfigService)),
|
|
618
|
+
__param(9, Inject(getQueueToken(MAINTENANCE_QUEUE_NAME))),
|
|
619
|
+
__param(10, Optional()),
|
|
620
|
+
__param(10, Inject(ENSURE_RUN_FAILS))
|
|
758
621
|
], RunQueue);
|
|
759
622
|
export { RunQueue };
|
|
760
623
|
/** The conversation's owner is active and can still see its agent (`isAccessible`'s rule). */
|