@capybara-hq/kernel 0.1.0-rc.23 → 0.1.0-rc.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bootstrap.json +25 -25
- package/dist/admin/admin.module.d.ts +2 -1
- package/dist/admin/admin.module.js +7 -3
- package/dist/admin/outbox-dead.controller.d.ts +23 -0
- package/dist/admin/outbox-dead.controller.js +62 -0
- package/dist/admin/outbox-dead.schemas.d.ts +8 -0
- package/dist/admin/outbox-dead.schemas.js +18 -0
- package/dist/admin/outbox-dead.service.d.ts +66 -0
- package/dist/admin/outbox-dead.service.js +179 -0
- package/dist/admin/overview.service.d.ts +40 -5
- package/dist/admin/overview.service.js +41 -7
- package/dist/agents/from-template.js +1 -1
- package/dist/background/finish.d.ts +5 -2
- package/dist/background/finish.js +14 -3
- package/dist/background/launch.d.ts +2 -1
- package/dist/background/launch.js +5 -2
- package/dist/background/store.d.ts +8 -0
- package/dist/background/store.js +24 -0
- package/dist/catalog/admin-catalog.controller.d.ts +21 -2
- package/dist/catalog/admin-catalog.controller.js +45 -6
- package/dist/catalog/admin-catalog.schemas.d.ts +5 -0
- package/dist/catalog/admin-catalog.schemas.js +8 -0
- package/dist/catalog/catalog.module.d.ts +2 -1
- package/dist/catalog/catalog.module.js +5 -1
- package/dist/catalog/marketplaces.service.d.ts +1 -1
- package/dist/catalog/platform-import-update.d.ts +50 -0
- package/dist/catalog/platform-import-update.js +116 -0
- package/dist/catalog/plugin-import.d.ts +36 -1
- package/dist/catalog/plugin-import.js +23 -1
- package/dist/catalog/plugin-upgrade.d.ts +18 -0
- package/dist/catalog/plugin-upgrade.js +19 -0
- package/dist/channels/channels.module.d.ts +5 -2
- package/dist/channels/channels.module.js +9 -2
- package/dist/channels/deliver-job.d.ts +25 -0
- package/dist/channels/deliver-job.js +25 -0
- package/dist/channels/inbound.d.ts +1 -1
- package/dist/channels/inbound.js +2 -15
- package/dist/channels/jobs.d.ts +28 -18
- package/dist/channels/jobs.js +63 -26
- package/dist/config/schema.d.ts +0 -3
- package/dist/config/schema.js +11 -20
- package/dist/conversations/agent-tree.d.ts +16 -0
- package/dist/conversations/agent-tree.js +36 -0
- package/dist/conversations/conversations.service.d.ts +44 -14
- package/dist/conversations/conversations.service.js +52 -38
- package/dist/conversations/event-log.d.ts +7 -10
- package/dist/conversations/event-log.js +24 -15
- package/dist/conversations/events.js +2 -1
- package/dist/conversations/queue.service.d.ts +8 -3
- package/dist/conversations/queue.service.js +37 -13
- package/dist/conversations/titles.d.ts +21 -8
- package/dist/conversations/titles.js +42 -15
- package/dist/db/kernel-schema.d.ts +26 -8
- package/dist/devices/devices.service.d.ts +12 -3
- package/dist/devices/devices.service.js +18 -3
- package/dist/errors.d.ts +11 -0
- package/dist/errors.js +11 -0
- package/dist/evals/run-replay.js +14 -8
- package/dist/extensions/extensions.module.js +3 -0
- package/dist/extensions/lifecycle.d.ts +6 -1
- package/dist/extensions/lifecycle.js +9 -2
- package/dist/extensions/usage-subscribers.d.ts +43 -0
- package/dist/extensions/usage-subscribers.js +108 -0
- package/dist/http/api.module.js +2 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/infra/cli/kernel-cli.module.js +5 -0
- package/dist/infra/events/kernel-events.d.ts +9 -35
- package/dist/infra/outbox/outbox-relay.module.d.ts +6 -0
- package/dist/infra/outbox/outbox-relay.module.js +18 -0
- package/dist/infra/outbox/outbox.dispatcher.d.ts +103 -0
- package/dist/infra/outbox/outbox.dispatcher.js +310 -0
- package/dist/infra/outbox/outbox.module.d.ts +24 -0
- package/dist/infra/outbox/outbox.module.js +50 -0
- package/dist/infra/outbox/outbox.relay.d.ts +31 -0
- package/dist/infra/outbox/outbox.relay.js +76 -0
- package/dist/infra/outbox/outbox.service.d.ts +51 -0
- package/dist/infra/outbox/outbox.service.js +131 -0
- package/dist/infra/outbox/registry.d.ts +120 -0
- package/dist/infra/outbox/registry.js +108 -0
- package/dist/infra/queue/queue-names.d.ts +8 -0
- package/dist/infra/queue/queue-names.js +16 -0
- package/dist/kernel.module.js +3 -0
- package/dist/local-devices/device-policy.service.d.ts +4 -0
- package/dist/local-devices/device-policy.service.js +61 -3
- package/dist/local-devices/machines.d.ts +13 -5
- package/dist/local-devices/machines.js +45 -7
- package/dist/local-devices/policy-summary.d.ts +44 -1
- package/dist/local-devices/policy-summary.js +69 -0
- package/dist/models/embeddings.d.ts +9 -9
- package/dist/models/embeddings.js +11 -18
- package/dist/models/models.module.js +3 -0
- package/dist/orgs/orgs.service.js +7 -0
- package/dist/overrides.d.ts +20 -3
- package/dist/overrides.js +10 -2
- package/dist/platform/exclusions.js +5 -0
- package/dist/platform/ops/admin.ops.js +18 -2
- package/dist/platform/ops/catalog-admin.ops.d.ts +3 -1
- package/dist/platform/ops/catalog-admin.ops.js +32 -4
- package/dist/platform/ops/outbox.ops.d.ts +21 -0
- package/dist/platform/ops/outbox.ops.js +125 -0
- package/dist/platform/ops/schedules.ops.js +40 -1
- package/dist/platform/platform-ops.module.d.ts +2 -1
- package/dist/platform/platform-ops.module.js +3 -0
- package/dist/platform/route-schemas.js +3 -0
- package/dist/roles/worker.module.js +9 -4
- package/dist/runtime/langgraph-loop.js +13 -5
- package/dist/runtime/limits.d.ts +15 -0
- package/dist/runtime/limits.js +25 -0
- package/dist/runtime/loop.d.ts +8 -3
- package/dist/runtime/loop.js +2 -1
- package/dist/runtime/maintenance.d.ts +4 -2
- package/dist/runtime/maintenance.js +17 -2
- package/dist/runtime/model-call.d.ts +14 -16
- package/dist/runtime/model-call.js +14 -24
- package/dist/runtime/queue-lost-jobs.d.ts +24 -0
- package/dist/runtime/queue-lost-jobs.js +51 -0
- package/dist/runtime/run-control.d.ts +42 -5
- package/dist/runtime/run-control.js +66 -12
- package/dist/runtime/run-finish.d.ts +39 -12
- package/dist/runtime/run-finish.js +97 -14
- package/dist/runtime/run-jobs.d.ts +95 -13
- package/dist/runtime/run-jobs.js +199 -28
- package/dist/runtime/run-orphan-check.job.d.ts +142 -0
- package/dist/runtime/run-orphan-check.job.js +339 -0
- package/dist/runtime/run-processors.d.ts +16 -9
- package/dist/runtime/run-processors.js +1 -1
- package/dist/runtime/run-queue.d.ts +64 -55
- package/dist/runtime/run-queue.js +147 -284
- package/dist/runtime/run-steps.d.ts +1 -2
- package/dist/runtime/run-steps.js +8 -11
- package/dist/runtime/run-worker.d.ts +6 -17
- package/dist/runtime/run-worker.js +32 -34
- package/dist/runtime/runtime.module.d.ts +2 -3
- package/dist/runtime/runtime.module.js +5 -6
- package/dist/runtime/step-log.d.ts +16 -4
- package/dist/runtime/step-log.js +26 -6
- package/dist/runtime/stream.controller.d.ts +2 -4
- package/dist/runtime/stream.controller.js +5 -11
- package/dist/runtime/subagents/child.js +1 -1
- package/dist/runtime/subagents/definition.d.ts +2 -1
- package/dist/runtime/subagents/definition.js +4 -2
- package/dist/runtime/subagents/deliveries.d.ts +6 -3
- package/dist/runtime/subagents/deliveries.js +16 -7
- package/dist/runtime/subagents/dispatch.d.ts +12 -12
- package/dist/runtime/subagents/dispatch.js +13 -38
- package/dist/runtime/subagents/messages.d.ts +5 -1
- package/dist/runtime/subagents/messages.js +11 -3
- package/dist/runtime/subagents/skill-fork.d.ts +3 -3
- package/dist/runtime/subagents/skill-fork.js +5 -5
- package/dist/runtime/subagents/subagents.service.js +1 -1
- package/dist/runtime/subagents/tree-jobs.d.ts +61 -0
- package/dist/runtime/subagents/tree-jobs.js +86 -0
- package/dist/runtime/usage-report-jobs.d.ts +23 -48
- package/dist/runtime/usage-report-jobs.js +67 -119
- package/dist/runtime/usage-report.d.ts +28 -77
- package/dist/runtime/usage-report.js +35 -213
- package/dist/runtime/waits.d.ts +48 -198
- package/dist/runtime/waits.js +103 -805
- package/dist/sandbox/sandbox.service.d.ts +17 -4
- package/dist/sandbox/sandbox.service.js +55 -14
- package/dist/schedules/calendar.d.ts +134 -0
- package/dist/schedules/calendar.js +410 -0
- package/dist/schedules/fire.d.ts +1 -3
- package/dist/schedules/fire.js +5 -10
- package/dist/schedules/run-tracker.d.ts +0 -5
- package/dist/schedules/run-tracker.js +0 -13
- package/dist/schedules/scan.job.js +15 -20
- package/dist/schedules/schedule-math.d.ts +42 -0
- package/dist/schedules/schedule-math.js +52 -0
- package/dist/schedules/scheduled-tasks.service.d.ts +16 -0
- package/dist/schedules/scheduled-tasks.service.js +53 -0
- package/dist/schedules/schedules.controller.d.ts +5 -1
- package/dist/schedules/schedules.controller.js +15 -1
- package/dist/schedules/schedules.schemas.d.ts +35 -0
- package/dist/schedules/schedules.schemas.js +27 -0
- package/dist/schedules/store.d.ts +5 -0
- package/dist/schedules/store.js +20 -0
- package/dist/tools/approvals.service.d.ts +26 -7
- package/dist/tools/approvals.service.js +88 -10
- package/dist/tools/builtin-names.d.ts +5 -3
- package/dist/tools/builtin-names.js +12 -4
- package/dist/tools/builtins/builtin-tools.d.ts +1 -2
- package/dist/tools/builtins/builtin-tools.js +21 -17
- package/dist/tools/builtins/sandbox/bash.js +9 -3
- package/dist/tools/builtins/sandbox/common.d.ts +18 -1
- package/dist/tools/builtins/sandbox/common.js +14 -1
- package/dist/tools/builtins/sandbox/index.d.ts +2 -1
- package/dist/tools/builtins/sandbox/index.js +4 -1
- package/dist/tools/builtins/sandbox/task-list.d.ts +12 -0
- package/dist/tools/builtins/sandbox/task-list.js +130 -0
- package/dist/tools/builtins/sandbox/task-output.d.ts +8 -1
- package/dist/tools/builtins/sandbox/task-output.js +21 -3
- package/dist/tools/builtins/web/extract.d.ts +2 -4
- package/dist/tools/builtins/web/extract.js +2 -5
- package/dist/tools/pipeline.d.ts +1 -1
- package/dist/tools/pipeline.js +1 -1
- package/dist/tools/questions.service.d.ts +15 -6
- package/dist/tools/questions.service.js +20 -8
- package/dist/tools/registry.d.ts +4 -2
- package/dist/tools/registry.js +4 -3
- package/dist/tools/tools.module.js +4 -3
- package/package.json +3 -3
- package/src/migrations/0118_device_policy_ops.sql +6 -0
- package/src/migrations/0119_outbox.sql +22 -0
- package/src/migrations/0120_drop_usage_report_sweep.sql +7 -0
|
@@ -16,7 +16,9 @@ export declare const MAINTENANCE_CONCURRENCY = 5;
|
|
|
16
16
|
* The one consumer of `cap-kernel-maintenance` (decision R11 of the Nest kernel spec): dispatches
|
|
17
17
|
* each one-off job by name to its domain (`wait-timeout` → {@link Waits}, `sandbox-destroy-for` and
|
|
18
18
|
* `sandbox-reset-files` → {@link SandboxService}, `drift-connector` → {@link ToolDrift}, `directory-sync` →
|
|
19
|
-
* {@link DirectorySync}, `bg-wake` → {@link RunQueue.ensureRun}, `
|
|
19
|
+
* {@link DirectorySync}, `bg-wake` and `ensure-run` → {@link RunQueue.ensureRun}, `stop-tree` →
|
|
20
|
+
* {@link processStopTree} and `deliver-to-parent` → {@link processDeliverToParent} (both in
|
|
21
|
+
* runtime/subagents/tree-jobs.ts, over {@link Waits}), `owned-plugin-cleanup` →
|
|
20
22
|
* {@link OwnedPluginCleanup}, `sched-fire` → {@link ScheduledTaskFirer}). The name of a periodic job (a `@LockedCron` / `@LockedInterval` job,
|
|
21
23
|
* such as `sandbox-suspend-idle`) runs that job's body once ({@link ScheduledRuns.runNow}), as the
|
|
22
24
|
* maintenance handlers did when those jobs were schedulers on this queue: an operator's or a
|
|
@@ -34,7 +36,7 @@ export declare class MaintenanceProcessor extends WorkerHost<Worker> {
|
|
|
34
36
|
private readonly scheduled;
|
|
35
37
|
private readonly log;
|
|
36
38
|
private readonly queue;
|
|
37
|
-
constructor(waits: Pick<Waits, "processTimeout">, sandbox: Pick<SandboxService, "destroyDevice" | "resetFileSystem">, drift: Pick<ToolDrift, "check">, directory: Pick<DirectorySync, "runSyncJob">, runs: Pick<RunQueue, "ensureRun">, ownedCleanup: Pick<OwnedPluginCleanup, "run">, scheduledTasks: Pick<ScheduledTaskFirer, "fire">, scheduled: Pick<ScheduledRuns, "runNow">, log: Pick<KernelLogger, "error" | "warn">, queue: Pick<Queue, "removeJobScheduler">);
|
|
39
|
+
constructor(waits: Pick<Waits, "processTimeout" | "childFinished" | "stopChildren" | "stopDescendantsOfStopped" | "stopTree">, sandbox: Pick<SandboxService, "destroyDevice" | "resetFileSystem">, drift: Pick<ToolDrift, "check">, directory: Pick<DirectorySync, "runSyncJob">, runs: Pick<RunQueue, "ensureRun">, ownedCleanup: Pick<OwnedPluginCleanup, "run">, scheduledTasks: Pick<ScheduledTaskFirer, "fire">, scheduled: Pick<ScheduledRuns, "runNow">, log: Pick<KernelLogger, "error" | "warn">, queue: Pick<Queue, "removeJobScheduler">);
|
|
38
40
|
/**
|
|
39
41
|
* Removes the retired job schedulers (`RETIRED_SCHEDULER_IDS`, runtime/retired-schedulers.ts): every fixed-period
|
|
40
42
|
* maintenance job is a `@LockedCron` / `@LockedInterval` now. Idempotent; called once when a
|
|
@@ -31,8 +31,9 @@ import { DRIFT_CONNECTOR_JOB } from "../tools/job-ids.js";
|
|
|
31
31
|
import { ToolsModule } from "../tools/tools.module.js";
|
|
32
32
|
import { MAINTENANCE_QUEUE_NAME } from "./maintenance-queue.js";
|
|
33
33
|
import { RETIRED_SCHEDULER_IDS } from "./retired-schedulers.js";
|
|
34
|
-
import { RunQueue } from "./run-queue.js";
|
|
34
|
+
import { ENSURE_RUN_JOB, RunQueue } from "./run-queue.js";
|
|
35
35
|
import { RuntimeModule } from "./runtime.module.js";
|
|
36
|
+
import { DELIVER_TO_PARENT_JOB, processDeliverToParent, processStopTree, STOP_TREE_JOB, } from "./subagents/tree-jobs.js";
|
|
36
37
|
import { WAIT_TIMEOUT_JOB, Waits } from "./waits.js";
|
|
37
38
|
export { MAINTENANCE_QUEUE_NAME } from "./maintenance-queue.js";
|
|
38
39
|
/** The maintenance worker's concurrency (as the hand-made worker had). */
|
|
@@ -41,7 +42,9 @@ export const MAINTENANCE_CONCURRENCY = 5;
|
|
|
41
42
|
* The one consumer of `cap-kernel-maintenance` (decision R11 of the Nest kernel spec): dispatches
|
|
42
43
|
* each one-off job by name to its domain (`wait-timeout` → {@link Waits}, `sandbox-destroy-for` and
|
|
43
44
|
* `sandbox-reset-files` → {@link SandboxService}, `drift-connector` → {@link ToolDrift}, `directory-sync` →
|
|
44
|
-
* {@link DirectorySync}, `bg-wake` → {@link RunQueue.ensureRun}, `
|
|
45
|
+
* {@link DirectorySync}, `bg-wake` and `ensure-run` → {@link RunQueue.ensureRun}, `stop-tree` →
|
|
46
|
+
* {@link processStopTree} and `deliver-to-parent` → {@link processDeliverToParent} (both in
|
|
47
|
+
* runtime/subagents/tree-jobs.ts, over {@link Waits}), `owned-plugin-cleanup` →
|
|
45
48
|
* {@link OwnedPluginCleanup}, `sched-fire` → {@link ScheduledTaskFirer}). The name of a periodic job (a `@LockedCron` / `@LockedInterval` job,
|
|
46
49
|
* such as `sandbox-suspend-idle`) runs that job's body once ({@link ScheduledRuns.runNow}), as the
|
|
47
50
|
* maintenance handlers did when those jobs were schedulers on this queue: an operator's or a
|
|
@@ -113,6 +116,18 @@ let MaintenanceProcessor = class MaintenanceProcessor extends WorkerHost {
|
|
|
113
116
|
await this.runs.ensureRun(conversationId, { wakeRetry: (retry ?? 0) + 1 });
|
|
114
117
|
return;
|
|
115
118
|
}
|
|
119
|
+
// the ensure-run backstop of a message, wake or run end (transactional outbox spec §6)
|
|
120
|
+
case ENSURE_RUN_JOB:
|
|
121
|
+
await this.runs.ensureRun(job.data.conversationId);
|
|
122
|
+
return;
|
|
123
|
+
// the stops a run's end or a root's delete owes its sub-agent tree (outbox spec §6)
|
|
124
|
+
case STOP_TREE_JOB:
|
|
125
|
+
await processStopTree(this.waits, job.data);
|
|
126
|
+
return;
|
|
127
|
+
// a finished sub-agent run's delivery to its parent (transactional outbox spec §6)
|
|
128
|
+
case DELIVER_TO_PARENT_JOB:
|
|
129
|
+
await processDeliverToParent(this.waits, job.data);
|
|
130
|
+
return;
|
|
116
131
|
// a removed org-owned plugin's extension (plugin scope phase 3, P3-20)
|
|
117
132
|
case OWNED_PLUGIN_CLEANUP_JOB:
|
|
118
133
|
await this.ownedCleanup.run(job.data.itemId);
|
|
@@ -11,7 +11,6 @@ import type { SubagentParent } from "./loop.js";
|
|
|
11
11
|
import { MediaCache } from "./media-cache.js";
|
|
12
12
|
import { type BoundProvider, type ProviderRegistry } from "./providers.js";
|
|
13
13
|
import type { StepLog } from "./step-log.js";
|
|
14
|
-
import type { UsageReporter } from "./usage-report.js";
|
|
15
14
|
/** 单轮收尾的尾部提醒(模型可见,英文;spec 6 轮次额度收尾)。 */
|
|
16
15
|
export declare const WRAP_UP_REMINDER = "Step limit reached. Do not call tools. Summarize what you found and what remains for the user.";
|
|
17
16
|
/**
|
|
@@ -48,8 +47,11 @@ export type StreamSink = (delta: {
|
|
|
48
47
|
export interface ModelStepDeps {
|
|
49
48
|
db: KernelSchemaDb;
|
|
50
49
|
eventLog: EventLog;
|
|
51
|
-
/**
|
|
52
|
-
|
|
50
|
+
/**
|
|
51
|
+
* The step's transactional writes (a model call with its tool specs, a denial with its audit row,
|
|
52
|
+
* usage events with their owed `usage.report` deliveries, a sub-agent's counted usage).
|
|
53
|
+
*/
|
|
54
|
+
stepLog: Pick<StepLog, "appendCounted" | "appendReported" | "appendAudited" | "appendWithSpecs">;
|
|
53
55
|
registry: Pick<ProviderRegistry, "resolve">;
|
|
54
56
|
/** 工具注册表:每步开始时解析 agent 引用的工具(状态可能在步骤之间变化)。 */
|
|
55
57
|
tools: Pick<ToolRegistry, "forAgent">;
|
|
@@ -67,11 +69,6 @@ export interface ModelStepDeps {
|
|
|
67
69
|
* `conversationKind: "chat"` on the step input.
|
|
68
70
|
*/
|
|
69
71
|
hooks?: HookRunner;
|
|
70
|
-
/**
|
|
71
|
-
* `usage.report` delivery (P-17): each written `usage` event is enqueued (chat runs and full-run eval
|
|
72
|
-
* replays, which both log usage). `enqueue` never throws and returns at once without handlers.
|
|
73
|
-
*/
|
|
74
|
-
usageReport?: Pick<UsageReporter, "enqueue">;
|
|
75
72
|
}
|
|
76
73
|
/** 构建请求用到的附件来源。 */
|
|
77
74
|
export interface MediaSourceDeps {
|
|
@@ -121,7 +118,8 @@ export interface ModelStepInput {
|
|
|
121
118
|
parentRunId?: string;
|
|
122
119
|
/**
|
|
123
120
|
* The run's step limit (`LoopContext.limits.maxSteps`) for the turn-limit reminder; a child's
|
|
124
|
-
* may be below `runtime.maxSteps` (final review m4).
|
|
121
|
+
* may be below `runtime.maxSteps` (final review m4). Absent: the run has no step limit, and no
|
|
122
|
+
* reminder is sent.
|
|
125
123
|
*/
|
|
126
124
|
maxSteps?: number;
|
|
127
125
|
}
|
|
@@ -176,15 +174,15 @@ export declare function runModelStep(deps: ModelStepDeps, input: ModelStepInput)
|
|
|
176
174
|
export declare function runFinalWrapUp(deps: ModelStepDeps, input: ModelStepInput, reason?: WrapUpReason): Promise<ModelStepResult>;
|
|
177
175
|
/**
|
|
178
176
|
* Appends a model call's events (its reply or compaction outcome with its `usage`; also the usage
|
|
179
|
-
* of a `WebFetch` extraction call, `tools/builtins/web/extract.ts`)
|
|
180
|
-
*
|
|
181
|
-
*
|
|
182
|
-
*
|
|
183
|
-
* exactly once.
|
|
177
|
+
* of a `WebFetch` extraction call, `tools/builtins/web/extract.ts`) in one transaction that also
|
|
178
|
+
* owes the `usage.report` delivery of the usage events it newly wrote (transactional outbox spec
|
|
179
|
+
* §8.1). For a sub-agent's run (`input.parentRunId`) that transaction also adds their tokens to the
|
|
180
|
+
* budget run's `subagent_tokens_used` (sub-agents spec §8.3): a usage event already in the log (an
|
|
181
|
+
* idempotent hit after a crash) adds nothing, so each call counts and is reported exactly once.
|
|
184
182
|
*/
|
|
185
183
|
export declare function appendCounted(deps: {
|
|
186
|
-
eventLog: Pick<EventLog, "
|
|
187
|
-
stepLog: Pick<StepLog, "appendCounted">;
|
|
184
|
+
eventLog: Pick<EventLog, "notify">;
|
|
185
|
+
stepLog: Pick<StepLog, "appendCounted" | "appendReported">;
|
|
188
186
|
}, input: {
|
|
189
187
|
conversationId: string;
|
|
190
188
|
parentRunId?: string | undefined;
|
|
@@ -123,12 +123,6 @@ wrapUpReason) {
|
|
|
123
123
|
const { conversationId, runId, step, signal, sink } = input;
|
|
124
124
|
const existing = await findExisting(deps.eventLog, conversationId, runId, step);
|
|
125
125
|
if (existing) {
|
|
126
|
-
// resumed after a crash between writing usage and enqueueing it: enqueue again (job id dedupes)
|
|
127
|
-
if (deps.usageReport && existing.usage !== null) {
|
|
128
|
-
const u = await deps.eventLog.findByIdem(conversationId, idem.usage(runId, step));
|
|
129
|
-
if (u)
|
|
130
|
-
void deps.usageReport.enqueue(conversationId, u.seq);
|
|
131
|
-
}
|
|
132
126
|
// a crash between the assistant_message and clearing its draft leaves one behind
|
|
133
127
|
sink({ kind: "step_end", runId, step });
|
|
134
128
|
return existing;
|
|
@@ -230,8 +224,9 @@ wrapUpReason) {
|
|
|
230
224
|
}
|
|
231
225
|
}
|
|
232
226
|
const { outcome, ref } = await ladder(deps, input, events, runtime, tools, wrapUpReason, streamed);
|
|
233
|
-
// assistant_message
|
|
234
|
-
|
|
227
|
+
// assistant_message and usage in one append = one transaction, which also owes the usage's
|
|
228
|
+
// usage.report delivery (transactional outbox spec §8.1)
|
|
229
|
+
await appendCounted(deps, input, [
|
|
235
230
|
{
|
|
236
231
|
type: "assistant_message",
|
|
237
232
|
runId,
|
|
@@ -245,10 +240,6 @@ wrapUpReason) {
|
|
|
245
240
|
payload: { step, provider: ref.provider, model: ref.model, ...outcome.usage },
|
|
246
241
|
},
|
|
247
242
|
]);
|
|
248
|
-
// usage.report metering (never awaited on the run's path; enqueue logs and swallows failures)
|
|
249
|
-
const usageEvent = appended.find((e) => e.type === "usage");
|
|
250
|
-
if (deps.usageReport && usageEvent)
|
|
251
|
-
void deps.usageReport.enqueue(conversationId, usageEvent.seq);
|
|
252
243
|
sink({ kind: "step_end", runId, step });
|
|
253
244
|
return {
|
|
254
245
|
content: outcome.content,
|
|
@@ -277,18 +268,18 @@ wrapUpReason) {
|
|
|
277
268
|
}
|
|
278
269
|
/**
|
|
279
270
|
* Appends a model call's events (its reply or compaction outcome with its `usage`; also the usage
|
|
280
|
-
* of a `WebFetch` extraction call, `tools/builtins/web/extract.ts`)
|
|
281
|
-
*
|
|
282
|
-
*
|
|
283
|
-
*
|
|
284
|
-
* exactly once.
|
|
271
|
+
* of a `WebFetch` extraction call, `tools/builtins/web/extract.ts`) in one transaction that also
|
|
272
|
+
* owes the `usage.report` delivery of the usage events it newly wrote (transactional outbox spec
|
|
273
|
+
* §8.1). For a sub-agent's run (`input.parentRunId`) that transaction also adds their tokens to the
|
|
274
|
+
* budget run's `subagent_tokens_used` (sub-agents spec §8.3): a usage event already in the log (an
|
|
275
|
+
* idempotent hit after a crash) adds nothing, so each call counts and is reported exactly once.
|
|
285
276
|
*/
|
|
286
277
|
export async function appendCounted(deps, input, events) {
|
|
287
278
|
const { conversationId, parentRunId } = input;
|
|
288
|
-
if (parentRunId === undefined)
|
|
289
|
-
return deps.eventLog.append(conversationId, events);
|
|
290
279
|
const budgetRunId = input.budgetRunId ?? input.parent?.budgetRunId;
|
|
291
|
-
const out =
|
|
280
|
+
const out = parentRunId === undefined
|
|
281
|
+
? await deps.stepLog.appendReported(conversationId, events)
|
|
282
|
+
: await deps.stepLog.appendCounted(conversationId, { parentRunId, ...(budgetRunId !== undefined && { budgetRunId }) }, events);
|
|
292
283
|
if (out.toSeq !== null)
|
|
293
284
|
await deps.eventLog.notify(conversationId, out.toSeq);
|
|
294
285
|
return out.written;
|
|
@@ -691,9 +682,6 @@ async function maybeCompact(deps, input, events, runtime, tools) {
|
|
|
691
682
|
});
|
|
692
683
|
}
|
|
693
684
|
const written = await appendCounted(deps, input, batch);
|
|
694
|
-
const usageEvent = written.find((e) => e.type === "usage");
|
|
695
|
-
if (deps.usageReport && usageEvent)
|
|
696
|
-
void deps.usageReport.enqueue(conversationId, usageEvent.seq);
|
|
697
685
|
return mergeLocal(events, written);
|
|
698
686
|
}
|
|
699
687
|
/** Merges written events into the local list (idempotent hits already in it are not added twice). */
|
|
@@ -719,7 +707,9 @@ async function ladder(deps, input, events, runtime, tools, wrapUpReason, streame
|
|
|
719
707
|
? BUDGET_WRAP_UP_REMINDER
|
|
720
708
|
: wrapUp
|
|
721
709
|
? WRAP_UP_REMINDER
|
|
722
|
-
:
|
|
710
|
+
: input.maxSteps === undefined
|
|
711
|
+
? null
|
|
712
|
+
: turnLimitReminder(step, input.maxSteps);
|
|
723
713
|
let evs = events;
|
|
724
714
|
const rungs = [
|
|
725
715
|
{ ref: agent.model, fallback: false },
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { Redis } from "ioredis";
|
|
2
|
+
/** The Redis hash of the lost-jobs canary: `count` and `lastAt` (ISO). */
|
|
3
|
+
export declare const QUEUE_LOST_JOBS_KEY = "capk:queue-lost-jobs";
|
|
4
|
+
/** How long the canary stays after its last report (then the overview drops it). */
|
|
5
|
+
export declare const QUEUE_LOST_JOBS_TTL_SECONDS: number;
|
|
6
|
+
/** What the admin overview shows: how many run jobs the queue lost, and when it last did. */
|
|
7
|
+
export interface QueueLostJobsSummary {
|
|
8
|
+
count: number;
|
|
9
|
+
lastAt: string;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* The "queue lost jobs" canary (transactional outbox spec §8.3): the `run-orphan-check` found
|
|
13
|
+
* queued runs whose job is neither in BullMQ nor owed by the outbox, which the outbox makes
|
|
14
|
+
* impossible unless Redis lost data (its durability broke). Each report adds to a count in Redis
|
|
15
|
+
* that the admin overview shows as an attention item until a week after the last report.
|
|
16
|
+
*/
|
|
17
|
+
export declare class QueueLostJobs {
|
|
18
|
+
private readonly redis;
|
|
19
|
+
constructor(redis: Redis);
|
|
20
|
+
/** Adds `n` lost jobs to the canary (no-op for 0). */
|
|
21
|
+
report(n: number): Promise<void>;
|
|
22
|
+
/** The canary, or null when no job was lost in the last week. */
|
|
23
|
+
summary(): Promise<QueueLostJobsSummary | null>;
|
|
24
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
2
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
3
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
4
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
|
+
};
|
|
7
|
+
var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
8
|
+
return function (target, key) { decorator(target, key, paramIndex); }
|
|
9
|
+
};
|
|
10
|
+
import { Inject, Injectable } from "@nestjs/common";
|
|
11
|
+
import { REDIS } from "../infra/tokens.js";
|
|
12
|
+
/** The Redis hash of the lost-jobs canary: `count` and `lastAt` (ISO). */
|
|
13
|
+
export const QUEUE_LOST_JOBS_KEY = "capk:queue-lost-jobs";
|
|
14
|
+
/** How long the canary stays after its last report (then the overview drops it). */
|
|
15
|
+
export const QUEUE_LOST_JOBS_TTL_SECONDS = 7 * 24 * 3600;
|
|
16
|
+
/**
|
|
17
|
+
* The "queue lost jobs" canary (transactional outbox spec §8.3): the `run-orphan-check` found
|
|
18
|
+
* queued runs whose job is neither in BullMQ nor owed by the outbox, which the outbox makes
|
|
19
|
+
* impossible unless Redis lost data (its durability broke). Each report adds to a count in Redis
|
|
20
|
+
* that the admin overview shows as an attention item until a week after the last report.
|
|
21
|
+
*/
|
|
22
|
+
let QueueLostJobs = class QueueLostJobs {
|
|
23
|
+
redis;
|
|
24
|
+
constructor(redis) {
|
|
25
|
+
this.redis = redis;
|
|
26
|
+
}
|
|
27
|
+
/** Adds `n` lost jobs to the canary (no-op for 0). */
|
|
28
|
+
async report(n) {
|
|
29
|
+
if (n <= 0)
|
|
30
|
+
return;
|
|
31
|
+
await this.redis
|
|
32
|
+
.multi()
|
|
33
|
+
.hincrby(QUEUE_LOST_JOBS_KEY, "count", n)
|
|
34
|
+
.hset(QUEUE_LOST_JOBS_KEY, "lastAt", new Date().toISOString())
|
|
35
|
+
.expire(QUEUE_LOST_JOBS_KEY, QUEUE_LOST_JOBS_TTL_SECONDS)
|
|
36
|
+
.exec();
|
|
37
|
+
}
|
|
38
|
+
/** The canary, or null when no job was lost in the last week. */
|
|
39
|
+
async summary() {
|
|
40
|
+
const h = await this.redis.hgetall(QUEUE_LOST_JOBS_KEY);
|
|
41
|
+
const count = Number(h.count ?? 0);
|
|
42
|
+
if (!(count > 0) || h.lastAt === undefined)
|
|
43
|
+
return null;
|
|
44
|
+
return { count, lastAt: h.lastAt };
|
|
45
|
+
}
|
|
46
|
+
};
|
|
47
|
+
QueueLostJobs = __decorate([
|
|
48
|
+
Injectable(),
|
|
49
|
+
__param(0, Inject(REDIS))
|
|
50
|
+
], QueueLostJobs);
|
|
51
|
+
export { QueueLostJobs };
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { OutboxService } from "../infra/outbox/outbox.service.js";
|
|
2
2
|
import type { KernelLogger } from "../log.js";
|
|
3
3
|
import { RunAbort } from "./abort.js";
|
|
4
|
+
import { type ResumeOutcome, RunResume } from "./run-jobs.js";
|
|
4
5
|
import { RunQueue } from "./run-queue.js";
|
|
5
6
|
import { Waits } from "./waits.js";
|
|
6
7
|
/**
|
|
@@ -11,13 +12,49 @@ export declare class RunControl {
|
|
|
11
12
|
private readonly queue;
|
|
12
13
|
private readonly waits;
|
|
13
14
|
private readonly abort;
|
|
14
|
-
|
|
15
|
+
/** The resume of a waiting run in a decision's transaction (a reply's skips). */
|
|
16
|
+
private readonly resumes;
|
|
17
|
+
/** Owes a deleted root's tree stop in the delete's transaction. */
|
|
18
|
+
private readonly outbox;
|
|
15
19
|
private readonly log;
|
|
16
|
-
constructor(queue: Pick<RunQueue, "ensureRun" | "abortIfOrphaned" | "wakeHeldTree">, waits: Pick<Waits, "abortWaiting" | "childFinished" | "stopChildren" | "stopTree">, abort: Pick<RunAbort, "request">,
|
|
20
|
+
constructor(queue: Pick<RunQueue, "ensureRun" | "oweEnsureRun" | "abortIfOrphaned" | "wakeHeldTree" | "oweHeldTreeIn">, waits: Pick<Waits, "abortWaiting" | "childFinished" | "stopChildren" | "stopTree">, abort: Pick<RunAbort, "request">,
|
|
21
|
+
/** The resume of a waiting run in a decision's transaction (a reply's skips). */
|
|
22
|
+
resumes: Pick<RunResume, "resumeIn">,
|
|
23
|
+
/** Owes a deleted root's tree stop in the delete's transaction. */
|
|
24
|
+
outbox: Pick<OutboxService, "add">, log: Pick<KernelLogger, "error">);
|
|
17
25
|
/** 有未消费的消息且没有进行中的运行时投递一轮运行;返回 runId 或 null。 */
|
|
18
26
|
ensureRun(conversationId: string): Promise<string | null>;
|
|
27
|
+
/**
|
|
28
|
+
* The direct `ensureRun` right after a commit that owes the ensure-run backstop (a message, a
|
|
29
|
+
* queue change): a failure (Redis unreachable) is logged and never thrown, since the committed
|
|
30
|
+
* write is not undone and the backstop starts the run later (transactional outbox spec §6).
|
|
31
|
+
*/
|
|
32
|
+
ensureRunAfterCommit(conversationId: string): Promise<void>;
|
|
33
|
+
/**
|
|
34
|
+
* Owes the ensure-run backstop in the caller's transaction (transactional outbox spec §6): `seq`
|
|
35
|
+
* is the event it wrote (or the conversation's newest), `orgId` the conversation's.
|
|
36
|
+
*/
|
|
37
|
+
oweEnsureRun(conversationId: string, seq: number | string, orgId: string): Promise<boolean>;
|
|
19
38
|
/** After a root's queue was unpaused: the wakes it held for its tree ({@link RunQueue.wakeHeldTree}). */
|
|
20
39
|
wakeHeldTree(rootId: string): Promise<void>;
|
|
40
|
+
/**
|
|
41
|
+
* Owes, in the transaction that may release a root's hold (an unpause, a "send now"), the
|
|
42
|
+
* ensure-run backstop of every sub-agent the hold kept waiting ({@link RunQueue.oweHeldTreeIn});
|
|
43
|
+
* the caller's {@link wakeHeldTree} after the commit is the direct call.
|
|
44
|
+
*/
|
|
45
|
+
oweHeldTreeIn(rootId: string): Promise<void>;
|
|
46
|
+
/**
|
|
47
|
+
* In the caller's transaction, after it locked the decided rows (lock order: conversations,
|
|
48
|
+
* decided rows, runs): resumes the waiting run `runId` when it waits on nothing else
|
|
49
|
+
* ({@link RunResume.resumeIn}).
|
|
50
|
+
*/
|
|
51
|
+
resumeIn(runId: string, waitId: string): Promise<ResumeOutcome>;
|
|
52
|
+
/**
|
|
53
|
+
* Owes, in a root's delete transaction, the stop of its whole sub-agent tree (`stop-tree`,
|
|
54
|
+
* transactional outbox spec §6); the caller's {@link stopTree} after the commit is the direct
|
|
55
|
+
* call.
|
|
56
|
+
*/
|
|
57
|
+
oweStopDeletedTree(conversationId: string, byAccountId: string | null, orgId: string): Promise<boolean>;
|
|
21
58
|
/**
|
|
22
59
|
* 运行是孤儿(会话锁空闲、任务已不存在,没有 worker 会收到中止广播)时直接把它写成 aborted,返回 true;
|
|
23
60
|
* 否则返回 false。A stopped sub-agent then reports to its parent at once, and a stopped parent
|
|
@@ -26,8 +63,8 @@ export declare class RunControl {
|
|
|
26
63
|
abortIfOrphaned(conversationId: string, runId: string, byAccountId: string | null): Promise<boolean>;
|
|
27
64
|
/**
|
|
28
65
|
* 运行在等人处理(waiting)时直接中止:取消等待、写 `run_aborted`、删除 checkpoint,返回 true;
|
|
29
|
-
* 运行已不是 waiting(例如刚被续跑)→ false.
|
|
30
|
-
*
|
|
66
|
+
* 运行已不是 waiting(例如刚被续跑)→ false. The stop's transaction owes the IM delivery that
|
|
67
|
+
* updates its approval / question cards (its appends, transactional outbox spec §8.1).
|
|
31
68
|
*/
|
|
32
69
|
abortWaiting(conversationId: string, runId: string, byAccountId: string | null): Promise<boolean>;
|
|
33
70
|
/**
|
|
@@ -8,10 +8,12 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
|
8
8
|
return function (target, key) { decorator(target, key, paramIndex); }
|
|
9
9
|
};
|
|
10
10
|
import { Inject, Injectable } from "@nestjs/common";
|
|
11
|
-
import {
|
|
11
|
+
import { OutboxService } from "../infra/outbox/outbox.service.js";
|
|
12
12
|
import { KERNEL_LOGGER } from "../infra/tokens.js";
|
|
13
13
|
import { RunAbort } from "./abort.js";
|
|
14
|
+
import { RunResume } from "./run-jobs.js";
|
|
14
15
|
import { RunQueue } from "./run-queue.js";
|
|
16
|
+
import { oweStopDeletedTree } from "./subagents/tree-jobs.js";
|
|
15
17
|
import { Waits } from "./waits.js";
|
|
16
18
|
/**
|
|
17
19
|
* Starting and stopping runs from outside the run worker: the conversations API (a message, a
|
|
@@ -21,23 +23,76 @@ let RunControl = class RunControl {
|
|
|
21
23
|
queue;
|
|
22
24
|
waits;
|
|
23
25
|
abort;
|
|
24
|
-
|
|
26
|
+
resumes;
|
|
27
|
+
outbox;
|
|
25
28
|
log;
|
|
26
|
-
constructor(queue, waits, abort,
|
|
29
|
+
constructor(queue, waits, abort,
|
|
30
|
+
/** The resume of a waiting run in a decision's transaction (a reply's skips). */
|
|
31
|
+
resumes,
|
|
32
|
+
/** Owes a deleted root's tree stop in the delete's transaction. */
|
|
33
|
+
outbox, log) {
|
|
27
34
|
this.queue = queue;
|
|
28
35
|
this.waits = waits;
|
|
29
36
|
this.abort = abort;
|
|
30
|
-
this.
|
|
37
|
+
this.resumes = resumes;
|
|
38
|
+
this.outbox = outbox;
|
|
31
39
|
this.log = log;
|
|
32
40
|
}
|
|
33
41
|
/** 有未消费的消息且没有进行中的运行时投递一轮运行;返回 runId 或 null。 */
|
|
34
42
|
ensureRun(conversationId) {
|
|
35
43
|
return this.queue.ensureRun(conversationId);
|
|
36
44
|
}
|
|
45
|
+
/**
|
|
46
|
+
* The direct `ensureRun` right after a commit that owes the ensure-run backstop (a message, a
|
|
47
|
+
* queue change): a failure (Redis unreachable) is logged and never thrown, since the committed
|
|
48
|
+
* write is not undone and the backstop starts the run later (transactional outbox spec §6).
|
|
49
|
+
*/
|
|
50
|
+
async ensureRunAfterCommit(conversationId) {
|
|
51
|
+
try {
|
|
52
|
+
await this.queue.ensureRun(conversationId);
|
|
53
|
+
}
|
|
54
|
+
catch (err) {
|
|
55
|
+
this.log.error("cannot start the conversation's run now; the ensure-run backstop will", {
|
|
56
|
+
conversationId,
|
|
57
|
+
err,
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Owes the ensure-run backstop in the caller's transaction (transactional outbox spec §6): `seq`
|
|
63
|
+
* is the event it wrote (or the conversation's newest), `orgId` the conversation's.
|
|
64
|
+
*/
|
|
65
|
+
oweEnsureRun(conversationId, seq, orgId) {
|
|
66
|
+
return this.queue.oweEnsureRun(conversationId, seq, orgId);
|
|
67
|
+
}
|
|
37
68
|
/** After a root's queue was unpaused: the wakes it held for its tree ({@link RunQueue.wakeHeldTree}). */
|
|
38
69
|
wakeHeldTree(rootId) {
|
|
39
70
|
return this.queue.wakeHeldTree(rootId);
|
|
40
71
|
}
|
|
72
|
+
/**
|
|
73
|
+
* Owes, in the transaction that may release a root's hold (an unpause, a "send now"), the
|
|
74
|
+
* ensure-run backstop of every sub-agent the hold kept waiting ({@link RunQueue.oweHeldTreeIn});
|
|
75
|
+
* the caller's {@link wakeHeldTree} after the commit is the direct call.
|
|
76
|
+
*/
|
|
77
|
+
oweHeldTreeIn(rootId) {
|
|
78
|
+
return this.queue.oweHeldTreeIn(rootId);
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* In the caller's transaction, after it locked the decided rows (lock order: conversations,
|
|
82
|
+
* decided rows, runs): resumes the waiting run `runId` when it waits on nothing else
|
|
83
|
+
* ({@link RunResume.resumeIn}).
|
|
84
|
+
*/
|
|
85
|
+
resumeIn(runId, waitId) {
|
|
86
|
+
return this.resumes.resumeIn(runId, waitId);
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Owes, in a root's delete transaction, the stop of its whole sub-agent tree (`stop-tree`,
|
|
90
|
+
* transactional outbox spec §6); the caller's {@link stopTree} after the commit is the direct
|
|
91
|
+
* call.
|
|
92
|
+
*/
|
|
93
|
+
oweStopDeletedTree(conversationId, byAccountId, orgId) {
|
|
94
|
+
return oweStopDeletedTree(this.outbox, conversationId, byAccountId, orgId);
|
|
95
|
+
}
|
|
41
96
|
/**
|
|
42
97
|
* 运行是孤儿(会话锁空闲、任务已不存在,没有 worker 会收到中止广播)时直接把它写成 aborted,返回 true;
|
|
43
98
|
* 否则返回 false。A stopped sub-agent then reports to its parent at once, and a stopped parent
|
|
@@ -57,13 +112,11 @@ let RunControl = class RunControl {
|
|
|
57
112
|
}
|
|
58
113
|
/**
|
|
59
114
|
* 运行在等人处理(waiting)时直接中止:取消等待、写 `run_aborted`、删除 checkpoint,返回 true;
|
|
60
|
-
* 运行已不是 waiting(例如刚被续跑)→ false.
|
|
61
|
-
*
|
|
115
|
+
* 运行已不是 waiting(例如刚被续跑)→ false. The stop's transaction owes the IM delivery that
|
|
116
|
+
* updates its approval / question cards (its appends, transactional outbox spec §8.1).
|
|
62
117
|
*/
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
await this.events.emit("run.wait-stopped", { conversationId, runId });
|
|
66
|
-
return r;
|
|
118
|
+
abortWaiting(conversationId, runId, byAccountId) {
|
|
119
|
+
return this.waits.abortWaiting(conversationId, runId, byAccountId);
|
|
67
120
|
}
|
|
68
121
|
/**
|
|
69
122
|
* Stops a sub-agent with its descendants, or with `includeSelf: false` only the sub-agent
|
|
@@ -83,7 +136,8 @@ RunControl = __decorate([
|
|
|
83
136
|
__param(0, Inject(RunQueue)),
|
|
84
137
|
__param(1, Inject(Waits)),
|
|
85
138
|
__param(2, Inject(RunAbort)),
|
|
86
|
-
__param(3, Inject(
|
|
87
|
-
__param(4, Inject(
|
|
139
|
+
__param(3, Inject(RunResume)),
|
|
140
|
+
__param(4, Inject(OutboxService)),
|
|
141
|
+
__param(5, Inject(KERNEL_LOGGER))
|
|
88
142
|
], RunControl);
|
|
89
143
|
export { RunControl };
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { type AppendEvent, EventLog } from "../conversations/event-log.js";
|
|
2
2
|
import { KernelDb } from "../infra/database/kernel-db.js";
|
|
3
3
|
import { TransactionHost } from "../infra/database/transaction-host.js";
|
|
4
|
+
import { OutboxService } from "../infra/outbox/outbox.service.js";
|
|
4
5
|
import { ScheduledTaskRunTracker } from "../schedules/run-tracker.js";
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
import { ApprovalsService } from "../tools/approvals.service.js";
|
|
7
|
+
import { QuestionsService } from "../tools/questions.service.js";
|
|
8
|
+
import { type TerminalStatus } from "./run-jobs.js";
|
|
8
9
|
export interface FinishRunArgs {
|
|
9
10
|
conversationId: string;
|
|
10
11
|
runId: string;
|
|
@@ -18,15 +19,10 @@ export interface FinishRunArgs {
|
|
|
18
19
|
* 超时(max_duration)传 false:loop 没吸收的消息没有被回复,必须留给下一轮(R34)。
|
|
19
20
|
*/
|
|
20
21
|
floorToTrigger?: boolean;
|
|
21
|
-
/**
|
|
22
|
-
* Runs in the transaction once the run is known not to be terminal, before the terminal event:
|
|
23
|
-
* a parent's open `Agent` calls get their results here (sub-agents spec §12, review m1).
|
|
24
|
-
*/
|
|
25
|
-
beforeTerminal?: () => Promise<void>;
|
|
26
22
|
}
|
|
27
23
|
/**
|
|
28
|
-
* Writes runs' terminal states (one instance; the run worker, the run queue's orphan
|
|
29
|
-
* waits and the eval replays finish runs through it).
|
|
24
|
+
* Writes runs' terminal states (one instance; the run worker, the run queue's orphan abort, the
|
|
25
|
+
* `run-orphan-check`, the waits and the eval replays finish runs through it).
|
|
30
26
|
*/
|
|
31
27
|
export declare class RunFinish {
|
|
32
28
|
private readonly kdb;
|
|
@@ -34,20 +30,51 @@ export declare class RunFinish {
|
|
|
34
30
|
readonly txHost: TransactionHost;
|
|
35
31
|
/** A scheduled task's run follows its kernel run's end in the same transaction (spec §6). */
|
|
36
32
|
private readonly taskRuns;
|
|
33
|
+
/** The run's end owes the conversation's ensure-run backstop (transactional outbox spec §6). */
|
|
34
|
+
private readonly outbox;
|
|
35
|
+
/** A finished run's pending approvals are cancelled in its terminal transaction. */
|
|
36
|
+
private readonly approvals;
|
|
37
|
+
/** ... and its pending input requests. */
|
|
38
|
+
private readonly questions;
|
|
37
39
|
constructor(kdb: KernelDb, eventLog: EventLog, txHost: TransactionHost,
|
|
38
40
|
/** A scheduled task's run follows its kernel run's end in the same transaction (spec §6). */
|
|
39
|
-
taskRuns: Pick<ScheduledTaskRunTracker, "onRunFinished"
|
|
41
|
+
taskRuns: Pick<ScheduledTaskRunTracker, "onRunFinished">,
|
|
42
|
+
/** The run's end owes the conversation's ensure-run backstop (transactional outbox spec §6). */
|
|
43
|
+
outbox: Pick<OutboxService, "add">,
|
|
44
|
+
/** A finished run's pending approvals are cancelled in its terminal transaction. */
|
|
45
|
+
approvals: Pick<ApprovalsService, "cancelPendingIn">,
|
|
46
|
+
/** ... and its pending input requests. */
|
|
47
|
+
questions: Pick<QuestionsService, "cancelPendingIn">);
|
|
40
48
|
/**
|
|
41
49
|
* 在调用方的事务内写运行的终态:会话行锁 → 运行仍未结束才写终态事件、推进 `consumed_seq`、更新 `runs.status`。
|
|
50
|
+
* It also owes the conversation's ensure-run backstop under the terminal event's seq.
|
|
51
|
+
*
|
|
52
|
+
* A finished run waits on nothing: its pending approvals and input requests (one asked just
|
|
53
|
+
* before the run failed, while it was still `running`) are cancelled before its terminal event
|
|
54
|
+
* (`approval_decided` / `question_answered` with `cancelled`). Lock order, as every path that
|
|
55
|
+
* writes them: the conversation row, the run's approval and input request rows, the run row.
|
|
56
|
+
*
|
|
57
|
+
* Sub-agents (transactional outbox spec §6): before the terminal event the run's open fork
|
|
58
|
+
* invocation gets its report (`pairStoppedForks`; an `Agent` call has its result from its
|
|
59
|
+
* dispatch), so no terminal path leaves an open call. After it, the stops and the delivery the
|
|
60
|
+
* run's end causes are owed here (runtime/subagents/tree-jobs.ts), the caller still doing them
|
|
61
|
+
* right after its commit: `stop-tree` when an active fork child remains or a sub-agent run a
|
|
62
|
+
* person stopped has sub-agents below it, `deliver-to-parent` for a sub-agent's run. The
|
|
63
|
+
* delivery is a job, not written here: this transaction holds the child's conversation and run
|
|
64
|
+
* rows, and the parent's conversation row comes first in the dispatch's lock order (parent
|
|
65
|
+
* conversation, parent run, budget run, child conversation).
|
|
42
66
|
* 返回终态事件的 seq;运行已是终态(例如已被其他副本标记失败)→ 返回 null,什么都不写。
|
|
43
67
|
*
|
|
44
68
|
* `consumed_seq = GREATEST(consumed_seq, consumed, runs.trigger_seq)`:不回退,且失败 / 中止 / 完成都至少
|
|
45
69
|
* 消费到触发该运行的消息——否则结束时的 ensureRun 会对同一条消息无限重试(失败)或卡住会话。
|
|
46
70
|
* 超时(`floorToTrigger: false`)只取 `GREATEST(consumed_seq, consumed)`:没吸收的消息留给下一轮(R34)。
|
|
47
|
-
*
|
|
71
|
+
* Lock order as `RunQueue.ensureRun` and the decisions take it: the conversation row, the run's
|
|
72
|
+
* approval and input request rows, the run row; they never deadlock with each other.
|
|
48
73
|
* 调用方在提交后对非 null 的返回值调用 {@link notifyFinished}。
|
|
49
74
|
*/
|
|
50
75
|
finishIn(a: FinishRunArgs): Promise<number | null>;
|
|
76
|
+
/** {@link finishIn}'s owed stops and delivery of a run's sub-agent tree. */
|
|
77
|
+
private oweTreeWork;
|
|
51
78
|
/** 自开事务写终态并在提交后发布会话通知;返回值同 {@link finishIn}。 */
|
|
52
79
|
finish(a: FinishRunArgs): Promise<number | null>;
|
|
53
80
|
private finishOwn;
|