@bastani/atomic 0.9.14-alpha.3 → 0.9.14-alpha.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/dist/builtin/intercom/package.json +1 -1
- package/dist/builtin/mcp/package.json +1 -1
- package/dist/builtin/subagents/CHANGELOG.md +6 -0
- package/dist/builtin/subagents/package.json +1 -1
- package/dist/builtin/subagents/src/extension/schemas.ts +5 -0
- package/dist/builtin/subagents/src/runs/shared/long-running-guard.ts +3 -1
- package/dist/builtin/subagents/src/runs/shared/progress-trend.ts +69 -0
- package/dist/builtin/subagents/src/runs/shared/subagent-control.ts +12 -1
- package/dist/builtin/subagents/src/shared/types-results.ts +2 -0
- package/dist/builtin/web-access/CHANGELOG.md +10 -0
- package/dist/builtin/web-access/README.md +3 -6
- package/dist/builtin/web-access/index.ts +1 -2
- package/dist/builtin/web-access/package.json +1 -1
- package/dist/builtin/web-access/web-search-config.ts +2 -8
- package/dist/builtin/web-access/web-search-tool.ts +2 -7
- package/dist/builtin/web-access/web-search-workflow.ts +10 -0
- package/dist/builtin/workflows/CHANGELOG.md +46 -0
- package/dist/builtin/workflows/README.md +6 -5
- package/dist/builtin/workflows/builtin/adversarial-verification-prompts.ts +13 -5
- package/dist/builtin/workflows/builtin/adversarial-verification-runner.ts +376 -89
- package/dist/builtin/workflows/builtin/adversarial-verification.d.ts +30 -6
- package/dist/builtin/workflows/builtin/adversarial-verification.ts +14 -9
- package/dist/builtin/workflows/builtin/generate-and-filter-prompts.ts +26 -3
- package/dist/builtin/workflows/builtin/generate-and-filter-runner.ts +18 -14
- package/dist/builtin/workflows/builtin/goal-artifacts.ts +9 -8
- package/dist/builtin/workflows/builtin/goal-convergence.ts +87 -0
- package/dist/builtin/workflows/builtin/goal-ledger.ts +4 -0
- package/dist/builtin/workflows/builtin/goal-prompts.ts +2 -0
- package/dist/builtin/workflows/builtin/goal-reducer.ts +6 -1
- package/dist/builtin/workflows/builtin/goal-reverify.ts +305 -0
- package/dist/builtin/workflows/builtin/goal-runner.ts +75 -10
- package/dist/builtin/workflows/builtin/goal-schemas.ts +7 -0
- package/dist/builtin/workflows/builtin/goal-types.ts +6 -0
- package/dist/builtin/workflows/builtin/loop-until-done-runner.ts +94 -6
- package/dist/builtin/workflows/builtin/loop-until-done.d.ts +8 -0
- package/dist/builtin/workflows/builtin/loop-until-done.ts +15 -0
- package/dist/builtin/workflows/builtin/progress-scoring.ts +230 -0
- package/dist/builtin/workflows/builtin/ralph-core.ts +11 -0
- package/dist/builtin/workflows/builtin/ralph-review-gate.ts +1 -0
- package/dist/builtin/workflows/builtin/ralph-reviewer-prompt.ts +2 -0
- package/dist/builtin/workflows/builtin/ralph-runner.ts +60 -10
- package/dist/builtin/workflows/builtin/selection-math.ts +156 -0
- package/dist/builtin/workflows/builtin/shared-prompts.ts +5 -0
- package/dist/builtin/workflows/builtin/tournament-prompts.ts +57 -75
- package/dist/builtin/workflows/builtin/tournament-runner.ts +384 -178
- package/dist/builtin/workflows/builtin/tournament.d.ts +46 -17
- package/dist/builtin/workflows/builtin/tournament.ts +66 -32
- package/dist/builtin/workflows/builtin/verification-criteria.ts +330 -0
- package/dist/builtin/workflows/builtin/verification-prompts.ts +206 -0
- package/dist/builtin/workflows/builtin/verification-usage.ts +44 -0
- package/dist/builtin/workflows/package.json +1 -1
- package/dist/builtin/workflows/skills/create-spec/SKILL.md +90 -30
- package/dist/builtin/workflows/skills/show-me/LICENSE.txt +21 -0
- package/dist/builtin/workflows/skills/show-me/SKILL.md +143 -0
- package/dist/builtin/workflows/src/authoring/workflow.ts +8 -0
- package/dist/builtin/workflows/src/authoring.d.ts +1 -1
- package/dist/builtin/workflows/src/durable/completed-catalog.ts +5 -2
- package/dist/builtin/workflows/src/durable/dbos-envelope.ts +1 -1
- package/dist/builtin/workflows/src/durable/resume-eligibility.ts +5 -3
- package/dist/builtin/workflows/src/durable/run-timing.ts +41 -10
- package/dist/builtin/workflows/src/durable/tool-primitive.ts +24 -2
- package/dist/builtin/workflows/src/engine/options.ts +1 -0
- package/dist/builtin/workflows/src/engine/primitives/workflow.ts +12 -3
- package/dist/builtin/workflows/src/engine/run-budget.ts +308 -0
- package/dist/builtin/workflows/src/engine/run-returned-status.ts +8 -0
- package/dist/builtin/workflows/src/engine/run-tool-node-lifecycle.ts +6 -0
- package/dist/builtin/workflows/src/engine/run.ts +124 -2
- package/dist/builtin/workflows/src/engine/runtime.ts +9 -0
- package/dist/builtin/workflows/src/extension/config-file-loader.ts +6 -0
- package/dist/builtin/workflows/src/extension/config-loader.ts +24 -1
- package/dist/builtin/workflows/src/extension/dispatcher.ts +6 -5
- package/dist/builtin/workflows/src/extension/extension-runtime-state.ts +2 -0
- package/dist/builtin/workflows/src/extension/index.bundle.mjs +3033 -872
- package/dist/builtin/workflows/src/extension/lifecycle-notifications.ts +51 -4
- package/dist/builtin/workflows/src/extension/public-types.ts +3 -1
- package/dist/builtin/workflows/src/extension/runtime-durable-resume.ts +7 -1
- package/dist/builtin/workflows/src/extension/runtime.ts +22 -10
- package/dist/builtin/workflows/src/extension/workflow-module-loader.ts +5 -0
- package/dist/builtin/workflows/src/extension/workflow-prompts.ts +1 -0
- package/dist/builtin/workflows/src/extension/workflow-schema.ts +16 -0
- package/dist/builtin/workflows/src/extension/workflow-status-summary.ts +44 -1
- package/dist/builtin/workflows/src/extension/workflow-tool-content.ts +10 -1
- package/dist/builtin/workflows/src/extension/workflow-tool-control.ts +21 -9
- package/dist/builtin/workflows/src/runs/foreground/executor-continuation.ts +14 -0
- package/dist/builtin/workflows/src/runs/foreground/executor-lifecycle.ts +15 -4
- package/dist/builtin/workflows/src/runs/foreground/executor-stage-call.ts +62 -5
- package/dist/builtin/workflows/src/runs/foreground/executor-stage-factory.ts +4 -0
- package/dist/builtin/workflows/src/runs/foreground/executor-stage-types.ts +2 -0
- package/dist/builtin/workflows/src/runs/foreground/executor-types.ts +3 -1
- package/dist/builtin/workflows/src/runs/foreground/stage-runner-controller.ts +10 -1
- package/dist/builtin/workflows/src/shared/authoring-contract-stage.d.ts +1 -0
- package/dist/builtin/workflows/src/shared/authoring-contract-stage.ts +1 -0
- package/dist/builtin/workflows/src/shared/authoring-contract-ui.d.ts +7 -0
- package/dist/builtin/workflows/src/shared/authoring-contract-ui.ts +7 -0
- package/dist/builtin/workflows/src/shared/authoring-contract.d.ts +1 -0
- package/dist/builtin/workflows/src/shared/budget-meter.ts +34 -0
- package/dist/builtin/workflows/src/shared/budget.d.ts +67 -0
- package/dist/builtin/workflows/src/shared/budget.ts +127 -0
- package/dist/builtin/workflows/src/shared/persistence-restore-helpers.ts +92 -8
- package/dist/builtin/workflows/src/shared/persistence-restore.ts +11 -1
- package/dist/builtin/workflows/src/shared/persistence-session-entries.ts +15 -3
- package/dist/builtin/workflows/src/shared/returned-run-status.ts +35 -2
- package/dist/builtin/workflows/src/shared/store-public-types.ts +4 -1
- package/dist/builtin/workflows/src/shared/store-run-methods.ts +8 -1
- package/dist/builtin/workflows/src/shared/store-stage-methods.ts +1 -0
- package/dist/builtin/workflows/src/shared/store-types.ts +24 -0
- package/dist/builtin/workflows/src/shared/types.ts +3 -0
- package/dist/builtin/workflows/src/shared/workflow-artifacts.ts +1 -0
- package/dist/builtin/workflows/src/shared/workflow-authoring-types.d.ts +3 -0
- package/dist/builtin/workflows/src/shared/workflow-authoring-types.ts +3 -0
- package/dist/builtin/workflows/src/tui/graph-theme.ts +11 -0
- package/dist/builtin/workflows/src/tui/graph-view-render.ts +19 -10
- package/dist/builtin/workflows/src/tui/tool-detail.ts +45 -26
- package/dist/core/atomic-guide-command.d.ts.map +1 -1
- package/dist/core/atomic-guide-command.js +1 -0
- package/dist/core/atomic-guide-command.js.map +1 -1
- package/dist/core/extensions/ui-types.d.ts +13 -3
- package/dist/core/extensions/ui-types.d.ts.map +1 -1
- package/dist/core/extensions/ui-types.js +15 -3
- package/dist/core/extensions/ui-types.js.map +1 -1
- package/dist/core/slash-commands.d.ts.map +1 -1
- package/dist/core/slash-commands.js +33 -3
- package/dist/core/slash-commands.js.map +1 -1
- package/dist/main-deferred-startup.d.ts.map +1 -1
- package/dist/main-deferred-startup.js +6 -2
- package/dist/main-deferred-startup.js.map +1 -1
- package/dist/modes/interactive/interactive-startup.js +4 -0
- package/dist/modes/interactive/interactive-startup.js.map +1 -1
- package/dist/modes/interactive/interactive-tui.d.ts.map +1 -1
- package/dist/modes/interactive/interactive-tui.js +19 -1
- package/dist/modes/interactive/interactive-tui.js.map +1 -1
- package/dist/modes/interactive-engine/isolated-runtime.d.ts +7 -0
- package/dist/modes/interactive-engine/isolated-runtime.d.ts.map +1 -1
- package/dist/modes/interactive-engine/isolated-runtime.js +94 -37
- package/dist/modes/interactive-engine/isolated-runtime.js.map +1 -1
- package/dist/modes/rpc/rpc-client.d.ts +1 -0
- package/dist/modes/rpc/rpc-client.d.ts.map +1 -1
- package/dist/modes/rpc/rpc-client.js +15 -2
- package/dist/modes/rpc/rpc-client.js.map +1 -1
- package/dist/modes/rpc/rpc-input-scheduler.d.ts +3 -2
- package/dist/modes/rpc/rpc-input-scheduler.d.ts.map +1 -1
- package/dist/modes/rpc/rpc-input-scheduler.js +5 -2
- package/dist/modes/rpc/rpc-input-scheduler.js.map +1 -1
- package/docs/extensions.md +1 -1
- package/docs/quickstart.md +1 -0
- package/docs/skills.md +4 -0
- package/docs/workflows.md +75 -11
- package/npm-shrinkwrap.json +29 -29
- package/package.json +2 -2
|
@@ -62,6 +62,7 @@ export function createChildWorkflowRunner(input: {
|
|
|
62
62
|
throw new Error(workflowDefinitionRequirementMessage("ctx.workflow(definition)", child));
|
|
63
63
|
const childName = child.normalizedName;
|
|
64
64
|
const boundaryName = options.stageName ?? `workflow:${childName}`;
|
|
65
|
+
if (runtime.budget.enabled) await runtime.budget.stopAtBoundaryAsync(boundaryName);
|
|
65
66
|
const boundaryReplayKey = input.nextWorkflowBoundaryReplayKey(boundaryName);
|
|
66
67
|
const durableInvocation = input.consumeDurableInvocation?.();
|
|
67
68
|
const boundary = runtime.spawnStage(boundaryName, {
|
|
@@ -127,11 +128,14 @@ export function createChildWorkflowRunner(input: {
|
|
|
127
128
|
rootRunId: runtime.parentRootRunId ?? runtime.runId,
|
|
128
129
|
},
|
|
129
130
|
signal: childController.signal,
|
|
131
|
+
budget: options.budget,
|
|
132
|
+
rootBudget: runtime.rootBudget,
|
|
130
133
|
deferWorkflowStart: false,
|
|
131
134
|
...(durableInvocation !== undefined ? { durableScope: durableInvocation.scope } : {}),
|
|
132
135
|
});
|
|
133
136
|
boundary.observeChildRun(childRunPromise);
|
|
134
137
|
const childRun = await childRunPromise;
|
|
138
|
+
if (runtime.budget.enabled) await runtime.budget.stopAtBoundaryAsync(boundaryName);
|
|
135
139
|
runtime.exit.throwIfWorkflowExitSelected();
|
|
136
140
|
|
|
137
141
|
// A gracefully quit child is a suspension, not a child failure: carry the
|
|
@@ -143,7 +147,12 @@ export function createChildWorkflowRunner(input: {
|
|
|
143
147
|
) {
|
|
144
148
|
throw new WorkflowGracefulQuitError(childRunId, `child workflow "${childName}" (${child.name})`);
|
|
145
149
|
}
|
|
146
|
-
|
|
150
|
+
const childBudgetExceeded = childRun.result?.status === "budget_exceeded";
|
|
151
|
+
const childExitStatus = isWorkflowExitStatus(childRun.status) ? childRun.status : "blocked";
|
|
152
|
+
if (
|
|
153
|
+
(!isWorkflowExitStatus(childRun.status) && !childBudgetExceeded) ||
|
|
154
|
+
(childRun.status === "failed" && childRun.exited !== true)
|
|
155
|
+
) {
|
|
147
156
|
const failedChildStage = childRun.stages.find((stage) => stage.failureKind !== undefined);
|
|
148
157
|
throw new Error(
|
|
149
158
|
`atomic-workflows: child workflow "${childName}" (${child.name}) failed with status ${childRun.status}${childRun.error !== undefined ? `: ${childRun.error}` : ""}`,
|
|
@@ -159,12 +168,12 @@ export function createChildWorkflowRunner(input: {
|
|
|
159
168
|
}
|
|
160
169
|
|
|
161
170
|
const outputs = selectWorkflowOutputs(child, childRun.result);
|
|
162
|
-
const childExited = childRun.exited === true || childRun.status !== "completed";
|
|
171
|
+
const childExited = childBudgetExceeded || childRun.exited === true || childRun.status !== "completed";
|
|
163
172
|
const childResult: WorkflowChildResult<TChildOutputs> = childExited
|
|
164
173
|
? {
|
|
165
174
|
workflow: child.normalizedName,
|
|
166
175
|
runId: childRun.runId,
|
|
167
|
-
status:
|
|
176
|
+
status: childBudgetExceeded ? "blocked" : childExitStatus,
|
|
168
177
|
exited: true,
|
|
169
178
|
outputs: outputs as Partial<TChildOutputs>,
|
|
170
179
|
...(childRun.exitReason !== undefined ? { exitReason: childRun.exitReason } : {}),
|
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
import {
|
|
2
|
+
type BudgetReport,
|
|
3
|
+
type EffectiveBudget,
|
|
4
|
+
enforceDurationBudget,
|
|
5
|
+
enforceUsageBudget,
|
|
6
|
+
} from "../shared/budget.js";
|
|
7
|
+
import { meter_run, type RunMeters, type RunUsageTree } from "../shared/budget-meter.js";
|
|
8
|
+
import type { RunBudgetAccountingState, RunBudgetState, RunSnapshot } from "../shared/store-types.js";
|
|
9
|
+
import { elapsedRunMs } from "../shared/timing.js";
|
|
10
|
+
import type { WorkflowModelUsage } from "../shared/types.js";
|
|
11
|
+
export const BUDGET_WRAP_UP_PROMPT =
|
|
12
|
+
"The workflow budget is exhausted. Stop substantive work, summarize useful progress, identify remaining work or blockers, and leave a clear next step. Do not start any new stages.";
|
|
13
|
+
export interface BudgetExceededReport extends BudgetReport {
|
|
14
|
+
readonly frontierStage: string;
|
|
15
|
+
readonly wrapUpSummary?: string;
|
|
16
|
+
readonly wrapUpUsage?: WorkflowModelUsage;
|
|
17
|
+
}
|
|
18
|
+
export class WorkflowBudgetExceededError extends Error {
|
|
19
|
+
readonly report: BudgetExceededReport;
|
|
20
|
+
constructor(report: BudgetExceededReport) {
|
|
21
|
+
const unit = report.dimension === "duration" ? "ms" : report.dimension;
|
|
22
|
+
super(
|
|
23
|
+
`atomic-workflows: ${report.dimension} budget exceeded (${report.reading}${unit} / ${report.ceiling}${unit}) at ${report.frontierStage}`,
|
|
24
|
+
);
|
|
25
|
+
this.name = "WorkflowBudgetExceededError";
|
|
26
|
+
this.report = report;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
export interface BudgetCheckpointContinue {
|
|
30
|
+
readonly kind: "continue";
|
|
31
|
+
}
|
|
32
|
+
export interface BudgetCheckpointWarning {
|
|
33
|
+
readonly kind: "warn";
|
|
34
|
+
readonly report: BudgetReport;
|
|
35
|
+
}
|
|
36
|
+
export interface BudgetCheckpointWrapUp {
|
|
37
|
+
readonly kind: "wrap_up";
|
|
38
|
+
readonly report: BudgetReport;
|
|
39
|
+
}
|
|
40
|
+
export interface BudgetCheckpointExhausted {
|
|
41
|
+
readonly kind: "exhausted";
|
|
42
|
+
readonly report: BudgetExceededReport;
|
|
43
|
+
}
|
|
44
|
+
export type BudgetCheckpoint =
|
|
45
|
+
| BudgetCheckpointContinue
|
|
46
|
+
| BudgetCheckpointWarning
|
|
47
|
+
| BudgetCheckpointWrapUp
|
|
48
|
+
| BudgetCheckpointExhausted;
|
|
49
|
+
export interface RunBudgetController {
|
|
50
|
+
readonly enabled: boolean;
|
|
51
|
+
readonly checkpoint: (frontierStage?: string) => BudgetCheckpoint;
|
|
52
|
+
readonly registerWrapUp: (frontierStage: string, handler: () => Promise<never>) => () => void;
|
|
53
|
+
readonly deliverWrapUp: (frontierStage: string) => Promise<never>;
|
|
54
|
+
readonly finishWrapUp: (
|
|
55
|
+
frontierStage: string | undefined,
|
|
56
|
+
summary?: string,
|
|
57
|
+
usage?: WorkflowModelUsage,
|
|
58
|
+
delivered?: boolean,
|
|
59
|
+
) => WorkflowBudgetExceededError;
|
|
60
|
+
readonly rethrowIfSystemOwnedStop: (frontierStage?: string) => void;
|
|
61
|
+
readonly stopAtBoundary: (frontierStage?: string) => void;
|
|
62
|
+
readonly stopAtBoundaryAsync: (frontierStage?: string) => Promise<void>;
|
|
63
|
+
readonly awaitPendingWrapUp: () => Promise<WorkflowBudgetExceededError | undefined>;
|
|
64
|
+
}
|
|
65
|
+
const withFrontier = (
|
|
66
|
+
report: BudgetReport,
|
|
67
|
+
frontierStage: string | undefined,
|
|
68
|
+
summary?: string,
|
|
69
|
+
usage?: WorkflowModelUsage,
|
|
70
|
+
): BudgetExceededReport =>
|
|
71
|
+
Object.assign(
|
|
72
|
+
{ ...report, frontierStage: frontierStage ?? "workflow frontier" },
|
|
73
|
+
summary === undefined ? {} : { wrapUpSummary: summary },
|
|
74
|
+
usage === undefined ? {} : { wrapUpUsage: usage },
|
|
75
|
+
);
|
|
76
|
+
const usageFields = ["input", "output", "cacheRead", "cacheWrite", "cost"] as const,
|
|
77
|
+
counterFields = ["input", "output", "cacheRead", "cacheWrite"] as const;
|
|
78
|
+
const mapFields = <F extends string>(fields: readonly F[], fn: (field: F) => number = () => 0): Record<F, number> =>
|
|
79
|
+
Object.fromEntries(fields.map((field) => [field, fn(field)])) as Record<F, number>;
|
|
80
|
+
export function createRunBudgetController(input: {
|
|
81
|
+
readonly run: RunSnapshot;
|
|
82
|
+
readonly budget: EffectiveBudget;
|
|
83
|
+
readonly usageTree?: () => RunUsageTree;
|
|
84
|
+
readonly onWarning?: (report: BudgetReport) => void;
|
|
85
|
+
readonly rootBudget?: RunBudgetController;
|
|
86
|
+
}) {
|
|
87
|
+
const { run, budget } = input;
|
|
88
|
+
const ownEnabled = budget.maxDurationMs > 0 || budget.maxTokens > 0 || budget.maxCost > 0;
|
|
89
|
+
const usageEnabled = budget.maxTokens > 0 || budget.maxCost > 0;
|
|
90
|
+
const root = input.rootBudget?.enabled === true ? input.rootBudget : undefined;
|
|
91
|
+
const enabled = ownEnabled || root !== undefined;
|
|
92
|
+
let state: RunBudgetState | undefined = run.budgetState === undefined ? undefined : { ...run.budgetState };
|
|
93
|
+
const previousAccounting = state?.accounting;
|
|
94
|
+
const initial =
|
|
95
|
+
previousAccounting && usageEnabled ? meter_run(input.usageTree?.() ?? { run }, Date.now()) : undefined;
|
|
96
|
+
const initialAccounting = initial === undefined ? undefined : { ...initial.perCounter, cost: initial.cost };
|
|
97
|
+
let baseline = mapFields(usageFields, (field) => initialAccounting?.[field] ?? 0);
|
|
98
|
+
let charged: RunBudgetAccountingState = {
|
|
99
|
+
baseline,
|
|
100
|
+
tokens: previousAccounting?.tokens ?? 0,
|
|
101
|
+
cost: previousAccounting?.cost ?? 0,
|
|
102
|
+
perCounter: { ...(previousAccounting?.perCounter ?? mapFields(counterFields)) },
|
|
103
|
+
};
|
|
104
|
+
let exhaustedReport: BudgetReport | undefined;
|
|
105
|
+
let lastReports: readonly BudgetReport[] = [];
|
|
106
|
+
let wrapUpPromise: Promise<never> | undefined;
|
|
107
|
+
let rootWrapUpPending = false;
|
|
108
|
+
const handlers: Array<{ readonly frontierStage: string; readonly handler: () => Promise<never> }> = [];
|
|
109
|
+
const updateState = (patch: Partial<RunBudgetState>): void => {
|
|
110
|
+
state = { ...(state ?? {}), ...patch };
|
|
111
|
+
run.budgetState = state;
|
|
112
|
+
};
|
|
113
|
+
const accountUsage = (meters: RunMeters): RunMeters => {
|
|
114
|
+
const current = { ...meters.perCounter, cost: meters.cost };
|
|
115
|
+
const delta = mapFields(usageFields, (field) => Math.max(0, current[field] - baseline[field]));
|
|
116
|
+
baseline = mapFields(usageFields, (field) => Math.max(baseline[field], current[field]));
|
|
117
|
+
charged = {
|
|
118
|
+
baseline,
|
|
119
|
+
tokens: charged.tokens + delta.input + delta.output,
|
|
120
|
+
cost: charged.cost + delta.cost,
|
|
121
|
+
perCounter: mapFields(counterFields, (field) => charged.perCounter[field] + delta[field]),
|
|
122
|
+
};
|
|
123
|
+
updateState({ accounting: charged });
|
|
124
|
+
return Object.assign(meters, { tokens: charged.tokens, cost: charged.cost, perCounter: charged.perCounter });
|
|
125
|
+
};
|
|
126
|
+
const measure = (): RunMeters => {
|
|
127
|
+
const durationMs = elapsedRunMs(run);
|
|
128
|
+
if (!usageEnabled) return { durationMs, tokens: 0, cost: 0, perCounter: mapFields(counterFields) };
|
|
129
|
+
return { ...accountUsage(meter_run(input.usageTree?.() ?? { run }, Date.now())), durationMs };
|
|
130
|
+
};
|
|
131
|
+
const warningWasSent = (report: BudgetReport): boolean =>
|
|
132
|
+
report.dimension === "duration" ? state?.warned === true : state?.warnings?.[report.dimension] !== undefined;
|
|
133
|
+
const recordWarning = (report: BudgetReport): void => {
|
|
134
|
+
updateState({
|
|
135
|
+
...(report.dimension === "duration" ? { warned: true } : {}),
|
|
136
|
+
warning: report,
|
|
137
|
+
warnings: { ...(state?.warnings ?? {}), [report.dimension]: report },
|
|
138
|
+
});
|
|
139
|
+
input.onWarning?.(report);
|
|
140
|
+
};
|
|
141
|
+
const ceilings = { duration: budget.maxDurationMs, tokens: budget.maxTokens, cost: budget.maxCost };
|
|
142
|
+
const setReports = (reports: readonly BudgetReport[]): void => {
|
|
143
|
+
for (const report of reports) {
|
|
144
|
+
if (ceilings[report.dimension] <= 0) continue;
|
|
145
|
+
updateState({ [report.dimension]: report } as Partial<RunBudgetState>);
|
|
146
|
+
}
|
|
147
|
+
};
|
|
148
|
+
const finishWrapUp = (
|
|
149
|
+
frontierStage: string | undefined,
|
|
150
|
+
summary?: string,
|
|
151
|
+
usage?: WorkflowModelUsage,
|
|
152
|
+
delivered = true,
|
|
153
|
+
): WorkflowBudgetExceededError => {
|
|
154
|
+
updateState({
|
|
155
|
+
systemOwnedStop: true,
|
|
156
|
+
...(delivered ? { wrapUpDelivered: true, wrapUpCompleted: true } : {}),
|
|
157
|
+
...(delivered && summary !== undefined ? { wrapUpSummary: summary } : {}),
|
|
158
|
+
...(delivered && usage !== undefined ? { wrapUpUsage: usage } : {}),
|
|
159
|
+
});
|
|
160
|
+
const report =
|
|
161
|
+
exhaustedReport ??
|
|
162
|
+
lastReports[0] ??
|
|
163
|
+
state?.duration ??
|
|
164
|
+
state?.tokens ??
|
|
165
|
+
state?.cost ??
|
|
166
|
+
enforceDurationBudget(elapsedRunMs(run), budget).report;
|
|
167
|
+
setReports([report]);
|
|
168
|
+
return new WorkflowBudgetExceededError(
|
|
169
|
+
withFrontier(report, frontierStage, state?.wrapUpSummary, state?.wrapUpUsage),
|
|
170
|
+
);
|
|
171
|
+
};
|
|
172
|
+
const ownCheckpoint = (frontierStage?: string): BudgetCheckpoint => {
|
|
173
|
+
const usageCheck = (dimension: "tokens" | "cost", reading: number, ceiling: number, warned: boolean) =>
|
|
174
|
+
ceiling > 0 ? enforceUsageBudget(dimension, reading, ceiling, budget.warnAtPercent, { warned }) : undefined;
|
|
175
|
+
if (!ownEnabled) return { kind: "continue" };
|
|
176
|
+
const meters = measure();
|
|
177
|
+
const checks = [
|
|
178
|
+
enforceDurationBudget(meters.durationMs, budget, { warned: state?.warned }),
|
|
179
|
+
usageCheck("tokens", meters.tokens, budget.maxTokens, state?.warnings?.tokens !== undefined),
|
|
180
|
+
usageCheck("cost", meters.cost, budget.maxCost, state?.warnings?.cost !== undefined),
|
|
181
|
+
].filter((check): check is Exclude<typeof check, undefined> => check !== undefined);
|
|
182
|
+
lastReports = checks.map((check) => check.report);
|
|
183
|
+
setReports(lastReports);
|
|
184
|
+
let warning: BudgetReport | undefined;
|
|
185
|
+
for (const check of checks)
|
|
186
|
+
if (check.kind === "continue" && check.warning && !warningWasSent(check.report)) {
|
|
187
|
+
warning ??= check.report;
|
|
188
|
+
recordWarning(check.report);
|
|
189
|
+
}
|
|
190
|
+
const exhausted = checks.find((check) => check.kind === "exhausted");
|
|
191
|
+
if (exhausted === undefined)
|
|
192
|
+
return warning === undefined ? { kind: "continue" } : { kind: "warn", report: warning };
|
|
193
|
+
exhaustedReport ??= exhausted.report;
|
|
194
|
+
if (state?.wrapUpCompleted !== true) return { kind: "wrap_up", report: exhausted.report };
|
|
195
|
+
return {
|
|
196
|
+
kind: "exhausted",
|
|
197
|
+
report: withFrontier(exhaustedReport, frontierStage, state.wrapUpSummary, state.wrapUpUsage),
|
|
198
|
+
};
|
|
199
|
+
};
|
|
200
|
+
const boundaryCheckpoint = (
|
|
201
|
+
frontierStage?: string,
|
|
202
|
+
): { readonly check: BudgetCheckpoint; readonly owner?: RunBudgetController } => {
|
|
203
|
+
const rootCheck = root?.checkpoint(frontierStage);
|
|
204
|
+
if (rootCheck?.kind === "wrap_up") rootWrapUpPending = true;
|
|
205
|
+
if (rootCheck?.kind === "wrap_up" || rootCheck?.kind === "exhausted") return { check: rootCheck, owner: root };
|
|
206
|
+
return { check: ownCheckpoint(frontierStage) };
|
|
207
|
+
};
|
|
208
|
+
const checkpoint = (frontierStage?: string): BudgetCheckpoint => boundaryCheckpoint(frontierStage).check;
|
|
209
|
+
const finishBoundary = (owner: RunBudgetController | undefined, frontierStage: string | undefined): never => {
|
|
210
|
+
throw (
|
|
211
|
+
owner?.finishWrapUp(frontierStage, undefined, undefined, false) ??
|
|
212
|
+
finishWrapUp(frontierStage, undefined, undefined, false)
|
|
213
|
+
);
|
|
214
|
+
};
|
|
215
|
+
const deliverBoundary = (owner: RunBudgetController | undefined, frontierStage: string): void => {
|
|
216
|
+
void (owner?.deliverWrapUp(frontierStage) ?? deliverWrapUp(frontierStage));
|
|
217
|
+
};
|
|
218
|
+
const stopAtBoundary = (frontierStage?: string): void => {
|
|
219
|
+
const resolvedFrontierStage = frontierStage ?? handlers.at(-1)?.frontierStage;
|
|
220
|
+
const { check, owner } = boundaryCheckpoint(resolvedFrontierStage);
|
|
221
|
+
if (check.kind === "wrap_up") {
|
|
222
|
+
deliverBoundary(owner, resolvedFrontierStage ?? "workflow frontier");
|
|
223
|
+
finishBoundary(owner, resolvedFrontierStage);
|
|
224
|
+
}
|
|
225
|
+
if (check.kind === "exhausted") finishBoundary(owner, resolvedFrontierStage);
|
|
226
|
+
};
|
|
227
|
+
const rethrowIfSystemOwnedStop = (frontierStage?: string): void => {
|
|
228
|
+
root?.rethrowIfSystemOwnedStop(frontierStage);
|
|
229
|
+
if (state?.systemOwnedStop === true)
|
|
230
|
+
throw finishWrapUp(frontierStage, state.wrapUpSummary, state.wrapUpUsage, state.wrapUpCompleted === true);
|
|
231
|
+
};
|
|
232
|
+
const registerLocalWrapUp = (registration: {
|
|
233
|
+
readonly frontierStage: string;
|
|
234
|
+
readonly handler: () => Promise<never>;
|
|
235
|
+
}): (() => void) => {
|
|
236
|
+
handlers.push(registration);
|
|
237
|
+
return () => {
|
|
238
|
+
const index = handlers.indexOf(registration);
|
|
239
|
+
if (index >= 0) handlers.splice(index, 1);
|
|
240
|
+
};
|
|
241
|
+
};
|
|
242
|
+
const registerWrapUp = (frontierStage: string, handler: () => Promise<never>): (() => void) => {
|
|
243
|
+
const registration = { frontierStage, handler };
|
|
244
|
+
const unregisterOwn = ownEnabled ? registerLocalWrapUp(registration) : undefined;
|
|
245
|
+
const unregisterRoot = root
|
|
246
|
+
? root.registerWrapUp(frontierStage, async () => {
|
|
247
|
+
try {
|
|
248
|
+
return await handler();
|
|
249
|
+
} catch (error) {
|
|
250
|
+
if (!(error instanceof WorkflowBudgetExceededError)) throw error;
|
|
251
|
+
throw root.finishWrapUp(
|
|
252
|
+
error.report.frontierStage,
|
|
253
|
+
error.report.wrapUpSummary,
|
|
254
|
+
error.report.wrapUpUsage,
|
|
255
|
+
error.report.wrapUpSummary !== undefined,
|
|
256
|
+
);
|
|
257
|
+
}
|
|
258
|
+
})
|
|
259
|
+
: undefined;
|
|
260
|
+
return () => {
|
|
261
|
+
unregisterOwn?.();
|
|
262
|
+
unregisterRoot?.();
|
|
263
|
+
};
|
|
264
|
+
};
|
|
265
|
+
const deliverWrapUp = (frontierStage: string): Promise<never> => {
|
|
266
|
+
if (wrapUpPromise !== undefined) return wrapUpPromise;
|
|
267
|
+
if (rootWrapUpPending && root !== undefined) return root.deliverWrapUp(frontierStage);
|
|
268
|
+
if (state?.systemOwnedStop === true && state.wrapUpCompleted !== true)
|
|
269
|
+
throw finishWrapUp(frontierStage, undefined, undefined, false);
|
|
270
|
+
const registration = handlers.findLast((entry) => entry.frontierStage === frontierStage) ?? handlers.at(-1);
|
|
271
|
+
if (registration === undefined) throw finishWrapUp(frontierStage, undefined, undefined, false);
|
|
272
|
+
wrapUpPromise = registration.handler();
|
|
273
|
+
return wrapUpPromise;
|
|
274
|
+
};
|
|
275
|
+
const stopAtBoundaryAsync = async (frontierStage?: string): Promise<void> => {
|
|
276
|
+
if (root !== undefined) await root.stopAtBoundaryAsync(frontierStage);
|
|
277
|
+
const check = ownCheckpoint(frontierStage);
|
|
278
|
+
if (check.kind === "continue" || check.kind === "warn") return;
|
|
279
|
+
if (check.kind === "wrap_up") {
|
|
280
|
+
await deliverWrapUp(frontierStage ?? "workflow frontier");
|
|
281
|
+
return;
|
|
282
|
+
}
|
|
283
|
+
throw finishWrapUp(frontierStage, state?.wrapUpSummary, state?.wrapUpUsage, state?.wrapUpCompleted === true);
|
|
284
|
+
};
|
|
285
|
+
const awaitPendingWrapUp = async (): Promise<WorkflowBudgetExceededError | undefined> => {
|
|
286
|
+
const rootError = await root?.awaitPendingWrapUp();
|
|
287
|
+
if (rootError !== undefined) return rootError;
|
|
288
|
+
if (wrapUpPromise === undefined) return undefined;
|
|
289
|
+
try {
|
|
290
|
+
await wrapUpPromise;
|
|
291
|
+
} catch (error) {
|
|
292
|
+
if (error instanceof WorkflowBudgetExceededError) return error;
|
|
293
|
+
throw error;
|
|
294
|
+
}
|
|
295
|
+
return undefined;
|
|
296
|
+
};
|
|
297
|
+
return {
|
|
298
|
+
enabled,
|
|
299
|
+
checkpoint,
|
|
300
|
+
registerWrapUp,
|
|
301
|
+
deliverWrapUp,
|
|
302
|
+
finishWrapUp,
|
|
303
|
+
rethrowIfSystemOwnedStop,
|
|
304
|
+
stopAtBoundary,
|
|
305
|
+
stopAtBoundaryAsync,
|
|
306
|
+
awaitPendingWrapUp,
|
|
307
|
+
};
|
|
308
|
+
}
|
|
@@ -28,6 +28,14 @@ export function classifyReturnedRunStatus(
|
|
|
28
28
|
};
|
|
29
29
|
}
|
|
30
30
|
const returnedStatus = normalizeReturnedWorkflowStatus(result?.status);
|
|
31
|
+
if (
|
|
32
|
+
returnedStatus === "budget_exceeded" &&
|
|
33
|
+
(runSnapshot?.budgetState?.wrapUpCompleted !== true ||
|
|
34
|
+
runSnapshot.failureDisposition !== "active_blocked" ||
|
|
35
|
+
runSnapshot.failureRecoverability !== "recoverable")
|
|
36
|
+
)
|
|
37
|
+
// Only the system-owned budget door may produce this status.
|
|
38
|
+
return { status: "completed" };
|
|
31
39
|
if (returnedStatus === undefined || returnedStatus === "completed" || returnedStatus === "complete") {
|
|
32
40
|
return { status: "completed" };
|
|
33
41
|
}
|
|
@@ -5,6 +5,7 @@ import type { Store } from "../shared/store.js";
|
|
|
5
5
|
import type { RunSnapshot, ToolNodeSnapshot } from "../shared/store-types.js";
|
|
6
6
|
import type { WorkflowToolPrimitive } from "../shared/types.js";
|
|
7
7
|
import type { GraphFrontierTracker } from "./graph-inference.js";
|
|
8
|
+
import type { RunBudgetController } from "./run-budget.js";
|
|
8
9
|
import { durableRunTopology } from "./run-durable-topology.js";
|
|
9
10
|
import type { RunTerminalEventArbiter } from "./run-terminal-event.js";
|
|
10
11
|
import type { ToolAdmissionBoundary } from "./run-tool-admission-boundary.js";
|
|
@@ -71,6 +72,7 @@ export function createTrackedToolPrimitive(input: {
|
|
|
71
72
|
readonly sourceToReplayedNodeIds: Map<string, string>;
|
|
72
73
|
readonly toolControls: ToolControlRegistry;
|
|
73
74
|
readonly toolAdmission: ToolAdmissionBoundary;
|
|
75
|
+
readonly budget: RunBudgetController;
|
|
74
76
|
}): {
|
|
75
77
|
readonly tool: WorkflowToolPrimitive;
|
|
76
78
|
readonly admittedTools: AdmittedToolExecutionTracker;
|
|
@@ -96,6 +98,9 @@ export function createTrackedToolPrimitive(input: {
|
|
|
96
98
|
},
|
|
97
99
|
});
|
|
98
100
|
const lifecycle = createToolNodeLifecycle(input);
|
|
101
|
+
const budgetBoundary = input.budget.enabled
|
|
102
|
+
? (): Promise<void> => input.budget.stopAtBoundaryAsync(input.run.stages.at(-1)?.name)
|
|
103
|
+
: undefined;
|
|
99
104
|
const tool = createToolPrimitive({
|
|
100
105
|
workflowId: input.workflowId,
|
|
101
106
|
backend: input.backend,
|
|
@@ -128,6 +133,7 @@ export function createTrackedToolPrimitive(input: {
|
|
|
128
133
|
if (!admission.accepted) observedQuit ??= admission.error;
|
|
129
134
|
return admission;
|
|
130
135
|
},
|
|
136
|
+
...(budgetBoundary !== undefined ? { beforeToolCall: budgetBoundary, afterToolCall: budgetBoundary } : {}),
|
|
131
137
|
...lifecycle,
|
|
132
138
|
});
|
|
133
139
|
const abandonInFlightAsCancelled = (reason: unknown): readonly string[] => {
|
|
@@ -2,7 +2,7 @@ import type { DurableWorkflowBackend } from "../durable/backend.js";
|
|
|
2
2
|
import type { DurableChildInvocation } from "../durable/boundary-topology.js";
|
|
3
3
|
import { createDurableChildWorkflowPrimitive } from "../durable/child-primitive.js";
|
|
4
4
|
import { getDurableBackend } from "../durable/factory.js";
|
|
5
|
-
import { inheritedRunElapsedMs, recordRunTimingCheckpoint } from "../durable/run-timing.js";
|
|
5
|
+
import { inheritedRunElapsedMs, priorRunAccounting, recordRunTimingCheckpoint } from "../durable/run-timing.js";
|
|
6
6
|
import { ScopedDurableBackend } from "../durable/scoped-backend.js";
|
|
7
7
|
import {
|
|
8
8
|
createDurableStagePrimitive,
|
|
@@ -47,6 +47,8 @@ import { createStageScheduler } from "../runs/foreground/executor-scheduler.js";
|
|
|
47
47
|
import type { RunOpts, RunResult } from "../runs/foreground/executor-types.js";
|
|
48
48
|
import { stageControlRegistry as defaultStageControlRegistry } from "../runs/foreground/stage-control-registry.js";
|
|
49
49
|
import { createRunLimiter } from "../runs/shared/concurrency.js";
|
|
50
|
+
import { resolve_budget, type WorkflowBudget } from "../shared/budget.js";
|
|
51
|
+
import type { RunUsageTree } from "../shared/budget-meter.js";
|
|
50
52
|
import { appendRunStart } from "../shared/persistence-session-entries.js";
|
|
51
53
|
import { store as defaultStore } from "../shared/store.js";
|
|
52
54
|
import type { RunSnapshot } from "../shared/store-types.js";
|
|
@@ -67,6 +69,7 @@ import { createWorkflowTaskRunners } from "./primitives/task.js";
|
|
|
67
69
|
import { buildExitGatedUiContext } from "./primitives/ui.js";
|
|
68
70
|
import { createChildWorkflowRunner } from "./primitives/workflow.js";
|
|
69
71
|
import { createContinuationReplayIndex } from "./replay.js";
|
|
72
|
+
import { createRunBudgetController, WorkflowBudgetExceededError } from "./run-budget.js";
|
|
70
73
|
import { admitDurableRootRun, durableRootRegistrationForRun } from "./run-durable-admission.js";
|
|
71
74
|
import { finalizeDurableTerminalStatus } from "./run-durable-finalize.js";
|
|
72
75
|
import { createDurableStageSessionRecorder } from "./run-durable-stage-session.js";
|
|
@@ -130,6 +133,23 @@ export async function run<TInputs extends WorkflowInputValues, TRunInputs extend
|
|
|
130
133
|
|
|
131
134
|
const resolvedInputs = resolveAndValidateInputs(def.inputs, inputs, `workflow "${def.name}"`);
|
|
132
135
|
const runId = opts.runId ?? crypto.randomUUID();
|
|
136
|
+
// A budget OBJECT is always materialized by the shipped config defaults, so
|
|
137
|
+
// presence proves nothing. A positive limit on any dimension means this run
|
|
138
|
+
// is budgeted; otherwise take the unbudgeted path untouched (R10).
|
|
139
|
+
const declaresBudget = (budget: WorkflowBudget | undefined): boolean =>
|
|
140
|
+
(budget?.maxDurationMs ?? 0) > 0 || (budget?.maxTokens ?? 0) > 0 || (budget?.maxCost ?? 0) > 0;
|
|
141
|
+
const hasBudgetDeclaration =
|
|
142
|
+
declaresBudget(opts.budget) || declaresBudget(def.budget) || declaresBudget(opts.config?.budget);
|
|
143
|
+
const priorRun =
|
|
144
|
+
hasBudgetDeclaration &&
|
|
145
|
+
opts.rootBudget === undefined &&
|
|
146
|
+
opts.continuation === undefined &&
|
|
147
|
+
opts.runId !== undefined
|
|
148
|
+
? activeStore.runs().find((candidate) => candidate.id === runId)
|
|
149
|
+
: undefined;
|
|
150
|
+
const continuedBudget = opts.continuation?.source.budget ?? priorRun?.budget;
|
|
151
|
+
const runBudget = opts.budget === undefined ? continuedBudget : { ...(continuedBudget ?? {}), ...opts.budget };
|
|
152
|
+
const resolvedBudget = resolve_budget({ config: opts.config?.budget, definition: def.budget, run: runBudget });
|
|
133
153
|
const exitScope = Symbol(`workflow-exit:${runId}`);
|
|
134
154
|
const ownController = new AbortController();
|
|
135
155
|
const terminalEvents = createRunTerminalEventArbiter(runId);
|
|
@@ -165,6 +185,39 @@ export async function run<TInputs extends WorkflowInputValues, TRunInputs extend
|
|
|
165
185
|
? inheritedRunElapsedMs({ backend: durableBackend, runId, continuationSource: opts.continuation?.source })
|
|
166
186
|
: undefined;
|
|
167
187
|
const continuationOrigin = opts.continuation !== undefined ? opts.continuation.source.origin : opts.origin;
|
|
188
|
+
const snapshotField = (field: "maxTokens" | "maxCost"): number | undefined =>
|
|
189
|
+
resolvedBudget[field] > 0 || opts.budget?.[field] !== undefined || def.budget?.[field] !== undefined
|
|
190
|
+
? resolvedBudget[field]
|
|
191
|
+
: undefined;
|
|
192
|
+
const budgetSnapshot =
|
|
193
|
+
resolvedBudget.maxDurationMs > 0 || resolvedBudget.maxTokens > 0 || resolvedBudget.maxCost > 0
|
|
194
|
+
? {
|
|
195
|
+
maxDurationMs: resolvedBudget.maxDurationMs,
|
|
196
|
+
...(snapshotField("maxTokens") !== undefined ? { maxTokens: snapshotField("maxTokens") } : {}),
|
|
197
|
+
...(snapshotField("maxCost") !== undefined ? { maxCost: snapshotField("maxCost") } : {}),
|
|
198
|
+
warnAtPercent: resolvedBudget.warnAtPercent,
|
|
199
|
+
}
|
|
200
|
+
: undefined;
|
|
201
|
+
const durableAccounting =
|
|
202
|
+
hasBudgetDeclaration && opts.parentRun === undefined && opts.continuation === undefined && priorRun === undefined
|
|
203
|
+
? priorRunAccounting(durableBackend, runId)
|
|
204
|
+
: undefined;
|
|
205
|
+
const continuedBudgetState =
|
|
206
|
+
opts.continuation?.source.budgetState ??
|
|
207
|
+
priorRun?.budgetState ??
|
|
208
|
+
(durableAccounting === undefined ? undefined : { accounting: durableAccounting });
|
|
209
|
+
const sameBudget =
|
|
210
|
+
budgetSnapshot !== undefined &&
|
|
211
|
+
continuedBudget?.warnAtPercent === budgetSnapshot.warnAtPercent &&
|
|
212
|
+
(["maxDurationMs", "maxTokens", "maxCost"] as const).every(
|
|
213
|
+
(field) => (continuedBudget?.[field] ?? 0) === (budgetSnapshot[field] ?? 0),
|
|
214
|
+
);
|
|
215
|
+
const continuationBudgetState =
|
|
216
|
+
continuedBudgetState === undefined || (!sameBudget && continuedBudgetState.accounting === undefined)
|
|
217
|
+
? undefined
|
|
218
|
+
: sameBudget
|
|
219
|
+
? continuedBudgetState
|
|
220
|
+
: { accounting: continuedBudgetState.accounting };
|
|
168
221
|
const runSnapshot: RunSnapshot = {
|
|
169
222
|
id: runId,
|
|
170
223
|
name: def.name,
|
|
@@ -183,7 +236,9 @@ export async function run<TInputs extends WorkflowInputValues, TRunInputs extend
|
|
|
183
236
|
...(opts.continuation !== undefined
|
|
184
237
|
? {
|
|
185
238
|
resumedFromRunId: opts.continuation.source.id,
|
|
186
|
-
|
|
239
|
+
...(opts.continuation.resumeFromStageId !== undefined
|
|
240
|
+
? { resumeFromStageId: opts.continuation.resumeFromStageId }
|
|
241
|
+
: {}),
|
|
187
242
|
}
|
|
188
243
|
: {}),
|
|
189
244
|
// A continuation keeps the attribution of the run it continues rather than
|
|
@@ -196,7 +251,40 @@ export async function run<TInputs extends WorkflowInputValues, TRunInputs extend
|
|
|
196
251
|
? { resumeActor: opts.resumeActor, resumeSource: "run_control" as const }
|
|
197
252
|
: {}),
|
|
198
253
|
...(inheritedElapsedMs !== undefined ? { accumulatedDurationMs: inheritedElapsedMs } : {}),
|
|
254
|
+
...(budgetSnapshot !== undefined ? { budget: budgetSnapshot } : {}),
|
|
255
|
+
...(continuationBudgetState !== undefined ? { budgetState: continuationBudgetState } : {}),
|
|
256
|
+
};
|
|
257
|
+
const usageTree = (): RunUsageTree => {
|
|
258
|
+
const snapshots = activeStore.runs();
|
|
259
|
+
const childrenByParent = new Map<string, RunSnapshot[]>();
|
|
260
|
+
for (const snapshot of snapshots) {
|
|
261
|
+
if (snapshot.parentRunId === undefined) continue;
|
|
262
|
+
const children = childrenByParent.get(snapshot.parentRunId) ?? [];
|
|
263
|
+
children.push(snapshot);
|
|
264
|
+
childrenByParent.set(snapshot.parentRunId, children);
|
|
265
|
+
}
|
|
266
|
+
const build = (run: RunSnapshot): RunUsageTree => {
|
|
267
|
+
const children = (childrenByParent.get(run.id) ?? []).map(build);
|
|
268
|
+
return children.length === 0 ? { run } : { run, children };
|
|
269
|
+
};
|
|
270
|
+
return build(snapshots.find((snapshot) => snapshot.id === runSnapshot.id) ?? runSnapshot);
|
|
199
271
|
};
|
|
272
|
+
const budget = createRunBudgetController({
|
|
273
|
+
run: runSnapshot,
|
|
274
|
+
budget: resolvedBudget,
|
|
275
|
+
usageTree,
|
|
276
|
+
rootBudget: opts.rootBudget,
|
|
277
|
+
onWarning: (report) => {
|
|
278
|
+
activeStore.recordNotice({
|
|
279
|
+
id: `workflow-budget-warning:${runId}:${report.dimension}`,
|
|
280
|
+
runId,
|
|
281
|
+
level: "warning",
|
|
282
|
+
message: `Workflow "${def.name}" is at ${report.percent.toFixed(1)}% of its ${report.dimension} budget (${report.reading} / ${report.ceiling}).`,
|
|
283
|
+
createdAt: Date.now(),
|
|
284
|
+
});
|
|
285
|
+
},
|
|
286
|
+
});
|
|
287
|
+
const rootBudget = opts.rootBudget ?? budget;
|
|
200
288
|
const classifiedFailures = new Map<unknown, WorkflowFailure>();
|
|
201
289
|
const classifyExecutorFailure = (error: unknown): WorkflowFailure => {
|
|
202
290
|
const cached = classifiedFailures.get(error);
|
|
@@ -231,6 +319,8 @@ export async function run<TInputs extends WorkflowInputValues, TRunInputs extend
|
|
|
231
319
|
...(runSnapshot.accumulatedDurationMs !== undefined
|
|
232
320
|
? { accumulatedDurationMs: runSnapshot.accumulatedDurationMs }
|
|
233
321
|
: {}),
|
|
322
|
+
...(runSnapshot.budget !== undefined ? { budget: runSnapshot.budget } : {}),
|
|
323
|
+
...(runSnapshot.budgetState !== undefined ? { budgetState: runSnapshot.budgetState } : {}),
|
|
234
324
|
ts: runSnapshot.startedAt,
|
|
235
325
|
});
|
|
236
326
|
}
|
|
@@ -342,6 +432,7 @@ export async function run<TInputs extends WorkflowInputValues, TRunInputs extend
|
|
|
342
432
|
onStageStart: opts.onStageStart,
|
|
343
433
|
onStageEnd: opts.onStageEnd,
|
|
344
434
|
onStageSession: opts.onStageSession,
|
|
435
|
+
rootBudget,
|
|
345
436
|
durableBackend,
|
|
346
437
|
durableRootBackend: rootBackend,
|
|
347
438
|
};
|
|
@@ -367,6 +458,8 @@ export async function run<TInputs extends WorkflowInputValues, TRunInputs extend
|
|
|
367
458
|
worktreeSymlinkDirectories: opts.config?.worktree?.symlinkDirectories,
|
|
368
459
|
exit,
|
|
369
460
|
classifyExecutorFailure,
|
|
461
|
+
rootBudget,
|
|
462
|
+
budget,
|
|
370
463
|
});
|
|
371
464
|
const workflowBoundaryReplayCounts = new Map<string, number>();
|
|
372
465
|
const nextWorkflowBoundaryReplayKey = (name: string): string => {
|
|
@@ -422,6 +515,7 @@ export async function run<TInputs extends WorkflowInputValues, TRunInputs extend
|
|
|
422
515
|
sourceToReplayedNodeIds,
|
|
423
516
|
toolControls,
|
|
424
517
|
toolAdmission,
|
|
518
|
+
budget,
|
|
425
519
|
});
|
|
426
520
|
let selectedAdmittedToolFailure: ReturnType<typeof admittedTools.firstFailure>;
|
|
427
521
|
/**
|
|
@@ -571,8 +665,14 @@ export async function run<TInputs extends WorkflowInputValues, TRunInputs extend
|
|
|
571
665
|
},
|
|
572
666
|
});
|
|
573
667
|
if (opts.deferWorkflowStart === true) opts.onWorkflowStartReady?.();
|
|
668
|
+
const sourceFrontierStage = opts.continuation?.source;
|
|
669
|
+
const sourceFrontierId = sourceFrontierStage?.failedStageId ?? opts.continuation?.resumeFromStageId;
|
|
670
|
+
const startupFrontierStage =
|
|
671
|
+
sourceFrontierStage?.stages.find((stage) => stage.id === sourceFrontierId)?.name ?? "workflow frontier";
|
|
672
|
+
if (budget.enabled) await budget.stopAtBoundaryAsync(startupFrontierStage);
|
|
574
673
|
const rawResult = await runWorkflowDefinitionCallback(def.name, runId, () => def.run(ctx));
|
|
575
674
|
await admittedTools.closeAndDrain();
|
|
675
|
+
budget.rethrowIfSystemOwnedStop(runSnapshot.stages.at(-1)?.name ?? startupFrontierStage);
|
|
576
676
|
const normalTerminalEvent = terminalEvents.winner();
|
|
577
677
|
if (normalTerminalEvent?.kind === "cancellation") {
|
|
578
678
|
const selectedExit = findWorkflowExitSignal(normalTerminalEvent.reason, exitScope);
|
|
@@ -632,6 +732,28 @@ export async function run<TInputs extends WorkflowInputValues, TRunInputs extend
|
|
|
632
732
|
findWorkflowGracefulQuit(ownController.signal.reason);
|
|
633
733
|
if (gracefulQuit !== undefined) return suspendForGracefulQuit(gracefulQuit);
|
|
634
734
|
await admittedTools.closeAndDrain();
|
|
735
|
+
if (err instanceof WorkflowBudgetExceededError) {
|
|
736
|
+
const pendingBudgetError = await budget.awaitPendingWrapUp();
|
|
737
|
+
const selectedBudgetError = pendingBudgetError ?? err;
|
|
738
|
+
const report = selectedBudgetError.report;
|
|
739
|
+
const frontierStageId =
|
|
740
|
+
runSnapshot.stages.find((stage) => stage.name === report.frontierStage)?.id ??
|
|
741
|
+
runSnapshot.stages.at(-1)?.id;
|
|
742
|
+
for (const stage of runSnapshot.stages) scheduler.blockKnownNonTerminalDescendants(stage.id);
|
|
743
|
+
const result: WorkflowOutputValues = { status: "budget_exceeded", ...report };
|
|
744
|
+
return recordActiveBlockedFailure(runId, runSnapshot, activeStore, opts.persistence, {
|
|
745
|
+
errorMessage: selectedBudgetError.message,
|
|
746
|
+
failureKind: "unknown",
|
|
747
|
+
failureCode: "unknown",
|
|
748
|
+
failureRecoverability: "recoverable",
|
|
749
|
+
failureDisposition: "active_blocked",
|
|
750
|
+
failureMessage: selectedBudgetError.message,
|
|
751
|
+
...(frontierStageId !== undefined ? { failedStageId: frontierStageId } : {}),
|
|
752
|
+
resumable: true,
|
|
753
|
+
result,
|
|
754
|
+
budgetState: runSnapshot.budgetState,
|
|
755
|
+
});
|
|
756
|
+
}
|
|
635
757
|
const observedAdmittedToolFailure = selectedAdmittedToolFailure ?? admittedTools.uniqueFailureFor(err);
|
|
636
758
|
const selectedExit =
|
|
637
759
|
findWorkflowExitSignal(err, exitScope) ?? findWorkflowExitSignal(ownController.signal.reason, exitScope);
|
|
@@ -19,6 +19,7 @@ import type { RunSnapshot } from "../shared/store-types.js";
|
|
|
19
19
|
import type { StageOptions } from "../shared/types.js";
|
|
20
20
|
import type { GraphFrontierTracker } from "./graph-inference.js";
|
|
21
21
|
import type { EngineChildRunOptions, EngineStageRuntimeOptions, EngineWorkflowBoundaryOptions } from "./options.js";
|
|
22
|
+
import type { RunBudgetController } from "./run-budget.js";
|
|
22
23
|
|
|
23
24
|
export interface EngineRuntimeInput {
|
|
24
25
|
readonly runId: string;
|
|
@@ -41,6 +42,8 @@ export interface EngineRuntimeInput {
|
|
|
41
42
|
readonly worktreeSymlinkDirectories?: readonly string[];
|
|
42
43
|
readonly stageRegistry: StageControlRegistry;
|
|
43
44
|
readonly exit: WorkflowExitManager;
|
|
45
|
+
readonly budget: RunBudgetController;
|
|
46
|
+
readonly rootBudget: RunBudgetController;
|
|
44
47
|
readonly classifyExecutorFailure: LiveStageRuntime["classifyExecutorFailure"];
|
|
45
48
|
}
|
|
46
49
|
|
|
@@ -83,6 +86,8 @@ export class EngineRuntime {
|
|
|
83
86
|
readonly exit: WorkflowExitManager;
|
|
84
87
|
readonly inputRuntimeDefaults: Partial<StageOptions>;
|
|
85
88
|
readonly workflowInvocationCwd: string;
|
|
89
|
+
readonly budget: RunBudgetController;
|
|
90
|
+
readonly rootBudget: RunBudgetController;
|
|
86
91
|
readonly gitWorktreeSetupCache: GitWorktreeSetupCache;
|
|
87
92
|
readonly worktreeSymlinkDirectories?: readonly string[];
|
|
88
93
|
|
|
@@ -110,6 +115,8 @@ export class EngineRuntime {
|
|
|
110
115
|
this.exit = input.exit;
|
|
111
116
|
this.inputRuntimeDefaults = input.inputRuntimeDefaults;
|
|
112
117
|
this.workflowInvocationCwd = input.workflowInvocationCwd;
|
|
118
|
+
this.budget = input.budget;
|
|
119
|
+
this.rootBudget = input.rootBudget;
|
|
113
120
|
this.gitWorktreeSetupCache = input.gitWorktreeSetupCache;
|
|
114
121
|
this.worktreeSymlinkDirectories = input.worktreeSymlinkDirectories;
|
|
115
122
|
|
|
@@ -130,6 +137,7 @@ export class EngineRuntime {
|
|
|
130
137
|
workflowInvocationCwd: input.workflowInvocationCwd,
|
|
131
138
|
gitWorktreeSetupCache: input.gitWorktreeSetupCache,
|
|
132
139
|
stageRegistry: input.stageRegistry,
|
|
140
|
+
budget: input.budget,
|
|
133
141
|
exit: input.exit,
|
|
134
142
|
classifyExecutorFailure: input.classifyExecutorFailure,
|
|
135
143
|
createMcpScope: (stageId, options) => this.createMcpScope(stageId, options),
|
|
@@ -164,6 +172,7 @@ export class EngineRuntime {
|
|
|
164
172
|
options?: StageOptions,
|
|
165
173
|
failFastScope?: ParallelFailFastScope,
|
|
166
174
|
): StageContextWithMeta => {
|
|
175
|
+
if (this.budget.enabled) this.budget.stopAtBoundary();
|
|
167
176
|
const handle = this.spawnStage(name, {
|
|
168
177
|
kind: "agent",
|
|
169
178
|
...(options !== undefined ? { options } : {}),
|