@sweny-ai/core 0.1.199 → 0.1.200
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/budget.d.ts +77 -11
- package/dist/budget.js +132 -16
- package/dist/cli/renderer.js +5 -0
- package/dist/cli/run-history.d.ts +12 -0
- package/dist/cli/run-history.js +16 -0
- package/dist/cli/run-output.d.ts +16 -1
- package/dist/cli/run-output.js +46 -4
- package/dist/cli/runs.js +38 -1
- package/dist/cli/ticket.js +7 -2
- package/dist/eval/index.d.ts +10 -0
- package/dist/eval/index.js +27 -6
- package/dist/eval/judge.d.ts +3 -0
- package/dist/eval/judge.js +12 -2
- package/dist/executor.js +276 -31
- package/dist/harness/acp.js +4 -1
- package/dist/harness/claude-code.d.ts +3 -1
- package/dist/harness/claude-code.js +4 -1
- package/dist/harness/codex.js +3 -1
- package/dist/harness/compat.js +1 -0
- package/dist/harness/index.d.ts +1 -1
- package/dist/harness/index.js +1 -1
- package/dist/harness/pi.js +3 -1
- package/dist/harness/prompts.d.ts +11 -1
- package/dist/harness/prompts.js +21 -2
- package/dist/harness/types.d.ts +9 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.js +2 -2
- package/dist/journal.d.ts +48 -10
- package/dist/journal.js +164 -16
- package/dist/safe-outputs.d.ts +9 -0
- package/dist/safe-outputs.js +41 -26
- package/dist/skills/linear.js +3 -1
- package/dist/studio.js +2 -1
- package/dist/types.d.ts +34 -1
- package/package.json +1 -1
package/dist/cli/ticket.js
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* METADATA ONLY, like the receipt line: counts, durations, tokens, cost,
|
|
7
7
|
* policy facts. Never prompts, node outputs, tool inputs, or model prose.
|
|
8
8
|
*/
|
|
9
|
-
import { formatBudgetAmount, formatCost, formatRouteCounts, formatReceiptDuration, formatTokenCount, policyParts, renderReceiptLine, } from "./run-output.js";
|
|
9
|
+
import { formatBudgetAmount, formatCost, formatRouteCounts, formatSpendSplit, formatReceiptDuration, formatTokenCount, policyParts, renderReceiptLine, } from "./run-output.js";
|
|
10
10
|
import { createPaint } from "./style.js";
|
|
11
11
|
import { colorEnabled, glyphsFor, richOutput, supportsUnicode, terminalColumns, ticketBoxFor } from "./terminal.js";
|
|
12
12
|
/** Widest ticket, in columns, frame included. */
|
|
@@ -52,8 +52,11 @@ export function stampWords(summary) {
|
|
|
52
52
|
}
|
|
53
53
|
return { words, strong };
|
|
54
54
|
}
|
|
55
|
+
const NBSP = "\u00a0";
|
|
55
56
|
/** Field rows, label then value, metadata only. */
|
|
56
57
|
function ticketFields(s) {
|
|
58
|
+
// A no-break space keeps each kind with its amount when the row wraps (put back after wrapping).
|
|
59
|
+
const spend = formatSpendSplit(s.spend, NBSP);
|
|
57
60
|
const failed = Math.max(0, s.nodesTotal - s.nodesOk - s.nodesSkipped);
|
|
58
61
|
const nodes = [
|
|
59
62
|
`${s.nodesOk}/${s.nodesTotal}`,
|
|
@@ -67,6 +70,7 @@ function ticketFields(s) {
|
|
|
67
70
|
["duration", formatReceiptDuration(s.durationMs)],
|
|
68
71
|
...(s.tokens !== undefined ? [["tokens", formatTokenCount(s.tokens)]] : []),
|
|
69
72
|
...(s.costUsd !== undefined ? [["cost", formatCost(s.costUsd)]] : []),
|
|
73
|
+
...(spend ? [["spend", spend]] : []),
|
|
70
74
|
...(s.harness !== undefined && s.harness !== "claude-code" ? [["harness", s.harness]] : []),
|
|
71
75
|
...(s.degraded && s.degraded.length > 0 ? [["degraded", s.degraded.join(", ")]] : []),
|
|
72
76
|
...(s.routes && s.routes.total > 0 ? [["routes", formatRouteCounts(s.routes)]] : []),
|
|
@@ -154,7 +158,8 @@ export function renderTicket(data, opts = {}) {
|
|
|
154
158
|
lines.push(row(""));
|
|
155
159
|
// Fields, aligned: label column then value column; long values wrap under the value.
|
|
156
160
|
for (const [label, value] of ticketFields(s)) {
|
|
157
|
-
wrap(value, valueWidth).forEach((
|
|
161
|
+
wrap(value, valueWidth).forEach((raw, i) => {
|
|
162
|
+
const part = raw.replaceAll(NBSP, " ");
|
|
158
163
|
const lab = (i === 0 ? label : "").padEnd(LABEL_WIDTH);
|
|
159
164
|
lines.push(row(lab + part, paint.muted(lab) + part));
|
|
160
165
|
});
|
package/dist/eval/index.d.ts
CHANGED
|
@@ -10,6 +10,16 @@ export interface EvaluateAllOptions {
|
|
|
10
10
|
node?: Node;
|
|
11
11
|
/** The workflow being executed. Used to resolve workflow-level `judge_model` overrides. */
|
|
12
12
|
workflow?: Workflow;
|
|
13
|
+
/**
|
|
14
|
+
* Run the evaluators one at a time instead of together. The executor sets it
|
|
15
|
+
* when a spend budget is on, so each judge's usage is counted (and a
|
|
16
|
+
* crossing stops the rest) before the next one starts.
|
|
17
|
+
*/
|
|
18
|
+
sequential?: boolean;
|
|
19
|
+
/** The run's abort signal: checked before each judge when sequential, and forwarded to judge calls. */
|
|
20
|
+
signal?: AbortSignal;
|
|
21
|
+
/** Per-call timeout forwarded to judge calls. */
|
|
22
|
+
timeoutMs?: number;
|
|
13
23
|
}
|
|
14
24
|
/**
|
|
15
25
|
* Run every declared evaluator against the node's result and return one
|
package/dist/eval/index.js
CHANGED
|
@@ -41,11 +41,28 @@ export async function evaluateAll(evaluators, result, opts = {}) {
|
|
|
41
41
|
throw new Error(`evaluator '${orphanJudge.name}' is kind: judge but no Claude client was provided to evaluateAll`);
|
|
42
42
|
}
|
|
43
43
|
// Evaluators are independent, so they run concurrently (#337): N judges on
|
|
44
|
-
// a node cost one round-trip of wall time, not N.
|
|
45
|
-
//
|
|
46
|
-
//
|
|
47
|
-
//
|
|
48
|
-
|
|
44
|
+
// a node cost one round-trip of wall time, not N. With `sequential` (a spend
|
|
45
|
+
// budget is on) they run one at a time, so each judge's spend is counted
|
|
46
|
+
// before the next starts, and an abort stops the rest. Results keep
|
|
47
|
+
// declaration order. Every evaluator settles before an error surfaces, and
|
|
48
|
+
// the first rejection in declaration order is rethrown, so the outcome never
|
|
49
|
+
// depends on which call finishes first.
|
|
50
|
+
const settled = [];
|
|
51
|
+
if (opts.sequential) {
|
|
52
|
+
for (const e of evaluators) {
|
|
53
|
+
if (opts.signal?.aborted)
|
|
54
|
+
throw new Error("workflow aborted");
|
|
55
|
+
try {
|
|
56
|
+
settled.push({ status: "fulfilled", value: await evaluateOne(e, result, opts) });
|
|
57
|
+
}
|
|
58
|
+
catch (reason) {
|
|
59
|
+
settled.push({ status: "rejected", reason });
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
else {
|
|
64
|
+
settled.push(...(await Promise.allSettled(evaluators.map((e) => evaluateOne(e, result, opts)))));
|
|
65
|
+
}
|
|
49
66
|
const out = [];
|
|
50
67
|
for (const s of settled) {
|
|
51
68
|
if (s.status === "rejected")
|
|
@@ -66,7 +83,11 @@ async function evaluateOne(e, result, opts) {
|
|
|
66
83
|
}
|
|
67
84
|
if (e.kind === "judge" && opts.claude) {
|
|
68
85
|
const model = resolveJudgeModel(e, opts.node, opts.workflow, opts.claude.defaultJudgeModel);
|
|
69
|
-
const verdict = await evaluateJudge(e, result, opts.claude, {
|
|
86
|
+
const verdict = await evaluateJudge(e, result, opts.claude, {
|
|
87
|
+
model,
|
|
88
|
+
...(opts.signal ? { signal: opts.signal } : {}),
|
|
89
|
+
...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}),
|
|
90
|
+
});
|
|
70
91
|
return { name: e.name, kind: "judge", pass: verdict.pass, reasoning: capReasoning(verdict.reasoning) };
|
|
71
92
|
}
|
|
72
93
|
return null;
|
package/dist/eval/judge.d.ts
CHANGED
|
@@ -2,6 +2,9 @@ import type { Claude, Evaluator, NodeResult } from "../types.js";
|
|
|
2
2
|
export interface JudgeOptions {
|
|
3
3
|
/** Resolved judge model (evaluator > node > workflow > package default). */
|
|
4
4
|
model: string;
|
|
5
|
+
/** The run's abort signal and per-call timeout, forwarded to the judge call. */
|
|
6
|
+
signal?: AbortSignal;
|
|
7
|
+
timeoutMs?: number;
|
|
5
8
|
}
|
|
6
9
|
interface ParsedResponse {
|
|
7
10
|
verdict: string | null;
|
package/dist/eval/judge.js
CHANGED
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
// For a typical 4-node workflow with 2 judges per node, that's 8 extra
|
|
14
14
|
// calls per run. The workflow-level `judge_budget` (default 50) gives
|
|
15
15
|
// authors a load-time warning when the total stretches past expected.
|
|
16
|
+
import { BudgetRefusal } from "../budget.js";
|
|
16
17
|
/**
|
|
17
18
|
* Build the structured judge prompt.
|
|
18
19
|
*
|
|
@@ -122,10 +123,19 @@ export async function evaluateJudge(evaluator, result, claude, opts) {
|
|
|
122
123
|
instruction: prompt,
|
|
123
124
|
context: {},
|
|
124
125
|
model: opts.model,
|
|
126
|
+
...(opts.signal ? { signal: opts.signal } : {}),
|
|
127
|
+
...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}),
|
|
125
128
|
});
|
|
126
129
|
}
|
|
127
|
-
catch {
|
|
128
|
-
//
|
|
130
|
+
catch (err) {
|
|
131
|
+
// A budget refusal is no answer at all: it reaches the executor, which
|
|
132
|
+
// stops the node on the budget instead of recording a parse failure.
|
|
133
|
+
if (err instanceof BudgetRefusal)
|
|
134
|
+
throw err;
|
|
135
|
+
// An aborted run is no answer either: never retried as a parse failure.
|
|
136
|
+
if (opts.signal?.aborted)
|
|
137
|
+
throw err;
|
|
138
|
+
// Treat other throws as parse failure for retry purposes; the second attempt
|
|
129
139
|
// either succeeds or surfaces the same path through the same channel.
|
|
130
140
|
continue;
|
|
131
141
|
}
|
package/dist/executor.js
CHANGED
|
@@ -30,7 +30,7 @@ import { toolRunContext } from "./run-context.js";
|
|
|
30
30
|
import { fenceUntrusted, fenceUntrustedJson } from "./untrusted.js";
|
|
31
31
|
import { asClaude } from "./harness/compat.js";
|
|
32
32
|
import { budgetGate, isToolClass, policyGate, resolveHarnessPolicy } from "./harness/policy.js";
|
|
33
|
-
import { BudgetGuard, describeOverrun, minLimits, spendOf, toLimits } from "./budget.js";
|
|
33
|
+
import { BudgetGuard, BudgetRefusal, describeOverrun, minLimits, spendOf, toLimits } from "./budget.js";
|
|
34
34
|
import { createRunDecider, wantsDecider } from "./decider.js";
|
|
35
35
|
import { redact, runSecretValues } from "./journal.js";
|
|
36
36
|
import { buildNodePolicy, resolveNodePermissions } from "./node-policy.js";
|
|
@@ -90,6 +90,7 @@ export async function execute(workflow, input, options) {
|
|
|
90
90
|
};
|
|
91
91
|
results.set(at.node, result);
|
|
92
92
|
trace.steps.push({ node: at.node, status: "failed", iteration: at.iteration });
|
|
93
|
+
at.syncSpend?.();
|
|
93
94
|
safeObserve(options.observer, { type: "node:exit", node: at.node, result }, logger);
|
|
94
95
|
logger.error(` ${err.message}`, { node: at.node });
|
|
95
96
|
journalDegraded();
|
|
@@ -130,6 +131,130 @@ async function executeRun(workflow, input, options, results, trace, at) {
|
|
|
130
131
|
// Spend budgets (#449): the run ceiling is the lower of the workflow's
|
|
131
132
|
// `budget` and the caller's (`--max-tokens`, `--max-cost`).
|
|
132
133
|
const budgetGuard = new BudgetGuard(minLimits(toLimits(workflow.budget), toLimits(options.budget)));
|
|
134
|
+
// The executor's own completions (judge, route, retry reflection, safe-output
|
|
135
|
+
// screen) report usage into the same guard as node runs, under their kind,
|
|
136
|
+
// and into the journal so a resume keeps them. Every call passes a gate
|
|
137
|
+
// first: once a ceiling is reached (or a usage record could not be
|
|
138
|
+
// journaled) the call is refused, not made. `sink` counts one completion's
|
|
139
|
+
// total and keeps any breach for the caller; it never throws into the
|
|
140
|
+
// adapter, so a journal failure inside it is held in `meterFault` and
|
|
141
|
+
// rethrown by the executor at the next step.
|
|
142
|
+
let meterFault;
|
|
143
|
+
const throwIfMeterFault = () => {
|
|
144
|
+
if (meterFault)
|
|
145
|
+
throw meterFault;
|
|
146
|
+
};
|
|
147
|
+
const metered = (refuse, sink) => {
|
|
148
|
+
const gate = () => {
|
|
149
|
+
throwIfMeterFault();
|
|
150
|
+
const stop = refuse();
|
|
151
|
+
if (stop)
|
|
152
|
+
throw new BudgetRefusal(stop);
|
|
153
|
+
};
|
|
154
|
+
return {
|
|
155
|
+
defaultJudgeModel: claude.defaultJudgeModel,
|
|
156
|
+
run: (req) => claude.run(req),
|
|
157
|
+
evaluate: async (opts) => {
|
|
158
|
+
gate();
|
|
159
|
+
return claude.evaluate({ ...opts, onUsage: sink });
|
|
160
|
+
},
|
|
161
|
+
ask: async (opts) => {
|
|
162
|
+
gate();
|
|
163
|
+
return claude.ask({ ...opts, onUsage: sink });
|
|
164
|
+
},
|
|
165
|
+
};
|
|
166
|
+
};
|
|
167
|
+
const journalMeter = (kind, node, iteration, u) => {
|
|
168
|
+
if (!journal?.meter || meterFault)
|
|
169
|
+
return;
|
|
170
|
+
try {
|
|
171
|
+
journal.meter(kind, node, iteration, u);
|
|
172
|
+
}
|
|
173
|
+
catch (err) {
|
|
174
|
+
meterFault = err;
|
|
175
|
+
}
|
|
176
|
+
};
|
|
177
|
+
const syncSpend = () => {
|
|
178
|
+
if (budgetGuard.hasMeteredCalls)
|
|
179
|
+
trace.spend = budgetGuard.breakdown();
|
|
180
|
+
};
|
|
181
|
+
at.syncSpend = syncSpend;
|
|
182
|
+
// A route call that crossed the run budget, or was refused because the run
|
|
183
|
+
// budget was already spent. The next node is then refused at its start; a
|
|
184
|
+
// route that ends the run fails the node it left.
|
|
185
|
+
let routeBreach;
|
|
186
|
+
let routingFrom;
|
|
187
|
+
const routeClaude = metered(() => (routeBreach ??= budgetGuard.runExhausted()), (u) => {
|
|
188
|
+
const b = budgetGuard.meter("route", u);
|
|
189
|
+
if (routingFrom)
|
|
190
|
+
journalMeter("route", routingFrom.node, routingFrom.iteration, u);
|
|
191
|
+
syncSpend();
|
|
192
|
+
if (b) {
|
|
193
|
+
routeBreach ??= b;
|
|
194
|
+
logger.warn(` ${describeOverrun(b)}; the next node does not start`);
|
|
195
|
+
}
|
|
196
|
+
});
|
|
197
|
+
// Routing from `from`. `budgetStop` says the run ended there on the budget:
|
|
198
|
+
// the caller then writes no route record, so the journal keeps the visit
|
|
199
|
+
// ended but unrouted. A resume (with a larger cap) then routes a visit that
|
|
200
|
+
// succeeded again, like a crash between node:end and the route; a visit
|
|
201
|
+
// that had failed (`on_fail: continue`) is run again.
|
|
202
|
+
const advance = async (from) => {
|
|
203
|
+
routeBreach = undefined;
|
|
204
|
+
routingFrom = { node: from, iteration: nodeRunCounts.get(from) ?? 1 };
|
|
205
|
+
// No paid routing once the run budget is spent: the decider (an HTTP
|
|
206
|
+
// model call) is skipped like the agent's route evaluation, which the gate refuses.
|
|
207
|
+
const spentBeforeRouting = budgetGuard.runExhausted();
|
|
208
|
+
if (spentBeforeRouting && routing.decider && !trace.deciderOff)
|
|
209
|
+
trace.deciderOff = "run budget spent";
|
|
210
|
+
const routeOpts = spentBeforeRouting ? { ...routing, decider: null } : routing;
|
|
211
|
+
let routed;
|
|
212
|
+
let refused = false;
|
|
213
|
+
try {
|
|
214
|
+
routed = await advanceFromNode(workflow, from, results, input, routeClaude, observer, edgeCounts, logger, trace, routeOpts);
|
|
215
|
+
}
|
|
216
|
+
catch (err) {
|
|
217
|
+
throwIfMeterFault();
|
|
218
|
+
if (!(err instanceof BudgetRefusal))
|
|
219
|
+
throw err;
|
|
220
|
+
logger.warn(` ${describeOverrun(err.breach)}; not routing from '${from}'`, { node: from });
|
|
221
|
+
routed = { next: null };
|
|
222
|
+
refused = true;
|
|
223
|
+
}
|
|
224
|
+
finally {
|
|
225
|
+
routingFrom = undefined;
|
|
226
|
+
}
|
|
227
|
+
throwIfMeterFault();
|
|
228
|
+
const breach = routeBreach;
|
|
229
|
+
routeBreach = undefined;
|
|
230
|
+
if (breach && routed.next === null) {
|
|
231
|
+
// No node is left to refuse: the stop lands on the node the route left.
|
|
232
|
+
const prior = results.get(from);
|
|
233
|
+
if (prior && !prior.budget) {
|
|
234
|
+
const error = describeOverrun(breach, from);
|
|
235
|
+
// A node that had already failed keeps its own error; the budget reason has its own field.
|
|
236
|
+
const priorError = prior.data?.error;
|
|
237
|
+
const amended = {
|
|
238
|
+
...prior,
|
|
239
|
+
status: "failed",
|
|
240
|
+
data: { ...prior.data, error: priorError ?? error, budget_error: error, budget_exceeded: true },
|
|
241
|
+
budget: breach,
|
|
242
|
+
};
|
|
243
|
+
results.set(from, amended);
|
|
244
|
+
// The trace and observers already saw the visit end: they get the amended result as an amendment, not a second exit.
|
|
245
|
+
const iteration = nodeRunCounts.get(from) ?? 1;
|
|
246
|
+
const step = [...trace.steps].reverse().find((st) => st.node === from && st.iteration === iteration);
|
|
247
|
+
if (step)
|
|
248
|
+
step.status = "failed";
|
|
249
|
+
safeObserve(observer, { type: "node:amended", node: from, result: amended }, logger);
|
|
250
|
+
}
|
|
251
|
+
// Routing already announced its own end (an invalid target); a refused route announced nothing.
|
|
252
|
+
if (refused)
|
|
253
|
+
safeObserve(observer, { type: "route", from, to: "(end)", reason: "budget exceeded" }, logger);
|
|
254
|
+
return { ...routed, budgetStop: true };
|
|
255
|
+
}
|
|
256
|
+
return routed;
|
|
257
|
+
};
|
|
133
258
|
const harnessPolicy = options.harnessPolicy ?? resolveHarnessPolicy(runEnv);
|
|
134
259
|
// #473: a checkout that persisted a git credential (actions/checkout's
|
|
135
260
|
// default). Each harness masks it for read-only and staged nodes, or reports
|
|
@@ -279,8 +404,9 @@ async function executeRun(workflow, input, options, results, trace, at) {
|
|
|
279
404
|
logger.info(` replayed from the run journal: ${replay.result.status}`, { node: currentId });
|
|
280
405
|
const from = currentId;
|
|
281
406
|
if (replay.next === undefined) {
|
|
282
|
-
const routed = await
|
|
283
|
-
|
|
407
|
+
const routed = await advance(from);
|
|
408
|
+
if (!routed.budgetStop)
|
|
409
|
+
journal?.route(from, routed.next, routed.rung, decider?.counters);
|
|
284
410
|
currentId = routed.next;
|
|
285
411
|
}
|
|
286
412
|
else {
|
|
@@ -304,6 +430,27 @@ async function executeRun(workflow, input, options, results, trace, at) {
|
|
|
304
430
|
continue;
|
|
305
431
|
}
|
|
306
432
|
journal?.nodeStart(currentId, iteration);
|
|
433
|
+
// The run budget is already spent (by a route call, or by the node before
|
|
434
|
+
// a skipped one): this node neither starts nor routes on, whatever its
|
|
435
|
+
// `requires` say, so a skipped node cannot carry the run past the stop.
|
|
436
|
+
// A checkpointed visit (resume) already ran its agent: its writes still apply.
|
|
437
|
+
const runSpent = replay?.kind === "checkpoint" ? undefined : budgetGuard.runExhausted();
|
|
438
|
+
if (runSpent) {
|
|
439
|
+
const error = describeOverrun(runSpent, currentId);
|
|
440
|
+
const result = {
|
|
441
|
+
status: "failed",
|
|
442
|
+
data: { error, budget_exceeded: true },
|
|
443
|
+
toolCalls: [],
|
|
444
|
+
budget: runSpent,
|
|
445
|
+
};
|
|
446
|
+
results.set(currentId, result);
|
|
447
|
+
journal?.nodeEnd(currentId, iteration, result, writeState);
|
|
448
|
+
trace.steps.push({ node: currentId, status: "failed", iteration });
|
|
449
|
+
safeObserve(observer, { type: "node:exit", node: currentId, result }, logger);
|
|
450
|
+
logger.warn(` ${error}; not starting the node`, { node: currentId });
|
|
451
|
+
safeObserve(observer, { type: "route", from: currentId, to: "(end)", reason: "node failed (budget exceeded)" }, logger);
|
|
452
|
+
break;
|
|
453
|
+
}
|
|
307
454
|
// Gather tools from the node's skills, then apply the node's optional
|
|
308
455
|
// skill-tool filter (`tools.allow` / `tools.deny`). Filtered tools are
|
|
309
456
|
// never registered for the run, so the model cannot see or call them.
|
|
@@ -406,8 +553,9 @@ async function executeRun(workflow, input, options, results, trace, at) {
|
|
|
406
553
|
}
|
|
407
554
|
// Apply normal routing rules (dry run gate + resolveNext).
|
|
408
555
|
// TODO: dedupe with requires path — see advanceFromNode helper below
|
|
409
|
-
const routed = await
|
|
410
|
-
|
|
556
|
+
const routed = await advance(currentId);
|
|
557
|
+
if (!routed.budgetStop)
|
|
558
|
+
journal?.route(currentId, routed.next, routed.rung, decider?.counters);
|
|
411
559
|
currentId = routed.next;
|
|
412
560
|
continue;
|
|
413
561
|
}
|
|
@@ -496,12 +644,40 @@ async function executeRun(workflow, input, options, results, trace, at) {
|
|
|
496
644
|
// run's. The gate says whether the harness can keep them; a strict policy
|
|
497
645
|
// refuses the node when it cannot report the budgeted unit at all.
|
|
498
646
|
const nodeBudget = budgetGuard.node(toLimits(node.budget));
|
|
647
|
+
// Resume: this visit's spend before the crash counts against its node
|
|
648
|
+
// budget again (the run total already has it from the seed), whether the
|
|
649
|
+
// write stage is replayed or the visit runs again.
|
|
650
|
+
const visitPrior = replay?.kind === "checkpoint" ? replay.spent : replay ? undefined : journal?.visitSpent?.(currentId, iteration);
|
|
651
|
+
if (visitPrior)
|
|
652
|
+
nodeBudget.restore(visitPrior);
|
|
499
653
|
const budgetOn = budgetGuard.active(nodeBudget.limits);
|
|
500
654
|
const budgetCheck = budgetOn && options.harness
|
|
501
655
|
? budgetGate(options.harness.capabilities, minLimits(nodeBudget.limits, budgetGuard.runLimits), permissions.strict || harnessPolicy === "strict")
|
|
502
656
|
: undefined;
|
|
503
657
|
// A budget stop: fail_soft and on_fail: continue never apply, the run halts.
|
|
504
658
|
let budgetStop;
|
|
659
|
+
// This visit's own completions (judge scoring, retry reflection, output
|
|
660
|
+
// screen) count against the node and the run; a crossing is kept here.
|
|
661
|
+
let meterBreach;
|
|
662
|
+
const visitId = currentId;
|
|
663
|
+
const visitClaude = (kind) => metered(
|
|
664
|
+
// Latched: once refused, every later call of this visit is refused too.
|
|
665
|
+
() => (meterBreach ??= budgetOn ? nodeBudget.exhausted() : undefined), (u) => {
|
|
666
|
+
const b = nodeBudget.meter(kind, u);
|
|
667
|
+
journalMeter(kind, visitId, iteration, u);
|
|
668
|
+
syncSpend();
|
|
669
|
+
meterBreach ??= b;
|
|
670
|
+
});
|
|
671
|
+
const judgeClaude = visitClaude("judge");
|
|
672
|
+
const reflectClaude = visitClaude("reflect");
|
|
673
|
+
const screenClaude = visitClaude("screen");
|
|
674
|
+
// A ceiling reached before a paid executor call (judges, reflection): the
|
|
675
|
+
// node fails with the budget reason instead of making the call.
|
|
676
|
+
const stopAtCeiling = (r, b) => {
|
|
677
|
+
const error = describeOverrun(b, visitId);
|
|
678
|
+
logger.warn(` ${error}`, { node: currentId });
|
|
679
|
+
return { ...r, status: "failed", data: { ...r.data, error, budget_exceeded: true }, budget: b };
|
|
680
|
+
};
|
|
505
681
|
let previous;
|
|
506
682
|
while (true) {
|
|
507
683
|
if (replay?.kind === "checkpoint") {
|
|
@@ -509,6 +685,8 @@ async function executeRun(workflow, input, options, results, trace, at) {
|
|
|
509
685
|
// write intents; the write stage below skips what the journal shows was already applied.
|
|
510
686
|
({ result, agentRunFailed, attempt } = replay);
|
|
511
687
|
intents.push(...replay.intents);
|
|
688
|
+
// The visit's own spend before the crash counts against its node budget
|
|
689
|
+
// again (the run total already has it from the seed).
|
|
512
690
|
break;
|
|
513
691
|
}
|
|
514
692
|
const refusal = gate?.refuse ?? budgetCheck?.refuse;
|
|
@@ -562,7 +740,6 @@ async function executeRun(workflow, input, options, results, trace, at) {
|
|
|
562
740
|
}
|
|
563
741
|
const journalUsage = journal?.usage
|
|
564
742
|
? (u) => {
|
|
565
|
-
liveUsage = { ...liveUsage, ...u };
|
|
566
743
|
if (journalFault)
|
|
567
744
|
return;
|
|
568
745
|
try {
|
|
@@ -574,12 +751,12 @@ async function executeRun(workflow, input, options, results, trace, at) {
|
|
|
574
751
|
}
|
|
575
752
|
}
|
|
576
753
|
: undefined;
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
754
|
+
// Live usage is always collected: with no limit it still feeds the split by kind.
|
|
755
|
+
const onUsage = (u) => {
|
|
756
|
+
liveUsage = { ...liveUsage, ...u };
|
|
757
|
+
attemptBudget?.onUsage(u);
|
|
758
|
+
journalUsage?.(u);
|
|
759
|
+
};
|
|
583
760
|
try {
|
|
584
761
|
result = await claude.run({
|
|
585
762
|
nodeId: currentId,
|
|
@@ -597,13 +774,17 @@ async function executeRun(workflow, input, options, results, trace, at) {
|
|
|
597
774
|
...(nodePolicy ? { policy: nodePolicy } : {}),
|
|
598
775
|
...(readOnlyNode || stageOutputs ? {} : { mcpServers: skillMcpServers }),
|
|
599
776
|
...(readOnlyNode ? { readOnly: true } : {}),
|
|
600
|
-
|
|
777
|
+
onUsage,
|
|
601
778
|
onProgress: (message) => {
|
|
602
779
|
safeObserve(observer, { type: "node:progress", node: currentId, message }, logger);
|
|
603
780
|
},
|
|
604
781
|
});
|
|
605
782
|
// Commit the attempt's spend before anything here can throw (#549); `finish` below reads the breach.
|
|
606
|
-
attemptBudget
|
|
783
|
+
if (attemptBudget)
|
|
784
|
+
attemptBudget.settle(result.usage);
|
|
785
|
+
// No limit to enforce: still count the run, so the split by kind is complete.
|
|
786
|
+
else
|
|
787
|
+
budgetGuard.meter("node", attemptSpend(result.usage, liveUsage));
|
|
607
788
|
}
|
|
608
789
|
catch (err) {
|
|
609
790
|
if (journalFault)
|
|
@@ -741,13 +922,37 @@ async function executeRun(workflow, input, options, results, trace, at) {
|
|
|
741
922
|
result.evals = [failure];
|
|
742
923
|
}
|
|
743
924
|
else {
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
925
|
+
// With a budget on, judges run one at a time, so a crossing (or a
|
|
926
|
+
// ceiling already reached) refuses the rest instead of all starting at once.
|
|
927
|
+
let evalResults;
|
|
928
|
+
try {
|
|
929
|
+
evalResults = await evaluateAll(node.eval, result, {
|
|
930
|
+
aliases: toolAliases,
|
|
931
|
+
claude: judgeClaude,
|
|
932
|
+
node,
|
|
933
|
+
workflow,
|
|
934
|
+
sequential: budgetOn,
|
|
935
|
+
...(signal ? { signal } : {}),
|
|
936
|
+
...(timeoutMs !== undefined ? { timeoutMs } : {}),
|
|
937
|
+
});
|
|
938
|
+
}
|
|
939
|
+
catch (err) {
|
|
940
|
+
throwIfMeterFault();
|
|
941
|
+
if (!(err instanceof BudgetRefusal))
|
|
942
|
+
throw err;
|
|
943
|
+
}
|
|
944
|
+
throwIfMeterFault();
|
|
945
|
+
if (evalResults)
|
|
946
|
+
result.evals = evalResults;
|
|
947
|
+
if (meterBreach || !evalResults) {
|
|
948
|
+
// Judge spend crossed a ceiling, or a judge was refused at one (the
|
|
949
|
+
// refusal latched it): the node fails whatever the verdicts said.
|
|
950
|
+
const stop = meterBreach ?? nodeBudget.exhausted() ?? budgetGuard.runExhausted();
|
|
951
|
+
result = stop ? stopAtCeiling(result, stop) : result;
|
|
952
|
+
agentRunFailed = true;
|
|
953
|
+
budgetStop = stop;
|
|
954
|
+
break;
|
|
955
|
+
}
|
|
751
956
|
outcome = aggregateEval(evalResults, node.eval_policy ?? "all_pass");
|
|
752
957
|
}
|
|
753
958
|
if (outcome.pass)
|
|
@@ -786,13 +991,21 @@ async function executeRun(workflow, input, options, results, trace, at) {
|
|
|
786
991
|
evalFailures: outcome.failures,
|
|
787
992
|
toolCalls: result.toolCalls,
|
|
788
993
|
nodeInstruction: resolvedInstruction,
|
|
789
|
-
claude,
|
|
994
|
+
claude: reflectClaude,
|
|
790
995
|
logger,
|
|
791
996
|
context,
|
|
792
997
|
model: nodeModel,
|
|
793
998
|
signal,
|
|
794
999
|
timeoutMs,
|
|
795
1000
|
});
|
|
1001
|
+
throwIfMeterFault();
|
|
1002
|
+
// A reflection refused at a ceiling (or one whose spend crossed it): no further attempt.
|
|
1003
|
+
if (meterBreach) {
|
|
1004
|
+
result = stopAtCeiling(result, meterBreach);
|
|
1005
|
+
agentRunFailed = true;
|
|
1006
|
+
budgetStop = meterBreach;
|
|
1007
|
+
break;
|
|
1008
|
+
}
|
|
796
1009
|
currentInstruction = `${preamble}\n\n---\n\n${instruction}`;
|
|
797
1010
|
safeObserve(observer, { type: "node:retry", node: currentId, attempt: attempt + 1, reason: outcome.error ?? "eval failed", preamble }, logger);
|
|
798
1011
|
attempt++;
|
|
@@ -840,19 +1053,35 @@ async function executeRun(workflow, input, options, results, trace, at) {
|
|
|
840
1053
|
state: writeState,
|
|
841
1054
|
logger,
|
|
842
1055
|
input: runInput,
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
1056
|
+
...(journal?.reconcile
|
|
1057
|
+
? {
|
|
1058
|
+
reconcile: (sk, tool, args, ctx) => journal.reconcile(visitId, iteration, sk, tool, args, ctx),
|
|
1059
|
+
}
|
|
1060
|
+
: {}),
|
|
1061
|
+
screen: async (writes) => {
|
|
1062
|
+
const verdict = await screenClaude.ask({
|
|
1063
|
+
instruction: SAFE_OUTPUT_SCREEN_INSTRUCTION,
|
|
1064
|
+
context: { writes },
|
|
1065
|
+
model: nodeModel,
|
|
1066
|
+
signal,
|
|
1067
|
+
timeoutMs,
|
|
1068
|
+
});
|
|
1069
|
+
// A screen whose spend crossed a ceiling, or whose meter record the
|
|
1070
|
+
// journal could not write, allows nothing: the writes are vetoed.
|
|
1071
|
+
return meterBreach || meterFault ? null : verdict;
|
|
1072
|
+
},
|
|
850
1073
|
});
|
|
1074
|
+
throwIfMeterFault();
|
|
851
1075
|
result = { ...result, outputs: stage.receipts };
|
|
852
1076
|
if (stage.error) {
|
|
853
1077
|
logger.warn(` ${stage.error}`, { node: currentId });
|
|
854
1078
|
result = { ...result, status: "failed", data: { ...result.data, error: stage.error } };
|
|
855
1079
|
}
|
|
1080
|
+
if (meterBreach && !budgetStop) {
|
|
1081
|
+
// The screen's spend crossed a ceiling: the node fails and the run halts.
|
|
1082
|
+
result = stopAtCeiling(result, meterBreach);
|
|
1083
|
+
budgetStop = meterBreach;
|
|
1084
|
+
}
|
|
856
1085
|
}
|
|
857
1086
|
else if (intents.length > 0) {
|
|
858
1087
|
result = {
|
|
@@ -885,17 +1114,33 @@ async function executeRun(workflow, input, options, results, trace, at) {
|
|
|
885
1114
|
}
|
|
886
1115
|
// Dry run gate + routing — shared with requires path via advanceFromNode helper.
|
|
887
1116
|
const routedFrom = currentId;
|
|
888
|
-
const routed = await
|
|
889
|
-
|
|
1117
|
+
const routed = await advance(routedFrom);
|
|
1118
|
+
if (!routed.budgetStop)
|
|
1119
|
+
journal?.route(routedFrom, routed.next, routed.rung, decider?.counters);
|
|
890
1120
|
currentId = routed.next;
|
|
891
1121
|
}
|
|
892
1122
|
safeObserve(observer, {
|
|
893
1123
|
type: "workflow:end",
|
|
894
1124
|
results: Object.fromEntries(results),
|
|
895
1125
|
}, logger);
|
|
1126
|
+
syncSpend();
|
|
896
1127
|
return { results, trace };
|
|
897
1128
|
}
|
|
898
1129
|
// ─── Internals ───────────────────────────────────────────────────
|
|
1130
|
+
/**
|
|
1131
|
+
* An attempt's spend as usage: per unit, the larger of the final report and
|
|
1132
|
+
* the last live one (what `BudgetAttempt.settle` commits), so the split is the
|
|
1133
|
+
* same with and without a limit. Undefined when neither reported anything.
|
|
1134
|
+
*/
|
|
1135
|
+
function attemptSpend(final, live) {
|
|
1136
|
+
const f = spendOf(final);
|
|
1137
|
+
const l = spendOf(live);
|
|
1138
|
+
const tokens = f.tokens === undefined && l.tokens === undefined ? undefined : Math.max(f.tokens ?? 0, l.tokens ?? 0);
|
|
1139
|
+
const costUsd = f.costUsd === undefined && l.costUsd === undefined ? undefined : Math.max(f.costUsd ?? 0, l.costUsd ?? 0);
|
|
1140
|
+
if (tokens === undefined && costUsd === undefined)
|
|
1141
|
+
return undefined;
|
|
1142
|
+
return { ...(tokens !== undefined ? { inputTokens: tokens } : {}), ...(costUsd !== undefined ? { costUsd } : {}) };
|
|
1143
|
+
}
|
|
899
1144
|
/**
|
|
900
1145
|
* Thrown when a conditional routing decision could not be made and there is
|
|
901
1146
|
* no explicit default (unconditional) edge to fall back to. Fails closed: the
|
package/dist/harness/acp.js
CHANGED
|
@@ -51,7 +51,7 @@ import { agentEnvOverlay, applyEnvOverlay, finishAgentEnv, heldCredentials, over
|
|
|
51
51
|
import { ACP_CAPABILITIES } from "./capabilities.js";
|
|
52
52
|
import { policyGate, resolveHarnessPolicy, TOOL_CLASSES } from "./policy.js";
|
|
53
53
|
import { gitCredentialPolicy } from "../git-credentials.js";
|
|
54
|
-
import { buildNodePrompt } from "./prompts.js";
|
|
54
|
+
import { buildNodePrompt, reportCompletionUsage } from "./prompts.js";
|
|
55
55
|
import { makeAbort } from "./abort.js";
|
|
56
56
|
import { parseToolResultContent, schemaMismatches, summarizeToolError, tryParseJSON } from "./parse.js";
|
|
57
57
|
import { startToolBridge } from "./tool-bridge/server.js";
|
|
@@ -937,6 +937,9 @@ export class AcpHarness {
|
|
|
937
937
|
}
|
|
938
938
|
if (conv.agentVersion)
|
|
939
939
|
this.agentVersion = conv.agentVersion;
|
|
940
|
+
// A completion that failed was still billed: report before the outcome.
|
|
941
|
+
if (conv.costUsd !== undefined)
|
|
942
|
+
reportCompletionUsage(req.onUsage, { costUsd: conv.costUsd }, this.logger);
|
|
940
943
|
if (conv.spawnError)
|
|
941
944
|
return failClosed(`the ACP agent could not start: ${conv.spawnError}`);
|
|
942
945
|
if (conv.abortReason === "timeout")
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* This is the ONLY supported LLM backend. The whole point of sweny
|
|
9
9
|
* is to use headless Claude Code - never the raw Anthropic API.
|
|
10
10
|
*/
|
|
11
|
-
import type { Claude, Tool, ToolContext, Logger, McpServerConfig } from "../types.js";
|
|
11
|
+
import type { Claude, Tool, ToolContext, NodeUsage, Logger, McpServerConfig } from "../types.js";
|
|
12
12
|
import { type HarnessEnvOptions, type SandboxMode } from "../agent-env.js";
|
|
13
13
|
import type { AgentHarness, HarnessCapabilities, HarnessCompleteRequest, HarnessInfo, HarnessRunRequest, HarnessRunResult, NodePolicy, ToolClass } from "./types.js";
|
|
14
14
|
import { type HarnessPolicyMode } from "./policy.js";
|
|
@@ -250,6 +250,7 @@ export declare class ClaudeCodeHarness implements Claude, AgentHarness {
|
|
|
250
250
|
timeoutMs?: number;
|
|
251
251
|
/** Caller-supplied abort signal. Aborting it interrupts the query. */
|
|
252
252
|
signal?: AbortSignal;
|
|
253
|
+
onUsage?: (usage: NodeUsage) => void;
|
|
253
254
|
}): Promise<string | null>;
|
|
254
255
|
/** Free-text completion. Delegates to core {@link coreAsk} over {@link complete}. */
|
|
255
256
|
ask(opts: {
|
|
@@ -260,6 +261,7 @@ export declare class ClaudeCodeHarness implements Claude, AgentHarness {
|
|
|
260
261
|
timeoutMs?: number;
|
|
261
262
|
/** Caller-supplied abort signal. Aborting it interrupts the query. */
|
|
262
263
|
signal?: AbortSignal;
|
|
264
|
+
onUsage?: (usage: NodeUsage) => void;
|
|
263
265
|
}): Promise<string>;
|
|
264
266
|
}
|
|
265
267
|
/** @deprecated Use {@link ClaudeCodeHarnessOptions}. */
|
|
@@ -16,7 +16,7 @@ import { AGENT_AUTH_VARS, agentEnvOverlay, applyEnvOverlay, finishAgentEnv, over
|
|
|
16
16
|
import { gitCredentialGap, policyGate } from "./policy.js";
|
|
17
17
|
import { gitCredentialPolicy } from "../git-credentials.js";
|
|
18
18
|
import { CLAUDE_CODE_CAPABILITIES } from "./capabilities.js";
|
|
19
|
-
import { ask as coreAsk, evaluate as coreEvaluate, buildEvaluatePrompt, buildNodePrompt } from "./prompts.js";
|
|
19
|
+
import { ask as coreAsk, evaluate as coreEvaluate, buildEvaluatePrompt, buildNodePrompt, reportCompletionUsage, } from "./prompts.js";
|
|
20
20
|
import { jsonSchemaToZodShape, toolErrorToMcpResult, toolOutputToMcpResult } from "./tool-bridge/protocol.js";
|
|
21
21
|
import { startToolBridge } from "./tool-bridge/server.js";
|
|
22
22
|
import { parseToolResultContent, summarizeToolError, tryParseJSON } from "./parse.js";
|
|
@@ -832,6 +832,9 @@ export class ClaudeCodeHarness {
|
|
|
832
832
|
if (message.type === "result") {
|
|
833
833
|
const resultMsg = message;
|
|
834
834
|
sawResult = true;
|
|
835
|
+
// A completion that failed was still billed: report before the outcome.
|
|
836
|
+
if (req.onUsage)
|
|
837
|
+
reportCompletionUsage(req.onUsage, extractUsage(resultMsg), this.logger);
|
|
835
838
|
if (resultMsg.subtype === "success" && "result" in resultMsg) {
|
|
836
839
|
response = resultMsg.result;
|
|
837
840
|
}
|