@sweny-ai/core 0.1.198 → 0.1.200
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-env.d.ts +94 -2
- package/dist/agent-env.js +144 -7
- package/dist/budget.d.ts +77 -11
- package/dist/budget.js +132 -16
- package/dist/cli/renderer.js +5 -0
- package/dist/cli/run-history.d.ts +12 -0
- package/dist/cli/run-history.js +16 -0
- package/dist/cli/run-output.d.ts +16 -1
- package/dist/cli/run-output.js +46 -4
- package/dist/cli/runs.js +38 -1
- package/dist/cli/ticket.js +7 -2
- package/dist/eval/index.d.ts +10 -0
- package/dist/eval/index.js +27 -6
- package/dist/eval/judge.d.ts +3 -0
- package/dist/eval/judge.js +12 -2
- package/dist/executor.d.ts +5 -1
- package/dist/executor.js +276 -31
- package/dist/harness/acp.d.ts +5 -2
- package/dist/harness/acp.js +11 -23
- package/dist/harness/auth.js +2 -1
- package/dist/harness/claude-code.d.ts +18 -13
- package/dist/harness/claude-code.js +34 -15
- package/dist/harness/codex.d.ts +22 -5
- package/dist/harness/codex.js +67 -11
- package/dist/harness/compat.js +1 -0
- package/dist/harness/index.d.ts +1 -1
- package/dist/harness/index.js +1 -1
- package/dist/harness/pi.d.ts +11 -5
- package/dist/harness/pi.js +27 -9
- package/dist/harness/prompts.d.ts +11 -1
- package/dist/harness/prompts.js +21 -2
- package/dist/harness/sandbox-wrapper.d.ts +13 -2
- package/dist/harness/sandbox-wrapper.js +29 -4
- package/dist/harness/types.d.ts +9 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.js +2 -2
- package/dist/journal.d.ts +48 -10
- package/dist/journal.js +164 -16
- package/dist/safe-outputs.d.ts +9 -0
- package/dist/safe-outputs.js +41 -26
- package/dist/skills/linear.js +3 -1
- package/dist/studio.js +2 -1
- package/dist/types.d.ts +34 -1
- package/package.json +1 -1
package/dist/budget.js
CHANGED
|
@@ -8,8 +8,14 @@
|
|
|
8
8
|
*
|
|
9
9
|
* Tokens are input + output tokens as the harness reports them (cache reads and
|
|
10
10
|
* writes are not counted, matching the run receipt). Cost is the harness's own
|
|
11
|
-
* reported USD, never estimated.
|
|
12
|
-
*
|
|
11
|
+
* reported USD, never estimated. Every model completion the executor itself
|
|
12
|
+
* makes is counted too, under its own `kind` (see {@link SpendKind}): judge
|
|
13
|
+
* scoring, route evaluation, retry reflection and the safe-output screen. A
|
|
14
|
+
* completion counts when its harness reports usage for it; one that reports
|
|
15
|
+
* none adds nothing (the same as a node run without usage). The run totals
|
|
16
|
+
* and the stop are over all kinds together, across a resume too; the split is
|
|
17
|
+
* `trace.spend`. Once a ceiling is reached no further executor completion
|
|
18
|
+
* starts. The decision model (`route_by: decider`) is not metered.
|
|
13
19
|
*
|
|
14
20
|
* Enforcement, per unit the harness can report:
|
|
15
21
|
* - live: the harness calls `onUsage` during the node; crossing a limit aborts
|
|
@@ -17,9 +23,34 @@
|
|
|
17
23
|
* - between nodes: usage on the finished result is checked; a crossing fails the
|
|
18
24
|
* node, and the run halts before the next node starts.
|
|
19
25
|
* - across a resume: the run journal records each attempt's spend (live and
|
|
20
|
-
* final), and
|
|
26
|
+
* final), and each metered completion (judge, route, reflection, screen)
|
|
27
|
+
* as its own record; `sweny workflow resume` seeds the run total with all
|
|
28
|
+
* of it, split by kind.
|
|
21
29
|
*/
|
|
30
|
+
/** The order kinds are listed in, wherever the split is shown. */
|
|
31
|
+
export const SPEND_KINDS = ["node", "judge", "route", "reflect", "screen"];
|
|
32
|
+
/**
|
|
33
|
+
* Thrown into an executor completion (judge, route, reflection, screen) the
|
|
34
|
+
* budget refuses: a ceiling was already reached, so the call is not made.
|
|
35
|
+
* Callers that swallow a failed completion (a judge's parse retry) must let
|
|
36
|
+
* this one through, so a refusal is never mistaken for a model answer.
|
|
37
|
+
*/
|
|
38
|
+
export class BudgetRefusal extends Error {
|
|
39
|
+
breach;
|
|
40
|
+
constructor(breach) {
|
|
41
|
+
super(describeOverrun(breach));
|
|
42
|
+
this.breach = breach;
|
|
43
|
+
this.name = "BudgetRefusal";
|
|
44
|
+
}
|
|
45
|
+
}
|
|
22
46
|
const isNum = (v) => typeof v === "number" && Number.isFinite(v);
|
|
47
|
+
/**
|
|
48
|
+
* Slack for comparing accumulated floats: 0.1 + 0.2 is 0.30000000000000004,
|
|
49
|
+
* which must not read as over a $0.30 limit. Far below a token or a cent.
|
|
50
|
+
*/
|
|
51
|
+
const EPSILON = 1e-9;
|
|
52
|
+
/** Cost as shown: accumulated float noise rounded off (10 decimal places). */
|
|
53
|
+
const roundCost = (n) => Math.round(n * 1e10) / 1e10;
|
|
23
54
|
export function toLimits(b) {
|
|
24
55
|
return {
|
|
25
56
|
...(b?.tokens !== undefined ? { tokens: b.tokens } : {}),
|
|
@@ -50,14 +81,14 @@ export function spendOf(u) {
|
|
|
50
81
|
function over(spent, limit, strictlyOver) {
|
|
51
82
|
if (limit === undefined)
|
|
52
83
|
return false;
|
|
53
|
-
return strictlyOver ? spent > limit : spent >= limit;
|
|
84
|
+
return strictlyOver ? spent > limit + EPSILON : spent >= limit - EPSILON;
|
|
54
85
|
}
|
|
55
86
|
function breachOf(scope, limits, spent, strictlyOver) {
|
|
56
87
|
if (over(spent.tokens, limits.tokens, strictlyOver)) {
|
|
57
88
|
return { scope, unit: "tokens", limit: limits.tokens, spent: spent.tokens };
|
|
58
89
|
}
|
|
59
90
|
if (over(spent.costUsd, limits.costUsd, strictlyOver)) {
|
|
60
|
-
return { scope, unit: "cost_usd", limit: limits.costUsd, spent: spent.costUsd };
|
|
91
|
+
return { scope, unit: "cost_usd", limit: limits.costUsd, spent: roundCost(spent.costUsd) };
|
|
61
92
|
}
|
|
62
93
|
return undefined;
|
|
63
94
|
}
|
|
@@ -74,12 +105,53 @@ export function describeOverrun(o, nodeId) {
|
|
|
74
105
|
export class BudgetGuard {
|
|
75
106
|
runLimits;
|
|
76
107
|
spent = { tokens: 0, costUsd: 0 };
|
|
108
|
+
kinds = new Map();
|
|
77
109
|
constructor(runLimits) {
|
|
78
110
|
this.runLimits = runLimits;
|
|
79
111
|
}
|
|
80
112
|
get totals() {
|
|
81
113
|
return { ...this.spent };
|
|
82
114
|
}
|
|
115
|
+
/**
|
|
116
|
+
* Spend by kind, in {@link SPEND_KINDS} order. Resumed spend is restored
|
|
117
|
+
* under its kind; a journal without meter records seeds `node` only.
|
|
118
|
+
*/
|
|
119
|
+
breakdown() {
|
|
120
|
+
const out = {};
|
|
121
|
+
for (const k of SPEND_KINDS) {
|
|
122
|
+
const v = this.kinds.get(k);
|
|
123
|
+
if (v)
|
|
124
|
+
out[k] = { tokens: v.tokens, costUsd: roundCost(v.costUsd), calls: v.calls };
|
|
125
|
+
}
|
|
126
|
+
return out;
|
|
127
|
+
}
|
|
128
|
+
/** True once any completion other than a node run reported usage. */
|
|
129
|
+
get hasMeteredCalls() {
|
|
130
|
+
return [...this.kinds.keys()].some((k) => k !== "node");
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Count one executor completion (judge, route, ...) against the run. Usage
|
|
134
|
+
* with no tokens and no cost counts nothing. Returns the breach this report
|
|
135
|
+
* caused, judged against the totals before it, like a live node report.
|
|
136
|
+
*/
|
|
137
|
+
meter(kind, usage, nodeBudget) {
|
|
138
|
+
const u = spendOf(usage);
|
|
139
|
+
if (u.tokens === undefined && u.costUsd === undefined)
|
|
140
|
+
return undefined;
|
|
141
|
+
const s = { tokens: u.tokens ?? 0, costUsd: u.costUsd ?? 0 };
|
|
142
|
+
const r = this.spent;
|
|
143
|
+
const breach = (nodeBudget
|
|
144
|
+
? breachOf("node", nodeBudget.limits, { tokens: nodeBudget.committed.tokens + s.tokens, costUsd: nodeBudget.committed.costUsd + s.costUsd }, true)
|
|
145
|
+
: undefined) ??
|
|
146
|
+
breachOf("run", this.runLimits, { tokens: r.tokens + s.tokens, costUsd: r.costUsd + s.costUsd }, true);
|
|
147
|
+
if (breach && kind !== "node")
|
|
148
|
+
breach.kind = kind;
|
|
149
|
+
if (nodeBudget)
|
|
150
|
+
nodeBudget.commit(s, kind, true);
|
|
151
|
+
else
|
|
152
|
+
this.commit(s, kind, true);
|
|
153
|
+
return breach;
|
|
154
|
+
}
|
|
83
155
|
/** True when any limit exists at all, so the executor can skip the machinery. */
|
|
84
156
|
active(nodeLimits) {
|
|
85
157
|
return hasLimits(this.runLimits) || hasLimits(nodeLimits);
|
|
@@ -92,17 +164,42 @@ export class BudgetGuard {
|
|
|
92
164
|
return new NodeBudget(this, nodeLimits);
|
|
93
165
|
}
|
|
94
166
|
/**
|
|
95
|
-
* Spend the run made before this process (a resumed run's journaled usage
|
|
96
|
-
* Counted against the run ceiling like any other
|
|
97
|
-
* resets the whole-run budget.
|
|
167
|
+
* Spend the run made before this process (a resumed run's journaled usage
|
|
168
|
+
* and metered completions). Counted against the run ceiling like any other
|
|
169
|
+
* spend, so a crash never resets the whole-run budget. Restored under its
|
|
170
|
+
* kind when the journal has the split (a journal written before meter
|
|
171
|
+
* records existed has node spend only). Seeded spend is not a call: `calls`
|
|
172
|
+
* is unchanged.
|
|
98
173
|
*/
|
|
99
174
|
seed(s) {
|
|
100
|
-
const ok = (n) => (Number.isFinite(n) && n > 0 ? n : 0);
|
|
101
|
-
|
|
175
|
+
const ok = (n) => (typeof n === "number" && Number.isFinite(n) && n > 0 ? n : 0);
|
|
176
|
+
if (!s.byKind) {
|
|
177
|
+
this.commit({ tokens: ok(s.tokens), costUsd: ok(s.costUsd) }, "node", false);
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
for (const k of SPEND_KINDS) {
|
|
181
|
+
const v = s.byKind[k];
|
|
182
|
+
if (v)
|
|
183
|
+
this.commit({ tokens: ok(v.tokens), costUsd: ok(v.costUsd) }, k, false);
|
|
184
|
+
}
|
|
102
185
|
}
|
|
103
|
-
/**
|
|
104
|
-
|
|
186
|
+
/**
|
|
187
|
+
* @internal `counted`: this commit is one completion that reported usage, so
|
|
188
|
+
* `calls` goes up. A commit that is not (seeded spend, an attempt that
|
|
189
|
+
* reported nothing) adds its spend only, and a zero one leaves no trace, so
|
|
190
|
+
* the split is the same with and without a limit.
|
|
191
|
+
*/
|
|
192
|
+
commit(s, kind, counted) {
|
|
105
193
|
this.spent = { tokens: this.spent.tokens + s.tokens, costUsd: this.spent.costUsd + s.costUsd };
|
|
194
|
+
const k = this.kinds.get(kind);
|
|
195
|
+
if (!k && !counted && s.tokens === 0 && s.costUsd === 0)
|
|
196
|
+
return;
|
|
197
|
+
const prev = k ?? { tokens: 0, costUsd: 0, calls: 0 };
|
|
198
|
+
this.kinds.set(kind, {
|
|
199
|
+
tokens: prev.tokens + s.tokens,
|
|
200
|
+
costUsd: prev.costUsd + s.costUsd,
|
|
201
|
+
calls: prev.calls + (counted ? 1 : 0),
|
|
202
|
+
});
|
|
106
203
|
}
|
|
107
204
|
}
|
|
108
205
|
export class NodeBudget {
|
|
@@ -120,14 +217,27 @@ export class NodeBudget {
|
|
|
120
217
|
attempt(outer) {
|
|
121
218
|
return new BudgetAttempt(this.guard, this, outer);
|
|
122
219
|
}
|
|
220
|
+
/**
|
|
221
|
+
* Spend this visit made before a resume (a checkpointed visit's journaled
|
|
222
|
+
* spend). It counts against the node's own limit only: the run total
|
|
223
|
+
* already holds it from the seed.
|
|
224
|
+
*/
|
|
225
|
+
restore(s) {
|
|
226
|
+
const ok = (n) => (Number.isFinite(n) && n > 0 ? n : 0);
|
|
227
|
+
this.spent = { tokens: this.spent.tokens + ok(s.tokens), costUsd: this.spent.costUsd + ok(s.costUsd) };
|
|
228
|
+
}
|
|
123
229
|
/** @internal */
|
|
124
230
|
get committed() {
|
|
125
231
|
return this.spent;
|
|
126
232
|
}
|
|
127
|
-
/**
|
|
128
|
-
|
|
233
|
+
/** Count one executor completion made during this visit (judge, retry reflection, screen). */
|
|
234
|
+
meter(kind, usage) {
|
|
235
|
+
return this.guard.meter(kind, usage, this);
|
|
236
|
+
}
|
|
237
|
+
/** @internal See {@link BudgetGuard.commit}. */
|
|
238
|
+
commit(s, kind, counted) {
|
|
129
239
|
this.spent = { tokens: this.spent.tokens + s.tokens, costUsd: this.spent.costUsd + s.costUsd };
|
|
130
|
-
this.guard.commit(s);
|
|
240
|
+
this.guard.commit(s, kind, counted);
|
|
131
241
|
}
|
|
132
242
|
}
|
|
133
243
|
/** One agent run. Its `signal` is the caller's signal plus a budget stop. */
|
|
@@ -138,6 +248,8 @@ export class BudgetAttempt {
|
|
|
138
248
|
controller = new AbortController();
|
|
139
249
|
live = { tokens: 0, costUsd: 0 };
|
|
140
250
|
liveUsage;
|
|
251
|
+
/** A live report carried tokens or cost. */
|
|
252
|
+
liveReported = false;
|
|
141
253
|
stopped;
|
|
142
254
|
settled;
|
|
143
255
|
onOuterAbort = () => this.controller.abort();
|
|
@@ -167,6 +279,8 @@ export class BudgetAttempt {
|
|
|
167
279
|
return;
|
|
168
280
|
this.liveUsage = { ...this.liveUsage, ...u };
|
|
169
281
|
const s = spendOf(u);
|
|
282
|
+
if (s.tokens !== undefined || s.costUsd !== undefined)
|
|
283
|
+
this.liveReported = true;
|
|
170
284
|
// Reports are cumulative; never let a partial report move spend backwards.
|
|
171
285
|
this.live = {
|
|
172
286
|
tokens: Math.max(this.live.tokens, s.tokens ?? 0),
|
|
@@ -204,7 +318,9 @@ export class BudgetAttempt {
|
|
|
204
318
|
};
|
|
205
319
|
const breach = this.stopped ?? this.check(spent);
|
|
206
320
|
this.settled = { breach };
|
|
207
|
-
|
|
321
|
+
// A call only when the attempt reported usage, live or final: an attempt
|
|
322
|
+
// that reported none is not counted, the same as without a limit.
|
|
323
|
+
this.node.commit(spent, "node", this.liveReported || f.tokens !== undefined || f.costUsd !== undefined);
|
|
208
324
|
return breach;
|
|
209
325
|
}
|
|
210
326
|
/** Settle with the attempt's final usage and return the breach, if any. */
|
package/dist/cli/renderer.js
CHANGED
|
@@ -234,6 +234,11 @@ export class DagRenderer {
|
|
|
234
234
|
s.finishedAt = Date.now();
|
|
235
235
|
break;
|
|
236
236
|
}
|
|
237
|
+
case "node:amended": {
|
|
238
|
+
const s = this.getOrCreate(event.node);
|
|
239
|
+
s.state = event.result.status === "failed" ? "failed" : "completed";
|
|
240
|
+
break;
|
|
241
|
+
}
|
|
237
242
|
// These events don't affect per-node rendering state
|
|
238
243
|
case "workflow:start":
|
|
239
244
|
case "workflow:end":
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
* know instead of guessing.
|
|
13
13
|
*/
|
|
14
14
|
import type { ExecutionEvent, ExecutionTrace, NodeResult, RouteRung, Workflow } from "../types.js";
|
|
15
|
+
import { type SpendKind } from "../budget.js";
|
|
15
16
|
export declare const RUN_HISTORY_SCHEMA_VERSION = 1;
|
|
16
17
|
export declare const RUN_HISTORY_DIR: string;
|
|
17
18
|
export declare const RUN_HISTORY_KEEP = 200;
|
|
@@ -58,6 +59,17 @@ export interface RunRecord {
|
|
|
58
59
|
};
|
|
59
60
|
/** Opinions the harness could not honor natively on some node, deduped. Absent when none. */
|
|
60
61
|
degraded?: string[];
|
|
62
|
+
/**
|
|
63
|
+
* Spend by kind (node, judge, route, reflect, screen), present when the
|
|
64
|
+
* executor's own completions reported usage. `totals` is then its sum, so a
|
|
65
|
+
* jump in `totals` that no node explains shows here.
|
|
66
|
+
*/
|
|
67
|
+
spend?: Partial<Record<SpendKind, RunKindSpend>>;
|
|
68
|
+
}
|
|
69
|
+
export interface RunKindSpend {
|
|
70
|
+
tokens: number;
|
|
71
|
+
cost_usd: number;
|
|
72
|
+
calls: number;
|
|
61
73
|
}
|
|
62
74
|
/** sha256 (hex) of the workflow as canonical JSON: keys sorted, undefined dropped. */
|
|
63
75
|
export declare function hashWorkflow(workflow: Workflow): string;
|
package/dist/cli/run-history.js
CHANGED
|
@@ -15,6 +15,7 @@ import crypto from "node:crypto";
|
|
|
15
15
|
import fs from "node:fs";
|
|
16
16
|
import path from "node:path";
|
|
17
17
|
import { summarizeRun } from "./run-output.js";
|
|
18
|
+
import { SPEND_KINDS } from "../budget.js";
|
|
18
19
|
import { openNoFollow } from "../safe-file.js";
|
|
19
20
|
export const RUN_HISTORY_SCHEMA_VERSION = 1;
|
|
20
21
|
export const RUN_HISTORY_DIR = path.join(".sweny", "runs");
|
|
@@ -63,10 +64,15 @@ export function createNodeTimer(now = Date.now) {
|
|
|
63
64
|
entered.set(event.node, now());
|
|
64
65
|
else if (event.type === "node:exit") {
|
|
65
66
|
const start = entered.get(event.node);
|
|
67
|
+
// Cleared on exit: a repeated exit can never count the same visit twice.
|
|
68
|
+
entered.delete(event.node);
|
|
66
69
|
if (start !== undefined)
|
|
67
70
|
durations.set(event.node, (durations.get(event.node) ?? 0) + (now() - start));
|
|
68
71
|
lastResults.set(event.node, event.result);
|
|
69
72
|
}
|
|
73
|
+
else if (event.type === "node:amended") {
|
|
74
|
+
lastResults.set(event.node, event.result);
|
|
75
|
+
}
|
|
70
76
|
},
|
|
71
77
|
};
|
|
72
78
|
}
|
|
@@ -117,8 +123,18 @@ export function buildRunRecord(i) {
|
|
|
117
123
|
},
|
|
118
124
|
...(harness ? { harness: { id: harness.id, version: harness.version } } : {}),
|
|
119
125
|
...(degraded.length > 0 ? { degraded } : {}),
|
|
126
|
+
...(summary.spend ? { spend: recordSpend(summary.spend) } : {}),
|
|
120
127
|
};
|
|
121
128
|
}
|
|
129
|
+
function recordSpend(spend) {
|
|
130
|
+
const out = {};
|
|
131
|
+
for (const k of SPEND_KINDS) {
|
|
132
|
+
const v = spend[k];
|
|
133
|
+
if (v)
|
|
134
|
+
out[k] = { tokens: v.tokens, cost_usd: v.costUsd, calls: v.calls };
|
|
135
|
+
}
|
|
136
|
+
return out;
|
|
137
|
+
}
|
|
122
138
|
// ── Write, prune, read ──────────────────────────────────────────
|
|
123
139
|
/** Atomic write: temp file in the same directory, fsync, rename. Returns the path, or null on failure. Never throws. */
|
|
124
140
|
export function writeRunRecord(record, cwd = process.cwd()) {
|
package/dist/cli/run-output.d.ts
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import { type ExecutionTrace, type Logger, type NodeResult, type Workflow } from "../types.js";
|
|
9
9
|
import { type NodeStatus } from "../mermaid.js";
|
|
10
|
+
import { type SpendKind } from "../budget.js";
|
|
10
11
|
export interface RunSummary {
|
|
11
12
|
ok: boolean;
|
|
12
13
|
nodesOk: number;
|
|
@@ -31,6 +32,12 @@ export interface RunSummary {
|
|
|
31
32
|
* knows none of it.
|
|
32
33
|
*/
|
|
33
34
|
policy?: RunPolicy;
|
|
35
|
+
/**
|
|
36
|
+
* Spend by kind (`node`, `judge`, `route`, `reflect`, `screen`), when the executor's own
|
|
37
|
+
* completions reported usage. `tokens` and `costUsd` above are then the sum over every kind
|
|
38
|
+
* (every attempt and visit), not over the node results. Absent otherwise.
|
|
39
|
+
*/
|
|
40
|
+
spend?: ExecutionTrace["spend"];
|
|
34
41
|
/** The spend budget a node crossed (#449). Absent when none was. */
|
|
35
42
|
budget?: {
|
|
36
43
|
node: string;
|
|
@@ -38,6 +45,8 @@ export interface RunSummary {
|
|
|
38
45
|
unit: "tokens" | "cost_usd";
|
|
39
46
|
limit: number;
|
|
40
47
|
spent: number;
|
|
48
|
+
/** The executor completion that crossed it (judge, route, ...); absent for a node run. */
|
|
49
|
+
kind?: SpendKind;
|
|
41
50
|
};
|
|
42
51
|
/**
|
|
43
52
|
* Who decided the run's conditional routes (#357), counts only: a `when`
|
|
@@ -89,8 +98,14 @@ export declare function formatTokenCount(n: number): string;
|
|
|
89
98
|
export declare function formatCost(usd: number): string;
|
|
90
99
|
/** `tokens 52k of 50k`: the overrun, in the budget's unit. */
|
|
91
100
|
export declare function formatBudgetAmount(b: NonNullable<RunSummary["budget"]>): string;
|
|
92
|
-
/** `budget exceeded: tokens 52k of 50k (run, at node gather)
|
|
101
|
+
/** `budget exceeded: tokens 52k of 50k (run, at node gather)`, plus `, judge` when an executor call crossed it. */
|
|
93
102
|
export declare function formatBudgetOverrun(b: NonNullable<RunSummary["budget"]>): string;
|
|
103
|
+
/**
|
|
104
|
+
* `node $0.15, judge $0.03, route <$0.01`: spend by kind in a fixed order
|
|
105
|
+
* (node, judge, route, reflect, screen), only the kinds present. In tokens
|
|
106
|
+
* when no kind reported a cost. Undefined when there is no split to show.
|
|
107
|
+
*/
|
|
108
|
+
export declare function formatSpendSplit(spend: RunSummary["spend"], sep?: string): string | undefined;
|
|
94
109
|
/** `5 (3 decider, 1 expr, 1 agent)`: zero rungs are left out. */
|
|
95
110
|
export declare function formatRouteCounts(r: RouteCounts): string;
|
|
96
111
|
/**
|
package/dist/cli/run-output.js
CHANGED
|
@@ -11,6 +11,7 @@ import { toMermaidBlock } from "../mermaid.js";
|
|
|
11
11
|
import { createPaint } from "./style.js";
|
|
12
12
|
import { colorEnabled, glyphs } from "./terminal.js";
|
|
13
13
|
import { trustedEnvValue } from "../startup-env.js";
|
|
14
|
+
import { SPEND_KINDS } from "../budget.js";
|
|
14
15
|
/** The short key of a `degraded` entry: the text before its first colon. */
|
|
15
16
|
export function degradedKey(entry) {
|
|
16
17
|
const i = entry.indexOf(":");
|
|
@@ -70,6 +71,19 @@ export function summarizeRun(results, durationMs, crashed = false, trace) {
|
|
|
70
71
|
}
|
|
71
72
|
if (staged > 0 || applied > 0)
|
|
72
73
|
policy.outputs = { staged, applied };
|
|
74
|
+
// The executor's own completions (judge, route, ...) are on no node result,
|
|
75
|
+
// and a node result holds only its last attempt. With the split present the
|
|
76
|
+
// totals are its sum, so they reconcile with it on one basis.
|
|
77
|
+
const split = Object.values(trace?.spend ?? {});
|
|
78
|
+
const extra = Object.keys(trace?.spend ?? {}).some((k) => k !== "node");
|
|
79
|
+
if (extra) {
|
|
80
|
+
const t = split.reduce((n, k) => n + (isNum(k.tokens) ? k.tokens : 0), 0);
|
|
81
|
+
const c = split.reduce((n, k) => n + (isNum(k.costUsd) ? k.costUsd : 0), 0);
|
|
82
|
+
if (t > 0 || tokens !== undefined)
|
|
83
|
+
tokens = t;
|
|
84
|
+
if (c > 0 || costUsd !== undefined)
|
|
85
|
+
costUsd = Math.round(c * 1e10) / 1e10;
|
|
86
|
+
}
|
|
73
87
|
return {
|
|
74
88
|
ok: !failed,
|
|
75
89
|
nodesOk,
|
|
@@ -83,6 +97,7 @@ export function summarizeRun(results, durationMs, crashed = false, trace) {
|
|
|
83
97
|
...(degraded.size > 0 ? { degraded: [...degraded] } : {}),
|
|
84
98
|
...(Object.keys(policy).length > 0 ? { policy } : {}),
|
|
85
99
|
...(budget ? { budget } : {}),
|
|
100
|
+
...(extra ? { spend: trace.spend } : {}),
|
|
86
101
|
...(trace && countRoutes(trace).total > 0 ? { routes: countRoutes(trace) } : {}),
|
|
87
102
|
...(trace?.deciderOff ? { deciderOff: trace.deciderOff } : {}),
|
|
88
103
|
};
|
|
@@ -160,9 +175,34 @@ export function formatBudgetAmount(b) {
|
|
|
160
175
|
const unit = b.unit === "cost_usd" ? "cost" : "tokens";
|
|
161
176
|
return `${unit} ${amount(b.spent)} of ${amount(b.limit)}`;
|
|
162
177
|
}
|
|
163
|
-
/** `budget exceeded: tokens 52k of 50k (run, at node gather)
|
|
178
|
+
/** `budget exceeded: tokens 52k of 50k (run, at node gather)`, plus `, judge` when an executor call crossed it. */
|
|
164
179
|
export function formatBudgetOverrun(b) {
|
|
165
|
-
return `budget exceeded: ${formatBudgetAmount(b)} (${b.scope}, at node ${b.node})`;
|
|
180
|
+
return `budget exceeded: ${formatBudgetAmount(b)} (${b.scope}, at node ${b.node}${b.kind ? `, ${b.kind}` : ""})`;
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* `node $0.15, judge $0.03, route <$0.01`: spend by kind in a fixed order
|
|
184
|
+
* (node, judge, route, reflect, screen), only the kinds present. In tokens
|
|
185
|
+
* when no kind reported a cost. Undefined when there is no split to show.
|
|
186
|
+
*/
|
|
187
|
+
export function formatSpendSplit(spend, sep = " ") {
|
|
188
|
+
const kinds = SPEND_KINDS.flatMap((k) => (spend?.[k] ? [[k, spend[k]]] : []));
|
|
189
|
+
if (kinds.length === 0)
|
|
190
|
+
return undefined;
|
|
191
|
+
const byCost = kinds.some(([, v]) => v.costUsd > 0);
|
|
192
|
+
return kinds.map(([k, v]) => `${k}${sep}${byCost ? formatCost(v.costUsd) : formatTokenCount(v.tokens)}`).join(", ");
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* `12k tokens (judge 2k)`: the executor's own kinds in brackets, on the token
|
|
196
|
+
* segment, or on the cost (`$0.18 (judge $0.03)`) when no tokens were reported.
|
|
197
|
+
*/
|
|
198
|
+
function ownSpendBracket(spend, unit) {
|
|
199
|
+
const own = SPEND_KINDS.flatMap((k) => {
|
|
200
|
+
const v = spend?.[k];
|
|
201
|
+
if (k === "node" || !v || !(v[unit] > 0))
|
|
202
|
+
return [];
|
|
203
|
+
return [`${k} ${unit === "tokens" ? formatTokenCount(v.tokens) : formatCost(v.costUsd)}`];
|
|
204
|
+
});
|
|
205
|
+
return own.length === 0 ? "" : ` (${own.join(", ")})`;
|
|
166
206
|
}
|
|
167
207
|
/** `5 (3 decider, 1 expr, 1 agent)`: zero rungs are left out. */
|
|
168
208
|
export function formatRouteCounts(r) {
|
|
@@ -182,8 +222,10 @@ export function formatReceipt(s) {
|
|
|
182
222
|
...(s.nodesSkipped > 0 ? [`${s.nodesSkipped} skipped`] : []),
|
|
183
223
|
`${s.toolCalls} tool ${s.toolCalls === 1 ? "call" : "calls"}`,
|
|
184
224
|
formatReceiptDuration(s.durationMs),
|
|
185
|
-
...(s.tokens !== undefined ? [`${formatTokenCount(s.tokens)} tokens`] : []),
|
|
186
|
-
...(s.costUsd !== undefined
|
|
225
|
+
...(s.tokens !== undefined ? [`${formatTokenCount(s.tokens)} tokens${ownSpendBracket(s.spend, "tokens")}`] : []),
|
|
226
|
+
...(s.costUsd !== undefined
|
|
227
|
+
? [`${formatCost(s.costUsd)}${s.tokens === undefined ? ownSpendBracket(s.spend, "costUsd") : ""}`]
|
|
228
|
+
: []),
|
|
187
229
|
...(s.harness !== undefined && s.harness !== "claude-code" ? [s.harness] : []),
|
|
188
230
|
...(s.degraded && s.degraded.length > 0 ? [`degraded: ${s.degraded.join(", ")}`] : []),
|
|
189
231
|
...(formatPolicySegment(s.policy) ? [formatPolicySegment(s.policy)] : []),
|
package/dist/cli/runs.js
CHANGED
|
@@ -5,7 +5,8 @@
|
|
|
5
5
|
* `color` is true (TTY and no NO_COLOR); the glyph set follows `unicode`.
|
|
6
6
|
* The diff always ends on one plain line with the answer counts.
|
|
7
7
|
*/
|
|
8
|
-
import { findRun, listRuns } from "./run-history.js";
|
|
8
|
+
import { findRun, listRuns, } from "./run-history.js";
|
|
9
|
+
import { SPEND_KINDS } from "../budget.js";
|
|
9
10
|
import { compareAnswers, readRunAnswers, } from "./run-answers.js";
|
|
10
11
|
import { formatCost, formatReceiptDuration, formatTokenCount } from "./run-output.js";
|
|
11
12
|
import { createPaint } from "./style.js";
|
|
@@ -84,8 +85,25 @@ export function printableRun(r) {
|
|
|
84
85
|
tokens: finiteOrNull(r.totals.tokens),
|
|
85
86
|
cost_usd: finiteOrNull(r.totals.cost_usd),
|
|
86
87
|
},
|
|
88
|
+
...(r.spend && typeof r.spend === "object" ? { spend: printableSpend(r.spend) } : {}),
|
|
87
89
|
};
|
|
88
90
|
}
|
|
91
|
+
/** The record's split, or its totals as all node spend when it has none. */
|
|
92
|
+
function splitOf(r) {
|
|
93
|
+
if (r.spend)
|
|
94
|
+
return r.spend;
|
|
95
|
+
return { node: { tokens: r.totals.tokens ?? 0, cost_usd: r.totals.cost_usd ?? 0, calls: 0 } };
|
|
96
|
+
}
|
|
97
|
+
/** Only the known kinds, own keys, finite numbers. */
|
|
98
|
+
function printableSpend(spend) {
|
|
99
|
+
const out = {};
|
|
100
|
+
for (const k of SPEND_KINDS) {
|
|
101
|
+
const v = Object.hasOwn(spend, k) ? spend[k] : undefined;
|
|
102
|
+
if (v && typeof v === "object")
|
|
103
|
+
out[k] = { tokens: finite(v.tokens), cost_usd: finite(v.cost_usd), calls: finite(v.calls) };
|
|
104
|
+
}
|
|
105
|
+
return out;
|
|
106
|
+
}
|
|
89
107
|
// ── sweny runs ──────────────────────────────────────────────────
|
|
90
108
|
export function formatRunsTable(records, color, nowMs = Date.now(), unicode = true) {
|
|
91
109
|
const ICON = iconsFor(glyphsFor(unicode));
|
|
@@ -312,6 +330,25 @@ export function formatRunDiff(rawA, rawB, color, opts = {}) {
|
|
|
312
330
|
for (const t of tot)
|
|
313
331
|
if (t)
|
|
314
332
|
out.push(` ${t}`);
|
|
333
|
+
// Spend by kind, when either run has the split: what moved the totals. A
|
|
334
|
+
// record without one (node-only, or written before the split) is all node spend.
|
|
335
|
+
if (a.spend || b.spend) {
|
|
336
|
+
const sa = splitOf(a);
|
|
337
|
+
const sb = splitOf(b);
|
|
338
|
+
const byCost = SPEND_KINDS.some((k) => (sa[k]?.cost_usd ?? 0) > 0 || (sb[k]?.cost_usd ?? 0) > 0);
|
|
339
|
+
for (const k of SPEND_KINDS) {
|
|
340
|
+
const va = sa[k];
|
|
341
|
+
const vb = sb[k];
|
|
342
|
+
if (!va && !vb)
|
|
343
|
+
continue;
|
|
344
|
+
const label = ` ${pad(k, 10)}`;
|
|
345
|
+
const line = byCost
|
|
346
|
+
? metric(label, va ? va.cost_usd : null, vb ? vb.cost_usd : null, formatCost, deltaCost, g.arrow)
|
|
347
|
+
: metric(label, va ? va.tokens : null, vb ? vb.tokens : null, formatTokenCount, deltaTokens, g.arrow);
|
|
348
|
+
if (line)
|
|
349
|
+
out.push(` ${line}`);
|
|
350
|
+
}
|
|
351
|
+
}
|
|
315
352
|
out.push("", " nodes");
|
|
316
353
|
const aNodes = new Map(a.nodes.map((n) => [n.id, n]));
|
|
317
354
|
const bNodes = new Map(b.nodes.map((n) => [n.id, n]));
|
package/dist/cli/ticket.js
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* METADATA ONLY, like the receipt line: counts, durations, tokens, cost,
|
|
7
7
|
* policy facts. Never prompts, node outputs, tool inputs, or model prose.
|
|
8
8
|
*/
|
|
9
|
-
import { formatBudgetAmount, formatCost, formatRouteCounts, formatReceiptDuration, formatTokenCount, policyParts, renderReceiptLine, } from "./run-output.js";
|
|
9
|
+
import { formatBudgetAmount, formatCost, formatRouteCounts, formatSpendSplit, formatReceiptDuration, formatTokenCount, policyParts, renderReceiptLine, } from "./run-output.js";
|
|
10
10
|
import { createPaint } from "./style.js";
|
|
11
11
|
import { colorEnabled, glyphsFor, richOutput, supportsUnicode, terminalColumns, ticketBoxFor } from "./terminal.js";
|
|
12
12
|
/** Widest ticket, in columns, frame included. */
|
|
@@ -52,8 +52,11 @@ export function stampWords(summary) {
|
|
|
52
52
|
}
|
|
53
53
|
return { words, strong };
|
|
54
54
|
}
|
|
55
|
+
const NBSP = "\u00a0";
|
|
55
56
|
/** Field rows, label then value, metadata only. */
|
|
56
57
|
function ticketFields(s) {
|
|
58
|
+
// A no-break space keeps each kind with its amount when the row wraps (put back after wrapping).
|
|
59
|
+
const spend = formatSpendSplit(s.spend, NBSP);
|
|
57
60
|
const failed = Math.max(0, s.nodesTotal - s.nodesOk - s.nodesSkipped);
|
|
58
61
|
const nodes = [
|
|
59
62
|
`${s.nodesOk}/${s.nodesTotal}`,
|
|
@@ -67,6 +70,7 @@ function ticketFields(s) {
|
|
|
67
70
|
["duration", formatReceiptDuration(s.durationMs)],
|
|
68
71
|
...(s.tokens !== undefined ? [["tokens", formatTokenCount(s.tokens)]] : []),
|
|
69
72
|
...(s.costUsd !== undefined ? [["cost", formatCost(s.costUsd)]] : []),
|
|
73
|
+
...(spend ? [["spend", spend]] : []),
|
|
70
74
|
...(s.harness !== undefined && s.harness !== "claude-code" ? [["harness", s.harness]] : []),
|
|
71
75
|
...(s.degraded && s.degraded.length > 0 ? [["degraded", s.degraded.join(", ")]] : []),
|
|
72
76
|
...(s.routes && s.routes.total > 0 ? [["routes", formatRouteCounts(s.routes)]] : []),
|
|
@@ -154,7 +158,8 @@ export function renderTicket(data, opts = {}) {
|
|
|
154
158
|
lines.push(row(""));
|
|
155
159
|
// Fields, aligned: label column then value column; long values wrap under the value.
|
|
156
160
|
for (const [label, value] of ticketFields(s)) {
|
|
157
|
-
wrap(value, valueWidth).forEach((
|
|
161
|
+
wrap(value, valueWidth).forEach((raw, i) => {
|
|
162
|
+
const part = raw.replaceAll(NBSP, " ");
|
|
158
163
|
const lab = (i === 0 ? label : "").padEnd(LABEL_WIDTH);
|
|
159
164
|
lines.push(row(lab + part, paint.muted(lab) + part));
|
|
160
165
|
});
|
package/dist/eval/index.d.ts
CHANGED
|
@@ -10,6 +10,16 @@ export interface EvaluateAllOptions {
|
|
|
10
10
|
node?: Node;
|
|
11
11
|
/** The workflow being executed. Used to resolve workflow-level `judge_model` overrides. */
|
|
12
12
|
workflow?: Workflow;
|
|
13
|
+
/**
|
|
14
|
+
* Run the evaluators one at a time instead of together. The executor sets it
|
|
15
|
+
* when a spend budget is on, so each judge's usage is counted (and a
|
|
16
|
+
* crossing stops the rest) before the next one starts.
|
|
17
|
+
*/
|
|
18
|
+
sequential?: boolean;
|
|
19
|
+
/** The run's abort signal: checked before each judge when sequential, and forwarded to judge calls. */
|
|
20
|
+
signal?: AbortSignal;
|
|
21
|
+
/** Per-call timeout forwarded to judge calls. */
|
|
22
|
+
timeoutMs?: number;
|
|
13
23
|
}
|
|
14
24
|
/**
|
|
15
25
|
* Run every declared evaluator against the node's result and return one
|
package/dist/eval/index.js
CHANGED
|
@@ -41,11 +41,28 @@ export async function evaluateAll(evaluators, result, opts = {}) {
|
|
|
41
41
|
throw new Error(`evaluator '${orphanJudge.name}' is kind: judge but no Claude client was provided to evaluateAll`);
|
|
42
42
|
}
|
|
43
43
|
// Evaluators are independent, so they run concurrently (#337): N judges on
|
|
44
|
-
// a node cost one round-trip of wall time, not N.
|
|
45
|
-
//
|
|
46
|
-
//
|
|
47
|
-
//
|
|
48
|
-
|
|
44
|
+
// a node cost one round-trip of wall time, not N. With `sequential` (a spend
|
|
45
|
+
// budget is on) they run one at a time, so each judge's spend is counted
|
|
46
|
+
// before the next starts, and an abort stops the rest. Results keep
|
|
47
|
+
// declaration order. Every evaluator settles before an error surfaces, and
|
|
48
|
+
// the first rejection in declaration order is rethrown, so the outcome never
|
|
49
|
+
// depends on which call finishes first.
|
|
50
|
+
const settled = [];
|
|
51
|
+
if (opts.sequential) {
|
|
52
|
+
for (const e of evaluators) {
|
|
53
|
+
if (opts.signal?.aborted)
|
|
54
|
+
throw new Error("workflow aborted");
|
|
55
|
+
try {
|
|
56
|
+
settled.push({ status: "fulfilled", value: await evaluateOne(e, result, opts) });
|
|
57
|
+
}
|
|
58
|
+
catch (reason) {
|
|
59
|
+
settled.push({ status: "rejected", reason });
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
else {
|
|
64
|
+
settled.push(...(await Promise.allSettled(evaluators.map((e) => evaluateOne(e, result, opts)))));
|
|
65
|
+
}
|
|
49
66
|
const out = [];
|
|
50
67
|
for (const s of settled) {
|
|
51
68
|
if (s.status === "rejected")
|
|
@@ -66,7 +83,11 @@ async function evaluateOne(e, result, opts) {
|
|
|
66
83
|
}
|
|
67
84
|
if (e.kind === "judge" && opts.claude) {
|
|
68
85
|
const model = resolveJudgeModel(e, opts.node, opts.workflow, opts.claude.defaultJudgeModel);
|
|
69
|
-
const verdict = await evaluateJudge(e, result, opts.claude, {
|
|
86
|
+
const verdict = await evaluateJudge(e, result, opts.claude, {
|
|
87
|
+
model,
|
|
88
|
+
...(opts.signal ? { signal: opts.signal } : {}),
|
|
89
|
+
...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}),
|
|
90
|
+
});
|
|
70
91
|
return { name: e.name, kind: "judge", pass: verdict.pass, reasoning: capReasoning(verdict.reasoning) };
|
|
71
92
|
}
|
|
72
93
|
return null;
|
package/dist/eval/judge.d.ts
CHANGED
|
@@ -2,6 +2,9 @@ import type { Claude, Evaluator, NodeResult } from "../types.js";
|
|
|
2
2
|
export interface JudgeOptions {
|
|
3
3
|
/** Resolved judge model (evaluator > node > workflow > package default). */
|
|
4
4
|
model: string;
|
|
5
|
+
/** The run's abort signal and per-call timeout, forwarded to the judge call. */
|
|
6
|
+
signal?: AbortSignal;
|
|
7
|
+
timeoutMs?: number;
|
|
5
8
|
}
|
|
6
9
|
interface ParsedResponse {
|
|
7
10
|
verdict: string | null;
|
package/dist/eval/judge.js
CHANGED
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
// For a typical 4-node workflow with 2 judges per node, that's 8 extra
|
|
14
14
|
// calls per run. The workflow-level `judge_budget` (default 50) gives
|
|
15
15
|
// authors a load-time warning when the total stretches past expected.
|
|
16
|
+
import { BudgetRefusal } from "../budget.js";
|
|
16
17
|
/**
|
|
17
18
|
* Build the structured judge prompt.
|
|
18
19
|
*
|
|
@@ -122,10 +123,19 @@ export async function evaluateJudge(evaluator, result, claude, opts) {
|
|
|
122
123
|
instruction: prompt,
|
|
123
124
|
context: {},
|
|
124
125
|
model: opts.model,
|
|
126
|
+
...(opts.signal ? { signal: opts.signal } : {}),
|
|
127
|
+
...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}),
|
|
125
128
|
});
|
|
126
129
|
}
|
|
127
|
-
catch {
|
|
128
|
-
//
|
|
130
|
+
catch (err) {
|
|
131
|
+
// A budget refusal is no answer at all: it reaches the executor, which
|
|
132
|
+
// stops the node on the budget instead of recording a parse failure.
|
|
133
|
+
if (err instanceof BudgetRefusal)
|
|
134
|
+
throw err;
|
|
135
|
+
// An aborted run is no answer either: never retried as a parse failure.
|
|
136
|
+
if (opts.signal?.aborted)
|
|
137
|
+
throw err;
|
|
138
|
+
// Treat other throws as parse failure for retry purposes; the second attempt
|
|
129
139
|
// either succeeds or surfaces the same path through the same channel.
|
|
130
140
|
continue;
|
|
131
141
|
}
|