@sweny-ai/core 0.1.199 → 0.1.200
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/budget.d.ts +77 -11
- package/dist/budget.js +132 -16
- package/dist/cli/renderer.js +5 -0
- package/dist/cli/run-history.d.ts +12 -0
- package/dist/cli/run-history.js +16 -0
- package/dist/cli/run-output.d.ts +16 -1
- package/dist/cli/run-output.js +46 -4
- package/dist/cli/runs.js +38 -1
- package/dist/cli/ticket.js +7 -2
- package/dist/eval/index.d.ts +10 -0
- package/dist/eval/index.js +27 -6
- package/dist/eval/judge.d.ts +3 -0
- package/dist/eval/judge.js +12 -2
- package/dist/executor.js +276 -31
- package/dist/harness/acp.js +4 -1
- package/dist/harness/claude-code.d.ts +3 -1
- package/dist/harness/claude-code.js +4 -1
- package/dist/harness/codex.js +3 -1
- package/dist/harness/compat.js +1 -0
- package/dist/harness/index.d.ts +1 -1
- package/dist/harness/index.js +1 -1
- package/dist/harness/pi.js +3 -1
- package/dist/harness/prompts.d.ts +11 -1
- package/dist/harness/prompts.js +21 -2
- package/dist/harness/types.d.ts +9 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.js +2 -2
- package/dist/journal.d.ts +48 -10
- package/dist/journal.js +164 -16
- package/dist/safe-outputs.d.ts +9 -0
- package/dist/safe-outputs.js +41 -26
- package/dist/skills/linear.js +3 -1
- package/dist/studio.js +2 -1
- package/dist/types.d.ts +34 -1
- package/package.json +1 -1
package/dist/budget.d.ts
CHANGED
|
@@ -8,8 +8,14 @@
|
|
|
8
8
|
*
|
|
9
9
|
* Tokens are input + output tokens as the harness reports them (cache reads and
|
|
10
10
|
* writes are not counted, matching the run receipt). Cost is the harness's own
|
|
11
|
-
* reported USD, never estimated.
|
|
12
|
-
*
|
|
11
|
+
* reported USD, never estimated. Every model completion the executor itself
|
|
12
|
+
* makes is counted too, under its own `kind` (see {@link SpendKind}): judge
|
|
13
|
+
* scoring, route evaluation, retry reflection and the safe-output screen. A
|
|
14
|
+
* completion counts when its harness reports usage for it; one that reports
|
|
15
|
+
* none adds nothing (the same as a node run without usage). The run totals
|
|
16
|
+
* and the stop are over all kinds together, across a resume too; the split is
|
|
17
|
+
* `trace.spend`. Once a ceiling is reached no further executor completion
|
|
18
|
+
* starts. The decision model (`route_by: decider`) is not metered.
|
|
13
19
|
*
|
|
14
20
|
* Enforcement, per unit the harness can report:
|
|
15
21
|
* - live: the harness calls `onUsage` during the node; crossing a limit aborts
|
|
@@ -17,7 +23,9 @@
|
|
|
17
23
|
* - between nodes: usage on the finished result is checked; a crossing fails the
|
|
18
24
|
* node, and the run halts before the next node starts.
|
|
19
25
|
* - across a resume: the run journal records each attempt's spend (live and
|
|
20
|
-
* final), and
|
|
26
|
+
* final), and each metered completion (judge, route, reflection, screen)
|
|
27
|
+
* as its own record; `sweny workflow resume` seeds the run total with all
|
|
28
|
+
* of it, split by kind.
|
|
21
29
|
*/
|
|
22
30
|
import type { NodeUsage } from "./types.js";
|
|
23
31
|
/** The `budget` field of a node or workflow. */
|
|
@@ -36,6 +44,20 @@ export interface Spend {
|
|
|
36
44
|
tokens: number;
|
|
37
45
|
costUsd: number;
|
|
38
46
|
}
|
|
47
|
+
/** Spend restored on a resume: the total, and its split by kind when the journal has one. */
|
|
48
|
+
export interface SeedSpend extends Spend {
|
|
49
|
+
byKind?: Partial<Record<SpendKind, Spend>>;
|
|
50
|
+
}
|
|
51
|
+
/** What a unit of spend paid for. `node` is an agent node run; the rest are the executor's own completions. */
|
|
52
|
+
export type SpendKind = "node" | "judge" | "route" | "reflect" | "screen";
|
|
53
|
+
/** The order kinds are listed in, wherever the split is shown. */
|
|
54
|
+
export declare const SPEND_KINDS: readonly SpendKind[];
|
|
55
|
+
/** Spend of one {@link SpendKind}. `calls` counts the completions that reported usage. */
|
|
56
|
+
export interface KindSpend extends Spend {
|
|
57
|
+
calls: number;
|
|
58
|
+
}
|
|
59
|
+
/** Spend by kind. A kind with no reported usage is absent. */
|
|
60
|
+
export type SpendBreakdown = Partial<Record<SpendKind, KindSpend>>;
|
|
39
61
|
export type BudgetUnit = "tokens" | "cost_usd";
|
|
40
62
|
/** Recorded on the node whose spend crossed a budget (`NodeResult.budget`). */
|
|
41
63
|
export interface BudgetOverrun {
|
|
@@ -45,6 +67,18 @@ export interface BudgetOverrun {
|
|
|
45
67
|
limit: number;
|
|
46
68
|
/** Spend at the moment of the stop (live) or at the end of the node. */
|
|
47
69
|
spent: number;
|
|
70
|
+
/** The executor completion whose spend crossed it (judge, route, ...). Absent for a node run. */
|
|
71
|
+
kind?: SpendKind;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Thrown into an executor completion (judge, route, reflection, screen) the
|
|
75
|
+
* budget refuses: a ceiling was already reached, so the call is not made.
|
|
76
|
+
* Callers that swallow a failed completion (a judge's parse retry) must let
|
|
77
|
+
* this one through, so a refusal is never mistaken for a model answer.
|
|
78
|
+
*/
|
|
79
|
+
export declare class BudgetRefusal extends Error {
|
|
80
|
+
readonly breach: BudgetOverrun;
|
|
81
|
+
constructor(breach: BudgetOverrun);
|
|
48
82
|
}
|
|
49
83
|
export declare function toLimits(b: Budget | undefined): SpendLimits;
|
|
50
84
|
/** The tighter of two limit sets, per unit. */
|
|
@@ -64,21 +98,43 @@ export declare function describeOverrun(o: BudgetOverrun, nodeId?: string): stri
|
|
|
64
98
|
export declare class BudgetGuard {
|
|
65
99
|
readonly runLimits: SpendLimits;
|
|
66
100
|
private spent;
|
|
101
|
+
private kinds;
|
|
67
102
|
constructor(runLimits: SpendLimits);
|
|
68
103
|
get totals(): Spend;
|
|
104
|
+
/**
|
|
105
|
+
* Spend by kind, in {@link SPEND_KINDS} order. Resumed spend is restored
|
|
106
|
+
* under its kind; a journal without meter records seeds `node` only.
|
|
107
|
+
*/
|
|
108
|
+
breakdown(): SpendBreakdown;
|
|
109
|
+
/** True once any completion other than a node run reported usage. */
|
|
110
|
+
get hasMeteredCalls(): boolean;
|
|
111
|
+
/**
|
|
112
|
+
* Count one executor completion (judge, route, ...) against the run. Usage
|
|
113
|
+
* with no tokens and no cost counts nothing. Returns the breach this report
|
|
114
|
+
* caused, judged against the totals before it, like a live node report.
|
|
115
|
+
*/
|
|
116
|
+
meter(kind: SpendKind, usage: NodeUsage | undefined, nodeBudget?: NodeBudget): BudgetOverrun | undefined;
|
|
69
117
|
/** True when any limit exists at all, so the executor can skip the machinery. */
|
|
70
118
|
active(nodeLimits: SpendLimits): boolean;
|
|
71
119
|
/** The run ceiling is already reached: no further attempt may start. */
|
|
72
120
|
runExhausted(): BudgetOverrun | undefined;
|
|
73
121
|
node(nodeLimits: SpendLimits): NodeBudget;
|
|
74
122
|
/**
|
|
75
|
-
* Spend the run made before this process (a resumed run's journaled usage
|
|
76
|
-
* Counted against the run ceiling like any other
|
|
77
|
-
* resets the whole-run budget.
|
|
123
|
+
* Spend the run made before this process (a resumed run's journaled usage
|
|
124
|
+
* and metered completions). Counted against the run ceiling like any other
|
|
125
|
+
* spend, so a crash never resets the whole-run budget. Restored under its
|
|
126
|
+
* kind when the journal has the split (a journal written before meter
|
|
127
|
+
* records existed has node spend only). Seeded spend is not a call: `calls`
|
|
128
|
+
* is unchanged.
|
|
78
129
|
*/
|
|
79
|
-
seed(s:
|
|
80
|
-
/**
|
|
81
|
-
|
|
130
|
+
seed(s: SeedSpend): void;
|
|
131
|
+
/**
|
|
132
|
+
* @internal `counted`: this commit is one completion that reported usage, so
|
|
133
|
+
* `calls` goes up. A commit that is not (seeded spend, an attempt that
|
|
134
|
+
* reported nothing) adds its spend only, and a zero one leaves no trace, so
|
|
135
|
+
* the split is the same with and without a limit.
|
|
136
|
+
*/
|
|
137
|
+
commit(s: Spend, kind: SpendKind, counted: boolean): void;
|
|
82
138
|
}
|
|
83
139
|
export declare class NodeBudget {
|
|
84
140
|
private readonly guard;
|
|
@@ -88,10 +144,18 @@ export declare class NodeBudget {
|
|
|
88
144
|
/** A limit is already reached before the next attempt (a retry after a spend-heavy attempt, or a spent run). */
|
|
89
145
|
exhausted(): BudgetOverrun | undefined;
|
|
90
146
|
attempt(outer?: AbortSignal): BudgetAttempt;
|
|
147
|
+
/**
|
|
148
|
+
* Spend this visit made before a resume (a checkpointed visit's journaled
|
|
149
|
+
* spend). It counts against the node's own limit only: the run total
|
|
150
|
+
* already holds it from the seed.
|
|
151
|
+
*/
|
|
152
|
+
restore(s: Spend): void;
|
|
91
153
|
/** @internal */
|
|
92
154
|
get committed(): Spend;
|
|
93
|
-
/**
|
|
94
|
-
|
|
155
|
+
/** Count one executor completion made during this visit (judge, retry reflection, screen). */
|
|
156
|
+
meter(kind: SpendKind, usage: NodeUsage | undefined): BudgetOverrun | undefined;
|
|
157
|
+
/** @internal See {@link BudgetGuard.commit}. */
|
|
158
|
+
commit(s: Spend, kind: SpendKind, counted: boolean): void;
|
|
95
159
|
}
|
|
96
160
|
/** One agent run. Its `signal` is the caller's signal plus a budget stop. */
|
|
97
161
|
export declare class BudgetAttempt {
|
|
@@ -101,6 +165,8 @@ export declare class BudgetAttempt {
|
|
|
101
165
|
private readonly controller;
|
|
102
166
|
private live;
|
|
103
167
|
private liveUsage;
|
|
168
|
+
/** A live report carried tokens or cost. */
|
|
169
|
+
private liveReported;
|
|
104
170
|
private stopped;
|
|
105
171
|
private settled;
|
|
106
172
|
private readonly onOuterAbort;
|
package/dist/budget.js
CHANGED
|
@@ -8,8 +8,14 @@
|
|
|
8
8
|
*
|
|
9
9
|
* Tokens are input + output tokens as the harness reports them (cache reads and
|
|
10
10
|
* writes are not counted, matching the run receipt). Cost is the harness's own
|
|
11
|
-
* reported USD, never estimated.
|
|
12
|
-
*
|
|
11
|
+
* reported USD, never estimated. Every model completion the executor itself
|
|
12
|
+
* makes is counted too, under its own `kind` (see {@link SpendKind}): judge
|
|
13
|
+
* scoring, route evaluation, retry reflection and the safe-output screen. A
|
|
14
|
+
* completion counts when its harness reports usage for it; one that reports
|
|
15
|
+
* none adds nothing (the same as a node run without usage). The run totals
|
|
16
|
+
* and the stop are over all kinds together, across a resume too; the split is
|
|
17
|
+
* `trace.spend`. Once a ceiling is reached no further executor completion
|
|
18
|
+
* starts. The decision model (`route_by: decider`) is not metered.
|
|
13
19
|
*
|
|
14
20
|
* Enforcement, per unit the harness can report:
|
|
15
21
|
* - live: the harness calls `onUsage` during the node; crossing a limit aborts
|
|
@@ -17,9 +23,34 @@
|
|
|
17
23
|
* - between nodes: usage on the finished result is checked; a crossing fails the
|
|
18
24
|
* node, and the run halts before the next node starts.
|
|
19
25
|
* - across a resume: the run journal records each attempt's spend (live and
|
|
20
|
-
* final), and
|
|
26
|
+
* final), and each metered completion (judge, route, reflection, screen)
|
|
27
|
+
* as its own record; `sweny workflow resume` seeds the run total with all
|
|
28
|
+
* of it, split by kind.
|
|
21
29
|
*/
|
|
30
|
+
/** The order kinds are listed in, wherever the split is shown. */
|
|
31
|
+
export const SPEND_KINDS = ["node", "judge", "route", "reflect", "screen"];
|
|
32
|
+
/**
|
|
33
|
+
* Thrown into an executor completion (judge, route, reflection, screen) the
|
|
34
|
+
* budget refuses: a ceiling was already reached, so the call is not made.
|
|
35
|
+
* Callers that swallow a failed completion (a judge's parse retry) must let
|
|
36
|
+
* this one through, so a refusal is never mistaken for a model answer.
|
|
37
|
+
*/
|
|
38
|
+
export class BudgetRefusal extends Error {
|
|
39
|
+
breach;
|
|
40
|
+
constructor(breach) {
|
|
41
|
+
super(describeOverrun(breach));
|
|
42
|
+
this.breach = breach;
|
|
43
|
+
this.name = "BudgetRefusal";
|
|
44
|
+
}
|
|
45
|
+
}
|
|
22
46
|
const isNum = (v) => typeof v === "number" && Number.isFinite(v);
|
|
47
|
+
/**
|
|
48
|
+
* Slack for comparing accumulated floats: 0.1 + 0.2 is 0.30000000000000004,
|
|
49
|
+
* which must not read as over a $0.30 limit. Far below a token or a cent.
|
|
50
|
+
*/
|
|
51
|
+
const EPSILON = 1e-9;
|
|
52
|
+
/** Cost as shown: accumulated float noise rounded off (10 decimal places). */
|
|
53
|
+
const roundCost = (n) => Math.round(n * 1e10) / 1e10;
|
|
23
54
|
export function toLimits(b) {
|
|
24
55
|
return {
|
|
25
56
|
...(b?.tokens !== undefined ? { tokens: b.tokens } : {}),
|
|
@@ -50,14 +81,14 @@ export function spendOf(u) {
|
|
|
50
81
|
function over(spent, limit, strictlyOver) {
|
|
51
82
|
if (limit === undefined)
|
|
52
83
|
return false;
|
|
53
|
-
return strictlyOver ? spent > limit : spent >= limit;
|
|
84
|
+
return strictlyOver ? spent > limit + EPSILON : spent >= limit - EPSILON;
|
|
54
85
|
}
|
|
55
86
|
function breachOf(scope, limits, spent, strictlyOver) {
|
|
56
87
|
if (over(spent.tokens, limits.tokens, strictlyOver)) {
|
|
57
88
|
return { scope, unit: "tokens", limit: limits.tokens, spent: spent.tokens };
|
|
58
89
|
}
|
|
59
90
|
if (over(spent.costUsd, limits.costUsd, strictlyOver)) {
|
|
60
|
-
return { scope, unit: "cost_usd", limit: limits.costUsd, spent: spent.costUsd };
|
|
91
|
+
return { scope, unit: "cost_usd", limit: limits.costUsd, spent: roundCost(spent.costUsd) };
|
|
61
92
|
}
|
|
62
93
|
return undefined;
|
|
63
94
|
}
|
|
@@ -74,12 +105,53 @@ export function describeOverrun(o, nodeId) {
|
|
|
74
105
|
export class BudgetGuard {
|
|
75
106
|
runLimits;
|
|
76
107
|
spent = { tokens: 0, costUsd: 0 };
|
|
108
|
+
kinds = new Map();
|
|
77
109
|
constructor(runLimits) {
|
|
78
110
|
this.runLimits = runLimits;
|
|
79
111
|
}
|
|
80
112
|
get totals() {
|
|
81
113
|
return { ...this.spent };
|
|
82
114
|
}
|
|
115
|
+
/**
|
|
116
|
+
* Spend by kind, in {@link SPEND_KINDS} order. Resumed spend is restored
|
|
117
|
+
* under its kind; a journal without meter records seeds `node` only.
|
|
118
|
+
*/
|
|
119
|
+
breakdown() {
|
|
120
|
+
const out = {};
|
|
121
|
+
for (const k of SPEND_KINDS) {
|
|
122
|
+
const v = this.kinds.get(k);
|
|
123
|
+
if (v)
|
|
124
|
+
out[k] = { tokens: v.tokens, costUsd: roundCost(v.costUsd), calls: v.calls };
|
|
125
|
+
}
|
|
126
|
+
return out;
|
|
127
|
+
}
|
|
128
|
+
/** True once any completion other than a node run reported usage. */
|
|
129
|
+
get hasMeteredCalls() {
|
|
130
|
+
return [...this.kinds.keys()].some((k) => k !== "node");
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Count one executor completion (judge, route, ...) against the run. Usage
|
|
134
|
+
* with no tokens and no cost counts nothing. Returns the breach this report
|
|
135
|
+
* caused, judged against the totals before it, like a live node report.
|
|
136
|
+
*/
|
|
137
|
+
meter(kind, usage, nodeBudget) {
|
|
138
|
+
const u = spendOf(usage);
|
|
139
|
+
if (u.tokens === undefined && u.costUsd === undefined)
|
|
140
|
+
return undefined;
|
|
141
|
+
const s = { tokens: u.tokens ?? 0, costUsd: u.costUsd ?? 0 };
|
|
142
|
+
const r = this.spent;
|
|
143
|
+
const breach = (nodeBudget
|
|
144
|
+
? breachOf("node", nodeBudget.limits, { tokens: nodeBudget.committed.tokens + s.tokens, costUsd: nodeBudget.committed.costUsd + s.costUsd }, true)
|
|
145
|
+
: undefined) ??
|
|
146
|
+
breachOf("run", this.runLimits, { tokens: r.tokens + s.tokens, costUsd: r.costUsd + s.costUsd }, true);
|
|
147
|
+
if (breach && kind !== "node")
|
|
148
|
+
breach.kind = kind;
|
|
149
|
+
if (nodeBudget)
|
|
150
|
+
nodeBudget.commit(s, kind, true);
|
|
151
|
+
else
|
|
152
|
+
this.commit(s, kind, true);
|
|
153
|
+
return breach;
|
|
154
|
+
}
|
|
83
155
|
/** True when any limit exists at all, so the executor can skip the machinery. */
|
|
84
156
|
active(nodeLimits) {
|
|
85
157
|
return hasLimits(this.runLimits) || hasLimits(nodeLimits);
|
|
@@ -92,17 +164,42 @@ export class BudgetGuard {
|
|
|
92
164
|
return new NodeBudget(this, nodeLimits);
|
|
93
165
|
}
|
|
94
166
|
/**
|
|
95
|
-
* Spend the run made before this process (a resumed run's journaled usage
|
|
96
|
-
* Counted against the run ceiling like any other
|
|
97
|
-
* resets the whole-run budget.
|
|
167
|
+
* Spend the run made before this process (a resumed run's journaled usage
|
|
168
|
+
* and metered completions). Counted against the run ceiling like any other
|
|
169
|
+
* spend, so a crash never resets the whole-run budget. Restored under its
|
|
170
|
+
* kind when the journal has the split (a journal written before meter
|
|
171
|
+
* records existed has node spend only). Seeded spend is not a call: `calls`
|
|
172
|
+
* is unchanged.
|
|
98
173
|
*/
|
|
99
174
|
seed(s) {
|
|
100
|
-
const ok = (n) => (Number.isFinite(n) && n > 0 ? n : 0);
|
|
101
|
-
|
|
175
|
+
const ok = (n) => (typeof n === "number" && Number.isFinite(n) && n > 0 ? n : 0);
|
|
176
|
+
if (!s.byKind) {
|
|
177
|
+
this.commit({ tokens: ok(s.tokens), costUsd: ok(s.costUsd) }, "node", false);
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
for (const k of SPEND_KINDS) {
|
|
181
|
+
const v = s.byKind[k];
|
|
182
|
+
if (v)
|
|
183
|
+
this.commit({ tokens: ok(v.tokens), costUsd: ok(v.costUsd) }, k, false);
|
|
184
|
+
}
|
|
102
185
|
}
|
|
103
|
-
/**
|
|
104
|
-
|
|
186
|
+
/**
|
|
187
|
+
* @internal `counted`: this commit is one completion that reported usage, so
|
|
188
|
+
* `calls` goes up. A commit that is not (seeded spend, an attempt that
|
|
189
|
+
* reported nothing) adds its spend only, and a zero one leaves no trace, so
|
|
190
|
+
* the split is the same with and without a limit.
|
|
191
|
+
*/
|
|
192
|
+
commit(s, kind, counted) {
|
|
105
193
|
this.spent = { tokens: this.spent.tokens + s.tokens, costUsd: this.spent.costUsd + s.costUsd };
|
|
194
|
+
const k = this.kinds.get(kind);
|
|
195
|
+
if (!k && !counted && s.tokens === 0 && s.costUsd === 0)
|
|
196
|
+
return;
|
|
197
|
+
const prev = k ?? { tokens: 0, costUsd: 0, calls: 0 };
|
|
198
|
+
this.kinds.set(kind, {
|
|
199
|
+
tokens: prev.tokens + s.tokens,
|
|
200
|
+
costUsd: prev.costUsd + s.costUsd,
|
|
201
|
+
calls: prev.calls + (counted ? 1 : 0),
|
|
202
|
+
});
|
|
106
203
|
}
|
|
107
204
|
}
|
|
108
205
|
export class NodeBudget {
|
|
@@ -120,14 +217,27 @@ export class NodeBudget {
|
|
|
120
217
|
attempt(outer) {
|
|
121
218
|
return new BudgetAttempt(this.guard, this, outer);
|
|
122
219
|
}
|
|
220
|
+
/**
|
|
221
|
+
* Spend this visit made before a resume (a checkpointed visit's journaled
|
|
222
|
+
* spend). It counts against the node's own limit only: the run total
|
|
223
|
+
* already holds it from the seed.
|
|
224
|
+
*/
|
|
225
|
+
restore(s) {
|
|
226
|
+
const ok = (n) => (Number.isFinite(n) && n > 0 ? n : 0);
|
|
227
|
+
this.spent = { tokens: this.spent.tokens + ok(s.tokens), costUsd: this.spent.costUsd + ok(s.costUsd) };
|
|
228
|
+
}
|
|
123
229
|
/** @internal */
|
|
124
230
|
get committed() {
|
|
125
231
|
return this.spent;
|
|
126
232
|
}
|
|
127
|
-
/**
|
|
128
|
-
|
|
233
|
+
/** Count one executor completion made during this visit (judge, retry reflection, screen). */
|
|
234
|
+
meter(kind, usage) {
|
|
235
|
+
return this.guard.meter(kind, usage, this);
|
|
236
|
+
}
|
|
237
|
+
/** @internal See {@link BudgetGuard.commit}. */
|
|
238
|
+
commit(s, kind, counted) {
|
|
129
239
|
this.spent = { tokens: this.spent.tokens + s.tokens, costUsd: this.spent.costUsd + s.costUsd };
|
|
130
|
-
this.guard.commit(s);
|
|
240
|
+
this.guard.commit(s, kind, counted);
|
|
131
241
|
}
|
|
132
242
|
}
|
|
133
243
|
/** One agent run. Its `signal` is the caller's signal plus a budget stop. */
|
|
@@ -138,6 +248,8 @@ export class BudgetAttempt {
|
|
|
138
248
|
controller = new AbortController();
|
|
139
249
|
live = { tokens: 0, costUsd: 0 };
|
|
140
250
|
liveUsage;
|
|
251
|
+
/** A live report carried tokens or cost. */
|
|
252
|
+
liveReported = false;
|
|
141
253
|
stopped;
|
|
142
254
|
settled;
|
|
143
255
|
onOuterAbort = () => this.controller.abort();
|
|
@@ -167,6 +279,8 @@ export class BudgetAttempt {
|
|
|
167
279
|
return;
|
|
168
280
|
this.liveUsage = { ...this.liveUsage, ...u };
|
|
169
281
|
const s = spendOf(u);
|
|
282
|
+
if (s.tokens !== undefined || s.costUsd !== undefined)
|
|
283
|
+
this.liveReported = true;
|
|
170
284
|
// Reports are cumulative; never let a partial report move spend backwards.
|
|
171
285
|
this.live = {
|
|
172
286
|
tokens: Math.max(this.live.tokens, s.tokens ?? 0),
|
|
@@ -204,7 +318,9 @@ export class BudgetAttempt {
|
|
|
204
318
|
};
|
|
205
319
|
const breach = this.stopped ?? this.check(spent);
|
|
206
320
|
this.settled = { breach };
|
|
207
|
-
|
|
321
|
+
// A call only when the attempt reported usage, live or final: an attempt
|
|
322
|
+
// that reported none is not counted, the same as without a limit.
|
|
323
|
+
this.node.commit(spent, "node", this.liveReported || f.tokens !== undefined || f.costUsd !== undefined);
|
|
208
324
|
return breach;
|
|
209
325
|
}
|
|
210
326
|
/** Settle with the attempt's final usage and return the breach, if any. */
|
package/dist/cli/renderer.js
CHANGED
|
@@ -234,6 +234,11 @@ export class DagRenderer {
|
|
|
234
234
|
s.finishedAt = Date.now();
|
|
235
235
|
break;
|
|
236
236
|
}
|
|
237
|
+
case "node:amended": {
|
|
238
|
+
const s = this.getOrCreate(event.node);
|
|
239
|
+
s.state = event.result.status === "failed" ? "failed" : "completed";
|
|
240
|
+
break;
|
|
241
|
+
}
|
|
237
242
|
// These events don't affect per-node rendering state
|
|
238
243
|
case "workflow:start":
|
|
239
244
|
case "workflow:end":
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
* know instead of guessing.
|
|
13
13
|
*/
|
|
14
14
|
import type { ExecutionEvent, ExecutionTrace, NodeResult, RouteRung, Workflow } from "../types.js";
|
|
15
|
+
import { type SpendKind } from "../budget.js";
|
|
15
16
|
export declare const RUN_HISTORY_SCHEMA_VERSION = 1;
|
|
16
17
|
export declare const RUN_HISTORY_DIR: string;
|
|
17
18
|
export declare const RUN_HISTORY_KEEP = 200;
|
|
@@ -58,6 +59,17 @@ export interface RunRecord {
|
|
|
58
59
|
};
|
|
59
60
|
/** Opinions the harness could not honor natively on some node, deduped. Absent when none. */
|
|
60
61
|
degraded?: string[];
|
|
62
|
+
/**
|
|
63
|
+
* Spend by kind (node, judge, route, reflect, screen), present when the
|
|
64
|
+
* executor's own completions reported usage. `totals` is then its sum, so a
|
|
65
|
+
* jump in `totals` that no node explains shows here.
|
|
66
|
+
*/
|
|
67
|
+
spend?: Partial<Record<SpendKind, RunKindSpend>>;
|
|
68
|
+
}
|
|
69
|
+
export interface RunKindSpend {
|
|
70
|
+
tokens: number;
|
|
71
|
+
cost_usd: number;
|
|
72
|
+
calls: number;
|
|
61
73
|
}
|
|
62
74
|
/** sha256 (hex) of the workflow as canonical JSON: keys sorted, undefined dropped. */
|
|
63
75
|
export declare function hashWorkflow(workflow: Workflow): string;
|
package/dist/cli/run-history.js
CHANGED
|
@@ -15,6 +15,7 @@ import crypto from "node:crypto";
|
|
|
15
15
|
import fs from "node:fs";
|
|
16
16
|
import path from "node:path";
|
|
17
17
|
import { summarizeRun } from "./run-output.js";
|
|
18
|
+
import { SPEND_KINDS } from "../budget.js";
|
|
18
19
|
import { openNoFollow } from "../safe-file.js";
|
|
19
20
|
export const RUN_HISTORY_SCHEMA_VERSION = 1;
|
|
20
21
|
export const RUN_HISTORY_DIR = path.join(".sweny", "runs");
|
|
@@ -63,10 +64,15 @@ export function createNodeTimer(now = Date.now) {
|
|
|
63
64
|
entered.set(event.node, now());
|
|
64
65
|
else if (event.type === "node:exit") {
|
|
65
66
|
const start = entered.get(event.node);
|
|
67
|
+
// Cleared on exit: a repeated exit can never count the same visit twice.
|
|
68
|
+
entered.delete(event.node);
|
|
66
69
|
if (start !== undefined)
|
|
67
70
|
durations.set(event.node, (durations.get(event.node) ?? 0) + (now() - start));
|
|
68
71
|
lastResults.set(event.node, event.result);
|
|
69
72
|
}
|
|
73
|
+
else if (event.type === "node:amended") {
|
|
74
|
+
lastResults.set(event.node, event.result);
|
|
75
|
+
}
|
|
70
76
|
},
|
|
71
77
|
};
|
|
72
78
|
}
|
|
@@ -117,8 +123,18 @@ export function buildRunRecord(i) {
|
|
|
117
123
|
},
|
|
118
124
|
...(harness ? { harness: { id: harness.id, version: harness.version } } : {}),
|
|
119
125
|
...(degraded.length > 0 ? { degraded } : {}),
|
|
126
|
+
...(summary.spend ? { spend: recordSpend(summary.spend) } : {}),
|
|
120
127
|
};
|
|
121
128
|
}
|
|
129
|
+
function recordSpend(spend) {
|
|
130
|
+
const out = {};
|
|
131
|
+
for (const k of SPEND_KINDS) {
|
|
132
|
+
const v = spend[k];
|
|
133
|
+
if (v)
|
|
134
|
+
out[k] = { tokens: v.tokens, cost_usd: v.costUsd, calls: v.calls };
|
|
135
|
+
}
|
|
136
|
+
return out;
|
|
137
|
+
}
|
|
122
138
|
// ── Write, prune, read ──────────────────────────────────────────
|
|
123
139
|
/** Atomic write: temp file in the same directory, fsync, rename. Returns the path, or null on failure. Never throws. */
|
|
124
140
|
export function writeRunRecord(record, cwd = process.cwd()) {
|
package/dist/cli/run-output.d.ts
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import { type ExecutionTrace, type Logger, type NodeResult, type Workflow } from "../types.js";
|
|
9
9
|
import { type NodeStatus } from "../mermaid.js";
|
|
10
|
+
import { type SpendKind } from "../budget.js";
|
|
10
11
|
export interface RunSummary {
|
|
11
12
|
ok: boolean;
|
|
12
13
|
nodesOk: number;
|
|
@@ -31,6 +32,12 @@ export interface RunSummary {
|
|
|
31
32
|
* knows none of it.
|
|
32
33
|
*/
|
|
33
34
|
policy?: RunPolicy;
|
|
35
|
+
/**
|
|
36
|
+
* Spend by kind (`node`, `judge`, `route`, `reflect`, `screen`), when the executor's own
|
|
37
|
+
* completions reported usage. `tokens` and `costUsd` above are then the sum over every kind
|
|
38
|
+
* (every attempt and visit), not over the node results. Absent otherwise.
|
|
39
|
+
*/
|
|
40
|
+
spend?: ExecutionTrace["spend"];
|
|
34
41
|
/** The spend budget a node crossed (#449). Absent when none was. */
|
|
35
42
|
budget?: {
|
|
36
43
|
node: string;
|
|
@@ -38,6 +45,8 @@ export interface RunSummary {
|
|
|
38
45
|
unit: "tokens" | "cost_usd";
|
|
39
46
|
limit: number;
|
|
40
47
|
spent: number;
|
|
48
|
+
/** The executor completion that crossed it (judge, route, ...); absent for a node run. */
|
|
49
|
+
kind?: SpendKind;
|
|
41
50
|
};
|
|
42
51
|
/**
|
|
43
52
|
* Who decided the run's conditional routes (#357), counts only: a `when`
|
|
@@ -89,8 +98,14 @@ export declare function formatTokenCount(n: number): string;
|
|
|
89
98
|
export declare function formatCost(usd: number): string;
|
|
90
99
|
/** `tokens 52k of 50k`: the overrun, in the budget's unit. */
|
|
91
100
|
export declare function formatBudgetAmount(b: NonNullable<RunSummary["budget"]>): string;
|
|
92
|
-
/** `budget exceeded: tokens 52k of 50k (run, at node gather)
|
|
101
|
+
/** `budget exceeded: tokens 52k of 50k (run, at node gather)`, plus `, judge` when an executor call crossed it. */
|
|
93
102
|
export declare function formatBudgetOverrun(b: NonNullable<RunSummary["budget"]>): string;
|
|
103
|
+
/**
|
|
104
|
+
* `node $0.15, judge $0.03, route <$0.01`: spend by kind in a fixed order
|
|
105
|
+
* (node, judge, route, reflect, screen), only the kinds present. In tokens
|
|
106
|
+
* when no kind reported a cost. Undefined when there is no split to show.
|
|
107
|
+
*/
|
|
108
|
+
export declare function formatSpendSplit(spend: RunSummary["spend"], sep?: string): string | undefined;
|
|
94
109
|
/** `5 (3 decider, 1 expr, 1 agent)`: zero rungs are left out. */
|
|
95
110
|
export declare function formatRouteCounts(r: RouteCounts): string;
|
|
96
111
|
/**
|
package/dist/cli/run-output.js
CHANGED
|
@@ -11,6 +11,7 @@ import { toMermaidBlock } from "../mermaid.js";
|
|
|
11
11
|
import { createPaint } from "./style.js";
|
|
12
12
|
import { colorEnabled, glyphs } from "./terminal.js";
|
|
13
13
|
import { trustedEnvValue } from "../startup-env.js";
|
|
14
|
+
import { SPEND_KINDS } from "../budget.js";
|
|
14
15
|
/** The short key of a `degraded` entry: the text before its first colon. */
|
|
15
16
|
export function degradedKey(entry) {
|
|
16
17
|
const i = entry.indexOf(":");
|
|
@@ -70,6 +71,19 @@ export function summarizeRun(results, durationMs, crashed = false, trace) {
|
|
|
70
71
|
}
|
|
71
72
|
if (staged > 0 || applied > 0)
|
|
72
73
|
policy.outputs = { staged, applied };
|
|
74
|
+
// The executor's own completions (judge, route, ...) are on no node result,
|
|
75
|
+
// and a node result holds only its last attempt. With the split present the
|
|
76
|
+
// totals are its sum, so they reconcile with it on one basis.
|
|
77
|
+
const split = Object.values(trace?.spend ?? {});
|
|
78
|
+
const extra = Object.keys(trace?.spend ?? {}).some((k) => k !== "node");
|
|
79
|
+
if (extra) {
|
|
80
|
+
const t = split.reduce((n, k) => n + (isNum(k.tokens) ? k.tokens : 0), 0);
|
|
81
|
+
const c = split.reduce((n, k) => n + (isNum(k.costUsd) ? k.costUsd : 0), 0);
|
|
82
|
+
if (t > 0 || tokens !== undefined)
|
|
83
|
+
tokens = t;
|
|
84
|
+
if (c > 0 || costUsd !== undefined)
|
|
85
|
+
costUsd = Math.round(c * 1e10) / 1e10;
|
|
86
|
+
}
|
|
73
87
|
return {
|
|
74
88
|
ok: !failed,
|
|
75
89
|
nodesOk,
|
|
@@ -83,6 +97,7 @@ export function summarizeRun(results, durationMs, crashed = false, trace) {
|
|
|
83
97
|
...(degraded.size > 0 ? { degraded: [...degraded] } : {}),
|
|
84
98
|
...(Object.keys(policy).length > 0 ? { policy } : {}),
|
|
85
99
|
...(budget ? { budget } : {}),
|
|
100
|
+
...(extra ? { spend: trace.spend } : {}),
|
|
86
101
|
...(trace && countRoutes(trace).total > 0 ? { routes: countRoutes(trace) } : {}),
|
|
87
102
|
...(trace?.deciderOff ? { deciderOff: trace.deciderOff } : {}),
|
|
88
103
|
};
|
|
@@ -160,9 +175,34 @@ export function formatBudgetAmount(b) {
|
|
|
160
175
|
const unit = b.unit === "cost_usd" ? "cost" : "tokens";
|
|
161
176
|
return `${unit} ${amount(b.spent)} of ${amount(b.limit)}`;
|
|
162
177
|
}
|
|
163
|
-
/** `budget exceeded: tokens 52k of 50k (run, at node gather)
|
|
178
|
+
/** `budget exceeded: tokens 52k of 50k (run, at node gather)`, plus `, judge` when an executor call crossed it. */
|
|
164
179
|
export function formatBudgetOverrun(b) {
|
|
165
|
-
return `budget exceeded: ${formatBudgetAmount(b)} (${b.scope}, at node ${b.node})`;
|
|
180
|
+
return `budget exceeded: ${formatBudgetAmount(b)} (${b.scope}, at node ${b.node}${b.kind ? `, ${b.kind}` : ""})`;
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* `node $0.15, judge $0.03, route <$0.01`: spend by kind in a fixed order
|
|
184
|
+
* (node, judge, route, reflect, screen), only the kinds present. In tokens
|
|
185
|
+
* when no kind reported a cost. Undefined when there is no split to show.
|
|
186
|
+
*/
|
|
187
|
+
export function formatSpendSplit(spend, sep = " ") {
|
|
188
|
+
const kinds = SPEND_KINDS.flatMap((k) => (spend?.[k] ? [[k, spend[k]]] : []));
|
|
189
|
+
if (kinds.length === 0)
|
|
190
|
+
return undefined;
|
|
191
|
+
const byCost = kinds.some(([, v]) => v.costUsd > 0);
|
|
192
|
+
return kinds.map(([k, v]) => `${k}${sep}${byCost ? formatCost(v.costUsd) : formatTokenCount(v.tokens)}`).join(", ");
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* `12k tokens (judge 2k)`: the executor's own kinds in brackets, on the token
|
|
196
|
+
* segment, or on the cost (`$0.18 (judge $0.03)`) when no tokens were reported.
|
|
197
|
+
*/
|
|
198
|
+
function ownSpendBracket(spend, unit) {
|
|
199
|
+
const own = SPEND_KINDS.flatMap((k) => {
|
|
200
|
+
const v = spend?.[k];
|
|
201
|
+
if (k === "node" || !v || !(v[unit] > 0))
|
|
202
|
+
return [];
|
|
203
|
+
return [`${k} ${unit === "tokens" ? formatTokenCount(v.tokens) : formatCost(v.costUsd)}`];
|
|
204
|
+
});
|
|
205
|
+
return own.length === 0 ? "" : ` (${own.join(", ")})`;
|
|
166
206
|
}
|
|
167
207
|
/** `5 (3 decider, 1 expr, 1 agent)`: zero rungs are left out. */
|
|
168
208
|
export function formatRouteCounts(r) {
|
|
@@ -182,8 +222,10 @@ export function formatReceipt(s) {
|
|
|
182
222
|
...(s.nodesSkipped > 0 ? [`${s.nodesSkipped} skipped`] : []),
|
|
183
223
|
`${s.toolCalls} tool ${s.toolCalls === 1 ? "call" : "calls"}`,
|
|
184
224
|
formatReceiptDuration(s.durationMs),
|
|
185
|
-
...(s.tokens !== undefined ? [`${formatTokenCount(s.tokens)} tokens`] : []),
|
|
186
|
-
...(s.costUsd !== undefined
|
|
225
|
+
...(s.tokens !== undefined ? [`${formatTokenCount(s.tokens)} tokens${ownSpendBracket(s.spend, "tokens")}`] : []),
|
|
226
|
+
...(s.costUsd !== undefined
|
|
227
|
+
? [`${formatCost(s.costUsd)}${s.tokens === undefined ? ownSpendBracket(s.spend, "costUsd") : ""}`]
|
|
228
|
+
: []),
|
|
187
229
|
...(s.harness !== undefined && s.harness !== "claude-code" ? [s.harness] : []),
|
|
188
230
|
...(s.degraded && s.degraded.length > 0 ? [`degraded: ${s.degraded.join(", ")}`] : []),
|
|
189
231
|
...(formatPolicySegment(s.policy) ? [formatPolicySegment(s.policy)] : []),
|
package/dist/cli/runs.js
CHANGED
|
@@ -5,7 +5,8 @@
|
|
|
5
5
|
* `color` is true (TTY and no NO_COLOR); the glyph set follows `unicode`.
|
|
6
6
|
* The diff always ends on one plain line with the answer counts.
|
|
7
7
|
*/
|
|
8
|
-
import { findRun, listRuns } from "./run-history.js";
|
|
8
|
+
import { findRun, listRuns, } from "./run-history.js";
|
|
9
|
+
import { SPEND_KINDS } from "../budget.js";
|
|
9
10
|
import { compareAnswers, readRunAnswers, } from "./run-answers.js";
|
|
10
11
|
import { formatCost, formatReceiptDuration, formatTokenCount } from "./run-output.js";
|
|
11
12
|
import { createPaint } from "./style.js";
|
|
@@ -84,8 +85,25 @@ export function printableRun(r) {
|
|
|
84
85
|
tokens: finiteOrNull(r.totals.tokens),
|
|
85
86
|
cost_usd: finiteOrNull(r.totals.cost_usd),
|
|
86
87
|
},
|
|
88
|
+
...(r.spend && typeof r.spend === "object" ? { spend: printableSpend(r.spend) } : {}),
|
|
87
89
|
};
|
|
88
90
|
}
|
|
91
|
+
/** The record's split, or its totals as all node spend when it has none. */
|
|
92
|
+
function splitOf(r) {
|
|
93
|
+
if (r.spend)
|
|
94
|
+
return r.spend;
|
|
95
|
+
return { node: { tokens: r.totals.tokens ?? 0, cost_usd: r.totals.cost_usd ?? 0, calls: 0 } };
|
|
96
|
+
}
|
|
97
|
+
/** Only the known kinds, own keys, finite numbers. */
|
|
98
|
+
function printableSpend(spend) {
|
|
99
|
+
const out = {};
|
|
100
|
+
for (const k of SPEND_KINDS) {
|
|
101
|
+
const v = Object.hasOwn(spend, k) ? spend[k] : undefined;
|
|
102
|
+
if (v && typeof v === "object")
|
|
103
|
+
out[k] = { tokens: finite(v.tokens), cost_usd: finite(v.cost_usd), calls: finite(v.calls) };
|
|
104
|
+
}
|
|
105
|
+
return out;
|
|
106
|
+
}
|
|
89
107
|
// ── sweny runs ──────────────────────────────────────────────────
|
|
90
108
|
export function formatRunsTable(records, color, nowMs = Date.now(), unicode = true) {
|
|
91
109
|
const ICON = iconsFor(glyphsFor(unicode));
|
|
@@ -312,6 +330,25 @@ export function formatRunDiff(rawA, rawB, color, opts = {}) {
|
|
|
312
330
|
for (const t of tot)
|
|
313
331
|
if (t)
|
|
314
332
|
out.push(` ${t}`);
|
|
333
|
+
// Spend by kind, when either run has the split: what moved the totals. A
|
|
334
|
+
// record without one (node-only, or written before the split) is all node spend.
|
|
335
|
+
if (a.spend || b.spend) {
|
|
336
|
+
const sa = splitOf(a);
|
|
337
|
+
const sb = splitOf(b);
|
|
338
|
+
const byCost = SPEND_KINDS.some((k) => (sa[k]?.cost_usd ?? 0) > 0 || (sb[k]?.cost_usd ?? 0) > 0);
|
|
339
|
+
for (const k of SPEND_KINDS) {
|
|
340
|
+
const va = sa[k];
|
|
341
|
+
const vb = sb[k];
|
|
342
|
+
if (!va && !vb)
|
|
343
|
+
continue;
|
|
344
|
+
const label = ` ${pad(k, 10)}`;
|
|
345
|
+
const line = byCost
|
|
346
|
+
? metric(label, va ? va.cost_usd : null, vb ? vb.cost_usd : null, formatCost, deltaCost, g.arrow)
|
|
347
|
+
: metric(label, va ? va.tokens : null, vb ? vb.tokens : null, formatTokenCount, deltaTokens, g.arrow);
|
|
348
|
+
if (line)
|
|
349
|
+
out.push(` ${line}`);
|
|
350
|
+
}
|
|
351
|
+
}
|
|
315
352
|
out.push("", " nodes");
|
|
316
353
|
const aNodes = new Map(a.nodes.map((n) => [n.id, n]));
|
|
317
354
|
const bNodes = new Map(b.nodes.map((n) => [n.id, n]));
|