@cohortapp/agent-sdk 2.18.13 → 2.18.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/maestro.mjs +38 -1
- package/docs/runbooks/fleet-rollout.md +58 -7
- package/docs/runbooks/recovery-and-failover.md +18 -0
- package/lib/assurance/batch.mjs +353 -0
- package/lib/assurance/first-reply.mjs +423 -0
- package/lib/assurance/notice-voice.mjs +357 -0
- package/lib/assurance/plan-note.mjs +43 -0
- package/lib/assurance/room-budget.mjs +55 -6
- package/lib/cadence-failure-class.mjs +245 -0
- package/lib/claude-bin.mjs +26 -7
- package/lib/cli/doctor-checks.mjs +149 -1
- package/lib/comms/send-gate.mjs +59 -0
- package/lib/diagnostics/alerts.mjs +33 -0
- package/lib/engine/agents/usage.mjs +45 -0
- package/lib/engine/budget.mjs +293 -29
- package/lib/engine/cli.mjs +54 -5
- package/lib/engine/loop.mjs +30 -0
- package/lib/engine/output/json.mjs +26 -0
- package/lib/engine/wire/errors.mjs +179 -0
- package/lib/engine/wire/search.mjs +44 -8
- package/lib/identity/persona.mjs +31 -2
- package/lib/org/quota.mjs +27 -0
- package/lib/session/config.mjs +4 -0
- package/lib/session/identity.mjs +71 -7
- package/lib/session/launch-failure.mjs +251 -0
- package/lib/session/resume-target.mjs +86 -0
- package/lib/telemetry/alerts.mjs +94 -0
- package/lib/telemetry/collect.mjs +155 -2
- package/lib/upgrade/pinned-drift.mjs +467 -0
- package/package.json +1 -1
- package/scaffold/config/alerts.yaml +7 -0
- package/scripts/ci/check-cadence-prompts-exist.mjs +96 -0
- package/scripts/ci/check.mjs +3 -0
- package/scripts/daemon/agent-daemon.mjs +75 -5
- package/scripts/daemon/assurance.mjs +709 -44
- package/scripts/daemon/cadence-consumer.mjs +281 -34
- package/scripts/daemon/deliver.mjs +109 -0
- package/scripts/daemon/dispatcher.mjs +21 -3
- package/scripts/daemon/inbox-deferral.mjs +102 -9
- package/scripts/daemon/session-lock.mjs +41 -1
- package/scripts/emergency-stop.sh +114 -13
- package/scripts/fleet/rollout.mjs +256 -10
- package/scripts/healthcheck.sh +131 -33
- package/scripts/local-triggers/autoupdate.sh +144 -11
- package/scripts/resume-operations.sh +101 -6
- package/scripts/session/supervisor.mjs +198 -5
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
19
|
import { addUsage, emptyUsage, toResultUsage } from "../wire/usage.mjs";
|
|
20
|
+
import { combineStated } from "../wire/errors.mjs";
|
|
20
21
|
|
|
21
22
|
/**
|
|
22
23
|
* @typedef {Object} UsageRecord
|
|
@@ -24,6 +25,10 @@ import { addUsage, emptyUsage, toResultUsage } from "../wire/usage.mjs";
|
|
|
24
25
|
* @property {import('../wire/usage.mjs').NormalisedUsage} usage
|
|
25
26
|
* @property {string|null} costMicros null = unknown (may have been billed with no figure)
|
|
26
27
|
* @property {string[]} requestIds
|
|
28
|
+
* @property {'off'|'shadow'|'on'|'unknown'|'mixed'|null} [enforcement]
|
|
29
|
+
* W16. What the gateway said ABOUT `costMicros` — only "on" means money moved.
|
|
30
|
+
* @property {string|null} [shadowCostMicros] what enforcement WOULD have charged; never spend
|
|
31
|
+
* @property {'ledger'|'unreadable'|'not_read'|'not_admitted'|'unknown'|'mixed'|null} [quotaSource]
|
|
27
32
|
*/
|
|
28
33
|
|
|
29
34
|
/**
|
|
@@ -40,6 +45,13 @@ export function recordFromOutcome(outcome, fallbackTier) {
|
|
|
40
45
|
usage: outcome.usage ?? emptyUsage("estimated"),
|
|
41
46
|
costMicros: outcome.costMicros != null ? outcome.costMicros : exact ? "0" : null,
|
|
42
47
|
requestIds: [...(outcome.requestIds ?? [])],
|
|
48
|
+
// W16. The qualifiers travel WITH the figure they qualify. A child's cost is
|
|
49
|
+
// summed into the parent's `costMicros`, so a child's `enforcement` must
|
|
50
|
+
// reach the same sum — otherwise the parent reports "on" beside a total that
|
|
51
|
+
// includes a child's shadow figure, which is the misread one layer up.
|
|
52
|
+
enforcement: outcome.enforcement ?? null,
|
|
53
|
+
shadowCostMicros: outcome.shadowCostMicros ?? null,
|
|
54
|
+
quotaSource: outcome.quotaSource ?? null,
|
|
43
55
|
};
|
|
44
56
|
}
|
|
45
57
|
|
|
@@ -53,8 +65,34 @@ function sumMicros(parts) {
|
|
|
53
65
|
return total.toString();
|
|
54
66
|
}
|
|
55
67
|
|
|
68
|
+
/**
|
|
69
|
+
* Sum the `shadow_cost_micros` figures that were reported, or null when none
|
|
70
|
+
* was. Unlike `costMicros` a missing counterfactual does not null the sum: it is
|
|
71
|
+
* not a charge, so a partial one is still worth reporting.
|
|
72
|
+
* @param {Array<string|null|undefined>} parts
|
|
73
|
+
*/
|
|
74
|
+
function sumShadowMicros(parts) {
|
|
75
|
+
let total = 0n;
|
|
76
|
+
let seen = false;
|
|
77
|
+
for (const p of parts) {
|
|
78
|
+
if (p == null) continue;
|
|
79
|
+
try {
|
|
80
|
+
total += BigInt(p);
|
|
81
|
+
seen = true;
|
|
82
|
+
} catch {
|
|
83
|
+
/* an unparseable counterfactual is simply not recorded */
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return seen ? total.toString() : null;
|
|
87
|
+
}
|
|
88
|
+
|
|
56
89
|
/**
|
|
57
90
|
* Combine the parent's record with its children's.
|
|
91
|
+
*
|
|
92
|
+
* W16: `enforcement` and `quotaSource` fold across parent AND children exactly
|
|
93
|
+
* as `costMicros` sums across them, so the qualifier and the figure it qualifies
|
|
94
|
+
* always describe the same scope. A parent served under `on` whose subagent was
|
|
95
|
+
* served under `shadow` reports `"mixed"`, never `"on"`.
|
|
58
96
|
* @param {UsageRecord} parent @param {UsageRecord[]} children
|
|
59
97
|
*/
|
|
60
98
|
export function aggregateUsage(parent, children) {
|
|
@@ -91,5 +129,12 @@ export function aggregateUsage(parent, children) {
|
|
|
91
129
|
requestIds: all.flatMap((r) => r.requestIds),
|
|
92
130
|
modelUsage,
|
|
93
131
|
cohortModelUsage: { byTier, total: { usage, costMicros } },
|
|
132
|
+
// The run-wide qualifiers for the run-wide `costMicros` above.
|
|
133
|
+
// `combineStated`, not `foldEnforcement`: each record's value is ALREADY
|
|
134
|
+
// folded, so it may be this client's own "mixed" — a word the gateway never
|
|
135
|
+
// sends and the raw reader would flatten to "unknown".
|
|
136
|
+
enforcement: all.reduce((m, r) => combineStated(m, r.enforcement), /** @type {any} */ (null)),
|
|
137
|
+
quotaSource: all.reduce((q, r) => combineStated(q, r.quotaSource), /** @type {any} */ (null)),
|
|
138
|
+
shadowCostMicros: sumShadowMicros(all.map((r) => r.shadowCostMicros)),
|
|
94
139
|
};
|
|
95
140
|
}
|
package/lib/engine/budget.mjs
CHANGED
|
@@ -53,12 +53,112 @@
|
|
|
53
53
|
* guess wearing a measurement's clothes — the exact failure CF-157 is about.
|
|
54
54
|
* `spent` therefore stays at the gateway's own figures, and only those.
|
|
55
55
|
*
|
|
56
|
+
* ## W16 — a figure the gateway reports is not the same as money that moved
|
|
57
|
+
*
|
|
58
|
+
* The gateway now states which billing mode produced each figure
|
|
59
|
+
* (`enforcement`, design §4.8, hq a60b9595). Under `shadow` a PRICED request
|
|
60
|
+
* settles for real and reports a real `cost_micros` while **no money moves**:
|
|
61
|
+
* a shadow request the ledger would have refused is admitted on an unfunded
|
|
62
|
+
* hold, served, and its settle posts an empty transaction. This meter used to
|
|
63
|
+
* add that figure to `spent` and enforce a cap against it — counting spend that
|
|
64
|
+
* never happened, against a cap that cannot fire. That is CF-157's own misread,
|
|
65
|
+
* surviving on the client after the server side was fixed.
|
|
66
|
+
*
|
|
67
|
+
* So a figure only becomes `spent` when the endpoint says `enforcement: "on"`.
|
|
68
|
+
* A figure reported under `shadow` or `off` is kept — as `unenforcedCostMicros`
|
|
69
|
+
* and `unenforcedRequests`, under names that cannot be read as spend — and adds
|
|
70
|
+
* a fourth cost-visibility state:
|
|
71
|
+
*
|
|
72
|
+
* unenforced figures arrived, and the endpoint states none of them is money.
|
|
73
|
+
* THE CAP CANNOT BIND — for a different reason than `blind`: not
|
|
74
|
+
* "we were not told the price" but "the price is not charged".
|
|
75
|
+
*
|
|
76
|
+
* `bindable()` is false for `unenforced` exactly as for `blind`, and `notice()`
|
|
77
|
+
* says which. Neither is reported as "$0 spent" alone: "nothing was spent" and
|
|
78
|
+
* "spending is not being enforced" are different statements.
|
|
79
|
+
*
|
|
80
|
+
* **When the endpoint states no mode at all, a figure IS counted as spend** —
|
|
81
|
+
* the pre-W16 behaviour. The choice is deliberate and it is the one asymmetry
|
|
82
|
+
* worth stating: over-counting produces a LOUD failure (the run stops and says
|
|
83
|
+
* the cap was reached, which an operator sees immediately), while under-counting
|
|
84
|
+
* produces the SILENT one (a cap that never fires, which is CF-157 itself).
|
|
85
|
+
* A never-seen `enforcement` word is therefore read as unknown, never as "on"
|
|
86
|
+
* — `snapshot().enforcement` reports `"unknown"` for it and `null` for silence
|
|
87
|
+
* — but unknown and silent both leave the cap armed. Only a stated `shadow` or
|
|
88
|
+
* `off` withholds a figure from spend.
|
|
89
|
+
*
|
|
90
|
+
* **…with one exception: silence AFTER the endpoint has named a mode inherits
|
|
91
|
+
* it.** A run talks to one `--base-url`, and a gateway's billing mode does not
|
|
92
|
+
* change without a deploy, so once this run has heard `shadow` from that
|
|
93
|
+
* endpoint, `shadow` is far better evidence about a later frameless request from
|
|
94
|
+
* it than the absence of a field is. Without this, a billed request whose frame
|
|
95
|
+
* never arrived — a stream that died after the headers, and every `searchWeb`
|
|
96
|
+
* refusal path, which has only headers to read — put a shadow figure straight
|
|
97
|
+
* into `spent`. The inheritance costs nothing against the cases the asymmetry
|
|
98
|
+
* above protects: a non-Cohort endpoint, a dropped-frame proxy and a gateway
|
|
99
|
+
* older than a60b9595 never name a mode at all, so they never inherit one.
|
|
100
|
+
* `snapshot().enforcement` still reports only what was STATED; the inherited
|
|
101
|
+
* decisions are counted separately as `inheritedModeRequests`.
|
|
102
|
+
*
|
|
103
|
+
* ## W16 — an empty quota is three different answers
|
|
104
|
+
*
|
|
105
|
+
* `quota_source` splits what an empty `quota` object used to collapse: the
|
|
106
|
+
* ledger answered and the seat has no included allowance (`ledger`), the read
|
|
107
|
+
* threw (`unreadable`), or nothing was read (`not_read`; `not_admitted` is
|
|
108
|
+
* "unknown"). A governor cannot tell "you have no allowance" from "we could not
|
|
109
|
+
* look", and those call for opposite behaviour. The meter records what was
|
|
110
|
+
* stated and `quotaNotice()` says it once per run, so an operator reading
|
|
111
|
+
* stderr learns which it was.
|
|
112
|
+
*
|
|
56
113
|
* Pure apart from its own counters; no I/O.
|
|
57
114
|
*
|
|
58
115
|
* @module lib/engine/budget
|
|
59
116
|
*/
|
|
60
117
|
|
|
61
118
|
import { addUsage, emptyUsage, readFrameUsage, totalTokens } from "./wire/usage.mjs";
|
|
119
|
+
import { foldEnforcement, foldQuotaSource, readEnforcement } from "./wire/errors.mjs";
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* The one-line answer to "what did an EMPTY quota mean?", or null when there is
|
|
123
|
+
* nothing to say. PURE and total.
|
|
124
|
+
*
|
|
125
|
+
* It lives outside {@link createBudgetMeter} because the meter only exists when
|
|
126
|
+
* `--max-budget-usd` was passed, and the operator who most needs this line — no
|
|
127
|
+
* cap, nothing pacing — is precisely the one who did not pass it. The meter's
|
|
128
|
+
* {@link createBudgetMeter} `quotaNotice` is the same text with a once-per-run
|
|
129
|
+
* latch; `cli.mjs` calls this directly so an unbudgeted run says it too.
|
|
130
|
+
*
|
|
131
|
+
* Silent when the quota actually answered (windows, credits or funding — there
|
|
132
|
+
* is then nothing ambiguous to explain) and silent when the endpoint stated no
|
|
133
|
+
* source at all (nothing new to say). Otherwise it names which of the three
|
|
134
|
+
* answers `{}` was, because "you have no allowance" and "we could not look" call
|
|
135
|
+
* for opposite behaviour from anything pacing against it.
|
|
136
|
+
*
|
|
137
|
+
* @param {'ledger'|'unreadable'|'not_read'|'not_admitted'|'unknown'|'mixed'|null|undefined} quotaSource
|
|
138
|
+
* @param {boolean} quotaAnswered
|
|
139
|
+
* @returns {string|null}
|
|
140
|
+
*/
|
|
141
|
+
export function quotaSourceNotice(quotaSource, quotaAnswered) {
|
|
142
|
+
if (quotaAnswered || quotaSource == null) return null;
|
|
143
|
+
const head = `cohort quota: this run saw no quota windows, and the endpoint states quota_source "${quotaSource}" — `;
|
|
144
|
+
if (quotaSource === "ledger") {
|
|
145
|
+
return (
|
|
146
|
+
head +
|
|
147
|
+
`the ledger ANSWERED: this seat has no included allowance (a money-only band reports no windows by design). ` +
|
|
148
|
+
`Nothing is wrong, and there is no window to pace against.`
|
|
149
|
+
);
|
|
150
|
+
}
|
|
151
|
+
if (quotaSource === "unreadable") {
|
|
152
|
+
return (
|
|
153
|
+
head +
|
|
154
|
+
`the quota read FAILED. The empty quota is a failed read, NOT "no allowance": do not read it as headroom.`
|
|
155
|
+
);
|
|
156
|
+
}
|
|
157
|
+
return (
|
|
158
|
+
head +
|
|
159
|
+
`no quota was read for this seat, so the empty quota states nothing about its allowance either way — read it as unknown.`
|
|
160
|
+
);
|
|
161
|
+
}
|
|
62
162
|
|
|
63
163
|
/**
|
|
64
164
|
* `--max-budget-usd` as integer micro-USD, or an error. PURE.
|
|
@@ -83,20 +183,56 @@ export function microsToUsdNumber(micros) {
|
|
|
83
183
|
* @param {{maxMicros:bigint}} o
|
|
84
184
|
*/
|
|
85
185
|
export function createBudgetMeter({ maxMicros }) {
|
|
186
|
+
/** Money that MOVED, by the gateway's own figures under `enforcement: "on"`. */
|
|
86
187
|
let spent = 0n;
|
|
188
|
+
/** Retail figures the endpoint reported while stating it enforces nothing. NEVER spend. */
|
|
189
|
+
let unenforcedCost = 0n;
|
|
190
|
+
/** `shadow_cost_micros`: what enforcement WOULD have charged. NEVER spend. */
|
|
191
|
+
let shadowCost = 0n;
|
|
192
|
+
let shadowCostSeen = false;
|
|
87
193
|
let unknownCost = false;
|
|
88
194
|
let requests = 0;
|
|
89
|
-
/** Billed requests
|
|
195
|
+
/** Billed requests whose figure counted toward the cap. */
|
|
90
196
|
let priced = 0;
|
|
91
|
-
/** Billed requests that carried NO cost figure — the shadow-
|
|
197
|
+
/** Billed requests that carried NO cost figure — the shadow-with-no-price-book signature. */
|
|
92
198
|
let unpriced = 0;
|
|
199
|
+
/** Billed requests that carried a figure the endpoint says is not money. */
|
|
200
|
+
let unenforced = 0;
|
|
201
|
+
/** The billing mode(s) the endpoint stated: a mode, 'unknown', 'mixed', or null for silence. */
|
|
202
|
+
let mode = /** @type {ReturnType<typeof foldEnforcement>} */ (null);
|
|
203
|
+
/**
|
|
204
|
+
* The last mode the endpoint actually NAMED (one of the three §4.8 words), or
|
|
205
|
+
* null before it named any. A later request that names none inherits this —
|
|
206
|
+
* see the module header. An unrecognised word does not replace it: it is not
|
|
207
|
+
* evidence about enforcement either way, and discarding a mode the endpoint
|
|
208
|
+
* did state would only make the seat count more shadow figures as spend.
|
|
209
|
+
*/
|
|
210
|
+
let lastStatedMode = /** @type {'off'|'shadow'|'on'|null} */ (null);
|
|
211
|
+
/** Requests whose billing mode was inherited from an earlier statement, not stated on the request itself. */
|
|
212
|
+
let inheritedMode = 0;
|
|
213
|
+
/** The quota source(s) stated, folded the same way. */
|
|
214
|
+
let quotaSource = /** @type {ReturnType<typeof foldQuotaSource>} */ (null);
|
|
215
|
+
/** True once any request's quota object actually said something. */
|
|
216
|
+
let quotaAnswered = false;
|
|
93
217
|
/** Exact token counts off the terminal frame (CF-158). Never priced. */
|
|
94
218
|
let tokens = emptyUsage("provider");
|
|
95
219
|
let tokenRequests = 0;
|
|
96
220
|
let noticed = false;
|
|
221
|
+
let quotaNoticed = false;
|
|
97
222
|
|
|
98
|
-
/** @returns {'exact'|'partial'|'blind'} */
|
|
99
|
-
const visibility = () =>
|
|
223
|
+
/** @returns {'exact'|'partial'|'blind'|'unenforced'} */
|
|
224
|
+
const visibility = () => {
|
|
225
|
+
// A figure that counted is the only thing that can make a cap bind.
|
|
226
|
+
if (priced > 0) return unpriced === 0 && unenforced === 0 ? "exact" : "partial";
|
|
227
|
+
// Figures arrived, and the endpoint says none of them is money.
|
|
228
|
+
if (unenforced > 0) return "unenforced";
|
|
229
|
+
// Nothing recorded yet reads as `exact`: nothing has contradicted the cap.
|
|
230
|
+
return unpriced === 0 ? "exact" : "blind";
|
|
231
|
+
};
|
|
232
|
+
const bindable = () => {
|
|
233
|
+
const v = visibility();
|
|
234
|
+
return v !== "blind" && v !== "unenforced";
|
|
235
|
+
};
|
|
100
236
|
|
|
101
237
|
return {
|
|
102
238
|
maxMicros,
|
|
@@ -111,11 +247,46 @@ export function createBudgetMeter({ maxMicros }) {
|
|
|
111
247
|
tokens = addUsage(tokens, u);
|
|
112
248
|
tokenRequests++;
|
|
113
249
|
}
|
|
250
|
+
// W16. Re-read through the wire's own total parsers rather than trusting
|
|
251
|
+
// the field: `record` is also handed results built by call sites that
|
|
252
|
+
// never went through `mergeCohort` (WebSearch, the fixtures), and an
|
|
253
|
+
// unrecognised word must land as unknown there too.
|
|
254
|
+
mode = foldEnforcement(mode, res.cohort?.enforcement);
|
|
255
|
+
quotaSource = foldQuotaSource(quotaSource, res.cohort?.quotaSource);
|
|
256
|
+
if (hasQuotaAnswer(res.cohort?.quota)) quotaAnswered = true;
|
|
257
|
+
const shadow = res.cohort?.shadowCostMicros;
|
|
258
|
+
if (shadow != null) {
|
|
259
|
+
try {
|
|
260
|
+
shadowCost += BigInt(shadow);
|
|
261
|
+
shadowCostSeen = true;
|
|
262
|
+
} catch {
|
|
263
|
+
/* an unparseable counterfactual is simply not recorded */
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
// Which mode governs THIS figure. A named mode governs itself; silence
|
|
267
|
+
// inherits the last mode this endpoint named, because within one run
|
|
268
|
+
// against one base URL the mode does not change without a deploy, and a
|
|
269
|
+
// frameless billed request (a stream that died after the headers, a
|
|
270
|
+
// search refusal) would otherwise put a shadow figure into spend. Silence
|
|
271
|
+
// before any statement, and an unrecognised word, leave the cap armed —
|
|
272
|
+
// see the module header for why that asymmetry is the safe one here.
|
|
273
|
+
const stated = readEnforcement(res.cohort?.enforcement);
|
|
274
|
+
if (stated) lastStatedMode = stated;
|
|
275
|
+
else if (res.cohort?.enforcement == null && lastStatedMode !== null) inheritedMode++;
|
|
276
|
+
const enforcing = stated ?? (res.cohort?.enforcement == null ? lastStatedMode : null);
|
|
277
|
+
const isMoney = enforcing !== "shadow" && enforcing !== "off";
|
|
114
278
|
const c = res.cohort?.costMicros;
|
|
115
279
|
if (c != null) {
|
|
116
280
|
try {
|
|
117
|
-
|
|
118
|
-
|
|
281
|
+
const micros = BigInt(c);
|
|
282
|
+
if (isMoney) {
|
|
283
|
+
spent += micros;
|
|
284
|
+
priced++;
|
|
285
|
+
} else {
|
|
286
|
+
// Kept, and kept out of spend: no money moved for this figure.
|
|
287
|
+
unenforcedCost += micros;
|
|
288
|
+
unenforced++;
|
|
289
|
+
}
|
|
119
290
|
return;
|
|
120
291
|
} catch {
|
|
121
292
|
/* an unparseable figure is an unknown one */
|
|
@@ -127,15 +298,20 @@ export function createBudgetMeter({ maxMicros }) {
|
|
|
127
298
|
unpriced++;
|
|
128
299
|
}
|
|
129
300
|
},
|
|
130
|
-
// A blind meter never reaches the cap: spent stays 0 because
|
|
131
|
-
// ever
|
|
132
|
-
// over by treating an unreported
|
|
301
|
+
// A blind or unenforced meter never reaches the cap: spent stays 0 because
|
|
302
|
+
// no money figure was ever counted. That is reported by
|
|
303
|
+
// bindable()/notice(), never papered over by treating an unreported — or
|
|
304
|
+
// unenforced — cost as zero spend.
|
|
133
305
|
exhausted: () => spent >= maxMicros,
|
|
134
306
|
spentMicros: () => spent,
|
|
135
|
-
/** 'exact' | 'partial' | 'blind' — see the module header. */
|
|
307
|
+
/** 'exact' | 'partial' | 'blind' | 'unenforced' — see the module header. */
|
|
136
308
|
costVisibility: visibility,
|
|
137
|
-
/** False when
|
|
138
|
-
bindable
|
|
309
|
+
/** False when no figure could count toward the cap, i.e. it cannot fire at all. */
|
|
310
|
+
bindable,
|
|
311
|
+
/** The billing mode the endpoint stated, or 'unknown'/'mixed'/null. */
|
|
312
|
+
enforcement: () => mode,
|
|
313
|
+
/** The quota source the endpoint stated, or 'unknown'/'mixed'/null. */
|
|
314
|
+
quotaSource: () => quotaSource,
|
|
139
315
|
/**
|
|
140
316
|
* A one-time, run-level warning when the cap cannot bind, else null.
|
|
141
317
|
* Self-clearing, so a caller may ask after every turn and the operator
|
|
@@ -143,35 +319,94 @@ export function createBudgetMeter({ maxMicros }) {
|
|
|
143
319
|
* @returns {string|null}
|
|
144
320
|
*/
|
|
145
321
|
notice() {
|
|
146
|
-
|
|
322
|
+
// A cap that cannot bind must say so — and so must a cap that binds on
|
|
323
|
+
// only PART of the run's figures. The second case is bindable, so it used
|
|
324
|
+
// to fall through this gate and say nothing at all: an operator saw a cap
|
|
325
|
+
// that looked armed against money that had not moved. That silence is the
|
|
326
|
+
// failure mode this lane exists to end, so `unenforced > 0` speaks too.
|
|
327
|
+
if (noticed || (bindable() && unenforced === 0)) return null;
|
|
147
328
|
noticed = true;
|
|
148
329
|
const observed =
|
|
149
330
|
tokenRequests > 0
|
|
150
331
|
? `${totalTokens(tokens)} tokens over ${tokenRequests} request(s)`
|
|
151
332
|
: "no token counts either";
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
// a non-Cohort Anthropic-compatible endpoint, a proxy that drops the
|
|
156
|
-
// header, or a gateway bug land here identically. Naming shadow
|
|
157
|
-
// billing as THE reason would be inferring a cause from an absence,
|
|
158
|
-
// which is the habit this lane exists to break.
|
|
159
|
-
`--max-budget-usd ($${microsToUsdNumber(maxMicros)}) is NOT being enforced: ` +
|
|
160
|
-
`the endpoint priced none of ${unpriced} billed request(s) (no cost_micros; shadow billing is the usual cause), ` +
|
|
161
|
-
`so the run reports $0 spent and the cap can never trigger. Measured instead: ${observed} ` +
|
|
333
|
+
const cap = `--max-budget-usd ($${microsToUsdNumber(maxMicros)}) is NOT being enforced: `;
|
|
334
|
+
const tail =
|
|
335
|
+
`Measured instead: ${observed} ` +
|
|
162
336
|
`— tokens are exact, but this SDK has no per-tier price table and does not convert them to dollars. ` +
|
|
163
|
-
`See docs/engine/eval.md §3 (CF-157)
|
|
337
|
+
`See docs/engine/eval.md §3 (CF-157).`;
|
|
338
|
+
if (priced > 0) {
|
|
339
|
+
// A MIXED run: some figures the endpoint says it charged for, some it
|
|
340
|
+
// says it did not. The cap IS armed — on the enforced subset only — so
|
|
341
|
+
// this does not claim it is unenforced; it names the split, because
|
|
342
|
+
// `spentMicros` here is not the whole of what the gateway reported.
|
|
343
|
+
return (
|
|
344
|
+
`--max-budget-usd ($${microsToUsdNumber(maxMicros)}) is being enforced on PART of this run only: ` +
|
|
345
|
+
`${priced} billed request(s) carried figures counted as spend, and ${unenforced} carried figures the ` +
|
|
346
|
+
`endpoint states are not money (enforcement "${mode}"; ${unenforcedCost.toString()} micro-USD reported, ` +
|
|
347
|
+
`deliberately kept out of spend), so the cap binds on a lower bound. ${tail}`
|
|
348
|
+
);
|
|
349
|
+
}
|
|
350
|
+
if (visibility() === "unenforced") {
|
|
351
|
+
// W16: the endpoint DID price these requests and says it charges
|
|
352
|
+
// nothing for them. Naming the mode here is not a diagnosis — it is
|
|
353
|
+
// quoting the field the gateway sent for exactly this purpose.
|
|
354
|
+
return (
|
|
355
|
+
cap +
|
|
356
|
+
`the endpoint priced ${unenforced} billed request(s) but states enforcement is "${mode}", so no money moved ` +
|
|
357
|
+
`for those figures (${unenforcedCost.toString()} micro-USD reported, deliberately kept out of spend) ` +
|
|
358
|
+
`and the cap can never trigger. ${tail}`
|
|
359
|
+
);
|
|
360
|
+
}
|
|
361
|
+
return (
|
|
362
|
+
// State the observation; do NOT diagnose the cause beyond what the
|
|
363
|
+
// endpoint itself stated. An absent cost_micros is all this meter can
|
|
364
|
+
// see, and --base-url is free-form: a non-Cohort Anthropic-compatible
|
|
365
|
+
// endpoint, a proxy that drops the header, or a gateway bug land here
|
|
366
|
+
// identically. Naming shadow billing as THE reason when the endpoint
|
|
367
|
+
// did not say so would be inferring a cause from an absence, which is
|
|
368
|
+
// the habit this lane exists to break.
|
|
369
|
+
cap +
|
|
370
|
+
`the endpoint priced none of ${unpriced} billed request(s) (no cost_micros; ` +
|
|
371
|
+
`${mode === null || mode === "unknown" ? "shadow billing is the usual cause" : `the endpoint states enforcement is "${mode}"`}), ` +
|
|
372
|
+
`so the run reports $0 spent and the cap can never trigger. ${tail}`
|
|
164
373
|
);
|
|
165
374
|
},
|
|
375
|
+
/**
|
|
376
|
+
* A one-time, run-level line naming what an EMPTY quota meant, else null.
|
|
377
|
+
* Self-clearing like {@link notice}.
|
|
378
|
+
*
|
|
379
|
+
* Silent when the gateway's quota actually answered (windows, credits or
|
|
380
|
+
* funding), and silent when no source was stated at all — there is then
|
|
381
|
+
* nothing new to say. Otherwise it distinguishes the three answers an empty
|
|
382
|
+
* `quota` used to collapse, because "you have no allowance" and "we could
|
|
383
|
+
* not look" call for opposite behaviour from anything pacing against it.
|
|
384
|
+
*
|
|
385
|
+
* The text is {@link quotaSourceNotice}, which is also what `cli.mjs` calls
|
|
386
|
+
* on an UNBUDGETED run — a meter-only line would never reach the operator
|
|
387
|
+
* who passed no cap, which is the common case. Two callers, one sentence, so
|
|
388
|
+
* they cannot drift; `cli.mjs` emits exactly one of them per run.
|
|
389
|
+
* @returns {string|null}
|
|
390
|
+
*/
|
|
391
|
+
quotaNotice() {
|
|
392
|
+
if (quotaNoticed) return null;
|
|
393
|
+
const line = quotaSourceNotice(quotaSource, quotaAnswered);
|
|
394
|
+
if (line === null) return null;
|
|
395
|
+
quotaNoticed = true;
|
|
396
|
+
return line;
|
|
397
|
+
},
|
|
166
398
|
/**
|
|
167
399
|
* The `cohort.budget` block of a result.
|
|
168
400
|
*
|
|
169
401
|
* **`spentUsd` / `spentMicros` / `exhausted` are only meaningful when
|
|
170
402
|
* `bindable` is true.** On a blind run they read `0` / `"0"` / `false`
|
|
171
|
-
* because the endpoint priced nothing
|
|
172
|
-
*
|
|
173
|
-
*
|
|
174
|
-
*
|
|
403
|
+
* because the endpoint priced nothing; on an `unenforced` one because the
|
|
404
|
+
* endpoint priced the requests and charges nothing for them. Neither means
|
|
405
|
+
* nothing was spent. Any consumer that shows or acts on the spend must read
|
|
406
|
+
* `bindable` (or `costVisibility`) in the same breath; `observedTokens` is
|
|
407
|
+
* the figure that is actually measured on such a run, and
|
|
408
|
+
* `unenforcedCostMicros` / `shadowCostMicros` are the money-shaped figures
|
|
409
|
+
* that were reported and are deliberately NOT spend.
|
|
175
410
|
*/
|
|
176
411
|
snapshot: () => ({
|
|
177
412
|
maxUsd: microsToUsdNumber(maxMicros),
|
|
@@ -182,13 +417,42 @@ export function createBudgetMeter({ maxMicros }) {
|
|
|
182
417
|
unknownCost,
|
|
183
418
|
requests,
|
|
184
419
|
costVisibility: visibility(),
|
|
185
|
-
bindable:
|
|
420
|
+
bindable: bindable(),
|
|
186
421
|
pricedRequests: priced,
|
|
187
422
|
unpricedRequests: unpriced,
|
|
423
|
+
// Requests whose billing mode was not stated on the request itself and was
|
|
424
|
+
// inherited from an earlier statement by the same endpoint in this run.
|
|
425
|
+
inheritedModeRequests: inheritedMode,
|
|
426
|
+
// W16. The mode the endpoint stated for this run: a mode, "mixed" when it
|
|
427
|
+
// changed mid-run, "unknown" when it stated a word this client does not
|
|
428
|
+
// know, null when it stated none. A null does NOT mean "on" — it means
|
|
429
|
+
// the figures above were counted on the endpoint's silence.
|
|
430
|
+
enforcement: mode,
|
|
431
|
+
// Figures reported under a mode that moves no money. Named so they cannot
|
|
432
|
+
// be mistaken for spend, and left in micros for the same reason.
|
|
433
|
+
unenforcedRequests: unenforced,
|
|
434
|
+
unenforcedCostMicros: unenforcedCost.toString(),
|
|
435
|
+
// `shadow_cost_micros`: what enforcement WOULD have charged. null when the
|
|
436
|
+
// endpoint reported none, which is every request in production today.
|
|
437
|
+
shadowCostMicros: shadowCostSeen ? shadowCost.toString() : null,
|
|
438
|
+
// Why `quota` was empty, when it was. null = the endpoint did not say.
|
|
439
|
+
quotaSource,
|
|
440
|
+
quotaAnswered,
|
|
188
441
|
// Exact token counts, or null when no frame reported any. Never dollars.
|
|
189
442
|
observedTokens: tokenRequests > 0 ? { ...tokens, total: totalTokens(tokens) } : null,
|
|
190
443
|
}),
|
|
191
444
|
};
|
|
192
445
|
}
|
|
193
446
|
|
|
447
|
+
/**
|
|
448
|
+
* Did the gateway's quota object actually say something? PURE. Used only to
|
|
449
|
+
* decide whether `quota_source` has anything to explain — an answered quota
|
|
450
|
+
* needs no explanation.
|
|
451
|
+
* @param {any} q
|
|
452
|
+
*/
|
|
453
|
+
function hasQuotaAnswer(q) {
|
|
454
|
+
if (!q || typeof q !== "object") return false;
|
|
455
|
+
return Object.keys(q.windows || {}).length > 0 || q.creditsRemainingMicros != null || q.funding != null;
|
|
456
|
+
}
|
|
457
|
+
|
|
194
458
|
/** @typedef {ReturnType<typeof createBudgetMeter>} BudgetMeter */
|
package/lib/engine/cli.mjs
CHANGED
|
@@ -124,7 +124,7 @@ import { createPromptCacheKeyState, promptCacheKeyMode } from "./wire/prompt-cac
|
|
|
124
124
|
// W4-A1: background-shell completion notices (CF-50); `cohort session` plugs in through `deps.host`.
|
|
125
125
|
import { createNotificationQueue, shellExitNotice } from "./session-runtime/notifications.mjs";
|
|
126
126
|
// W4-E1: --max-budget-usd.
|
|
127
|
-
import { parseBudgetUsd, createBudgetMeter } from "./budget.mjs";
|
|
127
|
+
import { parseBudgetUsd, createBudgetMeter, quotaSourceNotice } from "./budget.mjs";
|
|
128
128
|
// W4-E2: slash commands and skills invoked from a prompt (rows 14, 34).
|
|
129
129
|
import { discoverCommands, expandSlashCommand, resolveSlashCommand } from "./commands/index.mjs";
|
|
130
130
|
import { resolveAgentModel } from "./agents/definitions.mjs";
|
|
@@ -171,7 +171,9 @@ Flags:
|
|
|
171
171
|
--bare No user/project settings, hooks, instructions, skills, subagents or
|
|
172
172
|
project MCP; managed policy, --settings and --mcp-config still apply
|
|
173
173
|
--max-budget-usd <usd> Stop before the next model call once the run (subagents included) has
|
|
174
|
-
spent this much, by the gateway's cost figures (error_max_budget_usd)
|
|
174
|
+
spent this much, by the gateway's cost figures (error_max_budget_usd).
|
|
175
|
+
Only counts figures the gateway says it CHARGED: under shadow billing
|
|
176
|
+
the cap cannot bind, and stderr says so once per run
|
|
175
177
|
|
|
176
178
|
Environment:
|
|
177
179
|
COHORT_LLM_TOKEN Gateway credential (this or COHORT_LLM_TOKEN_HELPER)
|
|
@@ -690,6 +692,10 @@ async function runCliBody(argv, deps, inputs) {
|
|
|
690
692
|
const gatewayHeaders = { "x-cohort-surface": "engine", "x-cohort-session-id": session.id };
|
|
691
693
|
// W4-E1 (row 24): one spend meter for the run, its subagents and its workflow children.
|
|
692
694
|
const budget = o.maxBudgetMicros !== undefined ? createBudgetMeter({ maxMicros: o.maxBudgetMicros }) : null;
|
|
695
|
+
// W16: the quota discriminant is said once per RUN, not once per budgeted
|
|
696
|
+
// run — it rides the result, not the meter, so an operator who passed no cap
|
|
697
|
+
// still learns whether an empty quota was "no allowance" or a failed read.
|
|
698
|
+
let quotaNoticed = false;
|
|
693
699
|
// CF-156: a stalled stream ends on its own silence budget and says why on
|
|
694
700
|
// stderr. A stall or a mid-stream fault always reports — that is the
|
|
695
701
|
// evidence W13 did not have — while a merely slow stream reports only
|
|
@@ -1071,9 +1077,36 @@ async function runCliBody(argv, deps, inputs) {
|
|
|
1071
1077
|
history = outcome.messages;
|
|
1072
1078
|
|
|
1073
1079
|
const children = agentRuntime.drainRecords();
|
|
1074
|
-
|
|
1080
|
+
// W16: the three billing qualifiers go in beside the parent's own cost, and
|
|
1081
|
+
// come back folded across parent + children — so they describe the same
|
|
1082
|
+
// scope as the `costMicros` they qualify. Without this the result said
|
|
1083
|
+
// `enforcement: "on"` (the parent's) next to a total that included a
|
|
1084
|
+
// child's shadow figure.
|
|
1085
|
+
const total = aggregateUsage(
|
|
1086
|
+
{
|
|
1087
|
+
tier: outcome.modelTier || model,
|
|
1088
|
+
usage: outcome.usage,
|
|
1089
|
+
costMicros: outcome.costMicros,
|
|
1090
|
+
requestIds: outcome.requestIds,
|
|
1091
|
+
enforcement: outcome.enforcement,
|
|
1092
|
+
shadowCostMicros: outcome.shadowCostMicros,
|
|
1093
|
+
quotaSource: outcome.quotaSource,
|
|
1094
|
+
},
|
|
1095
|
+
children,
|
|
1096
|
+
);
|
|
1075
1097
|
const result = buildResult({
|
|
1076
|
-
outcome:
|
|
1098
|
+
outcome:
|
|
1099
|
+
children.length > 0
|
|
1100
|
+
? {
|
|
1101
|
+
...outcome,
|
|
1102
|
+
usage: total.usage,
|
|
1103
|
+
costMicros: total.costMicros,
|
|
1104
|
+
requestIds: total.requestIds,
|
|
1105
|
+
enforcement: total.enforcement,
|
|
1106
|
+
shadowCostMicros: total.shadowCostMicros,
|
|
1107
|
+
quotaSource: total.quotaSource,
|
|
1108
|
+
}
|
|
1109
|
+
: outcome,
|
|
1077
1110
|
sessionId: session.id,
|
|
1078
1111
|
durationMs: now() - turnStarted,
|
|
1079
1112
|
wire,
|
|
@@ -1087,7 +1120,10 @@ async function runCliBody(argv, deps, inputs) {
|
|
|
1087
1120
|
result.cohort.mcpServers = mcpServerStatuses();
|
|
1088
1121
|
result.cohort.modelUsage = total.cohortModelUsage;
|
|
1089
1122
|
result.cohort.todos = todoState.todos;
|
|
1090
|
-
|
|
1123
|
+
// W16: `enforcement` rides each child's own `costMicros` for the same
|
|
1124
|
+
// reason it rides the run's — a figure without its billing mode reads as
|
|
1125
|
+
// money, and a child may have been served under a different mode.
|
|
1126
|
+
result.cohort.agents = children.map((c) => ({ agentId: c.agentId, agentType: c.agentType, tier: c.tier, costMicros: c.costMicros, enforcement: c.enforcement ?? null, background: c.background, numTurns: c.numTurns, stop: c.stop }));
|
|
1091
1127
|
// Context stats for the session so far; `compactions` are this turn's.
|
|
1092
1128
|
const stats = manager.stats();
|
|
1093
1129
|
result.cohort.context = { ...stats, compactions: stats.compactions.slice(compactionsBefore) };
|
|
@@ -1099,6 +1135,19 @@ async function runCliBody(argv, deps, inputs) {
|
|
|
1099
1135
|
const budgetNotice = budget.notice();
|
|
1100
1136
|
if (budgetNotice) warn(budgetNotice);
|
|
1101
1137
|
}
|
|
1138
|
+
// W16: and say ONCE which of the three answers an empty quota was — "no
|
|
1139
|
+
// allowance" and "we could not look" call for opposite behaviour from
|
|
1140
|
+
// anything pacing against it, and `{}` used to collapse both. Outside the
|
|
1141
|
+
// `if (budget)` deliberately: the meter exists only under
|
|
1142
|
+
// --max-budget-usd, and the seat with no cap needs this line just as much.
|
|
1143
|
+
// `cohort.quota` is null unless a quota actually answered.
|
|
1144
|
+
if (!quotaNoticed) {
|
|
1145
|
+
const quotaNotice = quotaSourceNotice(result.cohort.quotaSource, result.cohort.quota != null);
|
|
1146
|
+
if (quotaNotice) {
|
|
1147
|
+
quotaNoticed = true;
|
|
1148
|
+
warn(quotaNotice);
|
|
1149
|
+
}
|
|
1150
|
+
}
|
|
1102
1151
|
if (slash.invoked) result.cohort.slashCommand = slash.invoked;
|
|
1103
1152
|
appendResult(session, result, now);
|
|
1104
1153
|
last = result;
|
package/lib/engine/loop.mjs
CHANGED
|
@@ -53,6 +53,7 @@ import { toolUses, messageText, stableStringify, userText } from "./messages.mjs
|
|
|
53
53
|
import { validateInput, repairToolJson } from "./tools/schema.mjs";
|
|
54
54
|
import { toolDefinitions } from "./tools/index.mjs";
|
|
55
55
|
import { emptyUsage, addUsage, readFrameUsage } from "./wire/usage.mjs";
|
|
56
|
+
import { foldEnforcement, foldQuotaSource } from "./wire/errors.mjs";
|
|
56
57
|
|
|
57
58
|
/**
|
|
58
59
|
* @typedef {{kind:'completed'} | {kind:'max_turns', maxTurns:number} | {kind:'wall_clock', wallClockMs:number}
|
|
@@ -247,6 +248,15 @@ async function runTurns(p, r) {
|
|
|
247
248
|
const requestIds = [];
|
|
248
249
|
let costTotal = 0n;
|
|
249
250
|
let costComplete = true;
|
|
251
|
+
// W16 (hq a60b9595): what the terminal frame says ABOUT those cost figures.
|
|
252
|
+
// `enforcement` qualifies `costMicros` — only under "on" is it money that
|
|
253
|
+
// moved — `shadowCostTotal` is the counterfactual charge and is never spend,
|
|
254
|
+
// and `quotaSource` says why `quota` is empty when it is. All three stay null
|
|
255
|
+
// when the endpoint says nothing, so silence never reads as an answer.
|
|
256
|
+
let enforcement = null;
|
|
257
|
+
let quotaSource = null;
|
|
258
|
+
let shadowCostTotal = 0n;
|
|
259
|
+
let shadowCostSeen = false;
|
|
250
260
|
let quota = null;
|
|
251
261
|
let modelTier = null;
|
|
252
262
|
let model = null;
|
|
@@ -300,6 +310,14 @@ async function runTurns(p, r) {
|
|
|
300
310
|
// turn and costComplete true was never billed (agents/usage.mjs).
|
|
301
311
|
costComplete,
|
|
302
312
|
quota,
|
|
313
|
+
// W16. `costMicros` above is what the gateway REPORTED; `enforcement` is
|
|
314
|
+
// whether that figure is money that moved. A consumer that sums the one
|
|
315
|
+
// without reading the other repeats the defect this lane fixed in the
|
|
316
|
+
// budget meter. `shadowCostMicros` is the counterfactual charge (null in
|
|
317
|
+
// production today) and `quotaSource` is why `quota` is empty when it is.
|
|
318
|
+
enforcement,
|
|
319
|
+
quotaSource,
|
|
320
|
+
shadowCostMicros: shadowCostSeen ? shadowCostTotal.toString() : null,
|
|
303
321
|
modelTier,
|
|
304
322
|
model,
|
|
305
323
|
apiMs,
|
|
@@ -331,6 +349,18 @@ async function runTurns(p, r) {
|
|
|
331
349
|
// rather than adding zeros.
|
|
332
350
|
const gu = readFrameUsage(res.cohort.usage);
|
|
333
351
|
if (gu) gatewayUsage = addUsage(gatewayUsage ?? emptyUsage("provider"), gu);
|
|
352
|
+
// W16. Folded rather than last-write-wins so a run whose gateway changed
|
|
353
|
+
// mode mid-flight reports "mixed" instead of whichever frame came last.
|
|
354
|
+
enforcement = foldEnforcement(enforcement, res.cohort.enforcement);
|
|
355
|
+
quotaSource = foldQuotaSource(quotaSource, res.cohort.quotaSource);
|
|
356
|
+
if (res.cohort.shadowCostMicros != null) {
|
|
357
|
+
try {
|
|
358
|
+
shadowCostTotal += BigInt(res.cohort.shadowCostMicros);
|
|
359
|
+
shadowCostSeen = true;
|
|
360
|
+
} catch {
|
|
361
|
+
/* an unparseable counterfactual is simply not recorded */
|
|
362
|
+
}
|
|
363
|
+
}
|
|
334
364
|
}
|
|
335
365
|
if (!res.ok) {
|
|
336
366
|
// A request that may have been billed has no cost figure, so the sum of
|