@cohortapp/agent-sdk 2.18.12 → 2.18.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/bin/maestro.mjs +38 -1
  2. package/docs/runbooks/fleet-rollout.md +14 -7
  3. package/docs/runbooks/recovery-and-failover.md +18 -0
  4. package/lib/cadence-failure-class.mjs +245 -0
  5. package/lib/claude-bin.mjs +26 -7
  6. package/lib/cli/doctor-checks.mjs +149 -1
  7. package/lib/comms/send-gate.mjs +6 -4
  8. package/lib/diagnostics/alerts.mjs +33 -0
  9. package/lib/engine/agents/usage.mjs +45 -0
  10. package/lib/engine/budget.mjs +293 -29
  11. package/lib/engine/cli.mjs +54 -5
  12. package/lib/engine/loop.mjs +30 -0
  13. package/lib/engine/output/json.mjs +26 -0
  14. package/lib/engine/wire/errors.mjs +179 -0
  15. package/lib/engine/wire/search.mjs +44 -8
  16. package/lib/identity/claude-md.mjs +107 -0
  17. package/lib/identity/disclosure-instructions.mjs +148 -0
  18. package/lib/identity/disclosure-scrub.mjs +207 -0
  19. package/lib/identity/persona.mjs +141 -6
  20. package/lib/org/inbound/conversation-frame.mjs +289 -0
  21. package/lib/org/inbound/directedness.mjs +27 -7
  22. package/lib/org/quota.mjs +27 -0
  23. package/lib/session/config.mjs +4 -0
  24. package/lib/session/identity.mjs +71 -7
  25. package/lib/session/launch-failure.mjs +251 -0
  26. package/lib/session/resume-target.mjs +86 -0
  27. package/lib/telemetry/collect.mjs +129 -0
  28. package/lib/upgrade/pinned-drift.mjs +467 -0
  29. package/package.json +1 -1
  30. package/plugins/maestro-skills/skills/persona-discipline.md +24 -2
  31. package/scaffold/config/alerts.yaml +7 -0
  32. package/scripts/ci/check-cadence-prompts-exist.mjs +96 -0
  33. package/scripts/ci/check.mjs +3 -0
  34. package/scripts/ci/run-tests.mjs +16 -2
  35. package/scripts/daemon/cadence-consumer.mjs +281 -34
  36. package/scripts/daemon/context-compiler.mjs +9 -1
  37. package/scripts/daemon/prompt-builder.mjs +219 -137
  38. package/scripts/daemon/responder.mjs +226 -26
  39. package/scripts/emergency-stop.sh +114 -13
  40. package/scripts/fleet/rollout.mjs +10 -3
  41. package/scripts/healthcheck.sh +131 -33
  42. package/scripts/resume-operations.sh +101 -6
  43. package/scripts/session/supervisor.mjs +198 -5
@@ -17,6 +17,7 @@
17
17
  */
18
18
 
19
19
  import { addUsage, emptyUsage, toResultUsage } from "../wire/usage.mjs";
20
+ import { combineStated } from "../wire/errors.mjs";
20
21
 
21
22
  /**
22
23
  * @typedef {Object} UsageRecord
@@ -24,6 +25,10 @@ import { addUsage, emptyUsage, toResultUsage } from "../wire/usage.mjs";
24
25
  * @property {import('../wire/usage.mjs').NormalisedUsage} usage
25
26
  * @property {string|null} costMicros null = unknown (may have been billed with no figure)
26
27
  * @property {string[]} requestIds
28
+ * @property {'off'|'shadow'|'on'|'unknown'|'mixed'|null} [enforcement]
29
+ * W16. What the gateway said ABOUT `costMicros` — only "on" means money moved.
30
+ * @property {string|null} [shadowCostMicros] what enforcement WOULD have charged; never spend
31
+ * @property {'ledger'|'unreadable'|'not_read'|'not_admitted'|'unknown'|'mixed'|null} [quotaSource]
27
32
  */
28
33
 
29
34
  /**
@@ -40,6 +45,13 @@ export function recordFromOutcome(outcome, fallbackTier) {
40
45
  usage: outcome.usage ?? emptyUsage("estimated"),
41
46
  costMicros: outcome.costMicros != null ? outcome.costMicros : exact ? "0" : null,
42
47
  requestIds: [...(outcome.requestIds ?? [])],
48
+ // W16. The qualifiers travel WITH the figure they qualify. A child's cost is
49
+ // summed into the parent's `costMicros`, so a child's `enforcement` must
50
+ // reach the same sum — otherwise the parent reports "on" beside a total that
51
+ // includes a child's shadow figure, which is the misread one layer up.
52
+ enforcement: outcome.enforcement ?? null,
53
+ shadowCostMicros: outcome.shadowCostMicros ?? null,
54
+ quotaSource: outcome.quotaSource ?? null,
43
55
  };
44
56
  }
45
57
 
@@ -53,8 +65,34 @@ function sumMicros(parts) {
53
65
  return total.toString();
54
66
  }
55
67
 
68
+ /**
69
+ * Sum the `shadow_cost_micros` figures that were reported, or null when none
70
+ * was. Unlike `costMicros` a missing counterfactual does not null the sum: it is
71
+ * not a charge, so a partial one is still worth reporting.
72
+ * @param {Array<string|null|undefined>} parts
73
+ */
74
+ function sumShadowMicros(parts) {
75
+ let total = 0n;
76
+ let seen = false;
77
+ for (const p of parts) {
78
+ if (p == null) continue;
79
+ try {
80
+ total += BigInt(p);
81
+ seen = true;
82
+ } catch {
83
+ /* an unparseable counterfactual is simply not recorded */
84
+ }
85
+ }
86
+ return seen ? total.toString() : null;
87
+ }
88
+
56
89
  /**
57
90
  * Combine the parent's record with its children's.
91
+ *
92
+ * W16: `enforcement` and `quotaSource` fold across parent AND children exactly
93
+ * as `costMicros` sums across them, so the qualifier and the figure it qualifies
94
+ * always describe the same scope. A parent served under `on` whose subagent was
95
+ * served under `shadow` reports `"mixed"`, never `"on"`.
58
96
  * @param {UsageRecord} parent @param {UsageRecord[]} children
59
97
  */
60
98
  export function aggregateUsage(parent, children) {
@@ -91,5 +129,12 @@ export function aggregateUsage(parent, children) {
91
129
  requestIds: all.flatMap((r) => r.requestIds),
92
130
  modelUsage,
93
131
  cohortModelUsage: { byTier, total: { usage, costMicros } },
132
+ // The run-wide qualifiers for the run-wide `costMicros` above.
133
+ // `combineStated`, not `foldEnforcement`: each record's value is ALREADY
134
+ // folded, so it may be this client's own "mixed" — a word the gateway never
135
+ // sends and the raw reader would flatten to "unknown".
136
+ enforcement: all.reduce((m, r) => combineStated(m, r.enforcement), /** @type {any} */ (null)),
137
+ quotaSource: all.reduce((q, r) => combineStated(q, r.quotaSource), /** @type {any} */ (null)),
138
+ shadowCostMicros: sumShadowMicros(all.map((r) => r.shadowCostMicros)),
94
139
  };
95
140
  }
@@ -53,12 +53,112 @@
53
53
  * guess wearing a measurement's clothes — the exact failure CF-157 is about.
54
54
  * `spent` therefore stays at the gateway's own figures, and only those.
55
55
  *
56
+ * ## W16 — a figure the gateway reports is not the same as money that moved
57
+ *
58
+ * The gateway now states which billing mode produced each figure
59
+ * (`enforcement`, design §4.8, hq a60b9595). Under `shadow` a PRICED request
60
+ * settles for real and reports a real `cost_micros` while **no money moves**:
61
+ * a shadow request the ledger would have refused is admitted on an unfunded
62
+ * hold, served, and its settle posts an empty transaction. This meter used to
63
+ * add that figure to `spent` and enforce a cap against it — counting spend that
64
+ * never happened, against a cap that cannot fire. That is CF-157's own misread,
65
+ * surviving on the client after the server side was fixed.
66
+ *
67
+ * So a figure only becomes `spent` when the endpoint says `enforcement: "on"`.
68
+ * A figure reported under `shadow` or `off` is kept — as `unenforcedCostMicros`
69
+ * and `unenforcedRequests`, under names that cannot be read as spend — and adds
70
+ * a fourth cost-visibility state:
71
+ *
72
+ * unenforced figures arrived, and the endpoint states none of them is money.
73
+ * THE CAP CANNOT BIND — for a different reason than `blind`: not
74
+ * "we were not told the price" but "the price is not charged".
75
+ *
76
+ * `bindable()` is false for `unenforced` exactly as for `blind`, and `notice()`
77
+ * says which. Neither is reported as "$0 spent" alone: "nothing was spent" and
78
+ * "spending is not being enforced" are different statements.
79
+ *
80
+ * **When the endpoint states no mode at all, a figure IS counted as spend** —
81
+ * the pre-W16 behaviour. The choice is deliberate and it is the one asymmetry
82
+ * worth stating: over-counting produces a LOUD failure (the run stops and says
83
+ * the cap was reached, which an operator sees immediately), while under-counting
84
+ * produces the SILENT one (a cap that never fires, which is CF-157 itself).
85
+ * A never-seen `enforcement` word is therefore read as unknown, never as "on"
86
+ * — `snapshot().enforcement` reports `"unknown"` for it and `null` for silence
87
+ * — but unknown and silent both leave the cap armed. Only a stated `shadow` or
88
+ * `off` withholds a figure from spend.
89
+ *
90
+ * **…with one exception: silence AFTER the endpoint has named a mode inherits
91
+ * it.** A run talks to one `--base-url`, and a gateway's billing mode does not
92
+ * change without a deploy, so once this run has heard `shadow` from that
93
+ * endpoint, `shadow` is far better evidence about a later frameless request from
94
+ * it than the absence of a field is. Without this, a billed request whose frame
95
+ * never arrived — a stream that died after the headers, and every `searchWeb`
96
+ * refusal path, which has only headers to read — put a shadow figure straight
97
+ * into `spent`. The inheritance costs nothing against the cases the asymmetry
98
+ * above protects: a non-Cohort endpoint, a dropped-frame proxy and a gateway
99
+ * older than a60b9595 never name a mode at all, so they never inherit one.
100
+ * `snapshot().enforcement` still reports only what was STATED; the inherited
101
+ * decisions are counted separately as `inheritedModeRequests`.
102
+ *
103
+ * ## W16 — an empty quota is three different answers
104
+ *
105
+ * `quota_source` splits what an empty `quota` object used to collapse: the
106
+ * ledger answered and the seat has no included allowance (`ledger`), the read
107
+ * threw (`unreadable`), or nothing was read (`not_read`; `not_admitted` is
108
+ * "unknown"). A governor cannot tell "you have no allowance" from "we could not
109
+ * look", and those call for opposite behaviour. The meter records what was
110
+ * stated and `quotaNotice()` says it once per run, so an operator reading
111
+ * stderr learns which it was.
112
+ *
56
113
  * Pure apart from its own counters; no I/O.
57
114
  *
58
115
  * @module lib/engine/budget
59
116
  */
60
117
 
61
118
  import { addUsage, emptyUsage, readFrameUsage, totalTokens } from "./wire/usage.mjs";
119
+ import { foldEnforcement, foldQuotaSource, readEnforcement } from "./wire/errors.mjs";
120
+
121
+ /**
122
+ * The one-line answer to "what did an EMPTY quota mean?", or null when there is
123
+ * nothing to say. PURE and total.
124
+ *
125
+ * It lives outside {@link createBudgetMeter} because the meter only exists when
126
+ * `--max-budget-usd` was passed, and the operator who most needs this line — no
127
+ * cap, nothing pacing — is precisely the one who did not pass it. The meter's
128
+ * {@link createBudgetMeter} `quotaNotice` is the same text with a once-per-run
129
+ * latch; `cli.mjs` calls this directly so an unbudgeted run says it too.
130
+ *
131
+ * Silent when the quota actually answered (windows, credits or funding — there
132
+ * is then nothing ambiguous to explain) and silent when the endpoint stated no
133
+ * source at all (nothing new to say). Otherwise it names which of the three
134
+ * answers `{}` was, because "you have no allowance" and "we could not look" call
135
+ * for opposite behaviour from anything pacing against it.
136
+ *
137
+ * @param {'ledger'|'unreadable'|'not_read'|'not_admitted'|'unknown'|'mixed'|null|undefined} quotaSource
138
+ * @param {boolean} quotaAnswered
139
+ * @returns {string|null}
140
+ */
141
+ export function quotaSourceNotice(quotaSource, quotaAnswered) {
142
+ if (quotaAnswered || quotaSource == null) return null;
143
+ const head = `cohort quota: this run saw no quota windows, and the endpoint states quota_source "${quotaSource}" — `;
144
+ if (quotaSource === "ledger") {
145
+ return (
146
+ head +
147
+ `the ledger ANSWERED: this seat has no included allowance (a money-only band reports no windows by design). ` +
148
+ `Nothing is wrong, and there is no window to pace against.`
149
+ );
150
+ }
151
+ if (quotaSource === "unreadable") {
152
+ return (
153
+ head +
154
+ `the quota read FAILED. The empty quota is a failed read, NOT "no allowance": do not read it as headroom.`
155
+ );
156
+ }
157
+ return (
158
+ head +
159
+ `no quota was read for this seat, so the empty quota states nothing about its allowance either way — read it as unknown.`
160
+ );
161
+ }
62
162
 
63
163
  /**
64
164
  * `--max-budget-usd` as integer micro-USD, or an error. PURE.
@@ -83,20 +183,56 @@ export function microsToUsdNumber(micros) {
83
183
  * @param {{maxMicros:bigint}} o
84
184
  */
85
185
  export function createBudgetMeter({ maxMicros }) {
186
+ /** Money that MOVED, by the gateway's own figures under `enforcement: "on"`. */
86
187
  let spent = 0n;
188
+ /** Retail figures the endpoint reported while stating it enforces nothing. NEVER spend. */
189
+ let unenforcedCost = 0n;
190
+ /** `shadow_cost_micros`: what enforcement WOULD have charged. NEVER spend. */
191
+ let shadowCost = 0n;
192
+ let shadowCostSeen = false;
87
193
  let unknownCost = false;
88
194
  let requests = 0;
89
- /** Billed requests that carried a cost figure. */
195
+ /** Billed requests whose figure counted toward the cap. */
90
196
  let priced = 0;
91
- /** Billed requests that carried NO cost figure — the shadow-mode signature. */
197
+ /** Billed requests that carried NO cost figure — the shadow-with-no-price-book signature. */
92
198
  let unpriced = 0;
199
+ /** Billed requests that carried a figure the endpoint says is not money. */
200
+ let unenforced = 0;
201
+ /** The billing mode(s) the endpoint stated: a mode, 'unknown', 'mixed', or null for silence. */
202
+ let mode = /** @type {ReturnType<typeof foldEnforcement>} */ (null);
203
+ /**
204
+ * The last mode the endpoint actually NAMED (one of the three §4.8 words), or
205
+ * null before it named any. A later request that names none inherits this —
206
+ * see the module header. An unrecognised word does not replace it: it is not
207
+ * evidence about enforcement either way, and discarding a mode the endpoint
208
+ * did state would only make the seat count more shadow figures as spend.
209
+ */
210
+ let lastStatedMode = /** @type {'off'|'shadow'|'on'|null} */ (null);
211
+ /** Requests whose billing mode was inherited from an earlier statement, not stated on the request itself. */
212
+ let inheritedMode = 0;
213
+ /** The quota source(s) stated, folded the same way. */
214
+ let quotaSource = /** @type {ReturnType<typeof foldQuotaSource>} */ (null);
215
+ /** True once any request's quota object actually said something. */
216
+ let quotaAnswered = false;
93
217
  /** Exact token counts off the terminal frame (CF-158). Never priced. */
94
218
  let tokens = emptyUsage("provider");
95
219
  let tokenRequests = 0;
96
220
  let noticed = false;
221
+ let quotaNoticed = false;
97
222
 
98
- /** @returns {'exact'|'partial'|'blind'} */
99
- const visibility = () => (unpriced === 0 ? "exact" : priced === 0 ? "blind" : "partial");
223
+ /** @returns {'exact'|'partial'|'blind'|'unenforced'} */
224
+ const visibility = () => {
225
+ // A figure that counted is the only thing that can make a cap bind.
226
+ if (priced > 0) return unpriced === 0 && unenforced === 0 ? "exact" : "partial";
227
+ // Figures arrived, and the endpoint says none of them is money.
228
+ if (unenforced > 0) return "unenforced";
229
+ // Nothing recorded yet reads as `exact`: nothing has contradicted the cap.
230
+ return unpriced === 0 ? "exact" : "blind";
231
+ };
232
+ const bindable = () => {
233
+ const v = visibility();
234
+ return v !== "blind" && v !== "unenforced";
235
+ };
100
236
 
101
237
  return {
102
238
  maxMicros,
@@ -111,11 +247,46 @@ export function createBudgetMeter({ maxMicros }) {
111
247
  tokens = addUsage(tokens, u);
112
248
  tokenRequests++;
113
249
  }
250
+ // W16. Re-read through the wire's own total parsers rather than trusting
251
+ // the field: `record` is also handed results built by call sites that
252
+ // never went through `mergeCohort` (WebSearch, the fixtures), and an
253
+ // unrecognised word must land as unknown there too.
254
+ mode = foldEnforcement(mode, res.cohort?.enforcement);
255
+ quotaSource = foldQuotaSource(quotaSource, res.cohort?.quotaSource);
256
+ if (hasQuotaAnswer(res.cohort?.quota)) quotaAnswered = true;
257
+ const shadow = res.cohort?.shadowCostMicros;
258
+ if (shadow != null) {
259
+ try {
260
+ shadowCost += BigInt(shadow);
261
+ shadowCostSeen = true;
262
+ } catch {
263
+ /* an unparseable counterfactual is simply not recorded */
264
+ }
265
+ }
266
+ // Which mode governs THIS figure. A named mode governs itself; silence
267
+ // inherits the last mode this endpoint named, because within one run
268
+ // against one base URL the mode does not change without a deploy, and a
269
+ // frameless billed request (a stream that died after the headers, a
270
+ // search refusal) would otherwise put a shadow figure into spend. Silence
271
+ // before any statement, and an unrecognised word, leave the cap armed —
272
+ // see the module header for why that asymmetry is the safe one here.
273
+ const stated = readEnforcement(res.cohort?.enforcement);
274
+ if (stated) lastStatedMode = stated;
275
+ else if (res.cohort?.enforcement == null && lastStatedMode !== null) inheritedMode++;
276
+ const enforcing = stated ?? (res.cohort?.enforcement == null ? lastStatedMode : null);
277
+ const isMoney = enforcing !== "shadow" && enforcing !== "off";
114
278
  const c = res.cohort?.costMicros;
115
279
  if (c != null) {
116
280
  try {
117
- spent += BigInt(c);
118
- priced++;
281
+ const micros = BigInt(c);
282
+ if (isMoney) {
283
+ spent += micros;
284
+ priced++;
285
+ } else {
286
+ // Kept, and kept out of spend: no money moved for this figure.
287
+ unenforcedCost += micros;
288
+ unenforced++;
289
+ }
119
290
  return;
120
291
  } catch {
121
292
  /* an unparseable figure is an unknown one */
@@ -127,15 +298,20 @@ export function createBudgetMeter({ maxMicros }) {
127
298
  unpriced++;
128
299
  }
129
300
  },
130
- // A blind meter never reaches the cap: spent stays 0 because no cost was
131
- // ever reported. That is reported by bindable()/notice(), never papered
132
- // over by treating an unreported cost as zero spend.
301
+ // A blind or unenforced meter never reaches the cap: spent stays 0 because
302
+ // no money figure was ever counted. That is reported by
303
+ // bindable()/notice(), never papered over by treating an unreported — or
304
+ // unenforced — cost as zero spend.
133
305
  exhausted: () => spent >= maxMicros,
134
306
  spentMicros: () => spent,
135
- /** 'exact' | 'partial' | 'blind' — see the module header. */
307
+ /** 'exact' | 'partial' | 'blind' | 'unenforced' — see the module header. */
136
308
  costVisibility: visibility,
137
- /** False when the gateway priced nothing, i.e. the cap cannot fire at all. */
138
- bindable: () => visibility() !== "blind",
309
+ /** False when no figure could count toward the cap, i.e. it cannot fire at all. */
310
+ bindable,
311
+ /** The billing mode the endpoint stated, or 'unknown'/'mixed'/null. */
312
+ enforcement: () => mode,
313
+ /** The quota source the endpoint stated, or 'unknown'/'mixed'/null. */
314
+ quotaSource: () => quotaSource,
139
315
  /**
140
316
  * A one-time, run-level warning when the cap cannot bind, else null.
141
317
  * Self-clearing, so a caller may ask after every turn and the operator
@@ -143,35 +319,94 @@ export function createBudgetMeter({ maxMicros }) {
143
319
  * @returns {string|null}
144
320
  */
145
321
  notice() {
146
- if (noticed || visibility() !== "blind") return null;
322
+ // A cap that cannot bind must say so — and so must a cap that binds on
323
+ // only PART of the run's figures. The second case is bindable, so it used
324
+ // to fall through this gate and say nothing at all: an operator saw a cap
325
+ // that looked armed against money that had not moved. That silence is the
326
+ // failure mode this lane exists to end, so `unenforced > 0` speaks too.
327
+ if (noticed || (bindable() && unenforced === 0)) return null;
147
328
  noticed = true;
148
329
  const observed =
149
330
  tokenRequests > 0
150
331
  ? `${totalTokens(tokens)} tokens over ${tokenRequests} request(s)`
151
332
  : "no token counts either";
152
- return (
153
- // State the observation; do NOT diagnose the cause. An absent
154
- // cost_micros is all this meter can see, and --base-url is free-form:
155
- // a non-Cohort Anthropic-compatible endpoint, a proxy that drops the
156
- // header, or a gateway bug land here identically. Naming shadow
157
- // billing as THE reason would be inferring a cause from an absence,
158
- // which is the habit this lane exists to break.
159
- `--max-budget-usd ($${microsToUsdNumber(maxMicros)}) is NOT being enforced: ` +
160
- `the endpoint priced none of ${unpriced} billed request(s) (no cost_micros; shadow billing is the usual cause), ` +
161
- `so the run reports $0 spent and the cap can never trigger. Measured instead: ${observed} ` +
333
+ const cap = `--max-budget-usd ($${microsToUsdNumber(maxMicros)}) is NOT being enforced: `;
334
+ const tail =
335
+ `Measured instead: ${observed} ` +
162
336
  `— tokens are exact, but this SDK has no per-tier price table and does not convert them to dollars. ` +
163
- `See docs/engine/eval.md §3 (CF-157).`
337
+ `See docs/engine/eval.md §3 (CF-157).`;
338
+ if (priced > 0) {
339
+ // A MIXED run: some figures the endpoint says it charged for, some it
340
+ // says it did not. The cap IS armed — on the enforced subset only — so
341
+ // this does not claim it is unenforced; it names the split, because
342
+ // `spentMicros` here is not the whole of what the gateway reported.
343
+ return (
344
+ `--max-budget-usd ($${microsToUsdNumber(maxMicros)}) is being enforced on PART of this run only: ` +
345
+ `${priced} billed request(s) carried figures counted as spend, and ${unenforced} carried figures the ` +
346
+ `endpoint states are not money (enforcement "${mode}"; ${unenforcedCost.toString()} micro-USD reported, ` +
347
+ `deliberately kept out of spend), so the cap binds on a lower bound. ${tail}`
348
+ );
349
+ }
350
+ if (visibility() === "unenforced") {
351
+ // W16: the endpoint DID price these requests and says it charges
352
+ // nothing for them. Naming the mode here is not a diagnosis — it is
353
+ // quoting the field the gateway sent for exactly this purpose.
354
+ return (
355
+ cap +
356
+ `the endpoint priced ${unenforced} billed request(s) but states enforcement is "${mode}", so no money moved ` +
357
+ `for those figures (${unenforcedCost.toString()} micro-USD reported, deliberately kept out of spend) ` +
358
+ `and the cap can never trigger. ${tail}`
359
+ );
360
+ }
361
+ return (
362
+ // State the observation; do NOT diagnose the cause beyond what the
363
+ // endpoint itself stated. An absent cost_micros is all this meter can
364
+ // see, and --base-url is free-form: a non-Cohort Anthropic-compatible
365
+ // endpoint, a proxy that drops the header, or a gateway bug land here
366
+ // identically. Naming shadow billing as THE reason when the endpoint
367
+ // did not say so would be inferring a cause from an absence, which is
368
+ // the habit this lane exists to break.
369
+ cap +
370
+ `the endpoint priced none of ${unpriced} billed request(s) (no cost_micros; ` +
371
+ `${mode === null || mode === "unknown" ? "shadow billing is the usual cause" : `the endpoint states enforcement is "${mode}"`}), ` +
372
+ `so the run reports $0 spent and the cap can never trigger. ${tail}`
164
373
  );
165
374
  },
375
+ /**
376
+ * A one-time, run-level line naming what an EMPTY quota meant, else null.
377
+ * Self-clearing like {@link notice}.
378
+ *
379
+ * Silent when the gateway's quota actually answered (windows, credits or
380
+ * funding), and silent when no source was stated at all — there is then
381
+ * nothing new to say. Otherwise it distinguishes the three answers an empty
382
+ * `quota` used to collapse, because "you have no allowance" and "we could
383
+ * not look" call for opposite behaviour from anything pacing against it.
384
+ *
385
+ * The text is {@link quotaSourceNotice}, which is also what `cli.mjs` calls
386
+ * on an UNBUDGETED run — a meter-only line would never reach the operator
387
+ * who passed no cap, which is the common case. Two callers, one sentence, so
388
+ * they cannot drift; `cli.mjs` emits exactly one of them per run.
389
+ * @returns {string|null}
390
+ */
391
+ quotaNotice() {
392
+ if (quotaNoticed) return null;
393
+ const line = quotaSourceNotice(quotaSource, quotaAnswered);
394
+ if (line === null) return null;
395
+ quotaNoticed = true;
396
+ return line;
397
+ },
166
398
  /**
167
399
  * The `cohort.budget` block of a result.
168
400
  *
169
401
  * **`spentUsd` / `spentMicros` / `exhausted` are only meaningful when
170
402
  * `bindable` is true.** On a blind run they read `0` / `"0"` / `false`
171
- * because the endpoint priced nothing — not because nothing was spent.
172
- * Any consumer that shows or acts on the spend must read `bindable` (or
173
- * `costVisibility`) in the same breath; `observedTokens` is the figure
174
- * that is actually measured on such a run.
403
+ * because the endpoint priced nothing; on an `unenforced` one because the
404
+ * endpoint priced the requests and charges nothing for them. Neither means
405
+ * nothing was spent. Any consumer that shows or acts on the spend must read
406
+ * `bindable` (or `costVisibility`) in the same breath; `observedTokens` is
407
+ * the figure that is actually measured on such a run, and
408
+ * `unenforcedCostMicros` / `shadowCostMicros` are the money-shaped figures
409
+ * that were reported and are deliberately NOT spend.
175
410
  */
176
411
  snapshot: () => ({
177
412
  maxUsd: microsToUsdNumber(maxMicros),
@@ -182,13 +417,42 @@ export function createBudgetMeter({ maxMicros }) {
182
417
  unknownCost,
183
418
  requests,
184
419
  costVisibility: visibility(),
185
- bindable: visibility() !== "blind",
420
+ bindable: bindable(),
186
421
  pricedRequests: priced,
187
422
  unpricedRequests: unpriced,
423
+ // Requests whose billing mode was not stated on the request itself and was
424
+ // inherited from an earlier statement by the same endpoint in this run.
425
+ inheritedModeRequests: inheritedMode,
426
+ // W16. The mode the endpoint stated for this run: a mode, "mixed" when it
427
+ // changed mid-run, "unknown" when it stated a word this client does not
428
+ // know, null when it stated none. A null does NOT mean "on" — it means
429
+ // the figures above were counted on the endpoint's silence.
430
+ enforcement: mode,
431
+ // Figures reported under a mode that moves no money. Named so they cannot
432
+ // be mistaken for spend, and left in micros for the same reason.
433
+ unenforcedRequests: unenforced,
434
+ unenforcedCostMicros: unenforcedCost.toString(),
435
+ // `shadow_cost_micros`: what enforcement WOULD have charged. null when the
436
+ // endpoint reported none, which is every request in production today.
437
+ shadowCostMicros: shadowCostSeen ? shadowCost.toString() : null,
438
+ // Why `quota` was empty, when it was. null = the endpoint did not say.
439
+ quotaSource,
440
+ quotaAnswered,
188
441
  // Exact token counts, or null when no frame reported any. Never dollars.
189
442
  observedTokens: tokenRequests > 0 ? { ...tokens, total: totalTokens(tokens) } : null,
190
443
  }),
191
444
  };
192
445
  }
193
446
 
447
+ /**
448
+ * Did the gateway's quota object actually say something? PURE. Used only to
449
+ * decide whether `quota_source` has anything to explain — an answered quota
450
+ * needs no explanation.
451
+ * @param {any} q
452
+ */
453
+ function hasQuotaAnswer(q) {
454
+ if (!q || typeof q !== "object") return false;
455
+ return Object.keys(q.windows || {}).length > 0 || q.creditsRemainingMicros != null || q.funding != null;
456
+ }
457
+
194
458
  /** @typedef {ReturnType<typeof createBudgetMeter>} BudgetMeter */
@@ -124,7 +124,7 @@ import { createPromptCacheKeyState, promptCacheKeyMode } from "./wire/prompt-cac
124
124
  // W4-A1: background-shell completion notices (CF-50); `cohort session` plugs in through `deps.host`.
125
125
  import { createNotificationQueue, shellExitNotice } from "./session-runtime/notifications.mjs";
126
126
  // W4-E1: --max-budget-usd.
127
- import { parseBudgetUsd, createBudgetMeter } from "./budget.mjs";
127
+ import { parseBudgetUsd, createBudgetMeter, quotaSourceNotice } from "./budget.mjs";
128
128
  // W4-E2: slash commands and skills invoked from a prompt (rows 14, 34).
129
129
  import { discoverCommands, expandSlashCommand, resolveSlashCommand } from "./commands/index.mjs";
130
130
  import { resolveAgentModel } from "./agents/definitions.mjs";
@@ -171,7 +171,9 @@ Flags:
171
171
  --bare No user/project settings, hooks, instructions, skills, subagents or
172
172
  project MCP; managed policy, --settings and --mcp-config still apply
173
173
  --max-budget-usd <usd> Stop before the next model call once the run (subagents included) has
174
- spent this much, by the gateway's cost figures (error_max_budget_usd)
174
+ spent this much, by the gateway's cost figures (error_max_budget_usd).
175
+ Only counts figures the gateway says it CHARGED: under shadow billing
176
+ the cap cannot bind, and stderr says so once per run
175
177
 
176
178
  Environment:
177
179
  COHORT_LLM_TOKEN Gateway credential (this or COHORT_LLM_TOKEN_HELPER)
@@ -690,6 +692,10 @@ async function runCliBody(argv, deps, inputs) {
690
692
  const gatewayHeaders = { "x-cohort-surface": "engine", "x-cohort-session-id": session.id };
691
693
  // W4-E1 (row 24): one spend meter for the run, its subagents and its workflow children.
692
694
  const budget = o.maxBudgetMicros !== undefined ? createBudgetMeter({ maxMicros: o.maxBudgetMicros }) : null;
695
+ // W16: the quota discriminant is said once per RUN, not once per budgeted
696
+ // run — it rides the result, not the meter, so an operator who passed no cap
697
+ // still learns whether an empty quota was "no allowance" or a failed read.
698
+ let quotaNoticed = false;
693
699
  // CF-156: a stalled stream ends on its own silence budget and says why on
694
700
  // stderr. A stall or a mid-stream fault always reports — that is the
695
701
  // evidence W13 did not have — while a merely slow stream reports only
@@ -1071,9 +1077,36 @@ async function runCliBody(argv, deps, inputs) {
1071
1077
  history = outcome.messages;
1072
1078
 
1073
1079
  const children = agentRuntime.drainRecords();
1074
- const total = aggregateUsage({ tier: outcome.modelTier || model, usage: outcome.usage, costMicros: outcome.costMicros, requestIds: outcome.requestIds }, children);
1080
+ // W16: the three billing qualifiers go in beside the parent's own cost, and
1081
+ // come back folded across parent + children — so they describe the same
1082
+ // scope as the `costMicros` they qualify. Without this the result said
1083
+ // `enforcement: "on"` (the parent's) next to a total that included a
1084
+ // child's shadow figure.
1085
+ const total = aggregateUsage(
1086
+ {
1087
+ tier: outcome.modelTier || model,
1088
+ usage: outcome.usage,
1089
+ costMicros: outcome.costMicros,
1090
+ requestIds: outcome.requestIds,
1091
+ enforcement: outcome.enforcement,
1092
+ shadowCostMicros: outcome.shadowCostMicros,
1093
+ quotaSource: outcome.quotaSource,
1094
+ },
1095
+ children,
1096
+ );
1075
1097
  const result = buildResult({
1076
- outcome: children.length > 0 ? { ...outcome, usage: total.usage, costMicros: total.costMicros, requestIds: total.requestIds } : outcome,
1098
+ outcome:
1099
+ children.length > 0
1100
+ ? {
1101
+ ...outcome,
1102
+ usage: total.usage,
1103
+ costMicros: total.costMicros,
1104
+ requestIds: total.requestIds,
1105
+ enforcement: total.enforcement,
1106
+ shadowCostMicros: total.shadowCostMicros,
1107
+ quotaSource: total.quotaSource,
1108
+ }
1109
+ : outcome,
1077
1110
  sessionId: session.id,
1078
1111
  durationMs: now() - turnStarted,
1079
1112
  wire,
@@ -1087,7 +1120,10 @@ async function runCliBody(argv, deps, inputs) {
1087
1120
  result.cohort.mcpServers = mcpServerStatuses();
1088
1121
  result.cohort.modelUsage = total.cohortModelUsage;
1089
1122
  result.cohort.todos = todoState.todos;
1090
- result.cohort.agents = children.map((c) => ({ agentId: c.agentId, agentType: c.agentType, tier: c.tier, costMicros: c.costMicros, background: c.background, numTurns: c.numTurns, stop: c.stop }));
1123
+ // W16: `enforcement` rides each child's own `costMicros` for the same
1124
+ // reason it rides the run's — a figure without its billing mode reads as
1125
+ // money, and a child may have been served under a different mode.
1126
+ result.cohort.agents = children.map((c) => ({ agentId: c.agentId, agentType: c.agentType, tier: c.tier, costMicros: c.costMicros, enforcement: c.enforcement ?? null, background: c.background, numTurns: c.numTurns, stop: c.stop }));
1091
1127
  // Context stats for the session so far; `compactions` are this turn's.
1092
1128
  const stats = manager.stats();
1093
1129
  result.cohort.context = { ...stats, compactions: stats.compactions.slice(compactionsBefore) };
@@ -1099,6 +1135,19 @@ async function runCliBody(argv, deps, inputs) {
1099
1135
  const budgetNotice = budget.notice();
1100
1136
  if (budgetNotice) warn(budgetNotice);
1101
1137
  }
1138
+ // W16: and say ONCE which of the three answers an empty quota was — "no
1139
+ // allowance" and "we could not look" call for opposite behaviour from
1140
+ // anything pacing against it, and `{}` used to collapse both. Outside the
1141
+ // `if (budget)` deliberately: the meter exists only under
1142
+ // --max-budget-usd, and the seat with no cap needs this line just as much.
1143
+ // `cohort.quota` is null unless a quota actually answered.
1144
+ if (!quotaNoticed) {
1145
+ const quotaNotice = quotaSourceNotice(result.cohort.quotaSource, result.cohort.quota != null);
1146
+ if (quotaNotice) {
1147
+ quotaNoticed = true;
1148
+ warn(quotaNotice);
1149
+ }
1150
+ }
1102
1151
  if (slash.invoked) result.cohort.slashCommand = slash.invoked;
1103
1152
  appendResult(session, result, now);
1104
1153
  last = result;
@@ -53,6 +53,7 @@ import { toolUses, messageText, stableStringify, userText } from "./messages.mjs
53
53
  import { validateInput, repairToolJson } from "./tools/schema.mjs";
54
54
  import { toolDefinitions } from "./tools/index.mjs";
55
55
  import { emptyUsage, addUsage, readFrameUsage } from "./wire/usage.mjs";
56
+ import { foldEnforcement, foldQuotaSource } from "./wire/errors.mjs";
56
57
 
57
58
  /**
58
59
  * @typedef {{kind:'completed'} | {kind:'max_turns', maxTurns:number} | {kind:'wall_clock', wallClockMs:number}
@@ -247,6 +248,15 @@ async function runTurns(p, r) {
247
248
  const requestIds = [];
248
249
  let costTotal = 0n;
249
250
  let costComplete = true;
251
+ // W16 (hq a60b9595): what the terminal frame says ABOUT those cost figures.
252
+ // `enforcement` qualifies `costMicros` — only under "on" is it money that
253
+ // moved — `shadowCostTotal` is the counterfactual charge and is never spend,
254
+ // and `quotaSource` says why `quota` is empty when it is. All three stay null
255
+ // when the endpoint says nothing, so silence never reads as an answer.
256
+ let enforcement = null;
257
+ let quotaSource = null;
258
+ let shadowCostTotal = 0n;
259
+ let shadowCostSeen = false;
250
260
  let quota = null;
251
261
  let modelTier = null;
252
262
  let model = null;
@@ -300,6 +310,14 @@ async function runTurns(p, r) {
300
310
  // turn and costComplete true was never billed (agents/usage.mjs).
301
311
  costComplete,
302
312
  quota,
313
+ // W16. `costMicros` above is what the gateway REPORTED; `enforcement` is
314
+ // whether that figure is money that moved. A consumer that sums the one
315
+ // without reading the other repeats the defect this lane fixed in the
316
+ // budget meter. `shadowCostMicros` is the counterfactual charge (null in
317
+ // production today) and `quotaSource` is why `quota` is empty when it is.
318
+ enforcement,
319
+ quotaSource,
320
+ shadowCostMicros: shadowCostSeen ? shadowCostTotal.toString() : null,
303
321
  modelTier,
304
322
  model,
305
323
  apiMs,
@@ -331,6 +349,18 @@ async function runTurns(p, r) {
331
349
  // rather than adding zeros.
332
350
  const gu = readFrameUsage(res.cohort.usage);
333
351
  if (gu) gatewayUsage = addUsage(gatewayUsage ?? emptyUsage("provider"), gu);
352
+ // W16. Folded rather than last-write-wins so a run whose gateway changed
353
+ // mode mid-flight reports "mixed" instead of whichever frame came last.
354
+ enforcement = foldEnforcement(enforcement, res.cohort.enforcement);
355
+ quotaSource = foldQuotaSource(quotaSource, res.cohort.quotaSource);
356
+ if (res.cohort.shadowCostMicros != null) {
357
+ try {
358
+ shadowCostTotal += BigInt(res.cohort.shadowCostMicros);
359
+ shadowCostSeen = true;
360
+ } catch {
361
+ /* an unparseable counterfactual is simply not recorded */
362
+ }
363
+ }
334
364
  }
335
365
  if (!res.ok) {
336
366
  // A request that may have been billed has no cost figure, so the sum of