@junghanacs/entwurf 0.16.0 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +5 -3
- package/CHANGELOG.md +337 -0
- package/README.md +8 -11
- package/VERIFY.md +8 -1
- package/demo/README.md +1 -1
- package/docs/acp-backend-rail.md +25 -14
- package/docs/setup-clean-host.md +24 -10
- package/mcp/entwurf-bridge/dist/pi-extensions/lib/acp/acp-client.js +1 -1
- package/mcp/entwurf-bridge/dist/pi-extensions/lib/acp/backend-adapter.js +34 -10
- package/package.json +10 -10
- package/pi-extensions/lib/acp/acp-client.ts +57 -4
- package/pi-extensions/lib/acp/backend-adapter.ts +78 -9
- package/pi-extensions/lib/acp/backend.ts +578 -18
- package/pi-extensions/lib/acp/claude-acp-launch.js +100 -0
- package/pi-extensions/lib/acp/event-mapper.ts +43 -6
- package/run.sh +129 -32
- package/scripts/check-acp-launch-namespace.ts +127 -0
- package/scripts/check-acp-prompt-lifecycle.ts +145 -2
- package/scripts/check-acp-stop-reason.ts +8 -2
- package/scripts/check-acp-usage-accounting.ts +1074 -0
- package/scripts/check-copilot-birth-hook.ts +28 -1
- package/scripts/check-gate-qualification.ts +4 -2
- package/scripts/check-omp-fresh-preflight.ts +27 -0
- package/scripts/check-setup-qualification.sh +40 -2
- package/scripts/copilot-bridge-oracle.sh +14 -6
- package/scripts/fake-copilot-vendor.sh +4 -2
- package/scripts/lib/pi-record-discovery.ts +47 -0
- package/scripts/mutants/acp-launch-namespace.json +34 -0
- package/scripts/mutants/acp-prompt-lifecycle.json +67 -2
- package/scripts/mutants/acp-stream-hooks.json +4 -2
- package/scripts/mutants/acp-usage-accounting.json +181 -0
- package/scripts/mutants/copilot-birth.json +3 -5
- package/scripts/mutants/pack-install.json +2 -2
- package/scripts/mutants/setup-verdict.json +35 -0
- package/scripts/omp-config-xdev.py +310 -0
- package/scripts/omp-config-xdev.sh +76 -0
- package/scripts/omp-tool-surface.py +61 -10
- package/scripts/raw-acp-child-exit-measure/README.md +285 -0
- package/scripts/raw-acp-child-exit-measure/acp-turn-population.py +89 -0
- package/scripts/raw-acp-child-exit-measure/reaper-correlation.py +47 -0
- package/scripts/smoke-acp-bundled-mcp-live.ts +2 -2
- package/scripts/smoke-acp-cortex-live.ts +2 -2
- package/scripts/smoke-acp-raw-turn-live.ts +1 -1
- package/scripts/smoke-acp-socket-citizen-live.ts +2 -2
- package/scripts/smoke-acp-v2-send-live.ts +2 -2
- package/scripts/smoke-entwurf-v2-matrix-live.ts +60 -10
- package/scripts/smoke-mux-lifecycle-live.ts +46 -2
- package/scripts/smoke-setup-verdict.sh +48 -3
|
@@ -0,0 +1,1074 @@
|
|
|
1
|
+
// Deterministic gate for the ACP USAGE ACCOUNTING contract (#93).
|
|
2
|
+
//
|
|
3
|
+
// WHAT THIS EXISTS TO STOP. A long-lived Claude ACP session's dashboard lied by
|
|
4
|
+
// 10-18x, measured on three independent live ledgers (oracle 18x / oracle 10.5x
|
|
5
|
+
// / thinkpad 14.4x, #93). Two defects produced that one number:
|
|
6
|
+
//
|
|
7
|
+
// 1. `PromptResponse.usage` — the only per-turn token carrier on the wire, and
|
|
8
|
+
// a ROUND-TRIP AGGREGATE rather than a per-request partition —
|
|
9
|
+
// was erased at the type boundary, so every ACP assistant message reported
|
|
10
|
+
// input/output/cacheRead/cacheWrite as 0. The cache-efficiency badge the
|
|
11
|
+
// operator wanted could not be computed at all: its inputs were zeroes.
|
|
12
|
+
// 2. The one number that DID survive, `usage_update.cost.amount`, is the
|
|
13
|
+
// backend's RUNNING SESSION TOTAL. It was assigned to a per-TURN field, and
|
|
14
|
+
// pi's footer sums per-turn costs. Summing a monotonically increasing series
|
|
15
|
+
// is how $24.261 was displayed as $444.370.
|
|
16
|
+
//
|
|
17
|
+
// The contract now:
|
|
18
|
+
//
|
|
19
|
+
// tokens are NOT written at all. ACP's only carrier is a per-turn round-trip
|
|
20
|
+
// AGGREGATE and pi's four fields mean ONE REQUEST's prompt shape;
|
|
21
|
+
// projecting one onto the other fired a false overflow (#93, measured
|
|
22
|
+
// 2026-09-02). Silence until the wire carries a per-request partition.
|
|
23
|
+
// cost is an ADJACENT DIFF of the backend's own running total, held on the
|
|
24
|
+
// BridgeSession. A missing total HOLDS the baseline (the amount lands
|
|
25
|
+
// in the next diff — misattributed by turn, exact by session). A
|
|
26
|
+
// DECREASING total rebaselines, attributes $0, and SAYS SO.
|
|
27
|
+
// totalTokens is CONTEXT OCCUPANCY, not a turn total, and is never overwritten
|
|
28
|
+
// with the turn aggregate — pi reads that field as the context gauge.
|
|
29
|
+
// a backend with no measured semantics (no sealsTurnAccounting) is sealed NOT AT
|
|
30
|
+
// ALL: cortex's emitted usage must be byte-identical to pre-#93.
|
|
31
|
+
//
|
|
32
|
+
// ORACLES, each independent of the subject:
|
|
33
|
+
// - the cost sum: THIS FILE constructs the cumulative series and asserts the
|
|
34
|
+
// emitted per-turn costs sum back to the final cumulative. That arithmetic is
|
|
35
|
+
// never performed by backend.ts.
|
|
36
|
+
// - the context gauge: pi's REAL `calculateContextTokens`, imported from
|
|
37
|
+
// @earendil-works/pi-coding-agent — the function that actually drives the
|
|
38
|
+
// footer percentage and auto-compaction, not our restatement of it.
|
|
39
|
+
// - the overflow verdict: pi's REAL `isContextOverflow`, imported from
|
|
40
|
+
// @earendil-works/pi-ai — the function that actually compacted the live
|
|
41
|
+
// session — driven by the incident's own numbers to scale.
|
|
42
|
+
//
|
|
43
|
+
// backend.ts imports its siblings with `.js` suffixes (the root/jiti runtime
|
|
44
|
+
// convention), so — like check-acp-prompt-lifecycle — we tsc-emit the project and
|
|
45
|
+
// import the COMPILED backend.js whose `.js` imports resolve to real siblings.
|
|
46
|
+
|
|
47
|
+
import { strict as assert } from "node:assert";
|
|
48
|
+
import { execFileSync } from "node:child_process";
|
|
49
|
+
import { copyFileSync, mkdirSync, mkdtempSync, rmdirSync, rmSync } from "node:fs";
|
|
50
|
+
import { tmpdir } from "node:os";
|
|
51
|
+
import { resolve } from "node:path";
|
|
52
|
+
import { pathToFileURL } from "node:url";
|
|
53
|
+
import type { Api, AssistantMessage, AssistantMessageEvent, Context, Message, Model } from "@earendil-works/pi-ai";
|
|
54
|
+
import { isContextOverflow } from "@earendil-works/pi-ai";
|
|
55
|
+
import { calculateContextTokens } from "@earendil-works/pi-coding-agent";
|
|
56
|
+
|
|
57
|
+
const claude = { id: "claude-sonnet-5" } as unknown as Model<Api>;
|
|
58
|
+
const cortex = { id: "cortex-auto" } as unknown as Model<Api>;
|
|
59
|
+
|
|
60
|
+
type Stream = AsyncIterable<AssistantMessageEvent>;
|
|
61
|
+
|
|
62
|
+
/** The wire shape of one ACP `PromptResponse.usage` (SDK `Usage`). */
|
|
63
|
+
interface WireUsage {
|
|
64
|
+
totalTokens: number;
|
|
65
|
+
inputTokens: number;
|
|
66
|
+
outputTokens: number;
|
|
67
|
+
cachedReadTokens?: number;
|
|
68
|
+
cachedWriteTokens?: number;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* One `_meta.quota.token_count` row. The field NAMES deliberately differ from
|
|
73
|
+
* `PromptResponse.usage`: cache reads are `cachedInputTokens` here because the
|
|
74
|
+
* shape is shared with codex-acp, and `cachedWriteTokens` is Claude's extra
|
|
75
|
+
* sibling (read at claude-agent-acp 0.73.0 `dist/acp-agent.js:5750-5765`).
|
|
76
|
+
* Reading a quota row with the `usage` field names silently yields zeros, so the
|
|
77
|
+
* fixture below spells the vendor's names out rather than reusing `WireUsage`.
|
|
78
|
+
*/
|
|
79
|
+
interface QuotaTokenCount {
|
|
80
|
+
totalTokens?: number;
|
|
81
|
+
inputTokens?: number;
|
|
82
|
+
cachedInputTokens?: number;
|
|
83
|
+
cachedWriteTokens?: number;
|
|
84
|
+
outputTokens?: number;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** What ONE turn of the fake backend reports. */
|
|
88
|
+
interface TurnScript {
|
|
89
|
+
usage?: WireUsage;
|
|
90
|
+
/** `_meta.quota.model_usage` — the vendor's ACCOUNTING-grade per-model totals,
|
|
91
|
+
* which also count Task subagents, sidechains and internal calls such as
|
|
92
|
+
* compaction. Present from claude-agent-acp 0.71.0 (`turnQuotaMeta`). */
|
|
93
|
+
modelUsage?: Array<{ model: string; token_count: QuotaTokenCount }>;
|
|
94
|
+
/** the backend's RUNNING SESSION TOTAL at the end of this turn, in USD */
|
|
95
|
+
cumulativeCostUsd?: number;
|
|
96
|
+
/** post-turn context occupancy (`usage_update.used`) */
|
|
97
|
+
occupancyTokens?: number;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const EMPTY_MCP_HASH = "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945";
|
|
101
|
+
const DEFAULT_RESOLVED_CONFIG: any = {
|
|
102
|
+
settingSources: [],
|
|
103
|
+
strictMcpConfig: true,
|
|
104
|
+
showToolNotifications: true,
|
|
105
|
+
mcpServers: [],
|
|
106
|
+
mcpServersHash: EMPTY_MCP_HASH,
|
|
107
|
+
tools: ["Read"],
|
|
108
|
+
skillPlugins: [],
|
|
109
|
+
permissionAllow: ["Read(*)"],
|
|
110
|
+
disallowedTools: [],
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
function makeFakeChild() {
|
|
114
|
+
const pipe = () => ({ destroy() {}, unref() {} });
|
|
115
|
+
return {
|
|
116
|
+
pid: 4242,
|
|
117
|
+
exitCode: null as number | null,
|
|
118
|
+
signalCode: null as NodeJS.Signals | null,
|
|
119
|
+
stdin: pipe(),
|
|
120
|
+
stdout: pipe(),
|
|
121
|
+
stderr: {
|
|
122
|
+
on() {},
|
|
123
|
+
once() {},
|
|
124
|
+
destroy() {},
|
|
125
|
+
unref() {},
|
|
126
|
+
},
|
|
127
|
+
kill() {
|
|
128
|
+
return true;
|
|
129
|
+
},
|
|
130
|
+
unref() {},
|
|
131
|
+
once() {},
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* One backend world whose turns are SCRIPTED.
|
|
137
|
+
*
|
|
138
|
+
* Each `prompt` call consumes the next TurnScript: it first pushes that turn's
|
|
139
|
+
* `usage_update` notification (the wire the running cost total actually arrives
|
|
140
|
+
* on — read at claude-agent-acp 0.73.0 `dist/acp-agent.js:2918-2933`), then answers the
|
|
141
|
+
* prompt with that turn's `PromptResponse.usage` (the wire the turn aggregate
|
|
142
|
+
* arrives on). Both orderings are the real one: the notification precedes the
|
|
143
|
+
* response, because the SDK emits it from the `result` message that ENDS the turn.
|
|
144
|
+
*/
|
|
145
|
+
function makeHarness(recordDir: string, scripts: TurnScript[]) {
|
|
146
|
+
const children: ReturnType<typeof makeFakeChild>[] = [];
|
|
147
|
+
let turnIndex = 0;
|
|
148
|
+
let newSessionCalls = 0;
|
|
149
|
+
|
|
150
|
+
const makeConnection = (handlers: any) => ({
|
|
151
|
+
initialize: async () => ({ agentCapabilities: {} }),
|
|
152
|
+
newSession: async () => {
|
|
153
|
+
newSessionCalls++;
|
|
154
|
+
return { sessionId: "ACP-1" };
|
|
155
|
+
},
|
|
156
|
+
setSessionConfigOption: async () => ({}),
|
|
157
|
+
prompt: async ({ sessionId }: any) => {
|
|
158
|
+
const script = scripts[turnIndex++] ?? {};
|
|
159
|
+
const update: Record<string, unknown> = { sessionUpdate: "usage_update" };
|
|
160
|
+
let notify = false;
|
|
161
|
+
if (typeof script.occupancyTokens === "number") {
|
|
162
|
+
update.used = script.occupancyTokens;
|
|
163
|
+
notify = true;
|
|
164
|
+
}
|
|
165
|
+
if (typeof script.cumulativeCostUsd === "number") {
|
|
166
|
+
update.cost = { amount: script.cumulativeCostUsd, currency: "USD" };
|
|
167
|
+
notify = true;
|
|
168
|
+
}
|
|
169
|
+
if (notify) await handlers.sessionUpdate({ update, sessionId });
|
|
170
|
+
return {
|
|
171
|
+
stopReason: "end_turn",
|
|
172
|
+
...(script.usage ? { usage: script.usage } : {}),
|
|
173
|
+
...(script.modelUsage ? { _meta: { quota: { model_usage: script.modelUsage } } } : {}),
|
|
174
|
+
};
|
|
175
|
+
},
|
|
176
|
+
cancel: () => {},
|
|
177
|
+
close: () => {},
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
return {
|
|
181
|
+
children,
|
|
182
|
+
get turnsPrompted() {
|
|
183
|
+
return turnIndex;
|
|
184
|
+
},
|
|
185
|
+
get newSessionCalls() {
|
|
186
|
+
return newSessionCalls;
|
|
187
|
+
},
|
|
188
|
+
deps: {
|
|
189
|
+
resolveLaunch: () => ({ command: "node", args: ["fake"] }),
|
|
190
|
+
ensureOverlay: () => {},
|
|
191
|
+
spawnChild: () => {
|
|
192
|
+
const c = makeFakeChild();
|
|
193
|
+
children.push(c);
|
|
194
|
+
return c;
|
|
195
|
+
},
|
|
196
|
+
createConnection: (_child: any, handlers: any) => makeConnection(handlers),
|
|
197
|
+
lifecyclePolicy: () => "process-scoped",
|
|
198
|
+
loadConfig: () => DEFAULT_RESOLVED_CONFIG,
|
|
199
|
+
now: () => "2026-09-01T00:00:00Z",
|
|
200
|
+
sessionDir: recordDir,
|
|
201
|
+
},
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
async function collect(stream: Stream): Promise<AssistantMessageEvent[]> {
|
|
206
|
+
const events: AssistantMessageEvent[] = [];
|
|
207
|
+
for await (const ev of stream) events.push(ev);
|
|
208
|
+
return events;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/** The sealed assistant message of a finished turn. */
|
|
212
|
+
function sealedMessage(events: AssistantMessageEvent[]): AssistantMessage {
|
|
213
|
+
const seal = events.filter((e) => e.type === "done" || e.type === "error") as any[];
|
|
214
|
+
assert.equal(seal.length, 1, "each turn seals exactly once");
|
|
215
|
+
assert.equal(seal[0].type, "done", `the scripted turn must end as a normal done, got ${seal[0].type}`);
|
|
216
|
+
return seal[0].message as AssistantMessage;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
const userCtx = (text: string): Context => ({ messages: [{ role: "user", content: text, timestamp: 0 }] }) as Context;
|
|
220
|
+
|
|
221
|
+
const zeroUsage = () => ({
|
|
222
|
+
input: 0,
|
|
223
|
+
output: 0,
|
|
224
|
+
cacheRead: 0,
|
|
225
|
+
cacheWrite: 0,
|
|
226
|
+
totalTokens: 0,
|
|
227
|
+
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
/** A reuse-shaped context: prior user, assistant, new user. */
|
|
231
|
+
function reuseCtx(prior: string, latest: string): Context {
|
|
232
|
+
return {
|
|
233
|
+
messages: [
|
|
234
|
+
{ role: "user", content: prior, timestamp: 0 },
|
|
235
|
+
{
|
|
236
|
+
role: "assistant",
|
|
237
|
+
content: [{ type: "text", text: "ok" }],
|
|
238
|
+
api: "x",
|
|
239
|
+
provider: "x",
|
|
240
|
+
model: "x",
|
|
241
|
+
usage: zeroUsage(),
|
|
242
|
+
stopReason: "stop",
|
|
243
|
+
timestamp: 0,
|
|
244
|
+
} as unknown as Message,
|
|
245
|
+
{ role: "user", content: latest, timestamp: 0 },
|
|
246
|
+
],
|
|
247
|
+
} as Context;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/** A third turn on the same reused session. */
|
|
251
|
+
function reuseCtx3(a: string, b: string, c: string): Context {
|
|
252
|
+
const assistant = {
|
|
253
|
+
role: "assistant",
|
|
254
|
+
content: [{ type: "text", text: "ok" }],
|
|
255
|
+
api: "x",
|
|
256
|
+
provider: "x",
|
|
257
|
+
model: "x",
|
|
258
|
+
usage: zeroUsage(),
|
|
259
|
+
stopReason: "stop",
|
|
260
|
+
timestamp: 0,
|
|
261
|
+
} as unknown as Message;
|
|
262
|
+
return {
|
|
263
|
+
messages: [
|
|
264
|
+
{ role: "user", content: a, timestamp: 0 },
|
|
265
|
+
assistant,
|
|
266
|
+
{ role: "user", content: b, timestamp: 0 },
|
|
267
|
+
assistant,
|
|
268
|
+
{ role: "user", content: c, timestamp: 0 },
|
|
269
|
+
],
|
|
270
|
+
} as Context;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
const TMP_EMIT = ".tmp-verify/acp-usage-accounting";
|
|
274
|
+
rmSync(TMP_EMIT, { recursive: true, force: true });
|
|
275
|
+
const recordDir = mkdtempSync(resolve(tmpdir(), "acp-usage-acct-"));
|
|
276
|
+
|
|
277
|
+
try {
|
|
278
|
+
execFileSync("node_modules/.bin/tsc", ["--outDir", TMP_EMIT, "--rootDir", ".", "--noEmit", "false"], {
|
|
279
|
+
stdio: "pipe",
|
|
280
|
+
});
|
|
281
|
+
const promptsOut = resolve(TMP_EMIT, "pi-extensions/lib/acp/prompts");
|
|
282
|
+
mkdirSync(promptsOut, { recursive: true });
|
|
283
|
+
copyFileSync("pi-extensions/lib/acp/prompts/engraving.md", resolve(promptsOut, "engraving.md"));
|
|
284
|
+
const backend = (await import(pathToFileURL(resolve(TMP_EMIT, "pi-extensions/lib/acp/backend.js")).href)) as any;
|
|
285
|
+
|
|
286
|
+
// ----------------------------------------------------------------------
|
|
287
|
+
// CELL 1 — the turn aggregate never reaches pi's per-request fields.
|
|
288
|
+
//
|
|
289
|
+
// This is the incident's shape: the four ACP totals belong on `usage.acp`, not
|
|
290
|
+
// on pi's request-shaped fields. Four distinct non-zero values keep a mutant
|
|
291
|
+
// from passing by cross-wiring fields; the incident-scale aggregate stays a
|
|
292
|
+
// deterministic receipt, while the live ledger remains a separate receipt.
|
|
293
|
+
// ----------------------------------------------------------------------
|
|
294
|
+
{
|
|
295
|
+
// The 2026-09-02 incident, to scale: ACP reported the turn's ROUND-TRIP
|
|
296
|
+
// AGGREGATE (cacheRead 4,185,084 over 21 requests) while the context that
|
|
297
|
+
// turn actually occupied was 223,516 on a 1,000,000 window.
|
|
298
|
+
const h = makeHarness(recordDir, [
|
|
299
|
+
{
|
|
300
|
+
usage: {
|
|
301
|
+
inputTokens: 42,
|
|
302
|
+
outputTokens: 17_676,
|
|
303
|
+
cachedReadTokens: 4_185_084,
|
|
304
|
+
cachedWriteTokens: 221_084,
|
|
305
|
+
totalTokens: 4_423_886,
|
|
306
|
+
},
|
|
307
|
+
cumulativeCostUsd: 1.5,
|
|
308
|
+
occupancyTokens: 223_516,
|
|
309
|
+
},
|
|
310
|
+
]);
|
|
311
|
+
const msg = sealedMessage(
|
|
312
|
+
await collect(backend.streamAcpTurn(claude, userCtx("turn one"), { sessionId: "usage-A" }, h.deps) as Stream),
|
|
313
|
+
);
|
|
314
|
+
|
|
315
|
+
// pi's REAL overflow detector — the function that actually fired on the live
|
|
316
|
+
// session — is the oracle for this claim, not our restatement of its
|
|
317
|
+
// arithmetic, and it is asserted FIRST so that a re-projected aggregate is
|
|
318
|
+
// caught BY THIS CLAIM rather than incidentally by the field comparison
|
|
319
|
+
// below. It reads `input + cacheRead` RAW and never consults totalTokens, so
|
|
320
|
+
// an honest totalTokens cannot rescue a projected aggregate.
|
|
321
|
+
assert.equal(
|
|
322
|
+
isContextOverflow(msg, 1_000_000),
|
|
323
|
+
false,
|
|
324
|
+
"[QK:ACP-TURN-AGGREGATE-NOT-PROJECTED] pi's own isContextOverflow must not fire on a turn whose context " +
|
|
325
|
+
"occupied 223,516 of a 1,000,000 window. With the aggregate projected it read 42 + 4,185,084 and " +
|
|
326
|
+
`compacted the session — the defect this cell exists to keep dead. Got usage: ${JSON.stringify(msg.usage)}`,
|
|
327
|
+
);
|
|
328
|
+
|
|
329
|
+
assert.deepEqual(
|
|
330
|
+
{
|
|
331
|
+
input: msg.usage.input,
|
|
332
|
+
output: msg.usage.output,
|
|
333
|
+
cacheRead: msg.usage.cacheRead,
|
|
334
|
+
cacheWrite: msg.usage.cacheWrite,
|
|
335
|
+
},
|
|
336
|
+
{ input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
337
|
+
"ACP's only token carrier is the SUM over this turn's API round " +
|
|
338
|
+
"trips; pi's four fields mean ONE REQUEST's prompt shape. Projecting the aggregate onto them is what " +
|
|
339
|
+
"compacted a live 223k session on a 1M window, invented two phantom cache misses and silenced the one " +
|
|
340
|
+
`real 195,177-token miss. No honest per-request value exists on this wire, so all four stay 0. Got: ${JSON.stringify(msg.usage)}`,
|
|
341
|
+
);
|
|
342
|
+
|
|
343
|
+
// pi's OWN reader of this message decides the context gauge and auto-compaction.
|
|
344
|
+
assert.equal(
|
|
345
|
+
calculateContextTokens(msg.usage),
|
|
346
|
+
223_516,
|
|
347
|
+
"[QK:ACP-CONTEXT-OCCUPANCY-PRESERVED] `totalTokens` is what pi reads as CONTEXT OCCUPANCY " +
|
|
348
|
+
"(calculateContextTokens returns it whenever it is non-zero), and the backend reports occupancy on " +
|
|
349
|
+
"usage_update.used — 223516 here. Overwriting it with this turn's aggregate (4423886) would send the " +
|
|
350
|
+
"status-line percentage and auto-compaction reading past the window on a session that fits inside it. " +
|
|
351
|
+
`Got: ${calculateContextTokens(msg.usage)}`,
|
|
352
|
+
);
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
// ----------------------------------------------------------------------
|
|
356
|
+
// CELL 1b — the turn's ACCOUNTING aggregate rides its own key.
|
|
357
|
+
//
|
|
358
|
+
// pi's four fields stay 0 (CELL 1); the vendor's four numbers must still reach
|
|
359
|
+
// the operator, on a key no pi reader mistakes for a per-request shape.
|
|
360
|
+
// ----------------------------------------------------------------------
|
|
361
|
+
{
|
|
362
|
+
const h = makeHarness(recordDir, [
|
|
363
|
+
{
|
|
364
|
+
usage: {
|
|
365
|
+
inputTokens: 42,
|
|
366
|
+
outputTokens: 17_676,
|
|
367
|
+
cachedReadTokens: 4_185_084,
|
|
368
|
+
cachedWriteTokens: 221_084,
|
|
369
|
+
totalTokens: 4_423_886,
|
|
370
|
+
},
|
|
371
|
+
cumulativeCostUsd: 1.5,
|
|
372
|
+
occupancyTokens: 223_516,
|
|
373
|
+
},
|
|
374
|
+
]);
|
|
375
|
+
const msg = sealedMessage(
|
|
376
|
+
await collect(backend.streamAcpTurn(claude, userCtx("turn one"), { sessionId: "usage-A2" }, h.deps) as Stream),
|
|
377
|
+
);
|
|
378
|
+
assert.deepEqual(
|
|
379
|
+
(msg.usage as unknown as { acp?: unknown }).acp,
|
|
380
|
+
{ input: 42, output: 17_676, cacheRead: 4_185_084, cacheWrite: 221_084 },
|
|
381
|
+
"[QK:ACP-TURN-ACCOUNTING-ATTACHED] the vendor's four turn totals must reach the message VERBATIM on their own " +
|
|
382
|
+
"key. Dropping them is the silence #93 refused: a session runs for hours on a cache-effect badge while a " +
|
|
383
|
+
"full prefix rewrite has already been paid for, and nothing says so. " +
|
|
384
|
+
`Got: ${JSON.stringify((msg.usage as unknown as { acp?: unknown }).acp)}`,
|
|
385
|
+
);
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
// ----------------------------------------------------------------------
|
|
389
|
+
// CELL 1e — the ACCOUNTING-GRADE numerator is PREFERRED over the main-loop one.
|
|
390
|
+
//
|
|
391
|
+
// Two token carriers arrive on the same response and they measure DIFFERENT
|
|
392
|
+
// scopes. `PromptResponse.usage` (== `_meta.quota.token_count`) is the MAIN
|
|
393
|
+
// AGENT LOOP only. `_meta.quota.model_usage` comes from `result.modelUsage` and
|
|
394
|
+
// also counts Task subagents, sidechains and INTERNAL CALLS SUCH AS COMPACTION;
|
|
395
|
+
// the vendor states its rows "can total more than `token_count`" and are "the
|
|
396
|
+
// fuller picture, not a decomposition of it" (read at claude-agent-acp 0.73.0
|
|
397
|
+
// `dist/acp-agent.js:5732-5738`).
|
|
398
|
+
//
|
|
399
|
+
// The wide one is required, not merely nicer, because the DENOMINATOR already
|
|
400
|
+
// has that scope: turn cost is the adjacent diff of the backend's running total,
|
|
401
|
+
// which includes those internal calls. Pairing a main-loop numerator with an
|
|
402
|
+
// all-inclusive denominator understates the cache-effect badge EXACTLY when
|
|
403
|
+
// compaction ran — and compaction is a live path again (#94). A silent fallback
|
|
404
|
+
// to the narrow carrier is therefore a misaccounting, not a degraded reading.
|
|
405
|
+
//
|
|
406
|
+
// The fixture drives BOTH carriers at once with deliberately different numbers,
|
|
407
|
+
// so a fallback cannot pass for a preference, and it supplies TWO rows whose sum
|
|
408
|
+
// matches neither row alone — so dropping the row summation fails here too.
|
|
409
|
+
// Row field names are the vendor's (`cachedInputTokens`), which is also why
|
|
410
|
+
// reading a quota row with the `usage` names would surface as zeros right here.
|
|
411
|
+
// ----------------------------------------------------------------------
|
|
412
|
+
{
|
|
413
|
+
const h = makeHarness(recordDir, [
|
|
414
|
+
{
|
|
415
|
+
// main-loop only — what a fallback would report
|
|
416
|
+
usage: {
|
|
417
|
+
inputTokens: 42,
|
|
418
|
+
outputTokens: 17_676,
|
|
419
|
+
cachedReadTokens: 4_185_084,
|
|
420
|
+
cachedWriteTokens: 221_084,
|
|
421
|
+
totalTokens: 4_423_886,
|
|
422
|
+
},
|
|
423
|
+
// accounting-grade — main loop PLUS an internal/compaction call
|
|
424
|
+
modelUsage: [
|
|
425
|
+
{
|
|
426
|
+
model: "claude-sonnet-5",
|
|
427
|
+
token_count: {
|
|
428
|
+
inputTokens: 42,
|
|
429
|
+
outputTokens: 17_676,
|
|
430
|
+
cachedInputTokens: 4_185_084,
|
|
431
|
+
cachedWriteTokens: 221_084,
|
|
432
|
+
},
|
|
433
|
+
},
|
|
434
|
+
{
|
|
435
|
+
model: "claude-haiku-5",
|
|
436
|
+
token_count: {
|
|
437
|
+
inputTokens: 8,
|
|
438
|
+
outputTokens: 1_324,
|
|
439
|
+
cachedInputTokens: 96_000,
|
|
440
|
+
cachedWriteTokens: 12_000,
|
|
441
|
+
},
|
|
442
|
+
},
|
|
443
|
+
],
|
|
444
|
+
cumulativeCostUsd: 1.5,
|
|
445
|
+
occupancyTokens: 223_516,
|
|
446
|
+
},
|
|
447
|
+
]);
|
|
448
|
+
const msg = sealedMessage(
|
|
449
|
+
await collect(backend.streamAcpTurn(claude, userCtx("turn one"), { sessionId: "usage-A5" }, h.deps) as Stream),
|
|
450
|
+
);
|
|
451
|
+
assert.deepEqual(
|
|
452
|
+
(msg.usage as unknown as { acp?: unknown }).acp,
|
|
453
|
+
{ input: 50, output: 19_000, cacheRead: 4_281_084, cacheWrite: 233_084 },
|
|
454
|
+
"[QK:ACP-ACCOUNTING-PREFERS-WIDEST] the accounting-grade `_meta.quota.model_usage` rows must be SUMMED and " +
|
|
455
|
+
"preferred over the main-loop `PromptResponse.usage`. Falling back to the narrow carrier while turn cost " +
|
|
456
|
+
"stays an all-inclusive diff understates the cache-effect badge exactly when compaction or a subagent ran — " +
|
|
457
|
+
"a silent misaccounting of the same class #93 exists to end. The main-loop-only answer would be " +
|
|
458
|
+
`{input:42,output:17676,cacheRead:4185084,cacheWrite:221084}. Got: ${JSON.stringify((msg.usage as unknown as { acp?: unknown }).acp)}`,
|
|
459
|
+
);
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
// ----------------------------------------------------------------------
|
|
463
|
+
// CELL 1c — a re-billed prefix is REPORTED, and its size is a proven bound.
|
|
464
|
+
//
|
|
465
|
+
// Both turns are the 2026-09-01 incident's own aggregates as entwurf saw them.
|
|
466
|
+
// Turn A ends at occupancy 195,627; turn B is the one after the 401-minute idle
|
|
467
|
+
// gap. Two ledger numbers describe that first request and they are NOT the same
|
|
468
|
+
// quantity: it WROTE 195,814 (cache creation) and it MISSED 195,177 (the cache
|
|
469
|
+
// read it would have got had the prefix survived — the identity break). They
|
|
470
|
+
// differ because the prompt grew a little between the two turns. The bound:
|
|
471
|
+
// 195,627 − (4 + 1,220) − max(0, 223,516 − 221,084) = 191,971
|
|
472
|
+
// is compared against the MISS, 195,177 — so the
|
|
473
|
+
// bound is BELOW the truth, which is what makes it a bound and not a guess.
|
|
474
|
+
// ----------------------------------------------------------------------
|
|
475
|
+
{
|
|
476
|
+
const h = makeHarness(recordDir, [
|
|
477
|
+
{
|
|
478
|
+
usage: {
|
|
479
|
+
inputTokens: 4,
|
|
480
|
+
outputTokens: 1_220,
|
|
481
|
+
cachedReadTokens: 387_477,
|
|
482
|
+
cachedWriteTokens: 2_085,
|
|
483
|
+
totalTokens: 390_786,
|
|
484
|
+
},
|
|
485
|
+
cumulativeCostUsd: 10,
|
|
486
|
+
occupancyTokens: 195_627,
|
|
487
|
+
},
|
|
488
|
+
{
|
|
489
|
+
usage: {
|
|
490
|
+
inputTokens: 42,
|
|
491
|
+
outputTokens: 17_676,
|
|
492
|
+
cachedReadTokens: 4_185_084,
|
|
493
|
+
cachedWriteTokens: 221_084,
|
|
494
|
+
totalTokens: 4_423_886,
|
|
495
|
+
},
|
|
496
|
+
cumulativeCostUsd: 14.745,
|
|
497
|
+
occupancyTokens: 223_516,
|
|
498
|
+
},
|
|
499
|
+
]);
|
|
500
|
+
const warm = sealedMessage(
|
|
501
|
+
await collect(backend.streamAcpTurn(claude, userCtx("warm NONCE"), { sessionId: "usage-M" }, h.deps) as Stream),
|
|
502
|
+
);
|
|
503
|
+
const afterGap = sealedMessage(
|
|
504
|
+
await collect(
|
|
505
|
+
backend.streamAcpTurn(
|
|
506
|
+
claude,
|
|
507
|
+
reuseCtx("warm NONCE", "after gap NONCE"),
|
|
508
|
+
{ sessionId: "usage-M" },
|
|
509
|
+
h.deps,
|
|
510
|
+
) as Stream,
|
|
511
|
+
),
|
|
512
|
+
);
|
|
513
|
+
assert.equal(h.children.length, 1, "both turns ran on ONE reused child — the prior-turn facts must survive it");
|
|
514
|
+
|
|
515
|
+
const textOf = (m: AssistantMessage): string => m.content.map((c) => (c.type === "text" ? c.text : "")).join("");
|
|
516
|
+
|
|
517
|
+
assert.ok(
|
|
518
|
+
!/cache miss/.test(textOf(warm)),
|
|
519
|
+
"the FIRST turn has no previous occupancy to compare against, so it must say nothing. A notice here would " +
|
|
520
|
+
`mean the bound fires on an unknown, which is a guess. Got: ${JSON.stringify(textOf(warm))}`,
|
|
521
|
+
);
|
|
522
|
+
assert.match(
|
|
523
|
+
textOf(afterGap),
|
|
524
|
+
/cache miss ≥192k re-billed/,
|
|
525
|
+
"[QK:ACP-CACHE-REBILL-REPORTED] a re-billed prefix must be SAID. Silence is not a modest reading, it is a " +
|
|
526
|
+
"false one — this turn's first request wrote 195,814 tokens of cache after a 401-minute idle gap, missing " +
|
|
527
|
+
"a 195,177-token read it would otherwise have had, and the operator saw nothing. The bound (191,971) " +
|
|
528
|
+
`rounds to 192k and sits below that miss, which is what makes it a bound. Got: ${JSON.stringify(textOf(afterGap))}`,
|
|
529
|
+
);
|
|
530
|
+
// $4.745 renders as $4.74, and the reason is worth stating so a future reader
|
|
531
|
+
// does not "fix" it: `toFixed(2)` rounds the Number it receives, while the
|
|
532
|
+
// adjacent subtraction is 4.744999999999999 rather than decimal 4.745.
|
|
533
|
+
// Money is shown to the cent; a sub-cent tail is not worth a wider field, and
|
|
534
|
+
// no number the backend never reported is invented either way.
|
|
535
|
+
assert.ok(
|
|
536
|
+
/this turn \$4\.74/.test(textOf(afterGap)),
|
|
537
|
+
"the notice must quote what the turn cost, taken from the SDK's own adjacent diff (14.745 − 10 = 4.745) and " +
|
|
538
|
+
`never from a local repricing. Got: ${JSON.stringify(textOf(afterGap))}`,
|
|
539
|
+
);
|
|
540
|
+
// The idle clause needs a real gap; a same-process gate cannot age the clock,
|
|
541
|
+
// so its absence here is correct and the 401m form stays a LIVE observation.
|
|
542
|
+
assert.ok(
|
|
543
|
+
!/0m idle/.test(textOf(afterGap)),
|
|
544
|
+
`a sub-minute gap must not be announced as idle. Got: ${JSON.stringify(textOf(afterGap))}`,
|
|
545
|
+
);
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
// ----------------------------------------------------------------------
|
|
549
|
+
// CELL 1d — NEGATIVE CONTROLS for the re-billed claim.
|
|
550
|
+
//
|
|
551
|
+
// The bound is arithmetic over four numbers; these are the inputs that would
|
|
552
|
+
// make it lie. Each one is reachable on this rail, not a thought experiment.
|
|
553
|
+
// ----------------------------------------------------------------------
|
|
554
|
+
{
|
|
555
|
+
// (i) A context SHRINK — organic compaction inside the child. Occupancy
|
|
556
|
+
// collapses while almost nothing is written. The raw bound reads 180,000;
|
|
557
|
+
// the clamp answers 1,000, which is under the floor, so NOTHING is said.
|
|
558
|
+
// (counterexample from gpt-5.6-sol, 2026-09-02)
|
|
559
|
+
const h = makeHarness(recordDir, [
|
|
560
|
+
{
|
|
561
|
+
usage: {
|
|
562
|
+
inputTokens: 500,
|
|
563
|
+
outputTokens: 500,
|
|
564
|
+
cachedReadTokens: 0,
|
|
565
|
+
cachedWriteTokens: 2_000,
|
|
566
|
+
totalTokens: 3_000,
|
|
567
|
+
},
|
|
568
|
+
cumulativeCostUsd: 1,
|
|
569
|
+
occupancyTokens: 200_000,
|
|
570
|
+
},
|
|
571
|
+
{
|
|
572
|
+
usage: {
|
|
573
|
+
inputTokens: 100,
|
|
574
|
+
outputTokens: 400,
|
|
575
|
+
cachedReadTokens: 0,
|
|
576
|
+
cachedWriteTokens: 1_000,
|
|
577
|
+
totalTokens: 1_500,
|
|
578
|
+
},
|
|
579
|
+
cumulativeCostUsd: 2,
|
|
580
|
+
occupancyTokens: 20_000,
|
|
581
|
+
},
|
|
582
|
+
]);
|
|
583
|
+
const textOf = (m: AssistantMessage): string => m.content.map((c) => (c.type === "text" ? c.text : "")).join("");
|
|
584
|
+
await collect(backend.streamAcpTurn(claude, userCtx("big NONCE"), { sessionId: "usage-N" }, h.deps) as Stream);
|
|
585
|
+
const shrunk = sealedMessage(
|
|
586
|
+
await collect(
|
|
587
|
+
backend.streamAcpTurn(
|
|
588
|
+
claude,
|
|
589
|
+
reuseCtx("big NONCE", "shrunk NONCE"),
|
|
590
|
+
{ sessionId: "usage-N" },
|
|
591
|
+
h.deps,
|
|
592
|
+
) as Stream,
|
|
593
|
+
),
|
|
594
|
+
);
|
|
595
|
+
assert.ok(
|
|
596
|
+
!/cache miss/.test(textOf(shrunk)),
|
|
597
|
+
"[QK:ACP-REBILL-NEVER-EXCEEDS-WRITE] a turn that WROTE 1,000 tokens cannot have re-billed 180,000. The raw " +
|
|
598
|
+
"telescoped bound says 200,000 − 1,000 − 19,000 = 180,000 because occupancy collapsed, but a re-billed " +
|
|
599
|
+
"prefix is paid for as cache creation, so the claim is capped by this turn's own cacheWrite. Without " +
|
|
600
|
+
`that cap an organic compaction announces a six-figure miss that never happened. Got: ${JSON.stringify(textOf(shrunk))}`,
|
|
601
|
+
);
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
// ----------------------------------------------------------------------
|
|
605
|
+
// CELL 1f — the bound's IO terms are MAIN-LOOP, never the wide accounting rows.
|
|
606
|
+
//
|
|
607
|
+
// Occupancy (`usage_update.used`) is MAIN-CONTEXT occupancy. The recurrence
|
|
608
|
+
// identity that produces the bound is a property of the MAIN AGENT LOOP's
|
|
609
|
+
// cache breakpoints. `_meta.quota.model_usage` is a WIDER scope: the vendor
|
|
610
|
+
// states those rows also count Task subagents, sidechains, and INTERNAL
|
|
611
|
+
// CALLS SUCH AS COMPACTION (read at claude-agent-acp 0.73.0
|
|
612
|
+
// `dist/acp-agent.js:5728-5748`). Mixing the two scopes inflates the bound
|
|
613
|
+
// through both remaining terms that mention cacheWrite:
|
|
614
|
+
// max(0, occupancy − cacheWrite) shrinks as wide cacheWrite grows, so
|
|
615
|
+
// less is subtracted; min(rawBound, cacheWrite) rises with it.
|
|
616
|
+
// Either path can push a WARM main prefix over the notice floor and attach
|
|
617
|
+
// this turn's all-inclusive dollar figure to a miss that did not happen
|
|
618
|
+
// on the prefix — the same class of false dollar assertion #93 exists to end.
|
|
619
|
+
//
|
|
620
|
+
// Constructed, not a live ledger. Turn 1 is a warm main context. Turn 2's
|
|
621
|
+
// MAIN LOOP writes 1,000 tokens of cache; the wide rows write 180,000
|
|
622
|
+
// because an internal/compaction call is included. Mixing those would
|
|
623
|
+
// announce "cache miss ≥174k" (200,000 − 1,000 − max(0, 205,000 − 180,000)
|
|
624
|
+
// = 174,000). The main-loop bound is negative, so nothing is said.
|
|
625
|
+
// `usage.acp` remains the WIDE totals — this cell does not walk back CELL 1e.
|
|
626
|
+
// ----------------------------------------------------------------------
|
|
627
|
+
{
|
|
628
|
+
const h = makeHarness(recordDir, [
|
|
629
|
+
{
|
|
630
|
+
usage: {
|
|
631
|
+
inputTokens: 500,
|
|
632
|
+
outputTokens: 500,
|
|
633
|
+
cachedReadTokens: 0,
|
|
634
|
+
cachedWriteTokens: 2_000,
|
|
635
|
+
totalTokens: 3_000,
|
|
636
|
+
},
|
|
637
|
+
cumulativeCostUsd: 1,
|
|
638
|
+
occupancyTokens: 200_000,
|
|
639
|
+
},
|
|
640
|
+
{
|
|
641
|
+
usage: {
|
|
642
|
+
inputTokens: 100,
|
|
643
|
+
outputTokens: 400,
|
|
644
|
+
cachedReadTokens: 0,
|
|
645
|
+
cachedWriteTokens: 1_000,
|
|
646
|
+
totalTokens: 1_500,
|
|
647
|
+
},
|
|
648
|
+
modelUsage: [
|
|
649
|
+
{
|
|
650
|
+
model: "claude-sonnet-5",
|
|
651
|
+
token_count: {
|
|
652
|
+
inputTokens: 100,
|
|
653
|
+
outputTokens: 400,
|
|
654
|
+
cachedInputTokens: 0,
|
|
655
|
+
cachedWriteTokens: 1_000,
|
|
656
|
+
},
|
|
657
|
+
},
|
|
658
|
+
{
|
|
659
|
+
model: "claude-haiku-5",
|
|
660
|
+
token_count: {
|
|
661
|
+
inputTokens: 8,
|
|
662
|
+
outputTokens: 1_324,
|
|
663
|
+
cachedInputTokens: 0,
|
|
664
|
+
cachedWriteTokens: 179_000,
|
|
665
|
+
},
|
|
666
|
+
},
|
|
667
|
+
],
|
|
668
|
+
cumulativeCostUsd: 5,
|
|
669
|
+
occupancyTokens: 205_000,
|
|
670
|
+
},
|
|
671
|
+
]);
|
|
672
|
+
const textOf = (m: AssistantMessage): string => m.content.map((c) => (c.type === "text" ? c.text : "")).join("");
|
|
673
|
+
await collect(
|
|
674
|
+
backend.streamAcpTurn(claude, userCtx("warm prefix NONCE"), { sessionId: "usage-S" }, h.deps) as Stream,
|
|
675
|
+
);
|
|
676
|
+
const mixed = sealedMessage(
|
|
677
|
+
await collect(
|
|
678
|
+
backend.streamAcpTurn(
|
|
679
|
+
claude,
|
|
680
|
+
reuseCtx("warm prefix NONCE", "compaction sidecar NONCE"),
|
|
681
|
+
{ sessionId: "usage-S" },
|
|
682
|
+
h.deps,
|
|
683
|
+
) as Stream,
|
|
684
|
+
),
|
|
685
|
+
);
|
|
686
|
+
assert.ok(
|
|
687
|
+
!/cache miss/.test(textOf(mixed)),
|
|
688
|
+
"[QK:ACP-REBILL-MAIN-LOOP-SCOPE] a warm main prefix that wrote 1,000 tokens of cache cannot announce a " +
|
|
689
|
+
"174k miss because an internal/compaction call wrote 179,000 more on the WIDE rows. Occupancy and the " +
|
|
690
|
+
"recurrence are main-loop; mixing them with wide cacheWrite inflates the bound through both remaining " +
|
|
691
|
+
"terms and attaches this turn's dollar figure to a miss that did not happen on the prefix. " +
|
|
692
|
+
`Got: ${JSON.stringify(textOf(mixed))}`,
|
|
693
|
+
);
|
|
694
|
+
assert.deepEqual(
|
|
695
|
+
(mixed.usage as unknown as { acp?: unknown }).acp,
|
|
696
|
+
{ input: 108, output: 1_724, cacheRead: 0, cacheWrite: 180_000 },
|
|
697
|
+
"…and usage.acp must still be the WIDE totals (CELL 1e). Narrowing the bound must not silently fall back " +
|
|
698
|
+
`the reporting numerator. Got: ${JSON.stringify((mixed.usage as unknown as { acp?: unknown }).acp)}`,
|
|
699
|
+
);
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
// ----------------------------------------------------------------------
|
|
703
|
+
// CELL 2 — two turns' costs SUM to the backend's session total.
|
|
704
|
+
//
|
|
705
|
+
// The series is cumulative and monotone, exactly as the live ledgers were.
|
|
706
|
+
// The subject never computes this sum; this file does.
|
|
707
|
+
// ----------------------------------------------------------------------
|
|
708
|
+
{
|
|
709
|
+
const C1 = 1.5;
|
|
710
|
+
const C2 = 2.75;
|
|
711
|
+
const h = makeHarness(recordDir, [
|
|
712
|
+
{ cumulativeCostUsd: C1, occupancyTokens: 100_000, usage: makeUsage(10) },
|
|
713
|
+
{ cumulativeCostUsd: C1 + C2, occupancyTokens: 180_000, usage: makeUsage(20) },
|
|
714
|
+
]);
|
|
715
|
+
const t1 = sealedMessage(
|
|
716
|
+
await collect(
|
|
717
|
+
backend.streamAcpTurn(claude, userCtx("first NONCE-1"), { sessionId: "usage-B" }, h.deps) as Stream,
|
|
718
|
+
),
|
|
719
|
+
);
|
|
720
|
+
const t2 = sealedMessage(
|
|
721
|
+
await collect(
|
|
722
|
+
backend.streamAcpTurn(
|
|
723
|
+
claude,
|
|
724
|
+
reuseCtx("first NONCE-1", "second NONCE-2"),
|
|
725
|
+
{ sessionId: "usage-B" },
|
|
726
|
+
h.deps,
|
|
727
|
+
) as Stream,
|
|
728
|
+
),
|
|
729
|
+
);
|
|
730
|
+
assert.equal(h.children.length, 1, "both turns ran on ONE reused child — the baseline must survive a reuse turn");
|
|
731
|
+
|
|
732
|
+
const perTurn = [t1.usage.cost.total, t2.usage.cost.total];
|
|
733
|
+
assert.equal(
|
|
734
|
+
round6(perTurn[0] + perTurn[1]),
|
|
735
|
+
round6(C1 + C2),
|
|
736
|
+
"[QK:ACP-TURN-COST-SUM-MATCHES-SDK] the backend reports a RUNNING SESSION TOTAL (its own ESTIMATE, not a bill), so a turn " +
|
|
737
|
+
"cost is the ADJACENT DIFF of consecutive totals — and the per-turn costs must then sum back to the " +
|
|
738
|
+
"final total. Assigning the cumulative to the turn field instead is the #93 defect: pi sums per-turn " +
|
|
739
|
+
`costs, which is how a $24.261 session was displayed as $444.370. Got per-turn ${JSON.stringify(perTurn)}`,
|
|
740
|
+
);
|
|
741
|
+
assert.deepEqual(
|
|
742
|
+
perTurn.map(round6),
|
|
743
|
+
[round6(C1), round6(C2)],
|
|
744
|
+
"…and each turn carries ITS OWN diff, not the running total",
|
|
745
|
+
);
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
// ----------------------------------------------------------------------
|
|
749
|
+
// CELL 3 — a turn with NO cost notification holds the baseline.
|
|
750
|
+
//
|
|
751
|
+
// Measured upstream: the result-path `usage_update` carries cost (read at
|
|
752
|
+
// claude-agent-acp 0.73.0 `dist/acp-agent.js:2913-2924`), while other
|
|
753
|
+
// `usage_update` paths can carry `used` without cost (for example the
|
|
754
|
+
// rate-limit path at `:3661-3671`). A live thinkpad ledger shows such turns
|
|
755
|
+
// really occur. The honest handling is to HOLD the baseline so the amount lands
|
|
756
|
+
// in the NEXT diff: misattributed by turn, exact by session. Rebaselining to 0
|
|
757
|
+
// there would double-count the whole prefix.
|
|
758
|
+
//
|
|
759
|
+
// The same turn also proves the OCCUPANCY carry-forward: with no notification
|
|
760
|
+
// there is no fresh `used`, and before this lane such a message was all-zero
|
|
761
|
+
// and pi skipped it. Now it carries accounting on `usage.acp` while pi's four
|
|
762
|
+
// fields remain zero, so a zero totalTokens would leave no direct occupancy for
|
|
763
|
+
// the gauge to use.
|
|
764
|
+
// ----------------------------------------------------------------------
|
|
765
|
+
{
|
|
766
|
+
const C1 = 1.5;
|
|
767
|
+
const C3 = 4.25;
|
|
768
|
+
const h = makeHarness(recordDir, [
|
|
769
|
+
{ cumulativeCostUsd: C1, occupancyTokens: 120_000, usage: makeUsage(10) },
|
|
770
|
+
{ usage: makeUsage(20) },
|
|
771
|
+
{ cumulativeCostUsd: C3, occupancyTokens: 260_000, usage: makeUsage(30) },
|
|
772
|
+
]);
|
|
773
|
+
const t1 = sealedMessage(
|
|
774
|
+
await collect(
|
|
775
|
+
backend.streamAcpTurn(claude, userCtx("first NONCE-1"), { sessionId: "usage-C" }, h.deps) as Stream,
|
|
776
|
+
),
|
|
777
|
+
);
|
|
778
|
+
const t2 = sealedMessage(
|
|
779
|
+
await collect(
|
|
780
|
+
backend.streamAcpTurn(
|
|
781
|
+
claude,
|
|
782
|
+
reuseCtx("first NONCE-1", "second NONCE-2"),
|
|
783
|
+
{ sessionId: "usage-C" },
|
|
784
|
+
h.deps,
|
|
785
|
+
) as Stream,
|
|
786
|
+
),
|
|
787
|
+
);
|
|
788
|
+
const t3 = sealedMessage(
|
|
789
|
+
await collect(
|
|
790
|
+
backend.streamAcpTurn(
|
|
791
|
+
claude,
|
|
792
|
+
reuseCtx3("first NONCE-1", "second NONCE-2", "third NONCE-3"),
|
|
793
|
+
{ sessionId: "usage-C" },
|
|
794
|
+
h.deps,
|
|
795
|
+
) as Stream,
|
|
796
|
+
),
|
|
797
|
+
);
|
|
798
|
+
assert.equal(h.children.length, 1, "all three turns ran on ONE reused child");
|
|
799
|
+
|
|
800
|
+
const perTurn = [t1.usage.cost.total, t2.usage.cost.total, t3.usage.cost.total].map(round6);
|
|
801
|
+
assert.deepEqual(
|
|
802
|
+
perTurn,
|
|
803
|
+
[round6(C1), 0, round6(C3 - C1)],
|
|
804
|
+
"[QK:ACP-COST-BASELINE-HELD-WHEN-MISSING] a turn whose cost notification never arrived must attribute $0 " +
|
|
805
|
+
"and HOLD the baseline — its real amount is still inside the backend's running total and the NEXT " +
|
|
806
|
+
"adjacent diff absorbs it. Zeroing or rebaselining there double-counts every dollar spent so far. " +
|
|
807
|
+
`Got ${JSON.stringify(perTurn)}`,
|
|
808
|
+
);
|
|
809
|
+
assert.equal(
|
|
810
|
+
round6(perTurn.reduce((a, b) => a + b, 0)),
|
|
811
|
+
round6(C3),
|
|
812
|
+
"…and the SESSION sum still reproduces the backend's final running total exactly despite the gap",
|
|
813
|
+
);
|
|
814
|
+
|
|
815
|
+
assert.equal(
|
|
816
|
+
calculateContextTokens(t2.usage),
|
|
817
|
+
120_000,
|
|
818
|
+
"[QK:ACP-CONTEXT-OCCUPANCY-CARRIED] a turn with no usage_update has no fresh occupancy, but it now DOES carry " +
|
|
819
|
+
"a token partition — so leaving totalTokens at 0 makes pi fall through to that partition and the context " +
|
|
820
|
+
"gauge drops from the real occupancy to one turn's delta. The last measurement must be carried forward " +
|
|
821
|
+
`(assigned, never summed). Got: ${calculateContextTokens(t2.usage)}`,
|
|
822
|
+
);
|
|
823
|
+
assert.equal(
|
|
824
|
+
calculateContextTokens(t3.usage),
|
|
825
|
+
260_000,
|
|
826
|
+
"…and a turn that DOES report occupancy uses the fresh value, never the carried one",
|
|
827
|
+
);
|
|
828
|
+
}
|
|
829
|
+
|
|
830
|
+
// ----------------------------------------------------------------------
|
|
831
|
+
// CELL 4 — a DECREASING session total is never silently absorbed.
|
|
832
|
+
//
|
|
833
|
+
// `conversation_reset` switches the session to a fresh transcript (read at
|
|
834
|
+
// claude-agent-acp 0.73.0 `dist/acp-agent.js:3675-3682`), but whether that
|
|
835
|
+
// changes `total_cost_usd` is an SDK-internal value we cannot observe here.
|
|
836
|
+
// A diff can therefore go negative in a session we are still holding.
|
|
837
|
+
// Absorbing it quietly would both misreport the turn and destroy the only
|
|
838
|
+
// observation that could settle the open question.
|
|
839
|
+
// ----------------------------------------------------------------------
|
|
840
|
+
{
|
|
841
|
+
const h = makeHarness(recordDir, [
|
|
842
|
+
{ cumulativeCostUsd: 4.0, occupancyTokens: 300_000, usage: makeUsage(10) },
|
|
843
|
+
{ cumulativeCostUsd: 1.0, occupancyTokens: 40_000, usage: makeUsage(20) },
|
|
844
|
+
{ cumulativeCostUsd: 1.5, occupancyTokens: 60_000, usage: makeUsage(30) },
|
|
845
|
+
]);
|
|
846
|
+
const t1 = sealedMessage(
|
|
847
|
+
await collect(
|
|
848
|
+
backend.streamAcpTurn(claude, userCtx("first NONCE-1"), { sessionId: "usage-D" }, h.deps) as Stream,
|
|
849
|
+
),
|
|
850
|
+
);
|
|
851
|
+
const t2events = await collect(
|
|
852
|
+
backend.streamAcpTurn(
|
|
853
|
+
claude,
|
|
854
|
+
reuseCtx("first NONCE-1", "second NONCE-2"),
|
|
855
|
+
{ sessionId: "usage-D" },
|
|
856
|
+
h.deps,
|
|
857
|
+
) as Stream,
|
|
858
|
+
);
|
|
859
|
+
const t2 = sealedMessage(t2events);
|
|
860
|
+
const t3 = sealedMessage(
|
|
861
|
+
await collect(
|
|
862
|
+
backend.streamAcpTurn(
|
|
863
|
+
claude,
|
|
864
|
+
reuseCtx3("first NONCE-1", "second NONCE-2", "third NONCE-3"),
|
|
865
|
+
{ sessionId: "usage-D" },
|
|
866
|
+
h.deps,
|
|
867
|
+
) as Stream,
|
|
868
|
+
),
|
|
869
|
+
);
|
|
870
|
+
|
|
871
|
+
const text = t2.content
|
|
872
|
+
.filter((b: any) => b.type === "text")
|
|
873
|
+
.map((b: any) => b.text)
|
|
874
|
+
.join("");
|
|
875
|
+
assert.equal(round6(t1.usage.cost.total), 4.0, "the first turn is the full running total (baseline starts empty)");
|
|
876
|
+
assert.ok(
|
|
877
|
+
round6(t2.usage.cost.total) === 0 && /cost baseline reset/.test(text) && text.includes("1.000000"),
|
|
878
|
+
"[QK:ACP-COST-RESET-NOT-SILENT] a running total that goes BACKWARDS must rebaseline, attribute $0 for that " +
|
|
879
|
+
"turn, AND tell the operator — with both numbers. Silently absorbing it would report a wrong turn cost " +
|
|
880
|
+
"and erase the only observation that can settle what a conversation reset does to the backend's total. " +
|
|
881
|
+
`Got cost=${t2.usage.cost.total} notice=${JSON.stringify(text)}`,
|
|
882
|
+
);
|
|
883
|
+
assert.equal(
|
|
884
|
+
round6(t3.usage.cost.total),
|
|
885
|
+
0.5,
|
|
886
|
+
"…and after the rebaseline the NEXT turn is measured against the new total (1.5 - 1.0), not the old one",
|
|
887
|
+
);
|
|
888
|
+
}
|
|
889
|
+
|
|
890
|
+
// ----------------------------------------------------------------------
|
|
891
|
+
// CELL 5 — a backend with no MEASURED semantics is sealed NOT AT ALL.
|
|
892
|
+
//
|
|
893
|
+
// backend.ts / event-mapper.ts are COMMON layer: cortex drives the same turn
|
|
894
|
+
// loop and the same mapper. Nobody has measured what cortex's ACP `usage`
|
|
895
|
+
// means (turn delta or session total? against which price table?), and ACP's
|
|
896
|
+
// own type is self-contradictory about it — the outer comment says "for this
|
|
897
|
+
// turn", the field comments say "across all turns/session". So cortex must
|
|
898
|
+
// keep exactly its pre-#93 output: the coarse mapper assignment, and no token
|
|
899
|
+
// projection, no baseline, no diff.
|
|
900
|
+
// ----------------------------------------------------------------------
|
|
901
|
+
// TWO turns, and the second one is what carries the claim: on turn ONE a
|
|
902
|
+
// diff-against-an-empty-baseline and the raw cumulative are the SAME number, so
|
|
903
|
+
// a single turn cannot tell "sealed" from "not sealed". Turn two separates them
|
|
904
|
+
// (raw 9.0 vs diff 1.75).
|
|
905
|
+
{
|
|
906
|
+
const h = makeHarness(recordDir, [
|
|
907
|
+
{
|
|
908
|
+
usage: {
|
|
909
|
+
inputTokens: 111,
|
|
910
|
+
outputTokens: 222,
|
|
911
|
+
cachedReadTokens: 333,
|
|
912
|
+
cachedWriteTokens: 444,
|
|
913
|
+
totalTokens: 1110,
|
|
914
|
+
},
|
|
915
|
+
cumulativeCostUsd: 7.25,
|
|
916
|
+
occupancyTokens: 90_000,
|
|
917
|
+
},
|
|
918
|
+
{
|
|
919
|
+
usage: { inputTokens: 11, outputTokens: 22, cachedReadTokens: 33, cachedWriteTokens: 44, totalTokens: 110 },
|
|
920
|
+
cumulativeCostUsd: 9.0,
|
|
921
|
+
occupancyTokens: 140_000,
|
|
922
|
+
},
|
|
923
|
+
]);
|
|
924
|
+
await collect(backend.streamAcpTurn(cortex, userCtx("cortex NONCE-1"), { sessionId: "usage-E" }, h.deps) as Stream);
|
|
925
|
+
const msg = sealedMessage(
|
|
926
|
+
await collect(
|
|
927
|
+
backend.streamAcpTurn(
|
|
928
|
+
cortex,
|
|
929
|
+
reuseCtx("cortex NONCE-1", "cortex NONCE-2"),
|
|
930
|
+
{ sessionId: "usage-E" },
|
|
931
|
+
h.deps,
|
|
932
|
+
) as Stream,
|
|
933
|
+
),
|
|
934
|
+
);
|
|
935
|
+
assert.equal(h.children.length, 1, "both cortex turns ran on ONE reused child");
|
|
936
|
+
assert.deepEqual(
|
|
937
|
+
{
|
|
938
|
+
input: msg.usage.input,
|
|
939
|
+
output: msg.usage.output,
|
|
940
|
+
cacheRead: msg.usage.cacheRead,
|
|
941
|
+
cacheWrite: msg.usage.cacheWrite,
|
|
942
|
+
total: msg.usage.cost.total,
|
|
943
|
+
totalTokens: msg.usage.totalTokens,
|
|
944
|
+
},
|
|
945
|
+
{ input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 9.0, totalTokens: 140_000 },
|
|
946
|
+
"[QK:ACP-CORTEX-USAGE-UNTOUCHED] the sealing must be gated on the ADAPTER declaring measured semantics, not " +
|
|
947
|
+
"on the common turn loop. A backend that declares nothing keeps its pre-#93 output exactly — " +
|
|
948
|
+
"projecting claude's measured semantics onto it would mint, in a second backend, the same unmeasured " +
|
|
949
|
+
`accounting this lane exists to end. Got: ${JSON.stringify(msg.usage)}`,
|
|
950
|
+
);
|
|
951
|
+
}
|
|
952
|
+
// ----------------------------------------------------------------------
|
|
953
|
+
// CELL 6 — the next turn may start THE INSTANT the previous one seals, and
|
|
954
|
+
// the cost baseline must survive that.
|
|
955
|
+
//
|
|
956
|
+
// Every other cell starts turn 2 after fully draining turn 1's stream. A real
|
|
957
|
+
// caller need not be that polite: pi's loop can begin the next turn as soon as
|
|
958
|
+
// it observes the terminal event. If a successfully completed process-scoped
|
|
959
|
+
// session were not yet discoverable at that moment, the next turn would find
|
|
960
|
+
// no session, open a second one, and start from an EMPTY baseline — the whole
|
|
961
|
+
// session's accounting would silently reset at a turn boundary.
|
|
962
|
+
//
|
|
963
|
+
// So this cell is the tightest caller expressible: turn 2 is launched from
|
|
964
|
+
// INSIDE the iteration, synchronously on the `done` event, before turn 1's
|
|
965
|
+
// stream has even finished draining. It holds today because the retention that
|
|
966
|
+
// follows the seal shares its synchronous run (no await between them) and the
|
|
967
|
+
// turn's in-flight claim covers the same span. Both are invariants a later edit
|
|
968
|
+
// could break WITHOUT breaking anything else — an inserted await would let this
|
|
969
|
+
// caller through the gap — which is exactly why the assertion is written here
|
|
970
|
+
// rather than left as a property of the current line order.
|
|
971
|
+
//
|
|
972
|
+
// Deliberately carries NO [QK:…] signature and no mutant. A QK claim is a
|
|
973
|
+
// promise that re-planting a CLOSED defect turns this red, and there is no
|
|
974
|
+
// reachable defect to re-plant: measured on this source, the seal cannot
|
|
975
|
+
// preempt the retention that shares its synchronous run, so the pre-existing
|
|
976
|
+
// line order passes this cell too. Labelling it anyway would mint a claim
|
|
977
|
+
// nothing can kill. It is a REGRESSION LOCK for a future edit, not evidence
|
|
978
|
+
// that a defect was closed.
|
|
979
|
+
// ----------------------------------------------------------------------
|
|
980
|
+
{
|
|
981
|
+
const C1 = 2.0;
|
|
982
|
+
const C2 = 3.5;
|
|
983
|
+
const h = makeHarness(recordDir, [
|
|
984
|
+
{ cumulativeCostUsd: C1, occupancyTokens: 100_000, usage: makeUsage(10) },
|
|
985
|
+
{ cumulativeCostUsd: C1 + C2, occupancyTokens: 210_000, usage: makeUsage(20) },
|
|
986
|
+
]);
|
|
987
|
+
|
|
988
|
+
const firstEvents: AssistantMessageEvent[] = [];
|
|
989
|
+
let secondTurn: Promise<AssistantMessageEvent[]> | undefined;
|
|
990
|
+
const firstStream = backend.streamAcpTurn(
|
|
991
|
+
claude,
|
|
992
|
+
userCtx("first NONCE-1"),
|
|
993
|
+
{ sessionId: "usage-F" },
|
|
994
|
+
h.deps,
|
|
995
|
+
) as Stream;
|
|
996
|
+
for await (const ev of firstStream) {
|
|
997
|
+
firstEvents.push(ev);
|
|
998
|
+
// The instant the turn seals — not after the loop, not on a later tick.
|
|
999
|
+
if ((ev.type === "done" || ev.type === "error") && !secondTurn) {
|
|
1000
|
+
secondTurn = collect(
|
|
1001
|
+
backend.streamAcpTurn(
|
|
1002
|
+
claude,
|
|
1003
|
+
reuseCtx("first NONCE-1", "second NONCE-2"),
|
|
1004
|
+
{ sessionId: "usage-F" },
|
|
1005
|
+
h.deps,
|
|
1006
|
+
) as Stream,
|
|
1007
|
+
);
|
|
1008
|
+
}
|
|
1009
|
+
}
|
|
1010
|
+
assert.ok(secondTurn, "the first turn must have sealed with a terminal event");
|
|
1011
|
+
const t1 = sealedMessage(firstEvents);
|
|
1012
|
+
const t2 = sealedMessage(await secondTurn);
|
|
1013
|
+
|
|
1014
|
+
assert.equal(
|
|
1015
|
+
h.children.length,
|
|
1016
|
+
1,
|
|
1017
|
+
"a turn started the instant the previous one sealed must find " +
|
|
1018
|
+
"the completed process-scoped session already discoverable and REUSE it. A second child here means the " +
|
|
1019
|
+
"next turn opened its own session, which also means it started from an empty cost baseline — a long " +
|
|
1020
|
+
`session's accounting would reset at a turn boundary. Got ${h.children.length} children`,
|
|
1021
|
+
);
|
|
1022
|
+
assert.equal(h.newSessionCalls, 1, "…and exactly one ACP session was ever created for that key");
|
|
1023
|
+
assert.deepEqual(
|
|
1024
|
+
[round6(t1.usage.cost.total), round6(t2.usage.cost.total)],
|
|
1025
|
+
[round6(C1), round6(C2)],
|
|
1026
|
+
"…and the baseline carried across that boundary, so turn 2 is its own diff and not the running total again",
|
|
1027
|
+
);
|
|
1028
|
+
}
|
|
1029
|
+
} finally {
|
|
1030
|
+
rmSync(TMP_EMIT, { recursive: true, force: true });
|
|
1031
|
+
try {
|
|
1032
|
+
// The qualification harness's work-surface hash walks ignored paths too, so
|
|
1033
|
+
// a leftover EMPTY parent dir reads as IMPURE tree drift even though git
|
|
1034
|
+
// porcelain is clean. Remove it when empty; a concurrent sibling gate's
|
|
1035
|
+
// emit keeps it alive and this rmdir simply fails.
|
|
1036
|
+
rmdirSync(".tmp-verify");
|
|
1037
|
+
} catch {
|
|
1038
|
+
// non-empty or already gone — fine either way
|
|
1039
|
+
}
|
|
1040
|
+
rmSync(recordDir, { recursive: true, force: true });
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
/** Distinct-but-uninteresting token partition for the cost-focused cells. */
|
|
1044
|
+
function makeUsage(seed: number): WireUsage {
|
|
1045
|
+
return {
|
|
1046
|
+
inputTokens: seed,
|
|
1047
|
+
outputTokens: seed * 2,
|
|
1048
|
+
cachedReadTokens: seed * 3,
|
|
1049
|
+
cachedWriteTokens: seed * 4,
|
|
1050
|
+
totalTokens: seed * 10,
|
|
1051
|
+
};
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
/** Float compare at 6 decimals — the numbers are USD sums, not bit patterns. */
|
|
1055
|
+
function round6(value: number): number {
|
|
1056
|
+
return Math.round(value * 1e6) / 1e6;
|
|
1057
|
+
}
|
|
1058
|
+
|
|
1059
|
+
console.log(
|
|
1060
|
+
"[check-acp-usage-accounting] ok — a Claude ACP turn's ROUND-TRIP AGGREGATE is never projected onto pi's four " +
|
|
1061
|
+
"per-request usage fields, so pi's own isContextOverflow does not fire on a session that fits its window; " +
|
|
1062
|
+
"the vendor's four turn totals still reach the operator VERBATIM on their own `usage.acp` key, taken from the " +
|
|
1063
|
+
"ACCOUNTING-GRADE `_meta.quota.model_usage` rows (summed) in preference to the main-loop-only " +
|
|
1064
|
+
"`PromptResponse.usage`, so the numerator's scope matches the all-inclusive cost denominator; a re-billed " +
|
|
1065
|
+
"prefix is REPORTED with a size that is a PROVEN LOWER BOUND (191,971 against the ledger's true 195,177) and " +
|
|
1066
|
+
"the turn's own SDK cost, computed from MAIN-LOOP usage so a wide compaction write cannot inflate it; " +
|
|
1067
|
+
"per-turn costs are ADJACENT DIFFS of the backend's running session total and sum back to it across " +
|
|
1068
|
+
"a reused session; a turn with no cost notification attributes $0 while HOLDING the baseline so the session sum " +
|
|
1069
|
+
"reproduces the backend's own cumulative ESTIMATE exactly (agreement with that carrier, never with a bill); a DECREASING total rebaselines, attributes $0 and says so to the operator with both numbers; " +
|
|
1070
|
+
"totalTokens stays CONTEXT OCCUPANCY (verified through pi's own calculateContextTokens) and is carried forward " +
|
|
1071
|
+
"when a turn reports none; a backend that declares no measured semantics — cortex — is sealed not at all; and a next turn " +
|
|
1072
|
+
"started from INSIDE the previous turn's terminal event still reuses the one session, so the cost baseline " +
|
|
1073
|
+
"survives the turn boundary",
|
|
1074
|
+
);
|