@junghanacs/entwurf 0.12.10 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -1
- package/BASELINE.md +45 -6
- package/CHANGELOG.md +16 -0
- package/CONTRIBUTING.md +4 -2
- package/DELIVERY.md +1 -1
- package/README.md +20 -5
- package/VERIFY.md +7 -4
- package/mcp/entwurf-bridge/dist/pi-extensions/lib/acp/backend-adapter.js +148 -5
- package/mcp/entwurf-bridge/dist/pi-extensions/lib/acp/config.js +16 -4
- package/mcp/entwurf-bridge/dist/pi-extensions/lib/acp/models.js +66 -7
- package/mcp/entwurf-bridge/dist/pi-extensions/lib/acp/overlay.js +190 -5
- package/mcp/entwurf-bridge/dist/pi-extensions/lib/acp/tool-surface.js +12 -4
- package/package.json +4 -2
- package/pi/settings.reference.json +1 -1
- package/pi-extensions/acp-provider.ts +20 -10
- package/pi-extensions/lib/acp/augment.ts +60 -2
- package/pi-extensions/lib/acp/backend-adapter.ts +183 -8
- package/pi-extensions/lib/acp/backend.ts +5 -1
- package/pi-extensions/lib/acp/config.ts +19 -5
- package/pi-extensions/lib/acp/engraving.ts +3 -1
- package/pi-extensions/lib/acp/event-mapper.ts +10 -3
- package/pi-extensions/lib/acp/models.ts +69 -7
- package/pi-extensions/lib/acp/overlay.ts +234 -5
- package/pi-extensions/lib/acp/tool-surface.ts +12 -4
- package/run.sh +152 -21
- package/scripts/check-acp-cortex.ts +668 -0
- package/scripts/check-acp-provider-surface.ts +50 -6
- package/scripts/check-acp-session-reuse.ts +64 -1
- package/scripts/check-gate-qualification.ts +2 -0
- package/scripts/check-probe-cli-shim.ts +879 -0
- package/scripts/check-probe-ordering.ts +2450 -0
- package/scripts/check-shell-quote.ts +4 -4
- package/scripts/fixtures/probe-cli-shim +20 -0
- package/scripts/fixtures/probe-mcp-server.ts +168 -12
- package/scripts/lib/probe-acp-turn.ts +207 -0
- package/scripts/lib/probe-cli-shim.ts +464 -0
- package/scripts/lib/probe-cli-target.ts +165 -0
- package/scripts/lib/probe-event-log.ts +383 -0
- package/scripts/lib/probe-verdict.ts +1213 -0
- package/scripts/mutants/acp-cortex.json +196 -0
- package/scripts/mutants/probe-ordering.json +1032 -0
- package/scripts/smoke-acp-cortex-live.ts +392 -0
- package/scripts/smoke-acp-ordering-probe-live.ts +848 -0
|
@@ -0,0 +1,2450 @@
|
|
|
1
|
+
// Deterministic gate for the §11-7 ordering probe (docs/acp-backend-rail.md).
|
|
2
|
+
//
|
|
3
|
+
// §11-7 allows the probe a raw client ONLY "bound by a gate asserting it issues
|
|
4
|
+
// the same calls, arguments, and order as the backend's real sequence" — THIS is
|
|
5
|
+
// that gate. Seven axes, no live API anywhere:
|
|
6
|
+
//
|
|
7
|
+
// 1) SAMENESS — driveProbeTurn over a recording fake connection must issue
|
|
8
|
+
// initialize → newSession → enforceModel → prompt with production-shaped
|
|
9
|
+
// arguments; the phase timeouts / call order / clientInfo / enforceModel
|
|
10
|
+
// wire shape / permission policy are pinned against backend.ts and
|
|
11
|
+
// backend-adapter.ts SOURCE, and the LIVE runner is pinned to inject the
|
|
12
|
+
// REAL emitted claudeAdapter + enrichMcpServersWithEnvelope — so
|
|
13
|
+
// production drift turns this gate red instead of silently unbinding the
|
|
14
|
+
// probe.
|
|
15
|
+
// 2) PHASE ATTRIBUTION — a failure names its wire phase (set-model included:
|
|
16
|
+
// §11-7 measured that dropping it misreads enforceModel stalls as C or D).
|
|
17
|
+
// 3) FIXTURE WIRE INSTRUMENTATION — the probe-mode fixture is spawned for
|
|
18
|
+
// real (child process, no API) and must stamp the §11-7 marker set,
|
|
19
|
+
// honor the injected startup delay, REQUIRE probeRunId, and stay
|
|
20
|
+
// byte-compatible with smoke-acp-mcp-live when the probe env is absent.
|
|
21
|
+
// 4) EVENT LOG INTEGRITY — the contract at the log's door, both layers: the
|
|
22
|
+
// writer may not have its own envelope rewritten by a payload key; a
|
|
23
|
+
// JSON-valid line carrying an unknown marker name or a broken sort axis is
|
|
24
|
+
// MALFORMED (→ the runner's INVALIDATED path), never a quiet event; and
|
|
25
|
+
// every PAYLOAD field the classifier judges on is typed there too, since a
|
|
26
|
+
// perfect envelope around `ok: "true"` reads as a phase failure and a wire
|
|
27
|
+
// marker naming no expected tool is not wire-availability at all.
|
|
28
|
+
// 4b) EVENT LOG STREAM INTEGRITY — the second door, one layer out: per writing
|
|
29
|
+
// pid, on the RAW APPEND ORDER, seq must strictly increase and the clock may
|
|
30
|
+
// not run backwards. Checked BEFORE the sort, because a post-sort check is
|
|
31
|
+
// circular — the comparator would have rewritten the order under examination.
|
|
32
|
+
// 5) VERDICT TRUTH TABLE — synthetic paired logs replay through the PURE
|
|
33
|
+
// classifier: P0/I0 outside the verdict space, phase-qualified D, the B
|
|
34
|
+
// promotion ladder (exact measured id only), C, and A's two-delay rule.
|
|
35
|
+
// 6) OBSERVATION WINDOW + RUNNER TOPOLOGY — what puts a run OUTSIDE the space:
|
|
36
|
+
// a window closed by child-exit with the marker unseen is CENSORED, never an
|
|
37
|
+
// MCP handshake/fixture/config attribution (the 2026-07-28 D2 misreading),
|
|
38
|
+
// while the SAME absence under a window held to its deadline is a real
|
|
39
|
+
// candidate. Runner-owned markers are exactly-once; repeatable ones are not
|
|
40
|
+
// swept in with them.
|
|
41
|
+
// 7) TWO AXES — (a) the server-wait observation and (b) the callability reading
|
|
42
|
+
// are reported separately, so a settled ordering fact is not buried by an
|
|
43
|
+
// unsettled failure axis. A keeps the newSession axis; B/C's causal window is
|
|
44
|
+
// promptStart, and the evidence carries both deltas plus the post-wire turn.
|
|
45
|
+
// 8) §11-7-c B-NAME-SNAPSHOT SEAM (consumer side) — the CLI-target
|
|
46
|
+
// precondition gate (ambient override refused, native-branch absolute
|
|
47
|
+
// target only, exact-allowlist scrub), the upstream launch-semantics
|
|
48
|
+
// inspector (synthetic-validated, applied to the installed dists), the
|
|
49
|
+
// shim event doors, and the snapshot verdict ladder (calibrated control,
|
|
50
|
+
// exactly-one ordinal binding, interval-ordered absence, roster-armed
|
|
51
|
+
// channel; B and B-name-snapshot never conflated). The PRODUCER (shim) is
|
|
52
|
+
// gated separately by check-probe-cli-shim, and the LIVE runner now ARMS the
|
|
53
|
+
// channel — 8d pins that wiring.
|
|
54
|
+
//
|
|
55
|
+
// Kill-proof, stated at its honest strength: scripts/mutants/probe-ordering.json
|
|
56
|
+
// qualifies 63 claims for THIS gate — each carries a [QK:...] signature appearing
|
|
57
|
+
// EXACTLY once below, and check-gate-qualification proves its mutant dies at that
|
|
58
|
+
// signature. (The lane also carries the §11-7-c PRODUCER claims, whose signatures
|
|
59
|
+
// live in check-probe-cli-shim.ts; one lane, two consuming gates.)
|
|
60
|
+
// [QK:*] tokens and qualified claims are 1:1 BY DESIGN: an assertion without a
|
|
61
|
+
// mutant carries a plain message, so "killed claim IDs, never assertion counts"
|
|
62
|
+
// stays readable. The remaining assertions are enforced-but-not-mutant-qualified.
|
|
63
|
+
// Properties that are review-pinned only (no deterministic mutant exists):
|
|
64
|
+
// - the write-CALLBACK timing of tools_list_response_forwarded (cross-process
|
|
65
|
+
// microsecond ordering; existence/attribution IS proven);
|
|
66
|
+
// - the fixture's errored-write path (EPIPE cannot be forced deterministically
|
|
67
|
+
// in this handshake; the callback rejects and never stamps — source-visible);
|
|
68
|
+
// - the writer's SINGLE clock read (a two-read millisecond straddle is a rare
|
|
69
|
+
// race, so it is pinned in source and made loud by the parser's ts↔tsMs
|
|
70
|
+
// equality rule, which IS mutant-qualified).
|
|
71
|
+
|
|
72
|
+
import { strict as assert } from "node:assert";
|
|
73
|
+
import { type ChildProcessByStdio, spawn } from "node:child_process";
|
|
74
|
+
import {
|
|
75
|
+
appendFileSync,
|
|
76
|
+
chmodSync,
|
|
77
|
+
existsSync,
|
|
78
|
+
mkdirSync,
|
|
79
|
+
mkdtempSync,
|
|
80
|
+
readFileSync,
|
|
81
|
+
rmSync,
|
|
82
|
+
statSync,
|
|
83
|
+
writeFileSync,
|
|
84
|
+
} from "node:fs";
|
|
85
|
+
import { createRequire } from "node:module";
|
|
86
|
+
import { tmpdir } from "node:os";
|
|
87
|
+
import { dirname, join, resolve } from "node:path";
|
|
88
|
+
import type { Readable, Writable } from "node:stream";
|
|
89
|
+
import { fileURLToPath } from "node:url";
|
|
90
|
+
import { PROTOCOL_VERSION } from "@agentclientprotocol/sdk";
|
|
91
|
+
import type { AcpConnectionLike } from "../pi-extensions/lib/acp/acp-client.ts";
|
|
92
|
+
import { resolveProviderConfig } from "../pi-extensions/lib/acp/config.ts";
|
|
93
|
+
import { terminateChild } from "./lib/acp-child-cleanup.ts";
|
|
94
|
+
import {
|
|
95
|
+
driveProbeTurn,
|
|
96
|
+
PROBE_PHASE_TIMEOUTS,
|
|
97
|
+
type ProbeAdapterSeam,
|
|
98
|
+
type ProbeMcpEnricher,
|
|
99
|
+
ProbePhaseError,
|
|
100
|
+
type ProbeTurnPhase,
|
|
101
|
+
} from "./lib/probe-acp-turn.ts";
|
|
102
|
+
import {
|
|
103
|
+
AMBIENT_OVERRIDE_ENV,
|
|
104
|
+
hashFileSha256,
|
|
105
|
+
PROBE_SHIM_ENV,
|
|
106
|
+
ProbeCliPreconditionError,
|
|
107
|
+
resolveProbeCliTarget,
|
|
108
|
+
SDK_SCRIPT_SUFFIXES,
|
|
109
|
+
SHIM_SCRUB_ENV_VARS,
|
|
110
|
+
} from "./lib/probe-cli-target.ts";
|
|
111
|
+
import {
|
|
112
|
+
appendProbeEvent,
|
|
113
|
+
PAYLOAD_CONTRACT_EVENTS,
|
|
114
|
+
PROBE_ENV,
|
|
115
|
+
PROBE_EVENTS,
|
|
116
|
+
PROBE_EXPECTED_TOOL,
|
|
117
|
+
type ProbeEvent,
|
|
118
|
+
type ProbeWindowReason,
|
|
119
|
+
RESERVED_EVENT_KEYS,
|
|
120
|
+
readProbeEvents,
|
|
121
|
+
} from "./lib/probe-event-log.ts";
|
|
122
|
+
import { classifyProbe, DELAY_WELL_BELOW_MS, type ProbeRunRecord, RUNNER_EXACTLY_ONCE } from "./lib/probe-verdict.ts";
|
|
123
|
+
|
|
124
|
+
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
|
125
|
+
const BACKEND_SRC = readFileSync(join(REPO_ROOT, "pi-extensions", "lib", "acp", "backend.ts"), "utf8");
|
|
126
|
+
const ADAPTER_SRC = readFileSync(join(REPO_ROOT, "pi-extensions", "lib", "acp", "backend-adapter.ts"), "utf8");
|
|
127
|
+
const RUNNER_SRC = readFileSync(join(REPO_ROOT, "scripts", "smoke-acp-ordering-probe-live.ts"), "utf8");
|
|
128
|
+
const EVENT_LOG_SRC = readFileSync(join(REPO_ROOT, "scripts", "lib", "probe-event-log.ts"), "utf8");
|
|
129
|
+
const VERDICT_SRC = readFileSync(join(REPO_ROOT, "scripts", "lib", "probe-verdict.ts"), "utf8");
|
|
130
|
+
const FIXTURE_PATH = join(REPO_ROOT, "scripts", "fixtures", "probe-mcp-server.ts");
|
|
131
|
+
const FIXTURE_SRC = readFileSync(FIXTURE_PATH, "utf8");
|
|
132
|
+
const tmp = mkdtempSync(join(tmpdir(), "check-probe-ordering-"));
|
|
133
|
+
|
|
134
|
+
// The REAL config resolver (config.ts is strip-types-clean) over a scratch
|
|
135
|
+
// operator settings file — the fixture entry must survive config→wire intact.
|
|
136
|
+
function makeConfigFixture(): ReturnType<typeof resolveProviderConfig> {
|
|
137
|
+
const cwd = join(tmp, "cfg");
|
|
138
|
+
mkdirSync(join(cwd, ".pi"), { recursive: true });
|
|
139
|
+
const projectSettingsPath = join(cwd, ".pi", "settings.json");
|
|
140
|
+
writeFileSync(
|
|
141
|
+
projectSettingsPath,
|
|
142
|
+
`${JSON.stringify({
|
|
143
|
+
entwurfProvider: {
|
|
144
|
+
mcpServers: { probe: { command: "node", args: ["probe.ts"], env: { X: "1" } } },
|
|
145
|
+
},
|
|
146
|
+
})}\n`,
|
|
147
|
+
);
|
|
148
|
+
return resolveProviderConfig({
|
|
149
|
+
cwd,
|
|
150
|
+
modelId: "claude-sonnet-5",
|
|
151
|
+
adapter: { resolveAdapterSettings: () => undefined } as never,
|
|
152
|
+
globalSettingsPath: join(cwd, ".pi", "no-global.json"),
|
|
153
|
+
projectSettingsPath,
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
interface RecordedCall {
|
|
158
|
+
method: string;
|
|
159
|
+
params: unknown;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
function makeRecordingConnection(calls: RecordedCall[]): AcpConnectionLike {
|
|
163
|
+
return {
|
|
164
|
+
initialize: async (params) => {
|
|
165
|
+
calls.push({ method: "initialize", params });
|
|
166
|
+
return { protocolVersion: PROTOCOL_VERSION };
|
|
167
|
+
},
|
|
168
|
+
newSession: async (params) => {
|
|
169
|
+
calls.push({ method: "newSession", params });
|
|
170
|
+
return { sessionId: "sess-gate" };
|
|
171
|
+
},
|
|
172
|
+
setSessionConfigOption: async (params) => {
|
|
173
|
+
calls.push({ method: "setSessionConfigOption", params });
|
|
174
|
+
return {};
|
|
175
|
+
},
|
|
176
|
+
prompt: async (params) => {
|
|
177
|
+
calls.push({ method: "prompt", params });
|
|
178
|
+
return { stopReason: "end_turn" };
|
|
179
|
+
},
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// ===========================================================================
|
|
184
|
+
// 1) SAMENESS — recorded sequence + production source pins
|
|
185
|
+
// ===========================================================================
|
|
186
|
+
{
|
|
187
|
+
const config = makeConfigFixture();
|
|
188
|
+
const calls: RecordedCall[] = [];
|
|
189
|
+
const logged: Array<{ event: string; payload: Record<string, unknown> }> = [];
|
|
190
|
+
const metaMarker = { modelId: "claude-sonnet-5", gateMeta: true };
|
|
191
|
+
const enrichedMarker = [{ name: "probe", command: "node", args: ["probe.ts"], env: [{ name: "X", value: "1" }] }];
|
|
192
|
+
let enrichArgs: { servers: unknown; envelope: unknown } | undefined;
|
|
193
|
+
let metaArgs: { params: unknown; carrier: unknown } | undefined;
|
|
194
|
+
const recordingAdapter: ProbeAdapterSeam = {
|
|
195
|
+
buildSessionMeta(params, carrier) {
|
|
196
|
+
metaArgs = { params, carrier };
|
|
197
|
+
return metaMarker;
|
|
198
|
+
},
|
|
199
|
+
async enforceModel(params) {
|
|
200
|
+
calls.push({ method: "enforceModel", params: { ...params, connection: "<connection>" } });
|
|
201
|
+
},
|
|
202
|
+
};
|
|
203
|
+
|
|
204
|
+
// PI_SESSION_ID would leak the host session into the envelope assertion —
|
|
205
|
+
// pin it to a deterministic value for the recording turn.
|
|
206
|
+
const prevPiSession = process.env.PI_SESSION_ID;
|
|
207
|
+
process.env.PI_SESSION_ID = "gate-pi-session";
|
|
208
|
+
try {
|
|
209
|
+
await driveProbeTurn(makeRecordingConnection(calls), {
|
|
210
|
+
cwd: "/scratch/gate",
|
|
211
|
+
modelId: "claude-sonnet-5",
|
|
212
|
+
nativeModelId: "claude-sonnet-5",
|
|
213
|
+
config,
|
|
214
|
+
carrier: "gate-carrier",
|
|
215
|
+
promptText: "gate prompt",
|
|
216
|
+
adapter: recordingAdapter,
|
|
217
|
+
enrichMcpServers: (servers, envelope) => {
|
|
218
|
+
enrichArgs = { servers, envelope };
|
|
219
|
+
return enrichedMarker as never;
|
|
220
|
+
},
|
|
221
|
+
log: (event, payload = {}) => logged.push({ event, payload }),
|
|
222
|
+
});
|
|
223
|
+
} finally {
|
|
224
|
+
if (prevPiSession === undefined) delete process.env.PI_SESSION_ID;
|
|
225
|
+
else process.env.PI_SESSION_ID = prevPiSession;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
assert.deepEqual(
|
|
229
|
+
calls.map((c) => c.method),
|
|
230
|
+
["initialize", "newSession", "enforceModel", "prompt"],
|
|
231
|
+
"probe issues the production wire sequence in order",
|
|
232
|
+
);
|
|
233
|
+
assert.deepEqual(
|
|
234
|
+
calls[0].params,
|
|
235
|
+
{
|
|
236
|
+
protocolVersion: PROTOCOL_VERSION,
|
|
237
|
+
clientCapabilities: {},
|
|
238
|
+
clientInfo: { name: "entwurf", version: "s2d" },
|
|
239
|
+
},
|
|
240
|
+
"initialize params match backend.ts byte-shape [QK:PROBE-SEQ-ORDER]",
|
|
241
|
+
);
|
|
242
|
+
const newSessionParams = calls[1].params as { cwd: string; mcpServers: unknown; _meta?: unknown };
|
|
243
|
+
assert.equal(newSessionParams.cwd, "/scratch/gate", "newSession carries the run cwd");
|
|
244
|
+
assert.equal(newSessionParams.mcpServers, enrichedMarker, "newSession sends EXACTLY the envelope-enriched servers");
|
|
245
|
+
assert.deepEqual(
|
|
246
|
+
enrichArgs,
|
|
247
|
+
{
|
|
248
|
+
servers: config.mcpServers,
|
|
249
|
+
envelope: { modelId: "claude-sonnet-5", piSessionId: "gate-pi-session" },
|
|
250
|
+
},
|
|
251
|
+
"enrichment sees the resolved servers + the production envelope",
|
|
252
|
+
);
|
|
253
|
+
assert.equal(config.mcpServers[0]?.name, "probe", "the fixture entry survives config resolution");
|
|
254
|
+
assert.equal(newSessionParams._meta, metaMarker, "newSession _meta is buildSessionMeta's result");
|
|
255
|
+
assert.deepEqual(
|
|
256
|
+
metaArgs,
|
|
257
|
+
{
|
|
258
|
+
params: { modelId: "claude-sonnet-5", nativeModelId: "claude-sonnet-5", config },
|
|
259
|
+
carrier: "gate-carrier",
|
|
260
|
+
},
|
|
261
|
+
"buildSessionMeta receives the production inputs incl. the carrier",
|
|
262
|
+
);
|
|
263
|
+
assert.deepEqual(
|
|
264
|
+
calls[2].params,
|
|
265
|
+
{
|
|
266
|
+
connection: "<connection>",
|
|
267
|
+
acpSessionId: "sess-gate",
|
|
268
|
+
modelId: "claude-sonnet-5",
|
|
269
|
+
nativeModelId: "claude-sonnet-5",
|
|
270
|
+
},
|
|
271
|
+
"enforceModel receives the production inputs (adapter owns the wire shape)",
|
|
272
|
+
);
|
|
273
|
+
const promptParams = calls[3].params as { sessionId: string; prompt: Array<{ type: string; text: string }> };
|
|
274
|
+
assert.equal(promptParams.sessionId, "sess-gate", "prompt binds the newSession sessionId");
|
|
275
|
+
assert.deepEqual(
|
|
276
|
+
promptParams.prompt,
|
|
277
|
+
[{ type: "text", text: "gate prompt" }],
|
|
278
|
+
"prompt block shape matches production",
|
|
279
|
+
);
|
|
280
|
+
|
|
281
|
+
// Set-model markers exist BETWEEN newSession end and prompt start — §11-7:
|
|
282
|
+
// dropping them misreads an enforceModel stall as C or D.
|
|
283
|
+
const eventOrder = logged.map((l) => l.event);
|
|
284
|
+
const nsEnd = eventOrder.indexOf(PROBE_EVENTS.newSessionEnd);
|
|
285
|
+
const smStart = eventOrder.indexOf(PROBE_EVENTS.setModelStart);
|
|
286
|
+
const smEnd = eventOrder.indexOf(PROBE_EVENTS.setModelEnd);
|
|
287
|
+
const pStart = eventOrder.indexOf(PROBE_EVENTS.promptStart);
|
|
288
|
+
assert.ok(
|
|
289
|
+
nsEnd !== -1 && smStart > nsEnd && smEnd > smStart && pStart > smEnd,
|
|
290
|
+
`set-model start/end are marked between newSession end and prompt start (got ${eventOrder.join(",")})`,
|
|
291
|
+
);
|
|
292
|
+
|
|
293
|
+
// §9-4 shape: an undefined session meta omits the `_meta` KEY entirely.
|
|
294
|
+
const calls2: RecordedCall[] = [];
|
|
295
|
+
await driveProbeTurn(makeRecordingConnection(calls2), {
|
|
296
|
+
cwd: "/scratch/gate2",
|
|
297
|
+
modelId: "claude-sonnet-5",
|
|
298
|
+
nativeModelId: "claude-sonnet-5",
|
|
299
|
+
config,
|
|
300
|
+
carrier: null,
|
|
301
|
+
promptText: "x",
|
|
302
|
+
adapter: { buildSessionMeta: () => undefined, enforceModel: async () => {} },
|
|
303
|
+
enrichMcpServers: (servers) => servers,
|
|
304
|
+
log: () => {},
|
|
305
|
+
});
|
|
306
|
+
assert.ok(
|
|
307
|
+
!Object.hasOwn(calls2.find((c) => c.method === "newSession")?.params as object, "_meta"),
|
|
308
|
+
"undefined session meta omits the _meta KEY entirely (§9-4)",
|
|
309
|
+
);
|
|
310
|
+
|
|
311
|
+
// --- production source pins ---------------------------------------------
|
|
312
|
+
const timeoutOf = (name: string): number => {
|
|
313
|
+
const m = new RegExp(`const ${name} = ([\\d_]+);`).exec(BACKEND_SRC);
|
|
314
|
+
assert.ok(m, `backend.ts declares ${name}`);
|
|
315
|
+
return Number(m[1].replaceAll("_", ""));
|
|
316
|
+
};
|
|
317
|
+
assert.deepEqual(
|
|
318
|
+
{
|
|
319
|
+
initializeMs: PROBE_PHASE_TIMEOUTS.initializeMs,
|
|
320
|
+
newSessionMs: PROBE_PHASE_TIMEOUTS.newSessionMs,
|
|
321
|
+
setModelMs: PROBE_PHASE_TIMEOUTS.setModelMs,
|
|
322
|
+
promptMs: PROBE_PHASE_TIMEOUTS.promptMs,
|
|
323
|
+
},
|
|
324
|
+
{
|
|
325
|
+
initializeMs: timeoutOf("INITIALIZE_TIMEOUT_MS"),
|
|
326
|
+
newSessionMs: timeoutOf("NEW_SESSION_TIMEOUT_MS"),
|
|
327
|
+
setModelMs: timeoutOf("SET_MODEL_TIMEOUT_MS"),
|
|
328
|
+
promptMs: timeoutOf("PROMPT_TIMEOUT_MS"),
|
|
329
|
+
},
|
|
330
|
+
"probe phase timeouts EQUAL backend.ts's — D is only readable against production boundaries [QK:PROBE-TIMEOUTS-MATCH-PRODUCTION]",
|
|
331
|
+
);
|
|
332
|
+
|
|
333
|
+
const idx = (needle: string): number => {
|
|
334
|
+
const i = BACKEND_SRC.indexOf(needle);
|
|
335
|
+
assert.ok(i !== -1, `backend.ts contains ${JSON.stringify(needle)}`);
|
|
336
|
+
return i;
|
|
337
|
+
};
|
|
338
|
+
const iInit = idx("connection.initialize({");
|
|
339
|
+
const iNew = idx("connection.newSession(newSessionArgs)");
|
|
340
|
+
const iEnforce = idx("adapter.enforceModel({");
|
|
341
|
+
const iPrompt = BACKEND_SRC.indexOf("connection.prompt({", iEnforce);
|
|
342
|
+
assert.ok(
|
|
343
|
+
iInit < iNew && iNew < iEnforce && iEnforce < iPrompt && iPrompt !== -1,
|
|
344
|
+
"backend.ts runNewTurn keeps initialize → newSession → enforceModel → prompt; the probe mirrors THIS sequence",
|
|
345
|
+
);
|
|
346
|
+
assert.ok(
|
|
347
|
+
BACKEND_SRC.includes('clientInfo: { name: "entwurf", version: "s2d" }'),
|
|
348
|
+
"backend.ts still sends the clientInfo the probe mirrors",
|
|
349
|
+
);
|
|
350
|
+
// The claude adapter's enforceModel wire shape — the LIVE probe executes THIS
|
|
351
|
+
// method (emitted twin), so its shape is pinned where it lives.
|
|
352
|
+
assert.ok(
|
|
353
|
+
ADAPTER_SRC.includes(
|
|
354
|
+
'await setConfig.call(connection, { sessionId: acpSessionId, configId: "model", value: nativeModelId });',
|
|
355
|
+
),
|
|
356
|
+
'claudeAdapter.enforceModel still sends setSessionConfigOption({configId:"model"})',
|
|
357
|
+
);
|
|
358
|
+
|
|
359
|
+
// Permission policy: the runner's copy must mirror backend.ts's private
|
|
360
|
+
// resolvePermissionResponse (approve-first-allow, empty → cancelled).
|
|
361
|
+
for (const [label, src] of [
|
|
362
|
+
["backend.ts", BACKEND_SRC],
|
|
363
|
+
["smoke-acp-ordering-probe-live.ts", RUNNER_SRC],
|
|
364
|
+
] as const) {
|
|
365
|
+
assert.ok(
|
|
366
|
+
src.includes('o.kind === "allow_once" || o.kind === "allow_always"') &&
|
|
367
|
+
src.includes('if (options.length === 0) return { outcome: { outcome: "cancelled" } };'),
|
|
368
|
+
`${label} carries the approve-first-allow permission policy`,
|
|
369
|
+
);
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
// The runner must drive THE gated sequence with the REAL emitted production
|
|
373
|
+
// modules — never a bare wire call or a re-implementation of its own.
|
|
374
|
+
assert.ok(
|
|
375
|
+
RUNNER_SRC.includes("driveProbeTurn(") &&
|
|
376
|
+
RUNNER_SRC.includes("adapterMod.claudeAdapter") &&
|
|
377
|
+
RUNNER_SRC.includes("configMod.enrichMcpServersWithEnvelope") &&
|
|
378
|
+
RUNNER_SRC.includes("adapter: claudeAdapter") &&
|
|
379
|
+
RUNNER_SRC.includes("enrichMcpServers: enrichMcpServersWithEnvelope") &&
|
|
380
|
+
!RUNNER_SRC.includes("connection.initialize(") &&
|
|
381
|
+
!RUNNER_SRC.includes("connection.newSession(") &&
|
|
382
|
+
!RUNNER_SRC.includes("connection.prompt("),
|
|
383
|
+
"the LIVE runner routes every wire call through driveProbeTurn with the emitted production adapter/config",
|
|
384
|
+
);
|
|
385
|
+
|
|
386
|
+
// The runtime No-such-tool marker may ride ONLY structured tool frames —
|
|
387
|
+
// scanning agent prose would let the model MINT the marker by saying the
|
|
388
|
+
// sentence, silently promoting prose to runtime evidence. Exactly one scan
|
|
389
|
+
// call site (plus its definition), and never inside the agent-chunk branch.
|
|
390
|
+
const scanSites = RUNNER_SRC.split("scanStructuredNoSuchTool").length - 1;
|
|
391
|
+
assert.ok(
|
|
392
|
+
scanSites === 2 && !/collectedText \+= t;[^}]*scanStructured/s.test(RUNNER_SRC),
|
|
393
|
+
"No-such-tool is scanned ONLY off structured tool frames, never agent prose [QK:PROBE-NO-PROSE-ERROR-SCAN]",
|
|
394
|
+
);
|
|
395
|
+
|
|
396
|
+
// A broken event log is run-invalidating on BOTH doors: a malformed line could
|
|
397
|
+
// be the very wire marker whose absence the classifier would then read as
|
|
398
|
+
// evidence, and a stream whose per-writer order cannot be trusted cannot carry
|
|
399
|
+
// an ordering verdict at all. The preserved classification.json must say
|
|
400
|
+
// INVALIDATED on its face, never a judgeable-looking thin-log verdict.
|
|
401
|
+
//
|
|
402
|
+
// Pinned as the WHOLE guard, not the substring `if (malformed.length > 0)`:
|
|
403
|
+
// that shorter form also matches the artifact-writing line right below it, so
|
|
404
|
+
// a mutant could disable the guard and still satisfy the pin. It did — this
|
|
405
|
+
// assertion SURVIVED its own mutant until the pin was tightened (2026-07-29).
|
|
406
|
+
// The post-delay slack is what lets a missing wire marker be read as evidence:
|
|
407
|
+
// an operator-shrinkable window could close early and then be called
|
|
408
|
+
// "deadline-sufficient". It is a constant, and no env may reach it.
|
|
409
|
+
assert.ok(
|
|
410
|
+
RUNNER_SRC.includes("const POST_DELAY_SLACK_MS = 5_000;") && !RUNNER_SRC.includes("PROBE_POST_DELAY_SLACK_MS"),
|
|
411
|
+
"the observation window's post-delay slack is a CONSTANT with no env override [QK:PROBE-WINDOW-SLACK-IS-CONSTANT]",
|
|
412
|
+
);
|
|
413
|
+
|
|
414
|
+
// The exit contract asks three separate questions. Failing the run on the
|
|
415
|
+
// composite verdict alone reported a pair that MEASURED its ordering axis as a
|
|
416
|
+
// failed run, purely because the callability axis had no marker.
|
|
417
|
+
assert.ok(
|
|
418
|
+
RUNNER_SRC.includes('classification.status.validity !== "valid" && classification.status.validity !== "partial"') &&
|
|
419
|
+
RUNNER_SRC.includes("do not read it as a complete series") &&
|
|
420
|
+
RUNNER_SRC.includes('classification.status.orderingMeasurement !== "measured"') &&
|
|
421
|
+
RUNNER_SRC.includes("NOT a claim about server wait behavior"),
|
|
422
|
+
"the runner's exit contract separates fatal validity from the two axes, and claims no server-wait conclusion [QK:RUNNER-EXIT-CONTRACT-SPLIT]",
|
|
423
|
+
);
|
|
424
|
+
|
|
425
|
+
assert.ok(
|
|
426
|
+
RUNNER_SRC.includes("if (malformed.length > 0 || sequenceViolations.length > 0) {") &&
|
|
427
|
+
RUNNER_SRC.includes("run INVALIDATED") &&
|
|
428
|
+
RUNNER_SRC.includes('verdict: "INVALIDATED"'),
|
|
429
|
+
"the runner refuses to judge a log with malformed lines OR per-writer order violations, and writes an INVALIDATED classification [QK:PROBE-MALFORMED-INVALIDATES]",
|
|
430
|
+
);
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
// ===========================================================================
|
|
434
|
+
// 2) PHASE ATTRIBUTION — a failure names its wire phase; timeouts are flagged
|
|
435
|
+
// ===========================================================================
|
|
436
|
+
{
|
|
437
|
+
const config = makeConfigFixture();
|
|
438
|
+
const passthroughAdapter: ProbeAdapterSeam = {
|
|
439
|
+
buildSessionMeta: () => undefined,
|
|
440
|
+
enforceModel: async () => {},
|
|
441
|
+
};
|
|
442
|
+
const identityEnrich: ProbeMcpEnricher = (servers) => [...servers];
|
|
443
|
+
const base = {
|
|
444
|
+
cwd: "/scratch/phase",
|
|
445
|
+
modelId: "claude-sonnet-5",
|
|
446
|
+
nativeModelId: "claude-sonnet-5",
|
|
447
|
+
config,
|
|
448
|
+
carrier: null as string | null,
|
|
449
|
+
promptText: "x",
|
|
450
|
+
adapter: passthroughAdapter,
|
|
451
|
+
enrichMcpServers: identityEnrich,
|
|
452
|
+
log: () => {},
|
|
453
|
+
};
|
|
454
|
+
const failAt = async (broken: Partial<AcpConnectionLike>, adapter?: ProbeAdapterSeam): Promise<ProbePhaseError> => {
|
|
455
|
+
const calls: RecordedCall[] = [];
|
|
456
|
+
const conn = { ...makeRecordingConnection(calls), ...broken };
|
|
457
|
+
try {
|
|
458
|
+
await driveProbeTurn(conn, { ...base, adapter: adapter ?? passthroughAdapter });
|
|
459
|
+
} catch (err) {
|
|
460
|
+
assert.ok(err instanceof ProbePhaseError, "turn failures are ProbePhaseError");
|
|
461
|
+
return err;
|
|
462
|
+
}
|
|
463
|
+
assert.fail("expected the broken connection to fail the turn");
|
|
464
|
+
};
|
|
465
|
+
|
|
466
|
+
const expectPhase = async (
|
|
467
|
+
phase: ProbeTurnPhase,
|
|
468
|
+
broken: Partial<AcpConnectionLike>,
|
|
469
|
+
adapter?: ProbeAdapterSeam,
|
|
470
|
+
): Promise<void> => {
|
|
471
|
+
const err = await failAt(broken, adapter);
|
|
472
|
+
assert.equal(err.phase, phase, `failure attributes to ${phase}`);
|
|
473
|
+
};
|
|
474
|
+
await expectPhase("initialize", { initialize: async () => Promise.reject(new Error("boom")) });
|
|
475
|
+
await expectPhase("newSession", { newSession: async () => Promise.reject(new Error("boom")) });
|
|
476
|
+
await expectPhase("newSession", { newSession: async () => ({}) }); // no sessionId
|
|
477
|
+
await expectPhase(
|
|
478
|
+
"enforceModel",
|
|
479
|
+
{},
|
|
480
|
+
{
|
|
481
|
+
buildSessionMeta: () => undefined,
|
|
482
|
+
enforceModel: async () => Promise.reject(new Error("boom")),
|
|
483
|
+
},
|
|
484
|
+
);
|
|
485
|
+
await expectPhase("prompt", { prompt: async () => Promise.reject(new Error("boom")) });
|
|
486
|
+
|
|
487
|
+
// Timeout path — a hung set-model must attribute to enforceModel AND flag timedOut.
|
|
488
|
+
const calls: RecordedCall[] = [];
|
|
489
|
+
try {
|
|
490
|
+
await driveProbeTurn(makeRecordingConnection(calls), {
|
|
491
|
+
...base,
|
|
492
|
+
adapter: { buildSessionMeta: () => undefined, enforceModel: () => new Promise<never>(() => {}) },
|
|
493
|
+
timeouts: { initializeMs: 1000, newSessionMs: 1000, setModelMs: 50, promptMs: 1000 },
|
|
494
|
+
});
|
|
495
|
+
assert.fail("hung set-model must time the turn out");
|
|
496
|
+
} catch (err) {
|
|
497
|
+
assert.ok(err instanceof ProbePhaseError, "timeout is a ProbePhaseError");
|
|
498
|
+
assert.equal(err.phase, "enforceModel", "timeout attributes to enforceModel");
|
|
499
|
+
assert.equal(err.timedOut, true, "timeout is flagged timedOut");
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
// ===========================================================================
|
|
504
|
+
// 3) FIXTURE WIRE INSTRUMENTATION — real child, raw JSON-RPC, no API
|
|
505
|
+
// ===========================================================================
|
|
506
|
+
|
|
507
|
+
interface FixtureChild {
|
|
508
|
+
child: ChildProcessByStdio<Writable, Readable, Readable>;
|
|
509
|
+
request(msg: Record<string, unknown>, timeoutMs?: number): Promise<Record<string, unknown>>;
|
|
510
|
+
notify(msg: Record<string, unknown>): void;
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
function spawnFixture(env: Record<string, string>): FixtureChild {
|
|
514
|
+
const child = spawn(process.execPath, ["--experimental-strip-types", FIXTURE_PATH], {
|
|
515
|
+
cwd: tmp,
|
|
516
|
+
env: { ...process.env, ...env },
|
|
517
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
518
|
+
}) as ChildProcessByStdio<Writable, Readable, Readable>;
|
|
519
|
+
let buffer = "";
|
|
520
|
+
const pending: Array<{ id: number; resolve: (msg: Record<string, unknown>) => void }> = [];
|
|
521
|
+
child.stdout.on("data", (chunk) => {
|
|
522
|
+
buffer += chunk.toString();
|
|
523
|
+
let nl = buffer.indexOf("\n");
|
|
524
|
+
while (nl !== -1) {
|
|
525
|
+
const line = buffer.slice(0, nl).trim();
|
|
526
|
+
buffer = buffer.slice(nl + 1);
|
|
527
|
+
if (line.length > 0) {
|
|
528
|
+
const msg = JSON.parse(line) as Record<string, unknown>;
|
|
529
|
+
const i = pending.findIndex((p) => p.id === msg.id);
|
|
530
|
+
if (i !== -1) pending.splice(i, 1)[0].resolve(msg);
|
|
531
|
+
}
|
|
532
|
+
nl = buffer.indexOf("\n");
|
|
533
|
+
}
|
|
534
|
+
});
|
|
535
|
+
return {
|
|
536
|
+
child,
|
|
537
|
+
request(msg, timeoutMs = 10_000) {
|
|
538
|
+
return new Promise((resolvePromise, reject) => {
|
|
539
|
+
const timer = setTimeout(
|
|
540
|
+
() => reject(new Error(`fixture did not answer ${msg.method} within ${timeoutMs}ms`)),
|
|
541
|
+
timeoutMs,
|
|
542
|
+
);
|
|
543
|
+
pending.push({
|
|
544
|
+
id: msg.id as number,
|
|
545
|
+
resolve: (m) => {
|
|
546
|
+
clearTimeout(timer);
|
|
547
|
+
resolvePromise(m);
|
|
548
|
+
},
|
|
549
|
+
});
|
|
550
|
+
child.stdin.write(`${JSON.stringify(msg)}\n`);
|
|
551
|
+
});
|
|
552
|
+
},
|
|
553
|
+
notify(msg) {
|
|
554
|
+
child.stdin.write(`${JSON.stringify(msg)}\n`);
|
|
555
|
+
},
|
|
556
|
+
};
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
const INIT_PARAMS = {
|
|
560
|
+
protocolVersion: "2024-11-05",
|
|
561
|
+
capabilities: {},
|
|
562
|
+
clientInfo: { name: "check-probe-ordering", version: "0" },
|
|
563
|
+
};
|
|
564
|
+
|
|
565
|
+
// --- probe mode: delay honored, marker set stamped, probeRunId required -----
|
|
566
|
+
{
|
|
567
|
+
const logPath = join(tmp, "probe-mode.ndjson");
|
|
568
|
+
const delayMs = 400;
|
|
569
|
+
const fx = spawnFixture({
|
|
570
|
+
[PROBE_ENV.eventLog]: logPath,
|
|
571
|
+
[PROBE_ENV.startupDelayMs]: String(delayMs),
|
|
572
|
+
[PROBE_ENV.runId]: "gate-run",
|
|
573
|
+
[PROBE_ENV.nonce]: "GATE_NONCE",
|
|
574
|
+
});
|
|
575
|
+
try {
|
|
576
|
+
// The injected delay sits before serving: this first request only answers
|
|
577
|
+
// after ≥ delayMs (generous wire timeout keeps slow machines green).
|
|
578
|
+
await fx.request({ jsonrpc: "2.0", id: 1, method: "initialize", params: INIT_PARAMS });
|
|
579
|
+
fx.notify({ jsonrpc: "2.0", method: "notifications/initialized" });
|
|
580
|
+
const list = await fx.request({ jsonrpc: "2.0", id: 2, method: "tools/list" });
|
|
581
|
+
const tools = (list.result as { tools: Array<{ name: string; inputSchema?: { required?: string[] } }> }).tools;
|
|
582
|
+
assert.equal(tools.length, 1, "fixture serves exactly one tool");
|
|
583
|
+
assert.equal(tools[0].name, "probe_nonce", "the tool is probe_nonce");
|
|
584
|
+
assert.ok(
|
|
585
|
+
tools[0].inputSchema?.required?.includes("probeRunId"),
|
|
586
|
+
"probe mode REQUIRES probeRunId [QK:PROBE-FIXTURE-RUNID-REQUIRED] — the §11-7 cross-layer join key",
|
|
587
|
+
);
|
|
588
|
+
|
|
589
|
+
const good = await fx.request({
|
|
590
|
+
jsonrpc: "2.0",
|
|
591
|
+
id: 3,
|
|
592
|
+
method: "tools/call",
|
|
593
|
+
params: { name: "probe_nonce", arguments: { probeRunId: "prb-gate" } },
|
|
594
|
+
});
|
|
595
|
+
const goodText = JSON.stringify(good.result ?? {});
|
|
596
|
+
assert.ok(goodText.includes("GATE_NONCE"), "a correlated call returns the nonce");
|
|
597
|
+
|
|
598
|
+
const bad = await fx.request({
|
|
599
|
+
jsonrpc: "2.0",
|
|
600
|
+
id: 4,
|
|
601
|
+
method: "tools/call",
|
|
602
|
+
params: { name: "probe_nonce", arguments: {} },
|
|
603
|
+
});
|
|
604
|
+
const badIsError = bad.error !== undefined || (bad.result as { isError?: boolean } | undefined)?.isError === true;
|
|
605
|
+
assert.ok(badIsError, "a call WITHOUT probeRunId is refused");
|
|
606
|
+
|
|
607
|
+
// Marker set + ordering + honored delay, read off the one shared log.
|
|
608
|
+
// (appendFileSync in the fixture is synchronous, but poll briefly anyway so
|
|
609
|
+
// a slow fs never turns a green wire into a red gate.)
|
|
610
|
+
let events: ProbeEvent[] = [];
|
|
611
|
+
for (let i = 0; i < 20; i++) {
|
|
612
|
+
({ events } = readProbeEvents(logPath));
|
|
613
|
+
if (events.some((e) => e.event === PROBE_EVENTS.fixtureToolsCallReplied)) break;
|
|
614
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
615
|
+
}
|
|
616
|
+
const names = events.map((e) => e.event);
|
|
617
|
+
for (const expected of [
|
|
618
|
+
PROBE_EVENTS.fixtureProcessStart,
|
|
619
|
+
PROBE_EVENTS.fixtureDelayStart,
|
|
620
|
+
PROBE_EVENTS.fixtureDelayEnd,
|
|
621
|
+
PROBE_EVENTS.fixtureTransportConnected,
|
|
622
|
+
PROBE_EVENTS.fixtureInitializeReceived,
|
|
623
|
+
PROBE_EVENTS.fixtureToolsListReceived,
|
|
624
|
+
PROBE_EVENTS.toolsListResponseForwarded,
|
|
625
|
+
PROBE_EVENTS.fixtureToolsCallReceived,
|
|
626
|
+
PROBE_EVENTS.fixtureToolsCallReplied,
|
|
627
|
+
]) {
|
|
628
|
+
assert.ok(names.includes(expected), `fixture stamps ${expected} [QK:PROBE-FIXTURE-WIRE-MARKER]`);
|
|
629
|
+
}
|
|
630
|
+
const delayStart = events.find((e) => e.event === PROBE_EVENTS.fixtureDelayStart);
|
|
631
|
+
const delayEnd = events.find((e) => e.event === PROBE_EVENTS.fixtureDelayEnd);
|
|
632
|
+
assert.ok(
|
|
633
|
+
delayStart && delayEnd && delayEnd.tsMs - delayStart.tsMs >= delayMs - 5,
|
|
634
|
+
`the startup delay is actually honored (${delayEnd?.tsMs}-${delayStart?.tsMs} vs ${delayMs})`,
|
|
635
|
+
);
|
|
636
|
+
const forwarded = events.find((e) => e.event === PROBE_EVENTS.toolsListResponseForwarded);
|
|
637
|
+
assert.ok(
|
|
638
|
+
Array.isArray(forwarded?.tools) && (forwarded.tools as string[]).includes("probe_nonce"),
|
|
639
|
+
"the forwarded marker names the expected tool",
|
|
640
|
+
);
|
|
641
|
+
const callReceived = events.find((e) => e.event === PROBE_EVENTS.fixtureToolsCallReceived);
|
|
642
|
+
assert.equal(callReceived?.probeRunId, "prb-gate", "the callability marker carries the correlation argument");
|
|
643
|
+
assert.ok(
|
|
644
|
+
(forwarded?.tsMs ?? 0) >= (delayEnd?.tsMs ?? Number.MAX_SAFE_INTEGER),
|
|
645
|
+
"nothing is forwarded before the injected delay elapses",
|
|
646
|
+
);
|
|
647
|
+
assert.ok(
|
|
648
|
+
events.every((e) => e.runId === "gate-run"),
|
|
649
|
+
"every fixture event carries the runId",
|
|
650
|
+
);
|
|
651
|
+
} finally {
|
|
652
|
+
await terminateChild(fx.child);
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
// --- legacy mode: byte-compatible surface for smoke-acp-mcp-live ------------
|
|
657
|
+
{
|
|
658
|
+
const legacyLog = join(tmp, "legacy-should-not-exist.ndjson");
|
|
659
|
+
const fx = spawnFixture({ [PROBE_ENV.nonce]: "LEGACY_NONCE" });
|
|
660
|
+
try {
|
|
661
|
+
await fx.request({ jsonrpc: "2.0", id: 1, method: "initialize", params: INIT_PARAMS });
|
|
662
|
+
fx.notify({ jsonrpc: "2.0", method: "notifications/initialized" });
|
|
663
|
+
const list = await fx.request({ jsonrpc: "2.0", id: 2, method: "tools/list" });
|
|
664
|
+
const tools = (list.result as { tools: Array<{ name: string; inputSchema?: { required?: string[] } }> }).tools;
|
|
665
|
+
assert.equal(tools[0]?.name, "probe_nonce", "legacy mode still serves probe_nonce");
|
|
666
|
+
assert.ok(
|
|
667
|
+
!tools[0].inputSchema?.required?.includes("probeRunId"),
|
|
668
|
+
"legacy mode does NOT require probeRunId (smoke-acp-mcp-live compat)",
|
|
669
|
+
);
|
|
670
|
+
const call = await fx.request({
|
|
671
|
+
jsonrpc: "2.0",
|
|
672
|
+
id: 3,
|
|
673
|
+
method: "tools/call",
|
|
674
|
+
params: { name: "probe_nonce", arguments: {} },
|
|
675
|
+
});
|
|
676
|
+
assert.ok(
|
|
677
|
+
JSON.stringify(call.result ?? {}).includes("LEGACY_NONCE"),
|
|
678
|
+
"legacy argument-less call still returns the nonce",
|
|
679
|
+
);
|
|
680
|
+
assert.ok(!existsSync(legacyLog), "legacy mode writes no event log");
|
|
681
|
+
} finally {
|
|
682
|
+
await terminateChild(fx.child);
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
// ===========================================================================
|
|
687
|
+
// 4) EVENT LOG INTEGRITY — the envelope contract at the door
|
|
688
|
+
// ===========================================================================
|
|
689
|
+
//
|
|
690
|
+
// The shared NDJSON log is the record the verdict is read off, so a line that
|
|
691
|
+
// merely parses as JSON is not yet an event. §11-7's two decisive reads are
|
|
692
|
+
// ABSENCE (a marker that never appears) and ORDERING (one tsMs axis across
|
|
693
|
+
// processes) — exactly what a typo'd marker name or a broken stamp corrupts
|
|
694
|
+
// while still looking healthy. Such a line must leave through the SAME door as
|
|
695
|
+
// a truncated one (malformed → the runner's INVALIDATED path above), and a
|
|
696
|
+
// caller's payload must never be able to rewrite the envelope it rides in.
|
|
697
|
+
{
|
|
698
|
+
// Writer round-trip: its own line clears the door, payload rides alongside,
|
|
699
|
+
// and the readable stamp agrees with the sort axis EXACTLY.
|
|
700
|
+
const okPath = join(tmp, "envelope-ok.ndjson");
|
|
701
|
+
appendProbeEvent(okPath, "env-run", PROBE_EVENTS.runStart, { note: "hello" });
|
|
702
|
+
const roundTrip = readProbeEvents(okPath);
|
|
703
|
+
assert.equal(roundTrip.malformed.length, 0, "the writer's own line clears the envelope contract");
|
|
704
|
+
assert.equal(roundTrip.events.length, 1, "…and lands as exactly one event");
|
|
705
|
+
const written = roundTrip.events[0];
|
|
706
|
+
assert.equal(Date.parse(written.ts), written.tsMs, "ts is DERIVED from tsMs — no millisecond straddle");
|
|
707
|
+
assert.equal(written.note, "hello", "payload rides alongside the envelope");
|
|
708
|
+
assert.ok(
|
|
709
|
+
EVENT_LOG_SRC.includes("const tsMs = Date.now();") && !EVENT_LOG_SRC.includes("ts: new Date().toISOString()"),
|
|
710
|
+
"the writer reads the clock ONCE and derives ts from it (a second read could straddle a millisecond)",
|
|
711
|
+
);
|
|
712
|
+
|
|
713
|
+
// Envelope authority: a payload key may never rewrite the run id, the marker
|
|
714
|
+
// name, or the sort axis of its own evidence line. The write REFUSES — a
|
|
715
|
+
// silent merge (either key order) would leave a lying line on disk.
|
|
716
|
+
const authPath = join(tmp, "envelope-authority.ndjson");
|
|
717
|
+
const refusedEveryKey = RESERVED_EVENT_KEYS.every((key) => {
|
|
718
|
+
try {
|
|
719
|
+
appendProbeEvent(authPath, "env-run", PROBE_EVENTS.runStart, { [key]: "hijacked" });
|
|
720
|
+
return false;
|
|
721
|
+
} catch {
|
|
722
|
+
return true;
|
|
723
|
+
}
|
|
724
|
+
});
|
|
725
|
+
assert.ok(
|
|
726
|
+
refusedEveryKey && !existsSync(authPath),
|
|
727
|
+
"a payload carrying a reserved envelope key is REFUSED and nothing is written [QK:PROBE-LOG-ENVELOPE-AUTHORITY]",
|
|
728
|
+
);
|
|
729
|
+
|
|
730
|
+
const STAMP_MS = 1_700_000_000_000;
|
|
731
|
+
const stamp = { seq: 0, pid: 1234, ts: new Date(STAMP_MS).toISOString(), tsMs: STAMP_MS, runId: "env-run" };
|
|
732
|
+
|
|
733
|
+
// Vocabulary: a name neither writer nor classifier knows is not a new event —
|
|
734
|
+
// it is a marker that went MISSING, and §11-7 reads absence as evidence.
|
|
735
|
+
const vocabPath = join(tmp, "envelope-vocab.ndjson");
|
|
736
|
+
appendFileSync(vocabPath, `${JSON.stringify({ ...stamp, event: "new_sesion_end" })}\n`, "utf8");
|
|
737
|
+
const vocab = readProbeEvents(vocabPath);
|
|
738
|
+
assert.ok(
|
|
739
|
+
vocab.events.length === 0 && vocab.malformed.length === 1,
|
|
740
|
+
"an unknown event name is a MISSING marker, never a quiet extra line [QK:PROBE-LOG-EVENT-VOCABULARY]",
|
|
741
|
+
);
|
|
742
|
+
|
|
743
|
+
// Structural envelope: the shared sort axis and its readable twin. A tsMs
|
|
744
|
+
// that is absent or non-numeric makes the comparator NaN and un-orders the
|
|
745
|
+
// whole log, so ran-ahead / ordering-kept would be read off file order.
|
|
746
|
+
// These lines deliberately carry an event with NO payload rule (run_start is
|
|
747
|
+
// forensic): the envelope checks must be what rejects them, or this claim
|
|
748
|
+
// would be silently proven by the payload layer instead of its own code.
|
|
749
|
+
const brokenPath = join(tmp, "envelope-broken.ndjson");
|
|
750
|
+
const brokenLines = [
|
|
751
|
+
{ ...stamp, event: PROBE_EVENTS.runStart, tsMs: undefined }, // no sort axis at all
|
|
752
|
+
{ ...stamp, event: PROBE_EVENTS.runStart, tsMs: "1700000000000" }, // string axis → NaN comparator
|
|
753
|
+
{ ...stamp, event: PROBE_EVENTS.runStart, ts: "not-a-timestamp" }, // readable twin unparseable
|
|
754
|
+
{ ...stamp, event: PROBE_EVENTS.runStart, tsMs: STAMP_MS + 1 }, // twin disagrees by 1ms
|
|
755
|
+
{ ...stamp, event: PROBE_EVENTS.runStart, seq: -1 }, // per-process counter
|
|
756
|
+
{ ...stamp, event: PROBE_EVENTS.runStart, pid: 0 }, // writer identity
|
|
757
|
+
{ ...stamp, event: PROBE_EVENTS.runStart, runId: "" }, // unattributable line
|
|
758
|
+
];
|
|
759
|
+
for (const line of brokenLines) appendFileSync(brokenPath, `${JSON.stringify(line)}\n`, "utf8");
|
|
760
|
+
const broken = readProbeEvents(brokenPath);
|
|
761
|
+
assert.ok(
|
|
762
|
+
broken.events.length === 0 && broken.malformed.length === brokenLines.length,
|
|
763
|
+
"a JSON-valid line with a broken envelope is MALFORMED, never an event [QK:PROBE-LOG-ENVELOPE-SCHEMA]",
|
|
764
|
+
);
|
|
765
|
+
|
|
766
|
+
// Payload contract: the envelope can be perfect and the line still a lie,
|
|
767
|
+
// because the classifier judges on payload. `ok === true` is FALSE for the
|
|
768
|
+
// string "true", so a corrupted phase end silently reads as a phase FAILURE
|
|
769
|
+
// (D / P0) instead of invalidating the run; an uncorrelatable id does the
|
|
770
|
+
// same toward absence. Every field the classifier reads is typed at the door.
|
|
771
|
+
const payloadPath = join(tmp, "payload-broken.ndjson");
|
|
772
|
+
const badPayloads = [
|
|
773
|
+
{ ...stamp, event: PROBE_EVENTS.newSessionEnd }, // no ok at all → reads as "failed"
|
|
774
|
+
{ ...stamp, event: PROBE_EVENTS.promptEnd, ok: "true" }, // string, not boolean
|
|
775
|
+
{ ...stamp, event: PROBE_EVENTS.initializeEnd, ok: true, timedOut: "yes" },
|
|
776
|
+
{ ...stamp, event: PROBE_EVENTS.setModelEnd, ok: false, error: 42 },
|
|
777
|
+
{ ...stamp, event: PROBE_EVENTS.acpNoSuchTool, toolId: 7 }, // the absence id
|
|
778
|
+
{ ...stamp, event: PROBE_EVENTS.acpToolCallObserved, providerToolId: "x" }, // no join key
|
|
779
|
+
{ ...stamp, event: PROBE_EVENTS.acpToolCallObserved, probeRunId: 9 },
|
|
780
|
+
{ ...stamp, event: PROBE_EVENTS.promptReply, carriesNonce: "yes" }, // the nonce echo
|
|
781
|
+
{ ...stamp, event: PROBE_EVENTS.fixtureToolsCallReceived, tool: 1, probeRunId: "p" },
|
|
782
|
+
];
|
|
783
|
+
for (const line of badPayloads) appendFileSync(payloadPath, `${JSON.stringify(line)}\n`, "utf8");
|
|
784
|
+
const payloadBroken = readProbeEvents(payloadPath);
|
|
785
|
+
assert.ok(
|
|
786
|
+
payloadBroken.events.length === 0 && payloadBroken.malformed.length === badPayloads.length,
|
|
787
|
+
"a valid envelope with a payload the classifier cannot judge on is MALFORMED [QK:PROBE-LOG-PAYLOAD-SCHEMA]",
|
|
788
|
+
);
|
|
789
|
+
|
|
790
|
+
// …and the readings the writer legitimately omits are NOT malformed: the
|
|
791
|
+
// fixture stamps an inbound tools/call before validating it, and an ACP frame
|
|
792
|
+
// may carry no name/title. Refusing those would manufacture INVALIDATED runs
|
|
793
|
+
// out of real model behavior (a call with no join key IS the absence reading).
|
|
794
|
+
const observedPath = join(tmp, "payload-observed.ndjson");
|
|
795
|
+
const observedLines = [
|
|
796
|
+
{ ...stamp, event: PROBE_EVENTS.fixtureToolsCallReceived, tool: PROBE_EXPECTED_TOOL }, // model called with no join key
|
|
797
|
+
{ ...stamp, event: PROBE_EVENTS.acpToolCallObserved, probeRunId: "prb-x" }, // no extractable provider id
|
|
798
|
+
{ ...stamp, event: PROBE_EVENTS.newSessionEnd, ok: false, timedOut: true, error: "boom" },
|
|
799
|
+
];
|
|
800
|
+
for (const line of observedLines) appendFileSync(observedPath, `${JSON.stringify(line)}\n`, "utf8");
|
|
801
|
+
const observed = readProbeEvents(observedPath);
|
|
802
|
+
assert.ok(
|
|
803
|
+
observed.malformed.length === 0 && observed.events.length === observedLines.length,
|
|
804
|
+
"a legitimately absent optional field is an OBSERVATION, not a corrupt line",
|
|
805
|
+
);
|
|
806
|
+
|
|
807
|
+
// The wire-availability proxy must NAME the expected tool. §11-7 defines the
|
|
808
|
+
// marker as the expected-tool tools/list frame reaching the pipe, so a marker
|
|
809
|
+
// carrying another list is not that event — and it drives ran-ahead, C, and
|
|
810
|
+
// the B branch. (The fixture only stamps it when the tool is present; this is
|
|
811
|
+
// the same check at the READING door, where the verdict is actually taken.)
|
|
812
|
+
const wirePath = join(tmp, "payload-wire.ndjson");
|
|
813
|
+
const wireLines = [
|
|
814
|
+
{ ...stamp, event: PROBE_EVENTS.toolsListResponseForwarded }, // no tools at all
|
|
815
|
+
{ ...stamp, event: PROBE_EVENTS.toolsListResponseForwarded, tools: [] }, // served nothing
|
|
816
|
+
{ ...stamp, event: PROBE_EVENTS.toolsListResponseForwarded, tools: ["other_tool"] },
|
|
817
|
+
{ ...stamp, event: PROBE_EVENTS.toolsListResponseForwarded, tools: [7] },
|
|
818
|
+
];
|
|
819
|
+
for (const line of wireLines) appendFileSync(wirePath, `${JSON.stringify(line)}\n`, "utf8");
|
|
820
|
+
const wire = readProbeEvents(wirePath);
|
|
821
|
+
assert.ok(
|
|
822
|
+
wire.events.length === 0 && wire.malformed.length === wireLines.length,
|
|
823
|
+
"a wire marker that does not name the expected tool is not wire-availability [QK:PROBE-LOG-WIRE-MARKER-NAMES-TOOL]",
|
|
824
|
+
);
|
|
825
|
+
const goodWirePath = join(tmp, "payload-wire-ok.ndjson");
|
|
826
|
+
appendFileSync(
|
|
827
|
+
goodWirePath,
|
|
828
|
+
`${JSON.stringify({ ...stamp, event: PROBE_EVENTS.toolsListResponseForwarded, tools: [PROBE_EXPECTED_TOOL, "x"] })}\n`,
|
|
829
|
+
"utf8",
|
|
830
|
+
);
|
|
831
|
+
assert.equal(readProbeEvents(goodWirePath).events.length, 1, "a marker naming the expected tool still passes");
|
|
832
|
+
|
|
833
|
+
// Coverage, stated as a hand-written literal (never read off the SUT): these
|
|
834
|
+
// are exactly the events the classifier judges payload on. A new classifier
|
|
835
|
+
// read without a door rule must turn this red rather than pass unnoticed.
|
|
836
|
+
assert.deepEqual(
|
|
837
|
+
[...PAYLOAD_CONTRACT_EVENTS].sort(),
|
|
838
|
+
[
|
|
839
|
+
PROBE_EVENTS.acpNoSuchTool,
|
|
840
|
+
PROBE_EVENTS.acpToolCallObserved,
|
|
841
|
+
PROBE_EVENTS.fixtureToolsCallReceived,
|
|
842
|
+
PROBE_EVENTS.initializeEnd,
|
|
843
|
+
PROBE_EVENTS.newSessionEnd,
|
|
844
|
+
PROBE_EVENTS.observationWindowEnd,
|
|
845
|
+
PROBE_EVENTS.promptEnd,
|
|
846
|
+
PROBE_EVENTS.promptReply,
|
|
847
|
+
PROBE_EVENTS.setModelEnd,
|
|
848
|
+
PROBE_EVENTS.toolsListResponseForwarded,
|
|
849
|
+
PROBE_EVENTS.shimBoot,
|
|
850
|
+
PROBE_EVENTS.shimPromptForwarded,
|
|
851
|
+
PROBE_EVENTS.shimInitSnapshot,
|
|
852
|
+
].sort(),
|
|
853
|
+
"the payload contract covers exactly the events the classifier judges on",
|
|
854
|
+
);
|
|
855
|
+
|
|
856
|
+
// One SSOT for the expected tool: fixture, parser, and classifier must not
|
|
857
|
+
// each keep a private literal that can drift apart in silence.
|
|
858
|
+
assert.ok(
|
|
859
|
+
EVENT_LOG_SRC.split('"probe_nonce"').length - 1 === 1 &&
|
|
860
|
+
!FIXTURE_SRC.includes('= "probe_nonce"') &&
|
|
861
|
+
!VERDICT_SRC.includes('= "probe_nonce"'),
|
|
862
|
+
"the expected tool name is defined ONCE (probe-event-log.ts) and imported by fixture and classifier",
|
|
863
|
+
);
|
|
864
|
+
|
|
865
|
+
// Truncation still behaves — the widened door did not lose the original one.
|
|
866
|
+
const mixedPath = join(tmp, "envelope-mixed.ndjson");
|
|
867
|
+
appendProbeEvent(mixedPath, "env-run", PROBE_EVENTS.runStart);
|
|
868
|
+
appendFileSync(mixedPath, '{"event":"run_end","runId":"env-run"\n', "utf8");
|
|
869
|
+
const mixed = readProbeEvents(mixedPath);
|
|
870
|
+
assert.ok(
|
|
871
|
+
mixed.events.length === 1 && mixed.malformed.length === 1,
|
|
872
|
+
"an unparseable line is still malformed and the good line still parses",
|
|
873
|
+
);
|
|
874
|
+
|
|
875
|
+
// The window close is judged payload: `reason` decides whether a missing wire
|
|
876
|
+
// marker is a reading or our own teardown, so an unknown reason must not fall
|
|
877
|
+
// through to whatever branch consumes it.
|
|
878
|
+
const winPath = join(tmp, "payload-window.ndjson");
|
|
879
|
+
const winStamp = { seq: 0, pid: 7, ts: new Date(2_000).toISOString(), tsMs: 2_000, runId: "w" };
|
|
880
|
+
appendFileSync(
|
|
881
|
+
winPath,
|
|
882
|
+
`${JSON.stringify({ ...winStamp, event: PROBE_EVENTS.observationWindowEnd, reason: "whenever", markerSeen: false })}\n`,
|
|
883
|
+
"utf8",
|
|
884
|
+
);
|
|
885
|
+
appendFileSync(
|
|
886
|
+
winPath,
|
|
887
|
+
`${JSON.stringify({ ...winStamp, seq: 1, tsMs: 2_001, ts: new Date(2_001).toISOString(), event: PROBE_EVENTS.observationWindowEnd, reason: "deadline", markerSeen: "no" })}\n`,
|
|
888
|
+
"utf8",
|
|
889
|
+
);
|
|
890
|
+
appendFileSync(
|
|
891
|
+
winPath,
|
|
892
|
+
`${JSON.stringify({ ...winStamp, seq: 2, tsMs: 2_002, ts: new Date(2_002).toISOString(), event: PROBE_EVENTS.observationWindowEnd, reason: "deadline", markerSeen: false })}\n`,
|
|
893
|
+
"utf8",
|
|
894
|
+
);
|
|
895
|
+
const win = readProbeEvents(winPath);
|
|
896
|
+
assert.ok(
|
|
897
|
+
win.malformed.length === 2 && win.events.length === 1,
|
|
898
|
+
"an unknown window reason and a non-boolean markerSeen are MALFORMED; the closed vocabulary passes [QK:PROBE-LOG-WINDOW-REASON-VOCAB]",
|
|
899
|
+
);
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
// ===========================================================================
|
|
903
|
+
// 4b) EVENT LOG STREAM INTEGRITY — per-writer order, checked on RAW APPEND ORDER
|
|
904
|
+
// ===========================================================================
|
|
905
|
+
{
|
|
906
|
+
const stream = (lines: Array<Record<string, unknown>>): ReturnType<typeof readProbeEvents> => {
|
|
907
|
+
const p = join(tmp, `stream-${Math.random().toString(36).slice(2)}.ndjson`);
|
|
908
|
+
for (const l of lines) appendFileSync(p, `${JSON.stringify(l)}\n`, "utf8");
|
|
909
|
+
return readProbeEvents(p);
|
|
910
|
+
};
|
|
911
|
+
const line = (pid: number, seq: number, tsMs: number, event = PROBE_EVENTS.acpToolCallRaw) => ({
|
|
912
|
+
seq,
|
|
913
|
+
pid,
|
|
914
|
+
ts: new Date(tsMs).toISOString(),
|
|
915
|
+
tsMs,
|
|
916
|
+
runId: "s",
|
|
917
|
+
event,
|
|
918
|
+
});
|
|
919
|
+
|
|
920
|
+
// Clean stream: two writers interleaved, each strictly increasing.
|
|
921
|
+
const clean = stream([line(1, 0, 100), line(2, 0, 101), line(1, 1, 102), line(2, 1, 103)]);
|
|
922
|
+
assert.equal(clean.sequenceViolations.length, 0, "interleaved writers, each monotonic, is a clean stream");
|
|
923
|
+
|
|
924
|
+
// GAPS are fine — the counter is process-wide, not file-wide, so a process
|
|
925
|
+
// writing to more than one log skips numbers here by construction.
|
|
926
|
+
const gapped = stream([line(1, 0, 100), line(1, 7, 101), line(1, 90, 102)]);
|
|
927
|
+
assert.equal(
|
|
928
|
+
gapped.sequenceViolations.length,
|
|
929
|
+
0,
|
|
930
|
+
"a per-pid seq GAP is not a violation — the counter is process-wide",
|
|
931
|
+
);
|
|
932
|
+
|
|
933
|
+
// A repeat is two lines claiming one slot: within a millisecond they cannot be
|
|
934
|
+
// ordered against each other at all.
|
|
935
|
+
const dupSeq = stream([line(1, 5, 100), line(1, 5, 100)]);
|
|
936
|
+
assert.ok(
|
|
937
|
+
dupSeq.sequenceViolations.length === 1 && /seq 5 does not exceed/.test(dupSeq.sequenceViolations[0]),
|
|
938
|
+
"a repeated per-pid seq is a stream violation [QK:PROBE-LOG-SEQ-STRICTLY-INCREASING]",
|
|
939
|
+
);
|
|
940
|
+
const backSeq = stream([line(1, 5, 100), line(1, 4, 101)]);
|
|
941
|
+
assert.equal(backSeq.sequenceViolations.length, 1, "a per-pid seq going backwards is a stream violation");
|
|
942
|
+
|
|
943
|
+
// THE claim, asserted BEFORE the clock rule below on purpose: validation runs
|
|
944
|
+
// on the RAW append order. This log sorts into perfect order — a post-sort
|
|
945
|
+
// check would see nothing — yet the file itself has pid 1 writing seq 9 before
|
|
946
|
+
// seq 2. (Ordering matters here: a check moved after the sort also perturbs the
|
|
947
|
+
// clock-regression case, so this assertion must be the one that fires.)
|
|
948
|
+
const outOfOrderInFile = stream([line(1, 9, 900), line(1, 2, 200)]);
|
|
949
|
+
assert.ok(
|
|
950
|
+
outOfOrderInFile.sequenceViolations.length > 0,
|
|
951
|
+
"per-writer order is judged on the RAW file order, not after the sort has rewritten it [QK:PROBE-LOG-RAW-ORDER-BEFORE-SORT]",
|
|
952
|
+
);
|
|
953
|
+
assert.deepEqual(
|
|
954
|
+
outOfOrderInFile.events.map((e) => e.seq),
|
|
955
|
+
[2, 9],
|
|
956
|
+
"the returned events are still sorted — the violation is reported, not repaired",
|
|
957
|
+
);
|
|
958
|
+
|
|
959
|
+
// One process reads one clock. A regression means the stamps were rewritten or
|
|
960
|
+
// the clock stepped — and those stamps ARE the ordering evidence.
|
|
961
|
+
const backTs = stream([line(1, 0, 500), line(1, 1, 499)]);
|
|
962
|
+
assert.ok(
|
|
963
|
+
backTs.sequenceViolations.length === 1 && /runs BACKWARDS/.test(backTs.sequenceViolations[0]),
|
|
964
|
+
"a per-pid tsMs regression is a stream violation [QK:PROBE-LOG-TS-NO-REGRESSION]",
|
|
965
|
+
);
|
|
966
|
+
// Different pids are independent: cross-process stamps are not comparable this
|
|
967
|
+
// way, and demanding it would flag every normal interleaving.
|
|
968
|
+
assert.equal(
|
|
969
|
+
stream([line(1, 0, 500), line(2, 0, 499)]).sequenceViolations.length,
|
|
970
|
+
0,
|
|
971
|
+
"a LOWER stamp from a DIFFERENT pid is not a regression",
|
|
972
|
+
);
|
|
973
|
+
|
|
974
|
+
// WRITER KEY is (runId, pid). The fixture is a fresh child per run and the OS
|
|
975
|
+
// reuses pids, so a later run's fixture can legitimately hold the same pid and
|
|
976
|
+
// restart its counter at 0. Keying on pid alone would call that healthy log
|
|
977
|
+
// corrupt — and cross-run ordering is not something any verdict reads.
|
|
978
|
+
const pidReuse = stream([
|
|
979
|
+
{ ...line(1, 0, 100), runId: "r1" },
|
|
980
|
+
{ ...line(1, 1, 101), runId: "r1" },
|
|
981
|
+
{ ...line(1, 0, 200), runId: "r2" },
|
|
982
|
+
{ ...line(1, 1, 201), runId: "r2" },
|
|
983
|
+
]);
|
|
984
|
+
assert.equal(
|
|
985
|
+
pidReuse.sequenceViolations.length,
|
|
986
|
+
0,
|
|
987
|
+
"a reused pid restarting its counter in a LATER run is not a violation — the writer key is (runId, pid) [QK:PROBE-LOG-WRITER-KEY-PER-RUN]",
|
|
988
|
+
);
|
|
989
|
+
// …but within one run the same pid is still held to the rule.
|
|
990
|
+
assert.equal(
|
|
991
|
+
stream([
|
|
992
|
+
{ ...line(1, 5, 100), runId: "r1" },
|
|
993
|
+
{ ...line(1, 5, 101), runId: "r1" },
|
|
994
|
+
]).sequenceViolations.length,
|
|
995
|
+
1,
|
|
996
|
+
"the same pid inside ONE run is still strictly increasing",
|
|
997
|
+
);
|
|
998
|
+
|
|
999
|
+
// Malformed lines never participate: they are not events, so they cannot
|
|
1000
|
+
// manufacture a sequence violation on top of their own refusal.
|
|
1001
|
+
const withMalformed = stream([line(1, 0, 100), { junk: true }, line(1, 1, 101)]);
|
|
1002
|
+
assert.ok(
|
|
1003
|
+
withMalformed.malformed.length === 1 && withMalformed.sequenceViolations.length === 0,
|
|
1004
|
+
"a malformed line is refused at the line door and does not also break the stream door",
|
|
1005
|
+
);
|
|
1006
|
+
}
|
|
1007
|
+
|
|
1008
|
+
// ===========================================================================
|
|
1009
|
+
// 5) VERDICT TRUTH TABLE — synthetic paired logs through the PURE classifier
|
|
1010
|
+
// ===========================================================================
|
|
1011
|
+
|
|
1012
|
+
let syntheticSeq = 0;
|
|
1013
|
+
function ev(runId: string, event: string, tsMs: number, payload: Record<string, unknown> = {}): ProbeEvent {
|
|
1014
|
+
return {
|
|
1015
|
+
seq: syntheticSeq++,
|
|
1016
|
+
pid: 1,
|
|
1017
|
+
ts: new Date(tsMs).toISOString(),
|
|
1018
|
+
tsMs,
|
|
1019
|
+
runId,
|
|
1020
|
+
event,
|
|
1021
|
+
...payload,
|
|
1022
|
+
} as ProbeEvent;
|
|
1023
|
+
}
|
|
1024
|
+
|
|
1025
|
+
interface SyntheticRunSpec {
|
|
1026
|
+
runId: string;
|
|
1027
|
+
probeRunId: string;
|
|
1028
|
+
base: number;
|
|
1029
|
+
/** newSession latency; wire marker fires at newSessionStart + wireAt. */
|
|
1030
|
+
nsLatency?: number;
|
|
1031
|
+
wireAt?: number | null; // null → no wire marker at all
|
|
1032
|
+
failPhase?: "initialize" | "newSession" | "enforceModel" | "prompt";
|
|
1033
|
+
fixtureCall?: boolean;
|
|
1034
|
+
fixtureCallAt?: number; // offset from newSessionStart
|
|
1035
|
+
providerToolId?: string;
|
|
1036
|
+
noSuchToolId?: string;
|
|
1037
|
+
nonceEchoed?: boolean;
|
|
1038
|
+
/** How the observation window closed. Default: inferred — the marker was seen
|
|
1039
|
+
* iff a wire marker exists, and the reason follows from that. `"omit"` drops
|
|
1040
|
+
* the marker entirely, which is a TOPOLOGY violation, not a soft default:
|
|
1041
|
+
* without it a missing wire marker cannot be told from our own teardown. */
|
|
1042
|
+
window?: { reason?: ProbeWindowReason; markerSeen?: boolean } | "omit";
|
|
1043
|
+
/** Extra runner-owned lines appended verbatim — used to build duplicate /
|
|
1044
|
+
* end-without-start topology corruptions. */
|
|
1045
|
+
extra?: (runId: string, base: number) => ProbeEvent[];
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
/** Close a synthetic run the way the runner does: the observation window is
|
|
1049
|
+
* stamped BEFORE run_end, on the success and the failure path alike. */
|
|
1050
|
+
function closeRun(out: ProbeEvent[], spec: SyntheticRunSpec, at: number, failed: boolean): ProbeEvent[] {
|
|
1051
|
+
const { runId } = spec;
|
|
1052
|
+
if (spec.window !== "omit") {
|
|
1053
|
+
// Read markerSeen off what the run ACTUALLY emitted, never off the spec: the
|
|
1054
|
+
// classifier now checks the self-reported flag against the log, so a helper
|
|
1055
|
+
// that guesses would manufacture incoherent fixtures (a run that failed at
|
|
1056
|
+
// initialize never reaches the wire marker, whatever `wireAt` says).
|
|
1057
|
+
const emittedWire = out.some((e) => e.event === PROBE_EVENTS.toolsListResponseForwarded);
|
|
1058
|
+
const markerSeen = spec.window?.markerSeen ?? emittedWire;
|
|
1059
|
+
const reason: ProbeWindowReason =
|
|
1060
|
+
spec.window?.reason ?? (failed ? "run-failed" : markerSeen ? "wire-marker" : "deadline");
|
|
1061
|
+
out.push(ev(runId, PROBE_EVENTS.observationWindowEnd, at, { reason, markerSeen }));
|
|
1062
|
+
}
|
|
1063
|
+
out.push(ev(runId, PROBE_EVENTS.runEnd, at + 1, { ok: !failed }));
|
|
1064
|
+
if (spec.extra) out.push(...spec.extra(runId, spec.base));
|
|
1065
|
+
return out;
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
function syntheticRun(spec: SyntheticRunSpec): ProbeEvent[] {
|
|
1069
|
+
const { runId, probeRunId, base } = spec;
|
|
1070
|
+
const nsStart = base + 100;
|
|
1071
|
+
const nsLatency = spec.nsLatency ?? 300;
|
|
1072
|
+
const out: ProbeEvent[] = [ev(runId, PROBE_EVENTS.runStart, base), ev(runId, PROBE_EVENTS.initializeStart, base)];
|
|
1073
|
+
if (spec.failPhase === "initialize") {
|
|
1074
|
+
out.push(ev(runId, PROBE_EVENTS.initializeEnd, base + 50, { ok: false, error: "init boom" }));
|
|
1075
|
+
return closeRun(out, spec, base + 60, true);
|
|
1076
|
+
}
|
|
1077
|
+
out.push(ev(runId, PROBE_EVENTS.initializeEnd, base + 50, { ok: true }));
|
|
1078
|
+
out.push(ev(runId, PROBE_EVENTS.newSessionStart, nsStart));
|
|
1079
|
+
if (spec.wireAt !== null)
|
|
1080
|
+
out.push(ev(runId, PROBE_EVENTS.toolsListResponseForwarded, nsStart + (spec.wireAt ?? 100)));
|
|
1081
|
+
if (spec.failPhase === "newSession") {
|
|
1082
|
+
out.push(
|
|
1083
|
+
ev(runId, PROBE_EVENTS.newSessionEnd, nsStart + nsLatency, { ok: false, timedOut: true, error: "ns boom" }),
|
|
1084
|
+
);
|
|
1085
|
+
return closeRun(out, spec, nsStart + nsLatency + 10, true);
|
|
1086
|
+
}
|
|
1087
|
+
out.push(ev(runId, PROBE_EVENTS.newSessionEnd, nsStart + nsLatency, { ok: true }));
|
|
1088
|
+
const smStart = nsStart + nsLatency + 10;
|
|
1089
|
+
out.push(ev(runId, PROBE_EVENTS.setModelStart, smStart));
|
|
1090
|
+
if (spec.failPhase === "enforceModel") {
|
|
1091
|
+
out.push(ev(runId, PROBE_EVENTS.setModelEnd, smStart + 30, { ok: false, error: "sm boom" }));
|
|
1092
|
+
return closeRun(out, spec, smStart + 40, true);
|
|
1093
|
+
}
|
|
1094
|
+
out.push(ev(runId, PROBE_EVENTS.setModelEnd, smStart + 30, { ok: true }));
|
|
1095
|
+
const pStart = smStart + 50;
|
|
1096
|
+
out.push(ev(runId, PROBE_EVENTS.promptStart, pStart));
|
|
1097
|
+
if (spec.providerToolId !== undefined) {
|
|
1098
|
+
out.push(
|
|
1099
|
+
ev(runId, PROBE_EVENTS.acpToolCallObserved, pStart + 100, { providerToolId: spec.providerToolId, probeRunId }),
|
|
1100
|
+
);
|
|
1101
|
+
}
|
|
1102
|
+
if (spec.fixtureCall) {
|
|
1103
|
+
out.push(
|
|
1104
|
+
ev(runId, PROBE_EVENTS.fixtureToolsCallReceived, nsStart + (spec.fixtureCallAt ?? 500), {
|
|
1105
|
+
tool: "probe_nonce",
|
|
1106
|
+
probeRunId,
|
|
1107
|
+
}),
|
|
1108
|
+
);
|
|
1109
|
+
}
|
|
1110
|
+
if (spec.noSuchToolId !== undefined) {
|
|
1111
|
+
out.push(ev(runId, PROBE_EVENTS.acpNoSuchTool, pStart + 150, { toolId: spec.noSuchToolId }));
|
|
1112
|
+
}
|
|
1113
|
+
if (spec.failPhase === "prompt") {
|
|
1114
|
+
out.push(ev(runId, PROBE_EVENTS.promptEnd, pStart + 200, { ok: false, error: "p boom" }));
|
|
1115
|
+
return closeRun(out, spec, pStart + 210, true);
|
|
1116
|
+
}
|
|
1117
|
+
// REAL writer order: driveProbeTurn stamps prompt_end and returns, THEN the
|
|
1118
|
+
// runner stamps prompt_reply. The synthetic corpus had these reversed until
|
|
1119
|
+
// the topology rule caught it (GPT review round 2, 2026-07-29).
|
|
1120
|
+
out.push(ev(runId, PROBE_EVENTS.promptEnd, pStart + 300, { ok: true }));
|
|
1121
|
+
out.push(ev(runId, PROBE_EVENTS.promptReply, pStart + 310, { carriesNonce: spec.nonceEchoed ?? true }));
|
|
1122
|
+
return closeRun(out, spec, pStart + 310, false);
|
|
1123
|
+
}
|
|
1124
|
+
|
|
1125
|
+
const PROVIDER_ID_MEASURED = "mcp__probe__probe_nonce";
|
|
1126
|
+
|
|
1127
|
+
function passingControl(base = 0): { record: ProbeRunRecord; events: ProbeEvent[] } {
|
|
1128
|
+
return {
|
|
1129
|
+
record: { runId: "ctl", role: "control", delayMs: 0, probeRunId: "prb-ctl", snapshotInstrumented: false },
|
|
1130
|
+
events: syntheticRun({
|
|
1131
|
+
runId: "ctl",
|
|
1132
|
+
probeRunId: "prb-ctl",
|
|
1133
|
+
base,
|
|
1134
|
+
wireAt: 50,
|
|
1135
|
+
nsLatency: 300,
|
|
1136
|
+
fixtureCall: true,
|
|
1137
|
+
providerToolId: PROVIDER_ID_MEASURED,
|
|
1138
|
+
nonceEchoed: true,
|
|
1139
|
+
}),
|
|
1140
|
+
};
|
|
1141
|
+
}
|
|
1142
|
+
|
|
1143
|
+
function intervention(
|
|
1144
|
+
runId: string,
|
|
1145
|
+
delayMs: number,
|
|
1146
|
+
base: number,
|
|
1147
|
+
spec: Partial<SyntheticRunSpec>,
|
|
1148
|
+
): { record: ProbeRunRecord; events: ProbeEvent[] } {
|
|
1149
|
+
const probeRunId = `prb-${runId}`;
|
|
1150
|
+
return {
|
|
1151
|
+
record: { runId, role: "intervention", delayMs, probeRunId, snapshotInstrumented: false },
|
|
1152
|
+
events: syntheticRun({ runId, probeRunId, base, ...spec }),
|
|
1153
|
+
};
|
|
1154
|
+
}
|
|
1155
|
+
|
|
1156
|
+
// --- P0: control failures invalidate the whole experiment -------------------
|
|
1157
|
+
{
|
|
1158
|
+
const ctl = {
|
|
1159
|
+
record: {
|
|
1160
|
+
runId: "ctl",
|
|
1161
|
+
role: "control",
|
|
1162
|
+
delayMs: 0,
|
|
1163
|
+
probeRunId: "prb-ctl",
|
|
1164
|
+
snapshotInstrumented: false,
|
|
1165
|
+
} as ProbeRunRecord,
|
|
1166
|
+
events: syntheticRun({ runId: "ctl", probeRunId: "prb-ctl", base: 0, failPhase: "initialize" }),
|
|
1167
|
+
};
|
|
1168
|
+
const d1 = intervention("d1", 2000, 10_000, { wireAt: 2100, nsLatency: 2400, fixtureCall: true });
|
|
1169
|
+
const res = classifyProbe([ctl.record, d1.record], [...ctl.events, ...d1.events]);
|
|
1170
|
+
assert.equal(res.verdict, "P0", "control initialize failure → P0");
|
|
1171
|
+
assert.equal(res.control.p0Reason, "initialize", "P0 carries reason=initialize");
|
|
1172
|
+
assert.equal(res.interventions.length, 0, "no intervention is judged under P0");
|
|
1173
|
+
|
|
1174
|
+
const ctl2 = {
|
|
1175
|
+
record: {
|
|
1176
|
+
runId: "ctl",
|
|
1177
|
+
role: "control",
|
|
1178
|
+
delayMs: 0,
|
|
1179
|
+
probeRunId: "prb-ctl",
|
|
1180
|
+
snapshotInstrumented: false,
|
|
1181
|
+
} as ProbeRunRecord,
|
|
1182
|
+
events: syntheticRun({
|
|
1183
|
+
runId: "ctl",
|
|
1184
|
+
probeRunId: "prb-ctl",
|
|
1185
|
+
base: 0,
|
|
1186
|
+
wireAt: 50,
|
|
1187
|
+
fixtureCall: false, // visible? unproven — never callable
|
|
1188
|
+
providerToolId: undefined,
|
|
1189
|
+
nonceEchoed: false,
|
|
1190
|
+
}),
|
|
1191
|
+
};
|
|
1192
|
+
const res2 = classifyProbe([ctl2.record, d1.record], [...ctl2.events, ...d1.events]);
|
|
1193
|
+
assert.equal(res2.verdict, "P0", "control without the callability marker → P0 [QK:VERDICT-P0-CONTROL-FAIL]");
|
|
1194
|
+
assert.equal(res2.control.p0Reason, "tool-unavailable", "P0 names tool-unavailable");
|
|
1195
|
+
}
|
|
1196
|
+
|
|
1197
|
+
// --- I0: intervention initialize failure is drift, never a D ----------------
|
|
1198
|
+
{
|
|
1199
|
+
const ctl = passingControl();
|
|
1200
|
+
const d1 = intervention("d1", 2000, 10_000, { failPhase: "initialize" });
|
|
1201
|
+
const res = classifyProbe([ctl.record, d1.record], [...ctl.events, ...d1.events]);
|
|
1202
|
+
assert.equal(res.verdict, "I0", "intervention initialize failure → I0, never D [QK:VERDICT-I0-NEVER-D]");
|
|
1203
|
+
}
|
|
1204
|
+
|
|
1205
|
+
// --- D: phase-qualified fail-loud readings ----------------------------------
|
|
1206
|
+
{
|
|
1207
|
+
for (const [phase, expected] of [
|
|
1208
|
+
["newSession", "D-newSession"],
|
|
1209
|
+
["enforceModel", "D-enforceModel"],
|
|
1210
|
+
["prompt", "D-prompt"],
|
|
1211
|
+
] as const) {
|
|
1212
|
+
const ctl = passingControl();
|
|
1213
|
+
const d1 = intervention("d1", 2000, 10_000, { failPhase: phase, wireAt: null });
|
|
1214
|
+
const res = classifyProbe([ctl.record, d1.record], [...ctl.events, ...d1.events]);
|
|
1215
|
+
assert.equal(res.verdict, expected, `${phase} failure → ${expected}`);
|
|
1216
|
+
}
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1219
|
+
// --- B promotion ladder — B is ONLY the marker-grade combination ------------
|
|
1220
|
+
{
|
|
1221
|
+
// (a) no wire marker at all → NOT B: the §11-7 promotion ladder files it as an
|
|
1222
|
+
// MCP handshake / fixture / config CANDIDATE and keeps unlisted combinations
|
|
1223
|
+
// inconclusive. Reading it as B would let a fixture that never served
|
|
1224
|
+
// manufacture a sufficiency verdict (GPT review 2026-07-28).
|
|
1225
|
+
const ctl = passingControl();
|
|
1226
|
+
const noWire = intervention("d1", 2000, 10_000, { wireAt: null, fixtureCall: false, nonceEchoed: false });
|
|
1227
|
+
const resA = classifyProbe([ctl.record, noWire.record], [...ctl.events, ...noWire.events]);
|
|
1228
|
+
assert.equal(
|
|
1229
|
+
resA.verdict,
|
|
1230
|
+
"inconclusive",
|
|
1231
|
+
"absence without the wire marker is a handshake/fixture/config candidate, never B [QK:VERDICT-NOWIRE-CANDIDATE]",
|
|
1232
|
+
);
|
|
1233
|
+
assert.equal(resA.promotable, false, "…and never promotes");
|
|
1234
|
+
|
|
1235
|
+
// (b) wire forwarded + no fixture call + No-such-tool naming the MEASURED id → promotable B.
|
|
1236
|
+
const exact = intervention("d1", 2000, 10_000, {
|
|
1237
|
+
wireAt: 2600,
|
|
1238
|
+
nsLatency: 300,
|
|
1239
|
+
fixtureCall: false,
|
|
1240
|
+
noSuchToolId: PROVIDER_ID_MEASURED,
|
|
1241
|
+
nonceEchoed: false,
|
|
1242
|
+
});
|
|
1243
|
+
const resB = classifyProbe([ctl.record, exact.record], [...ctl.events, ...exact.events]);
|
|
1244
|
+
assert.equal(
|
|
1245
|
+
resB.verdict,
|
|
1246
|
+
"B",
|
|
1247
|
+
"marker-complete absence reads B — the runtime No-such-tool ladder OWNS runtime-error runs; no other channel may stand in for it [QK:VERDICT-RUNTIME-B-LADDER-OWNS]",
|
|
1248
|
+
);
|
|
1249
|
+
assert.equal(resB.promotable, true, "exact measured-id No-such-tool promotes");
|
|
1250
|
+
|
|
1251
|
+
// (b2) same markers but the turn did NOT run ahead (wire forwarded BEFORE
|
|
1252
|
+
// newSession end) → NOT B: the delta-table B is "the delayed run puts the
|
|
1253
|
+
// turn ahead of wire-availability AND yields absence" — an exact-id absence
|
|
1254
|
+
// with ordering kept is a different, unlisted finding.
|
|
1255
|
+
const keptAbsence = intervention("k1", 2000, 250_000, {
|
|
1256
|
+
wireAt: 100,
|
|
1257
|
+
nsLatency: 2300,
|
|
1258
|
+
fixtureCall: false,
|
|
1259
|
+
noSuchToolId: PROVIDER_ID_MEASURED,
|
|
1260
|
+
nonceEchoed: false,
|
|
1261
|
+
});
|
|
1262
|
+
const resB2 = classifyProbe([ctl.record, keptAbsence.record], [...ctl.events, ...keptAbsence.events]);
|
|
1263
|
+
assert.equal(
|
|
1264
|
+
resB2.verdict,
|
|
1265
|
+
"inconclusive",
|
|
1266
|
+
"exact-id absence WITHOUT running ahead of wire-availability is not delta-B [QK:VERDICT-B-REQUIRES-RANAHEAD]",
|
|
1267
|
+
);
|
|
1268
|
+
assert.equal(resB2.promotable, false, "…and never promotes");
|
|
1269
|
+
|
|
1270
|
+
// (c) alias/bare-name No-such-tool → model/alias mismatch, NOT absence
|
|
1271
|
+
// evidence (§11-7 ladder: the real provider-bound id may have been in schema).
|
|
1272
|
+
const alias = intervention("d1", 2000, 10_000, {
|
|
1273
|
+
wireAt: 2600,
|
|
1274
|
+
nsLatency: 300,
|
|
1275
|
+
fixtureCall: false,
|
|
1276
|
+
noSuchToolId: "probe_nonce",
|
|
1277
|
+
nonceEchoed: false,
|
|
1278
|
+
});
|
|
1279
|
+
const resC = classifyProbe([ctl.record, alias.record], [...ctl.events, ...alias.events]);
|
|
1280
|
+
assert.equal(resC.verdict, "inconclusive", "alias-mismatch absence is not B [QK:VERDICT-B-PROMOTION-RULES]");
|
|
1281
|
+
assert.equal(resC.promotable, false, "alias mismatch never promotes");
|
|
1282
|
+
}
|
|
1283
|
+
|
|
1284
|
+
// --- model prose alone never reads as evidence ------------------------------
|
|
1285
|
+
{
|
|
1286
|
+
const ctl = passingControl();
|
|
1287
|
+
const prose = intervention("d1", 2000, 10_000, {
|
|
1288
|
+
wireAt: 2600,
|
|
1289
|
+
nsLatency: 300,
|
|
1290
|
+
fixtureCall: false,
|
|
1291
|
+
nonceEchoed: false, // model SAID the tool is missing; no marker, no error
|
|
1292
|
+
});
|
|
1293
|
+
const res = classifyProbe([ctl.record, prose.record], [...ctl.events, ...prose.events]);
|
|
1294
|
+
assert.equal(res.verdict, "inconclusive", "prose-only absence stays inconclusive");
|
|
1295
|
+
assert.equal(res.promotable, false, "prose never promotes");
|
|
1296
|
+
}
|
|
1297
|
+
|
|
1298
|
+
// --- C: ran ahead of wire-availability, later call succeeded ----------------
|
|
1299
|
+
{
|
|
1300
|
+
const ctl = passingControl();
|
|
1301
|
+
const late = intervention("d1", 2000, 10_000, {
|
|
1302
|
+
wireAt: 2600, // after newSession end (nsLatency 300) → ran ahead
|
|
1303
|
+
nsLatency: 300,
|
|
1304
|
+
fixtureCall: true,
|
|
1305
|
+
fixtureCallAt: 3000,
|
|
1306
|
+
nonceEchoed: true,
|
|
1307
|
+
});
|
|
1308
|
+
const res = classifyProbe([ctl.record, late.record], [...ctl.events, ...late.events]);
|
|
1309
|
+
assert.equal(res.verdict, "C", "ahead-of-wire + late success → C");
|
|
1310
|
+
}
|
|
1311
|
+
|
|
1312
|
+
// --- A: needs TWO distinct delays whose newSession excess tracks D ----------
|
|
1313
|
+
{
|
|
1314
|
+
const ctl = passingControl();
|
|
1315
|
+
// ordering kept: wire fires before newSession end; latency ≈ control + D.
|
|
1316
|
+
const d1 = intervention("d1", 2000, 10_000, {
|
|
1317
|
+
wireAt: 2050,
|
|
1318
|
+
nsLatency: 2300,
|
|
1319
|
+
fixtureCall: true,
|
|
1320
|
+
nonceEchoed: true,
|
|
1321
|
+
});
|
|
1322
|
+
const d2 = intervention("d2", 8000, 40_000, {
|
|
1323
|
+
wireAt: 8050,
|
|
1324
|
+
nsLatency: 8300,
|
|
1325
|
+
fixtureCall: true,
|
|
1326
|
+
nonceEchoed: true,
|
|
1327
|
+
});
|
|
1328
|
+
const one = classifyProbe([ctl.record, d1.record], [...ctl.events, ...d1.events]);
|
|
1329
|
+
assert.equal(one.verdict, "A-withheld", "one nonzero delay → wait verdict WITHHELD [QK:VERDICT-A-NEEDS-TWO-DELAYS]");
|
|
1330
|
+
const two = classifyProbe([ctl.record, d1.record, d2.record], [...ctl.events, ...d1.events, ...d2.events]);
|
|
1331
|
+
assert.equal(two.verdict, "A", "two tracking delays → A");
|
|
1332
|
+
|
|
1333
|
+
// ordering kept but latency does NOT track D → withheld (contradictory data).
|
|
1334
|
+
const flat1 = intervention("f1", 2000, 70_000, { wireAt: 100, nsLatency: 320, fixtureCall: true, nonceEchoed: true });
|
|
1335
|
+
const flat2 = intervention("f2", 8000, 90_000, { wireAt: 100, nsLatency: 340, fixtureCall: true, nonceEchoed: true });
|
|
1336
|
+
const flat = classifyProbe(
|
|
1337
|
+
[ctl.record, flat1.record, flat2.record],
|
|
1338
|
+
[...ctl.events, ...flat1.events, ...flat2.events],
|
|
1339
|
+
);
|
|
1340
|
+
assert.equal(flat.verdict, "A-withheld", "ordering without latency tracking stays withheld");
|
|
1341
|
+
|
|
1342
|
+
// Overshoot: excess grows with D but far EXCEEDS it (10s/16s for 2s/8s). A
|
|
1343
|
+
// floor-only check would call this "tracking"; the band must refuse — an
|
|
1344
|
+
// overshoot is some other stall, not wait-for-delay evidence.
|
|
1345
|
+
const over1 = intervention("o1", 2000, 110_000, {
|
|
1346
|
+
wireAt: 9950,
|
|
1347
|
+
nsLatency: 10_300,
|
|
1348
|
+
fixtureCall: true,
|
|
1349
|
+
nonceEchoed: true,
|
|
1350
|
+
});
|
|
1351
|
+
const over2 = intervention("o2", 8000, 140_000, {
|
|
1352
|
+
wireAt: 15_950,
|
|
1353
|
+
nsLatency: 16_300,
|
|
1354
|
+
fixtureCall: true,
|
|
1355
|
+
nonceEchoed: true,
|
|
1356
|
+
});
|
|
1357
|
+
const over = classifyProbe(
|
|
1358
|
+
[ctl.record, over1.record, over2.record],
|
|
1359
|
+
[...ctl.events, ...over1.events, ...over2.events],
|
|
1360
|
+
);
|
|
1361
|
+
assert.equal(
|
|
1362
|
+
over.verdict,
|
|
1363
|
+
"A-withheld",
|
|
1364
|
+
"excess overshooting the [0.8·D, D+slack] band is not tracking — A withheld [QK:VERDICT-A-TRACKING-BAND]",
|
|
1365
|
+
);
|
|
1366
|
+
|
|
1367
|
+
// In-band per point but NOT growing with D (4.0s → 6.4s excess for 2s → 8s
|
|
1368
|
+
// delays: ΔE=2.4s < 0.6·ΔD=3.6s) → withheld.
|
|
1369
|
+
const nog1 = intervention("g1", 2000, 170_000, {
|
|
1370
|
+
wireAt: 3950,
|
|
1371
|
+
nsLatency: 4_300,
|
|
1372
|
+
fixtureCall: true,
|
|
1373
|
+
nonceEchoed: true,
|
|
1374
|
+
});
|
|
1375
|
+
const nog2 = intervention("g2", 8000, 200_000, {
|
|
1376
|
+
wireAt: 6650,
|
|
1377
|
+
nsLatency: 6_700,
|
|
1378
|
+
fixtureCall: true,
|
|
1379
|
+
nonceEchoed: true,
|
|
1380
|
+
});
|
|
1381
|
+
const nog = classifyProbe([ctl.record, nog1.record, nog2.record], [...ctl.events, ...nog1.events, ...nog2.events]);
|
|
1382
|
+
assert.equal(nog.verdict, "A-withheld", "in-band but non-growing excess is not tracking — A withheld");
|
|
1383
|
+
}
|
|
1384
|
+
|
|
1385
|
+
// --- same-ms cross-process tie is unordered, never ordering evidence --------
|
|
1386
|
+
{
|
|
1387
|
+
// wire marker stamped in the SAME millisecond as newSession end (different
|
|
1388
|
+
// pids): the shared axis cannot order them — reading ordering-kept off the
|
|
1389
|
+
// tie would manufacture A-side evidence out of clock resolution.
|
|
1390
|
+
const ctl = passingControl();
|
|
1391
|
+
const tie = intervention("t1", 2000, 230_000, {
|
|
1392
|
+
wireAt: 2300,
|
|
1393
|
+
nsLatency: 2300,
|
|
1394
|
+
fixtureCall: true,
|
|
1395
|
+
nonceEchoed: true,
|
|
1396
|
+
});
|
|
1397
|
+
const res = classifyProbe([ctl.record, tie.record], [...ctl.events, ...tie.events]);
|
|
1398
|
+
assert.equal(
|
|
1399
|
+
res.verdict,
|
|
1400
|
+
"inconclusive",
|
|
1401
|
+
"a same-ms wire/newSession-end tie is unordered — neither kept nor ahead [QK:VERDICT-SAMEMS-AMBIGUOUS]",
|
|
1402
|
+
);
|
|
1403
|
+
|
|
1404
|
+
// A tie combined with an exact-id No-such-tool must not read B either —
|
|
1405
|
+
// unordered means ranAhead is unestablished, and B requires it.
|
|
1406
|
+
const tieAbsence = intervention("t2", 2000, 260_000, {
|
|
1407
|
+
wireAt: 2300,
|
|
1408
|
+
nsLatency: 2300,
|
|
1409
|
+
fixtureCall: false,
|
|
1410
|
+
noSuchToolId: PROVIDER_ID_MEASURED,
|
|
1411
|
+
nonceEchoed: false,
|
|
1412
|
+
});
|
|
1413
|
+
const res2 = classifyProbe([ctl.record, tieAbsence.record], [...ctl.events, ...tieAbsence.events]);
|
|
1414
|
+
assert.equal(res2.verdict, "inconclusive", "a same-ms tie plus exact-id absence still is not B");
|
|
1415
|
+
assert.equal(res2.promotable, false, "…and never promotes");
|
|
1416
|
+
}
|
|
1417
|
+
|
|
1418
|
+
// --- delay outside the §11-7 window can never read as D ---------------------
|
|
1419
|
+
{
|
|
1420
|
+
const ctl = passingControl();
|
|
1421
|
+
const wide = intervention("d1", DELAY_WELL_BELOW_MS, 10_000, { failPhase: "newSession", wireAt: null });
|
|
1422
|
+
const res = classifyProbe([ctl.record, wide.record], [...ctl.events, ...wide.events]);
|
|
1423
|
+
assert.equal(
|
|
1424
|
+
res.verdict,
|
|
1425
|
+
"inconclusive",
|
|
1426
|
+
`delay ≥ ${DELAY_WELL_BELOW_MS}ms is outside the experiment window — not a D`,
|
|
1427
|
+
);
|
|
1428
|
+
}
|
|
1429
|
+
|
|
1430
|
+
// ===========================================================================
|
|
1431
|
+
// 6) OBSERVATION WINDOW + RUNNER TOPOLOGY — what puts a run OUTSIDE the space
|
|
1432
|
+
// ===========================================================================
|
|
1433
|
+
{
|
|
1434
|
+
const ctl = passingControl();
|
|
1435
|
+
|
|
1436
|
+
// Without the window marker, a missing wire marker cannot be told apart from
|
|
1437
|
+
// our own teardown — so the run is not judged at all rather than judged
|
|
1438
|
+
// permissively. (Re-parsing any artifact written before the window protocol
|
|
1439
|
+
// lands here, which is exactly right: those logs cannot answer the question.)
|
|
1440
|
+
const noWindow = intervention("w0", 2000, 10_000, { wireAt: null, fixtureCall: false, window: "omit" });
|
|
1441
|
+
const resNoWindow = classifyProbe([ctl.record, noWindow.record], [...ctl.events, ...noWindow.events]);
|
|
1442
|
+
// Two independent nets cover this — the exactly-once inventory below AND the
|
|
1443
|
+
// explicit windowReason guard in the classifier — so no SINGLE mutation can
|
|
1444
|
+
// kill it and it carries no [QK:] token. The inventory itself is qualified.
|
|
1445
|
+
assert.ok(
|
|
1446
|
+
resNoWindow.verdict === "INVALIDATED" && resNoWindow.interventions[0].invalidReason === "topology",
|
|
1447
|
+
"a run with no observation-window marker is INVALIDATED for TOPOLOGY — absence cannot be told from our own teardown",
|
|
1448
|
+
);
|
|
1449
|
+
// The exactly-once inventory, as a HAND-WRITTEN literal (never read off the
|
|
1450
|
+
// module under test): dropping a member would retire a topology rule silently.
|
|
1451
|
+
assert.deepEqual(
|
|
1452
|
+
[...RUNNER_EXACTLY_ONCE].sort(),
|
|
1453
|
+
[PROBE_EVENTS.observationWindowEnd, PROBE_EVENTS.runEnd, PROBE_EVENTS.runStart].sort(),
|
|
1454
|
+
"the runner-owned exactly-once marker set is exactly run_start, the window close, and run_end [QK:VERDICT-RUNNER-EXACTLY-ONCE-INVENTORY]",
|
|
1455
|
+
);
|
|
1456
|
+
|
|
1457
|
+
// THE regression the first LIVE pair produced: D2's child was torn down while
|
|
1458
|
+
// the fixture was still inside its injected delay, so the wire marker could
|
|
1459
|
+
// never land — and a wire-marker-less run was filed as an MCP handshake /
|
|
1460
|
+
// fixture / config candidate. That is an attribution about the SERVER derived
|
|
1461
|
+
// from a fact about OUR teardown.
|
|
1462
|
+
const censored = intervention("w1", 8000, 40_000, {
|
|
1463
|
+
wireAt: null,
|
|
1464
|
+
fixtureCall: false,
|
|
1465
|
+
nonceEchoed: false,
|
|
1466
|
+
window: { reason: "child-exit", markerSeen: false },
|
|
1467
|
+
});
|
|
1468
|
+
const resCensored = classifyProbe([ctl.record, censored.record], [...ctl.events, ...censored.events]);
|
|
1469
|
+
assert.equal(
|
|
1470
|
+
resCensored.interventions[0].invalidReason,
|
|
1471
|
+
"observation-window-closed",
|
|
1472
|
+
"a window closed by child-exit with the marker unseen is CENSORED, not a handshake/fixture/config candidate [QK:VERDICT-CENSORED-NOT-CANDIDATE]",
|
|
1473
|
+
);
|
|
1474
|
+
assert.equal(
|
|
1475
|
+
resCensored.interventions[0].ordering,
|
|
1476
|
+
"censored",
|
|
1477
|
+
"…its (a) axis reads censored — a fact about the probe, not an ordering comparison",
|
|
1478
|
+
);
|
|
1479
|
+
assert.notEqual(
|
|
1480
|
+
resCensored.interventions[0].failure,
|
|
1481
|
+
"candidate-handshake",
|
|
1482
|
+
"…and its (b) axis refuses the attribution the first LIVE pair made",
|
|
1483
|
+
);
|
|
1484
|
+
assert.equal(resCensored.verdict, "INVALIDATED", "the pair's only intervention being censored invalidates the pair");
|
|
1485
|
+
// A fatal status still has to NAME what it discarded — `invalidRuns` is a
|
|
1486
|
+
// common field of the status contract, so leaving it empty on the fatal paths
|
|
1487
|
+
// would quietly lose the only record of which runs were thrown out and why.
|
|
1488
|
+
assert.deepEqual(
|
|
1489
|
+
resCensored.status.invalidRuns,
|
|
1490
|
+
[{ runId: "w1", reason: "observation-window-closed" }],
|
|
1491
|
+
"a fatal status still names the discarded run and its reason [QK:VERDICT-STATUS-NAMES-INVALID-RUNS]",
|
|
1492
|
+
);
|
|
1493
|
+
|
|
1494
|
+
// The SAME absence under a window we kept open to its deadline IS a reading:
|
|
1495
|
+
// the difference is entirely whether we looked long enough.
|
|
1496
|
+
const sufficient = intervention("w2", 8000, 70_000, {
|
|
1497
|
+
wireAt: null,
|
|
1498
|
+
fixtureCall: false,
|
|
1499
|
+
nonceEchoed: false,
|
|
1500
|
+
window: { reason: "deadline", markerSeen: false },
|
|
1501
|
+
});
|
|
1502
|
+
const resSufficient = classifyProbe([ctl.record, sufficient.record], [...ctl.events, ...sufficient.events]);
|
|
1503
|
+
assert.equal(
|
|
1504
|
+
resSufficient.interventions[0].failure,
|
|
1505
|
+
"candidate-handshake",
|
|
1506
|
+
"the same absence under a SUFFICIENT window is a handshake/fixture/config candidate (the mutant for this condition is VERDICT-CENSORED-NOT-CANDIDATE — one condition, one kill)",
|
|
1507
|
+
);
|
|
1508
|
+
assert.equal(resSufficient.verdict, "inconclusive", "…still not promotable, and still not B");
|
|
1509
|
+
|
|
1510
|
+
// Runner-owned markers are ours and exactly-once by construction: a duplicate
|
|
1511
|
+
// means the log describing the run is not the run.
|
|
1512
|
+
const dupEnd = intervention("w3", 2000, 100_000, {
|
|
1513
|
+
fixtureCall: true,
|
|
1514
|
+
extra: (runId, base) => [ev(runId, PROBE_EVENTS.runEnd, base + 5_000, { ok: true })],
|
|
1515
|
+
});
|
|
1516
|
+
const resDup = classifyProbe([ctl.record, dupEnd.record], [...ctl.events, ...dupEnd.events]);
|
|
1517
|
+
assert.equal(
|
|
1518
|
+
resDup.verdict,
|
|
1519
|
+
"INVALIDATED",
|
|
1520
|
+
"a duplicated runner-owned marker INVALIDATES the run [QK:VERDICT-RUNNER-TOPOLOGY-EXACTLY-ONCE]",
|
|
1521
|
+
);
|
|
1522
|
+
assert.match(resDup.interventions[0].evidence, /run_end appears 2 times/, "…and the evidence names the duplicate");
|
|
1523
|
+
|
|
1524
|
+
// An end with no start is the same defect seen from the other side.
|
|
1525
|
+
const orphanEnd = intervention("w4", 2000, 130_000, {
|
|
1526
|
+
failPhase: "initialize",
|
|
1527
|
+
extra: (runId, base) => [ev(runId, PROBE_EVENTS.promptEnd, base + 70, { ok: true })],
|
|
1528
|
+
});
|
|
1529
|
+
const resOrphan = classifyProbe([ctl.record, orphanEnd.record], [...ctl.events, ...orphanEnd.events]);
|
|
1530
|
+
assert.equal(
|
|
1531
|
+
resOrphan.verdict,
|
|
1532
|
+
"INVALIDATED",
|
|
1533
|
+
"a phase end with no start INVALIDATES the run — topology precedes I0",
|
|
1534
|
+
);
|
|
1535
|
+
assert.match(resOrphan.interventions[0].evidence, /prompt_end without prompt_start/, "…named exactly");
|
|
1536
|
+
|
|
1537
|
+
// Repeatable markers are NOT swept into the exactly-once rule: the model may
|
|
1538
|
+
// produce several tool-call frames and the client may re-request tools/list.
|
|
1539
|
+
const repeats = intervention("w5", 2000, 160_000, {
|
|
1540
|
+
fixtureCall: true,
|
|
1541
|
+
nsLatency: 2400,
|
|
1542
|
+
wireAt: 2100,
|
|
1543
|
+
extra: (runId, base) => [
|
|
1544
|
+
ev(runId, PROBE_EVENTS.acpToolCallRaw, base + 900, { kind: "tool_call_update", raw: "{}" }),
|
|
1545
|
+
ev(runId, PROBE_EVENTS.acpToolCallRaw, base + 901, { kind: "tool_call_update", raw: "{}" }),
|
|
1546
|
+
],
|
|
1547
|
+
});
|
|
1548
|
+
const resRepeats = classifyProbe([ctl.record, repeats.record], [...ctl.events, ...repeats.events]);
|
|
1549
|
+
assert.notEqual(
|
|
1550
|
+
resRepeats.verdict,
|
|
1551
|
+
"INVALIDATED",
|
|
1552
|
+
"repeatable forensic markers repeating is not a topology violation (enforced, not mutant-qualified: widening the exactly-once set fails every run at once, so no isolated mutant exists)",
|
|
1553
|
+
);
|
|
1554
|
+
}
|
|
1555
|
+
|
|
1556
|
+
// --- window marker coherence + phase production order -----------------------
|
|
1557
|
+
{
|
|
1558
|
+
const ctl = passingControl();
|
|
1559
|
+
|
|
1560
|
+
// The window marker is SELF-REPORTED. A close claiming the marker was seen,
|
|
1561
|
+
// in a run whose log has no wire marker, would walk a censored run straight
|
|
1562
|
+
// into the candidate branch — so the flag is checked against the log.
|
|
1563
|
+
const lying = intervention("c1", 2000, 10_000, {
|
|
1564
|
+
wireAt: null,
|
|
1565
|
+
fixtureCall: false,
|
|
1566
|
+
nonceEchoed: false,
|
|
1567
|
+
window: { reason: "wire-marker", markerSeen: true },
|
|
1568
|
+
});
|
|
1569
|
+
const resLying = classifyProbe([ctl.record, lying.record], [...ctl.events, ...lying.events]);
|
|
1570
|
+
assert.equal(
|
|
1571
|
+
resLying.verdict,
|
|
1572
|
+
"INVALIDATED",
|
|
1573
|
+
"a window close claiming markerSeen=true with no wire marker in the log is INVALIDATED, never a candidate [QK:VERDICT-WINDOW-MARKER-COHERENCE]",
|
|
1574
|
+
);
|
|
1575
|
+
assert.match(resLying.interventions[0].evidence, /contradicts its own evidence/, "…named as a self-contradiction");
|
|
1576
|
+
|
|
1577
|
+
// The SAME bar applies to the CONTROL. Coherence lives in the run's shared
|
|
1578
|
+
// validity list precisely so the baseline cannot claim a wire marker it never
|
|
1579
|
+
// logged — a control free to lie about its own window is not a baseline, and
|
|
1580
|
+
// every intervention is read as a delta against it.
|
|
1581
|
+
const lyingControl = {
|
|
1582
|
+
record: {
|
|
1583
|
+
runId: "ctl-lie",
|
|
1584
|
+
role: "control",
|
|
1585
|
+
delayMs: 0,
|
|
1586
|
+
probeRunId: "prb-ctl-lie",
|
|
1587
|
+
snapshotInstrumented: false,
|
|
1588
|
+
} as ProbeRunRecord,
|
|
1589
|
+
events: syntheticRun({
|
|
1590
|
+
runId: "ctl-lie",
|
|
1591
|
+
probeRunId: "prb-ctl-lie",
|
|
1592
|
+
base: 500_000,
|
|
1593
|
+
wireAt: null, // no wire marker in the log …
|
|
1594
|
+
fixtureCall: true,
|
|
1595
|
+
providerToolId: PROVIDER_ID_MEASURED,
|
|
1596
|
+
nonceEchoed: true,
|
|
1597
|
+
window: { reason: "wire-marker", markerSeen: true }, // … but the close claims one
|
|
1598
|
+
}),
|
|
1599
|
+
};
|
|
1600
|
+
const d1ok = intervention("d1ok", 2000, 540_000, { wireAt: 2100, nsLatency: 2400, fixtureCall: true });
|
|
1601
|
+
const resLyingCtl = classifyProbe([lyingControl.record, d1ok.record], [...lyingControl.events, ...d1ok.events]);
|
|
1602
|
+
assert.ok(
|
|
1603
|
+
resLyingCtl.verdict === "INVALIDATED" && resLyingCtl.control.pass === false,
|
|
1604
|
+
"a CONTROL whose window close contradicts its own log is INVALIDATED before P0 is even considered — the baseline is held to the same bar [QK:VERDICT-CONTROL-HELD-TO-COHERENCE]",
|
|
1605
|
+
);
|
|
1606
|
+
assert.equal(resLyingCtl.interventions.length, 0, "…and no intervention is judged against a baseline that lied");
|
|
1607
|
+
|
|
1608
|
+
// The reason must agree with the flag too: `deadline` and `child-exit` both
|
|
1609
|
+
// mean the marker did not arrive.
|
|
1610
|
+
const wrongReason = intervention("c2", 2000, 40_000, {
|
|
1611
|
+
wireAt: 2200,
|
|
1612
|
+
fixtureCall: true,
|
|
1613
|
+
window: { reason: "deadline", markerSeen: true },
|
|
1614
|
+
});
|
|
1615
|
+
const resWrong = classifyProbe([ctl.record, wrongReason.record], [...ctl.events, ...wrongReason.events]);
|
|
1616
|
+
assert.equal(resWrong.verdict, "INVALIDATED", "reason=deadline with markerSeen=true is an incoherent close");
|
|
1617
|
+
|
|
1618
|
+
// PHASE-TO-PHASE order, not just start<end inside each phase: a log whose
|
|
1619
|
+
// phases are transposed has every pair intact and still does not describe the
|
|
1620
|
+
// driver's sequence. Both fixtures below are otherwise WELL FORMED — they
|
|
1621
|
+
// differ from a healthy run in exactly one way, so the violation they trip is
|
|
1622
|
+
// unambiguous and their mutants cannot die on someone else's assertion.
|
|
1623
|
+
const transposed: ProbeEvent[] = [
|
|
1624
|
+
ev("x1", PROBE_EVENTS.runStart, 300_000),
|
|
1625
|
+
ev("x1", PROBE_EVENTS.initializeStart, 300_000),
|
|
1626
|
+
ev("x1", PROBE_EVENTS.initializeEnd, 300_050, { ok: true }),
|
|
1627
|
+
// prompt BEFORE newSession — each pair is well formed on its own
|
|
1628
|
+
ev("x1", PROBE_EVENTS.promptStart, 300_100),
|
|
1629
|
+
ev("x1", PROBE_EVENTS.promptEnd, 300_200, { ok: true }),
|
|
1630
|
+
ev("x1", PROBE_EVENTS.promptReply, 300_210, { carriesNonce: false }),
|
|
1631
|
+
ev("x1", PROBE_EVENTS.newSessionStart, 300_300),
|
|
1632
|
+
ev("x1", PROBE_EVENTS.newSessionEnd, 300_400, { ok: true }),
|
|
1633
|
+
ev("x1", PROBE_EVENTS.setModelStart, 300_500),
|
|
1634
|
+
ev("x1", PROBE_EVENTS.setModelEnd, 300_530, { ok: true }),
|
|
1635
|
+
ev("x1", PROBE_EVENTS.observationWindowEnd, 300_600, { reason: "deadline", markerSeen: false }),
|
|
1636
|
+
ev("x1", PROBE_EVENTS.runEnd, 300_601, { ok: true }),
|
|
1637
|
+
];
|
|
1638
|
+
const transposedRec: ProbeRunRecord = {
|
|
1639
|
+
runId: "x1",
|
|
1640
|
+
role: "intervention",
|
|
1641
|
+
delayMs: 2000,
|
|
1642
|
+
probeRunId: "prb-x1",
|
|
1643
|
+
snapshotInstrumented: false,
|
|
1644
|
+
};
|
|
1645
|
+
const resTrans = classifyProbe([ctl.record, transposedRec], [...ctl.events, ...transposed]);
|
|
1646
|
+
assert.equal(
|
|
1647
|
+
resTrans.verdict,
|
|
1648
|
+
"INVALIDATED",
|
|
1649
|
+
"transposed phases are a topology violation even though every start/end pair is intact [QK:VERDICT-PHASE-SEQUENTIAL]",
|
|
1650
|
+
);
|
|
1651
|
+
assert.match(
|
|
1652
|
+
resTrans.interventions[0].evidence,
|
|
1653
|
+
/phases are sequential/,
|
|
1654
|
+
"…named as a phase-sequencing violation (all four phases are present, so it is the ORDER that is wrong)",
|
|
1655
|
+
);
|
|
1656
|
+
|
|
1657
|
+
// The prefix rule catches the other shape: a phase that ran without the phases
|
|
1658
|
+
// before it. A failed run is a PREFIX of the production order, never a hole.
|
|
1659
|
+
const hole: ProbeEvent[] = [
|
|
1660
|
+
ev("x2", PROBE_EVENTS.runStart, 400_000),
|
|
1661
|
+
ev("x2", PROBE_EVENTS.initializeStart, 400_000),
|
|
1662
|
+
ev("x2", PROBE_EVENTS.initializeEnd, 400_050, { ok: true }),
|
|
1663
|
+
// newSession skipped entirely — enforceModel and prompt still ran
|
|
1664
|
+
ev("x2", PROBE_EVENTS.setModelStart, 400_100),
|
|
1665
|
+
ev("x2", PROBE_EVENTS.setModelEnd, 400_130, { ok: true }),
|
|
1666
|
+
ev("x2", PROBE_EVENTS.promptStart, 400_200),
|
|
1667
|
+
ev("x2", PROBE_EVENTS.promptEnd, 400_300, { ok: true }),
|
|
1668
|
+
ev("x2", PROBE_EVENTS.promptReply, 400_310, { carriesNonce: false }),
|
|
1669
|
+
ev("x2", PROBE_EVENTS.observationWindowEnd, 400_400, { reason: "deadline", markerSeen: false }),
|
|
1670
|
+
ev("x2", PROBE_EVENTS.runEnd, 400_401, { ok: true }),
|
|
1671
|
+
];
|
|
1672
|
+
const holeRec: ProbeRunRecord = {
|
|
1673
|
+
runId: "x2",
|
|
1674
|
+
role: "intervention",
|
|
1675
|
+
delayMs: 2000,
|
|
1676
|
+
probeRunId: "prb-x2",
|
|
1677
|
+
snapshotInstrumented: false,
|
|
1678
|
+
};
|
|
1679
|
+
const resHole = classifyProbe([ctl.record, holeRec], [...ctl.events, ...hole]);
|
|
1680
|
+
assert.equal(
|
|
1681
|
+
resHole.verdict,
|
|
1682
|
+
"INVALIDATED",
|
|
1683
|
+
"a skipped phase is a topology violation — a failed run is a PREFIX of the production order, never a hole [QK:VERDICT-PHASE-PRODUCTION-ORDER]",
|
|
1684
|
+
);
|
|
1685
|
+
assert.match(resHole.interventions[0].evidence, /prefix of the production order/, "…named as a prefix violation");
|
|
1686
|
+
}
|
|
1687
|
+
|
|
1688
|
+
// ===========================================================================
|
|
1689
|
+
// 7) TWO AXES — an ordering observation is not hidden by a missing (b) marker
|
|
1690
|
+
// ===========================================================================
|
|
1691
|
+
{
|
|
1692
|
+
const ctl = passingControl();
|
|
1693
|
+
|
|
1694
|
+
// The D1 shape, exactly as measured 2026-07-28: the turn is opened BEFORE the
|
|
1695
|
+
// tools reach the wire, the wire marker lands mid-turn, and the model never
|
|
1696
|
+
// attempts the call. (b) has no marker — model silence is not evidence — but
|
|
1697
|
+
// (a) is settled: this server did NOT wait. Reporting one verdict let the
|
|
1698
|
+
// missing (b) marker bury the (a) fact.
|
|
1699
|
+
const d1 = intervention("a1", 2000, 10_000, {
|
|
1700
|
+
nsLatency: 2000,
|
|
1701
|
+
wireAt: 2200, // promptStart is nsStart+2060, promptEnd nsStart+2360
|
|
1702
|
+
fixtureCall: false,
|
|
1703
|
+
nonceEchoed: false,
|
|
1704
|
+
});
|
|
1705
|
+
const res = classifyProbe([ctl.record, d1.record], [...ctl.events, ...d1.events]);
|
|
1706
|
+
const r = res.interventions[0];
|
|
1707
|
+
assert.equal(r.deltas.promptRanAhead, true, "promptStart precedes the wire marker");
|
|
1708
|
+
assert.equal(
|
|
1709
|
+
r.ordering,
|
|
1710
|
+
"prompt-request-ahead-of-wire",
|
|
1711
|
+
"(a) is a settled COMPARISON: we issued the prompt request before the wire marker landed — named for the comparison, never for a server-wait conclusion",
|
|
1712
|
+
);
|
|
1713
|
+
assert.equal(r.failure, "inconclusive", "(b) has no marker — model silence never promotes");
|
|
1714
|
+
assert.equal(res.verdict, "inconclusive", "the composite verdict stays inconclusive because (b) is unsettled");
|
|
1715
|
+
assert.equal(
|
|
1716
|
+
res.ordering.summary,
|
|
1717
|
+
"prompt-request-ahead-of-wire",
|
|
1718
|
+
"…and the (a) axis is reported on its OWN terms rather than being folded into that verdict [QK:VERDICT-ORDERING-AXIS-REPORTED]",
|
|
1719
|
+
);
|
|
1720
|
+
// Diagnosability: §11-7-b's first artifact classified D1 correctly and still
|
|
1721
|
+
// left a reader unable to SEE the ran-ahead or the turn time left after it.
|
|
1722
|
+
assert.match(r.evidence, /promptStart \d+ms BEFORE wire/, "the evidence exposes the prompt↔wire delta");
|
|
1723
|
+
assert.match(
|
|
1724
|
+
r.evidence,
|
|
1725
|
+
/\d+ms of turn remained after wire/,
|
|
1726
|
+
"the evidence exposes how much turn was left after the wire marker [QK:VERDICT-EVIDENCE-EXPOSES-DELTAS]",
|
|
1727
|
+
);
|
|
1728
|
+
|
|
1729
|
+
// The axis split is load-bearing for B/C, not cosmetic. Here the wire lands
|
|
1730
|
+
// AFTER newSession end but BEFORE the prompt is issued: the newSession axis
|
|
1731
|
+
// says "ran ahead", the causal window says the turn was opened against a wire
|
|
1732
|
+
// that was already available. Only the second one may decide B.
|
|
1733
|
+
const between = intervention("a2", 2000, 40_000, {
|
|
1734
|
+
nsLatency: 2000,
|
|
1735
|
+
wireAt: 2030, // newSessionEnd + 30, promptStart is +60
|
|
1736
|
+
fixtureCall: false,
|
|
1737
|
+
noSuchToolId: PROVIDER_ID_MEASURED,
|
|
1738
|
+
nonceEchoed: false,
|
|
1739
|
+
});
|
|
1740
|
+
const resBetween = classifyProbe([ctl.record, between.record], [...ctl.events, ...between.events]);
|
|
1741
|
+
const rb = resBetween.interventions[0];
|
|
1742
|
+
assert.equal(rb.deltas.newSessionRanAhead, true, "the newSession axis alone would call this ran-ahead");
|
|
1743
|
+
assert.equal(rb.deltas.promptRanAhead, false, "…but the turn was opened AFTER the wire was available");
|
|
1744
|
+
assert.equal(
|
|
1745
|
+
resBetween.verdict,
|
|
1746
|
+
"inconclusive",
|
|
1747
|
+
"exact-id absence with the wire available before the prompt is NOT delta-B — B's window is promptStart, not newSession end [QK:VERDICT-B-WINDOW-IS-PROMPT]",
|
|
1748
|
+
);
|
|
1749
|
+
assert.equal(
|
|
1750
|
+
rb.ordering,
|
|
1751
|
+
"wire-before-prompt-request",
|
|
1752
|
+
"…and (a) records the comparison: the wire marker landed before we issued the prompt request",
|
|
1753
|
+
);
|
|
1754
|
+
|
|
1755
|
+
// A keeps its own axis: `wire < newSessionEnd` plus latency scaling. Replacing
|
|
1756
|
+
// the single ran-ahead flag with the prompt axis would have silently broken it.
|
|
1757
|
+
const k1 = intervention("k1", 2000, 70_000, { wireAt: 100, nsLatency: 2200, fixtureCall: true });
|
|
1758
|
+
const k2 = intervention("k2", 8000, 100_000, { wireAt: 100, nsLatency: 8200, fixtureCall: true });
|
|
1759
|
+
const resA = classifyProbe([ctl.record, k1.record, k2.record], [...ctl.events, ...k1.events, ...k2.events]);
|
|
1760
|
+
assert.equal(resA.verdict, "A", "A still reads off the newSession axis with latency tracking");
|
|
1761
|
+
assert.equal(
|
|
1762
|
+
resA.ordering.summary,
|
|
1763
|
+
"wire-before-newSession-end",
|
|
1764
|
+
"…and the (a) summary is named for the comparison, deliberately NOT 'wait' — the wait verdict needs the scaling A adds",
|
|
1765
|
+
);
|
|
1766
|
+
}
|
|
1767
|
+
|
|
1768
|
+
// ===========================================================================
|
|
1769
|
+
// 8) §11-7-c B-name-snapshot seam — CONSUMER side (preconditions, doors,
|
|
1770
|
+
// verdict ladder). The PRODUCER (the CLI shim) is gated by
|
|
1771
|
+
// check-probe-cli-shim; these prove the contract it must satisfy, and 8d
|
|
1772
|
+
// pins the runner's arming — the order of the ambient refusal against the
|
|
1773
|
+
// deliberate injection, what is injected, and the instrument's runtime graph.
|
|
1774
|
+
// ===========================================================================
|
|
1775
|
+
|
|
1776
|
+
// --- 8a) CLI-target precondition seam: refusals are NAMED, never fallbacks --
|
|
1777
|
+
{
|
|
1778
|
+
const fakeBin = join(tmp, "fake-claude");
|
|
1779
|
+
writeFileSync(fakeBin, "#!/bin/sh\nexit 0\n");
|
|
1780
|
+
chmodSync(fakeBin, 0o755);
|
|
1781
|
+
const fakeScript = join(tmp, "fake-claude.mjs");
|
|
1782
|
+
writeFileSync(fakeScript, "process.exit(0);\n");
|
|
1783
|
+
chmodSync(fakeScript, 0o755);
|
|
1784
|
+
const fakeDir = join(tmp, "fake-claude-dir");
|
|
1785
|
+
mkdirSync(fakeDir, { recursive: true });
|
|
1786
|
+
const fakeNonExec = join(tmp, "fake-claude-noexec");
|
|
1787
|
+
writeFileSync(fakeNonExec, "#!/bin/sh\nexit 0\n");
|
|
1788
|
+
chmodSync(fakeNonExec, 0o644);
|
|
1789
|
+
|
|
1790
|
+
const reasonOf = async (env: Record<string, string | undefined>, target: string): Promise<string> => {
|
|
1791
|
+
try {
|
|
1792
|
+
await resolveProbeCliTarget({ env, resolveNative: async () => target });
|
|
1793
|
+
return "resolved";
|
|
1794
|
+
} catch (err) {
|
|
1795
|
+
return err instanceof ProbeCliPreconditionError ? err.reason : "unexpected-error";
|
|
1796
|
+
}
|
|
1797
|
+
};
|
|
1798
|
+
|
|
1799
|
+
assert.equal(
|
|
1800
|
+
await reasonOf({ [AMBIENT_OVERRIDE_ENV]: "/somewhere/claude" }, fakeBin),
|
|
1801
|
+
"ambient-override-present",
|
|
1802
|
+
"an ambient CLAUDE_CODE_EXECUTABLE is REFUSED before resolution — claudeCliPath() would return it verbatim [QK:PROBE-TARGET-AMBIENT-REFUSED]",
|
|
1803
|
+
);
|
|
1804
|
+
// KEY PRESENCE is the predicate: upstream's `??` treats "" as set and passes
|
|
1805
|
+
// it on while a truthy check treats it as unset — the probe refuses the
|
|
1806
|
+
// ambiguity instead of picking a side (GPT review 2026-07-29).
|
|
1807
|
+
assert.equal(
|
|
1808
|
+
await reasonOf({ [AMBIENT_OVERRIDE_ENV]: "" }, fakeBin),
|
|
1809
|
+
"ambient-override-present",
|
|
1810
|
+
"an EMPTY-string override is still a present key — refused, not treated as unset",
|
|
1811
|
+
);
|
|
1812
|
+
assert.equal(
|
|
1813
|
+
await reasonOf({}, "relative/claude"),
|
|
1814
|
+
"target-not-absolute",
|
|
1815
|
+
"a non-absolute resolved target is refused — it would resolve against the session cwd at spawn time",
|
|
1816
|
+
);
|
|
1817
|
+
assert.equal(
|
|
1818
|
+
await reasonOf({}, fakeScript),
|
|
1819
|
+
"target-script-suffix",
|
|
1820
|
+
"a script-suffixed target is refused — the SDK would take the node|bun branch, which this seam asserts against instead of reproducing [QK:PROBE-TARGET-NATIVE-BRANCH-ONLY]",
|
|
1821
|
+
);
|
|
1822
|
+
assert.equal(
|
|
1823
|
+
await reasonOf({}, join(tmp, "no-such-claude")),
|
|
1824
|
+
"target-missing",
|
|
1825
|
+
"a missing target is refused before a LIVE turn spends money on a spawn error",
|
|
1826
|
+
);
|
|
1827
|
+
assert.equal(
|
|
1828
|
+
await reasonOf({}, fakeDir),
|
|
1829
|
+
"target-not-regular-file",
|
|
1830
|
+
"a directory target is refused — existsSync alone would have taken it happy (GPT review 2026-07-29)",
|
|
1831
|
+
);
|
|
1832
|
+
assert.equal(
|
|
1833
|
+
await reasonOf({}, fakeNonExec),
|
|
1834
|
+
"target-not-executable",
|
|
1835
|
+
"a non-executable regular file is refused (X_OK) — it would fail only after the pair started spending",
|
|
1836
|
+
);
|
|
1837
|
+
const resolved = await resolveProbeCliTarget({ env: {}, resolveNative: async () => fakeBin });
|
|
1838
|
+
assert.ok(
|
|
1839
|
+
resolved.path === fakeBin && resolved.sha256 === hashFileSha256(fakeBin) && /^[0-9a-f]{64}$/.test(resolved.sha256),
|
|
1840
|
+
"an absolute, extensionless, executable regular file resolves with its content hash",
|
|
1841
|
+
);
|
|
1842
|
+
}
|
|
1843
|
+
|
|
1844
|
+
// --- 8b) upstream override semantics — inspector validated on synthetic
|
|
1845
|
+
// fixtures, THEN applied to the installed dists. node_modules can never be
|
|
1846
|
+
// a mutant subject (§11-7-c), so kill-power here is the synthetic
|
|
1847
|
+
// negatives, not a manifest entry.
|
|
1848
|
+
{
|
|
1849
|
+
const inspect = (acpSrc: string, sdkSrc: string): string[] => {
|
|
1850
|
+
const violations: string[] = [];
|
|
1851
|
+
if (!acpSrc.includes("export async function claudeCliPath()")) {
|
|
1852
|
+
violations.push("claudeCliPath export missing from acp-agent.js");
|
|
1853
|
+
}
|
|
1854
|
+
if (!/if \(process\.env\.CLAUDE_CODE_EXECUTABLE\) \{\s*return process\.env\.CLAUDE_CODE_EXECUTABLE;/.test(acpSrc)) {
|
|
1855
|
+
violations.push(
|
|
1856
|
+
"claudeCliPath no longer returns the ambient override VERBATIM — the refusal precondition's premise moved",
|
|
1857
|
+
);
|
|
1858
|
+
}
|
|
1859
|
+
const overrideLine = "pathToClaudeCodeExecutable: process.env.CLAUDE_CODE_EXECUTABLE ?? (await claudeCliPath())";
|
|
1860
|
+
const iOverride = acpSrc.indexOf(overrideLine);
|
|
1861
|
+
if (iOverride === -1) {
|
|
1862
|
+
violations.push("the env-??-claudeCliPath resolution at the query options literal is gone");
|
|
1863
|
+
} else {
|
|
1864
|
+
const iSpread = acpSrc.indexOf("...userProvidedOptions,");
|
|
1865
|
+
if (iSpread === -1 || iSpread > iOverride) {
|
|
1866
|
+
violations.push(
|
|
1867
|
+
"the explicit pathToClaudeCodeExecutable key no longer follows ...userProvidedOptions — the key-order assumption (override wins) is broken",
|
|
1868
|
+
);
|
|
1869
|
+
}
|
|
1870
|
+
}
|
|
1871
|
+
if (!sdkSrc.includes(JSON.stringify(SDK_SCRIPT_SUFFIXES))) {
|
|
1872
|
+
violations.push(
|
|
1873
|
+
`the SDK's script-suffix discriminator no longer equals ${JSON.stringify(SDK_SCRIPT_SUFFIXES)} — the native-branch assert would drift from upstream`,
|
|
1874
|
+
);
|
|
1875
|
+
}
|
|
1876
|
+
if (!sdkSrc.includes('?"bun":"node"')) {
|
|
1877
|
+
violations.push("the SDK's node|bun default-executable choice moved — the script branch premise changed");
|
|
1878
|
+
}
|
|
1879
|
+
// The no-shell proof is LOCALIZED to the spawn leaf: asserting only that
|
|
1880
|
+
// one spelling of `shell:!0` is absent SOMEWHERE in a megabyte of minified
|
|
1881
|
+
// source proves nothing — `shell:true`, a variable shell, or the leaf
|
|
1882
|
+
// moving entirely would all pass (GPT review 2026-07-29). Pin the
|
|
1883
|
+
// spawnLocalProcess definition window and judge the spawn options THERE.
|
|
1884
|
+
const iLeaf = sdkSrc.indexOf("spawnLocalProcess(");
|
|
1885
|
+
if (iLeaf === -1) {
|
|
1886
|
+
violations.push("the SDK's spawnLocalProcess leaf is gone — the spawn-shape premise has no anchor");
|
|
1887
|
+
} else {
|
|
1888
|
+
const leaf = sdkSrc.slice(iLeaf, iLeaf + 800);
|
|
1889
|
+
if (!leaf.includes('stdio:["pipe","pipe","pipe"]') || !leaf.includes("windowsHide:!0")) {
|
|
1890
|
+
violations.push("the spawn leaf no longer shows piped stdio + windowsHide — spawn semantics premise changed");
|
|
1891
|
+
}
|
|
1892
|
+
if (/\bshell\s*:/.test(leaf)) {
|
|
1893
|
+
violations.push("the spawn leaf carries a `shell:` option — the no-shell premise is broken");
|
|
1894
|
+
}
|
|
1895
|
+
}
|
|
1896
|
+
return violations;
|
|
1897
|
+
};
|
|
1898
|
+
|
|
1899
|
+
// Synthetic fixtures FIRST — an inspector that cannot see the defect it
|
|
1900
|
+
// exists for is no inspector.
|
|
1901
|
+
const goodAcp =
|
|
1902
|
+
"export async function claudeCliPath() {\n" +
|
|
1903
|
+
" if (process.env.CLAUDE_CODE_EXECUTABLE) {\n return process.env.CLAUDE_CODE_EXECUTABLE;\n }\n}\n" +
|
|
1904
|
+
"const options = {\n ...userProvidedOptions,\n" +
|
|
1905
|
+
" pathToClaudeCodeExecutable: process.env.CLAUDE_CODE_EXECUTABLE ?? (await claudeCliPath()),\n};\n";
|
|
1906
|
+
const goodSdk = `x=![".js",".mjs",".tsx",".ts",".jsx"].some((r)=>e.endsWith(r));y=Cs()?"bun":"node";spawnLocalProcess(e){let{command:t,args:r,cwd:n,env:o,signal:i}=e,s=cxe(t,r,{cwd:n,stdio:["pipe","pipe","pipe"],signal:i,env:o,windowsHide:!0})}`;
|
|
1907
|
+
assert.deepEqual(inspect(goodAcp, goodSdk), [], "inspector passes the correct synthetic fixture");
|
|
1908
|
+
const invertedAcp = goodAcp.replace(
|
|
1909
|
+
" ...userProvidedOptions,\n pathToClaudeCodeExecutable: process.env.CLAUDE_CODE_EXECUTABLE ?? (await claudeCliPath()),\n",
|
|
1910
|
+
" pathToClaudeCodeExecutable: process.env.CLAUDE_CODE_EXECUTABLE ?? (await claudeCliPath()),\n ...userProvidedOptions,\n",
|
|
1911
|
+
);
|
|
1912
|
+
assert.ok(
|
|
1913
|
+
inspect(invertedAcp, goodSdk).some((v) => v.includes("key-order")),
|
|
1914
|
+
"inspector catches userProvidedOptions inverted to win over the explicit override key",
|
|
1915
|
+
);
|
|
1916
|
+
assert.ok(
|
|
1917
|
+
inspect(goodAcp.replace("export async function claudeCliPath()", "async function claudeCliPath()"), goodSdk).some(
|
|
1918
|
+
(v) => v.includes("export missing"),
|
|
1919
|
+
),
|
|
1920
|
+
"inspector catches the claudeCliPath export disappearing",
|
|
1921
|
+
);
|
|
1922
|
+
assert.ok(
|
|
1923
|
+
inspect(goodAcp, goodSdk.replace('".mjs",', "")).some((v) => v.includes("discriminator")),
|
|
1924
|
+
"inspector catches the SDK suffix list drifting from the pinned SDK_SCRIPT_SUFFIXES",
|
|
1925
|
+
);
|
|
1926
|
+
// The no-shell negatives must be caught IN THE LEAF WINDOW, in both the
|
|
1927
|
+
// minified and the plain spelling — the global one-spelling absence check
|
|
1928
|
+
// this replaced passed them all.
|
|
1929
|
+
assert.ok(
|
|
1930
|
+
inspect(goodAcp, goodSdk.replace("windowsHide:!0", "windowsHide:!0,shell:!0")).some((v) =>
|
|
1931
|
+
v.includes("no-shell"),
|
|
1932
|
+
) &&
|
|
1933
|
+
inspect(goodAcp, goodSdk.replace("windowsHide:!0", "windowsHide:!0,shell:true")).some((v) =>
|
|
1934
|
+
v.includes("no-shell"),
|
|
1935
|
+
) &&
|
|
1936
|
+
inspect(goodAcp, goodSdk.replace("spawnLocalProcess(", "spawnElsewhere(")).some((v) => v.includes("anchor")),
|
|
1937
|
+
"inspector catches shell:!0 AND shell:true inside the spawn leaf, and the leaf disappearing entirely",
|
|
1938
|
+
);
|
|
1939
|
+
|
|
1940
|
+
// Installed dists — the actual §11-7-c load-bearing assumptions.
|
|
1941
|
+
const rootRequire = createRequire(resolve(REPO_ROOT, "package.json"));
|
|
1942
|
+
const acpPkgJson = rootRequire.resolve("@agentclientprotocol/claude-agent-acp/package.json");
|
|
1943
|
+
const acpDist = readFileSync(join(dirname(acpPkgJson), "dist", "acp-agent.js"), "utf8");
|
|
1944
|
+
const acpRequire = createRequire(acpPkgJson);
|
|
1945
|
+
const sdkEntry = acpRequire.resolve("@anthropic-ai/claude-agent-sdk");
|
|
1946
|
+
let sdkDist = readFileSync(sdkEntry, "utf8");
|
|
1947
|
+
if (!sdkDist.includes(JSON.stringify(SDK_SCRIPT_SUFFIXES))) {
|
|
1948
|
+
const sibling = join(dirname(sdkEntry), "sdk.mjs");
|
|
1949
|
+
if (existsSync(sibling)) sdkDist = readFileSync(sibling, "utf8");
|
|
1950
|
+
}
|
|
1951
|
+
assert.deepEqual(
|
|
1952
|
+
inspect(acpDist, sdkDist),
|
|
1953
|
+
[],
|
|
1954
|
+
"installed acp-agent.js + sdk.mjs still carry every §11-7-c launch-semantics assumption (claudeCliPath verbatim env, override key order, suffix discriminator, node|bun choice, piped no-shell spawn)",
|
|
1955
|
+
);
|
|
1956
|
+
}
|
|
1957
|
+
|
|
1958
|
+
// --- 8c) the scrub list is an EXACT allowlist, never a prefix pattern -------
|
|
1959
|
+
{
|
|
1960
|
+
assert.ok(
|
|
1961
|
+
SHIM_SCRUB_ENV_VARS.includes(AMBIENT_OVERRIDE_ENV) &&
|
|
1962
|
+
Object.values(PROBE_SHIM_ENV).every((v) => SHIM_SCRUB_ENV_VARS.includes(v)) &&
|
|
1963
|
+
SHIM_SCRUB_ENV_VARS.length === 1 + Object.values(PROBE_SHIM_ENV).length &&
|
|
1964
|
+
SHIM_SCRUB_ENV_VARS.every((v) => /^[A-Z][A-Z0-9_]*$/.test(v)),
|
|
1965
|
+
"the shim scrub list is the exact enumerated allowlist — the override plus every probe-private var by literal name, no wildcard/prefix semantics [QK:PROBE-SCRUB-EXACT-ALLOWLIST]",
|
|
1966
|
+
);
|
|
1967
|
+
// The shim env names must not collide with the fixture's — two processes,
|
|
1968
|
+
// two channels, one shared log.
|
|
1969
|
+
const fixtureVals = Object.values(PROBE_ENV) as string[];
|
|
1970
|
+
assert.ok(
|
|
1971
|
+
Object.values(PROBE_SHIM_ENV).every((v) => !fixtureVals.includes(v)),
|
|
1972
|
+
"shim env names are disjoint from the fixture's PROBE_ENV names",
|
|
1973
|
+
);
|
|
1974
|
+
}
|
|
1975
|
+
|
|
1976
|
+
// --- 8d) runner pins — one assert per claim, so each [QK:] names exactly what
|
|
1977
|
+
// its mutant kills (bundling four claims under one token let one mutant
|
|
1978
|
+
// stand in for all of them; GPT review 2026-07-29) ----------------------
|
|
1979
|
+
{
|
|
1980
|
+
assert.ok(
|
|
1981
|
+
RUNNER_SRC.includes("assertNoAmbientOverride(spawnEnv, `composed acp child env for ${runId}`);"),
|
|
1982
|
+
"the COMPOSED spawn env of every ACP child is asserted override-free — launch defaults / overlay overrides could inject what process.env did not carry [QK:RUNNER-TARGET-PRECONDITION-PINNED]",
|
|
1983
|
+
);
|
|
1984
|
+
assert.ok(
|
|
1985
|
+
RUNNER_SRC.includes("snapshotInstrumented: true") && !RUNNER_SRC.includes("snapshotInstrumented: false"),
|
|
1986
|
+
"the snapshot channel is ARMED and no run is left declaring otherwise — a roster mixing armed and unarmed runs " +
|
|
1987
|
+
"would let a run whose shim never reported in pass as an ordinary absence [QK:RUNNER-SNAPSHOT-CHANNEL-ARMED]",
|
|
1988
|
+
);
|
|
1989
|
+
// Pinned as the CONTIGUOUS roster-record shape: the same two stamps also ride
|
|
1990
|
+
// the run_start payload (forensics), so field-by-field includes() would stay
|
|
1991
|
+
// green with the roster copy deleted.
|
|
1992
|
+
assert.ok(
|
|
1993
|
+
RUNNER_SRC.includes(
|
|
1994
|
+
"snapshotInstrumented: true,\n\t\tcliTargetPath: CLI_TARGET.path,\n\t\tcliTargetSha256: CLI_TARGET.sha256,",
|
|
1995
|
+
),
|
|
1996
|
+
"the pair's expected CLI target identity rides EVERY roster record so the classifier can consume it (condition 5) [QK:RUNNER-TARGET-IDENTITY-IN-ROSTER]",
|
|
1997
|
+
);
|
|
1998
|
+
assert.ok(
|
|
1999
|
+
RUNNER_SRC.includes("rehash = hashFileSha256(CLI_TARGET.path);") &&
|
|
2000
|
+
RUNNER_SRC.includes('reason: "cli-target-drift"') &&
|
|
2001
|
+
RUNNER_SRC.includes('reason: "cli-target-unreadable"'),
|
|
2002
|
+
"the target is RE-HASHED after the pair, and both drift and unreadability write a named INVALIDATED classification [QK:RUNNER-DRIFT-REHASH-PINNED]",
|
|
2003
|
+
);
|
|
2004
|
+
assert.ok(
|
|
2005
|
+
RUNNER_SRC.includes("CLI_TARGET = await resolveProbeCliTarget({") &&
|
|
2006
|
+
RUNNER_SRC.includes('"@agentclientprotocol/claude-agent-acp/dist/acp-agent.js"') &&
|
|
2007
|
+
RUNNER_SRC.includes("`precondition-${err.reason}`"),
|
|
2008
|
+
"the runner resolves the target through upstream claudeCliPath BEFORE any run, and a precondition refusal writes a NAMED classification on the artifact (not stderr alone)",
|
|
2009
|
+
);
|
|
2010
|
+
// --- arming (§11-7-c CP3). The producer exists, so the channel is armed — and
|
|
2011
|
+
// arming is the point where two things can go quietly wrong: the order of
|
|
2012
|
+
// the checkpoint against the injection, and WHAT gets injected.
|
|
2013
|
+
assert.ok(
|
|
2014
|
+
RUNNER_SRC.includes(
|
|
2015
|
+
"assertNoAmbientOverride(spawnEnv, `composed acp child env for ${runId}`);\n" + "\t\t// ORDER IS THE CONTRACT",
|
|
2016
|
+
) &&
|
|
2017
|
+
RUNNER_SRC.indexOf("assertNoAmbientOverride(spawnEnv") <
|
|
2018
|
+
RUNNER_SRC.indexOf("spawnEnv[AMBIENT_OVERRIDE_ENV] = SHIM_TARGET.path;"),
|
|
2019
|
+
"the ambient-override refusal runs against the env as PRODUCTION composed it, and the probe installs its own " +
|
|
2020
|
+
"override only AFTER. Inverted, the checkpoint would inspect the override the probe itself just injected and " +
|
|
2021
|
+
"REFUSE every run — loudly, but for the wrong reason, and the operator's ambient environment would never be " +
|
|
2022
|
+
"examined at all [QK:RUNNER-ARMING-ORDER]",
|
|
2023
|
+
);
|
|
2024
|
+
assert.ok(
|
|
2025
|
+
RUNNER_SRC.includes(
|
|
2026
|
+
"spawnEnv[AMBIENT_OVERRIDE_ENV] = SHIM_TARGET.path;\n" +
|
|
2027
|
+
"\t\tspawnEnv[PROBE_SHIM_ENV.target] = CLI_TARGET.path;\n" +
|
|
2028
|
+
"\t\tspawnEnv[PROBE_SHIM_ENV.eventLog] = logPath;\n" +
|
|
2029
|
+
"\t\tspawnEnv[PROBE_SHIM_ENV.runId] = runId;",
|
|
2030
|
+
),
|
|
2031
|
+
"the injection is exactly four names: the override pointing at the SHIM, and the three probe-private vars the " +
|
|
2032
|
+
"shim reads — the target it must exec (resolved HERE, never by the shim), the shared log, and this run's id. " +
|
|
2033
|
+
"All four are on the shim's scrub list, so none of them reach the real CLI [QK:RUNNER-SHIM-OVERRIDE-EXACT]",
|
|
2034
|
+
);
|
|
2035
|
+
assert.ok(
|
|
2036
|
+
RUNNER_SRC.includes(
|
|
2037
|
+
"SHIM_TARGET = await resolveProbeCliTarget({ env: {}, resolveNative: async () => PROBE_SHIM });",
|
|
2038
|
+
) && RUNNER_SRC.includes("`precondition-shim-${err.reason}`"),
|
|
2039
|
+
"the instrument passes the SAME precondition asserts as the stimulus — absolute, native branch, present regular " +
|
|
2040
|
+
"file, executable — and a refusal is a NAMED classification on the artifact. A shim that fails any of those " +
|
|
2041
|
+
"either never runs or runs on the OTHER launch branch, and the pair would measure something else " +
|
|
2042
|
+
"[QK:RUNNER-SHIM-PRECONDITION-PINNED]",
|
|
2043
|
+
);
|
|
2044
|
+
// The path the runner points at is checked on DISK too, not just in source: a
|
|
2045
|
+
// pin proves the runner asks for the right file, not that the file can run.
|
|
2046
|
+
{
|
|
2047
|
+
// --- the instrument is a GRAPH. The launcher is a two-line delegate, so
|
|
2048
|
+
// "control and interventions shared one shim" is a claim about every local
|
|
2049
|
+
// module a fresh Node process reads — and the boot marker cannot see any of
|
|
2050
|
+
// it, because it reports the CLI target rather than the instrument.
|
|
2051
|
+
assert.ok(
|
|
2052
|
+
RUNNER_SRC.includes("SHIM_RUNTIME = hashShimRuntime();") &&
|
|
2053
|
+
RUNNER_SRC.includes("shimRehash = hashShimRuntime();") &&
|
|
2054
|
+
RUNNER_SRC.includes('reason: "shim-runtime-drift"') &&
|
|
2055
|
+
RUNNER_SRC.includes('reason: "shim-runtime-unreadable"') &&
|
|
2056
|
+
RUNNER_SRC.includes("JSON.stringify(shimRehash) !== JSON.stringify(SHIM_RUNTIME)"),
|
|
2057
|
+
"the instrument's runtime graph is pinned before the first run and RE-HASHED after the last, on its own axis " +
|
|
2058
|
+
"with its own two names — an edit to the implementation landing between control and intervention is " +
|
|
2059
|
+
"invisible to every other check, including the shim's own boot marker [QK:RUNNER-SHIM-RUNTIME-PINNED]",
|
|
2060
|
+
);
|
|
2061
|
+
{
|
|
2062
|
+
// The runner's list must equal the STATIC LOCAL IMPORT CLOSURE of the
|
|
2063
|
+
// launcher — derived here, never restated, or the list becomes a second
|
|
2064
|
+
// unverified copy that drifts the moment the implementation grows a helper.
|
|
2065
|
+
// node: builtins and package specifiers are out of scope (they are not
|
|
2066
|
+
// tracked files of ours); a DYNAMIC import inside the closure is refused
|
|
2067
|
+
// outright, because a graph that assembles itself at runtime cannot be
|
|
2068
|
+
// pinned at all.
|
|
2069
|
+
const localImportsOf = (file: string): string[] => {
|
|
2070
|
+
const src = readFileSync(file, "utf8");
|
|
2071
|
+
assert.ok(
|
|
2072
|
+
!/\bimport\s*\(/.test(src),
|
|
2073
|
+
`${file} carries a DYNAMIC import — the shim runtime graph would no longer be statically knowable, so it ` +
|
|
2074
|
+
"could not be pinned across a pair",
|
|
2075
|
+
);
|
|
2076
|
+
return [...src.matchAll(/^\s*import\s[^;]*?from\s+"(\.[^"]+)"|^\s*import\s+"(\.[^"]+)"/gm)]
|
|
2077
|
+
.map((m) => m[1] ?? m[2])
|
|
2078
|
+
.map((spec) => resolve(dirname(file), spec));
|
|
2079
|
+
};
|
|
2080
|
+
const launcher = join(REPO_ROOT, "scripts", "fixtures", "probe-cli-shim");
|
|
2081
|
+
const closure: string[] = [];
|
|
2082
|
+
const walk = (file: string): void => {
|
|
2083
|
+
if (closure.includes(file)) return;
|
|
2084
|
+
closure.push(file);
|
|
2085
|
+
for (const next of localImportsOf(file)) walk(next);
|
|
2086
|
+
};
|
|
2087
|
+
walk(launcher);
|
|
2088
|
+
const declared = [
|
|
2089
|
+
...RUNNER_SRC.matchAll(/^\t(?:PROBE_SHIM,|join\(REPO_ROOT, "scripts", "lib", "([^"]+)"\),)$/gm),
|
|
2090
|
+
].map((m) => (m[1] === undefined ? launcher : join(REPO_ROOT, "scripts", "lib", m[1])));
|
|
2091
|
+
assert.deepEqual(
|
|
2092
|
+
[...declared].sort(),
|
|
2093
|
+
[...closure].sort(),
|
|
2094
|
+
"the runner's pinned instrument list is EXACTLY the launcher's static local-import closure — a helper added " +
|
|
2095
|
+
"to the shim without being pinned would otherwise be free to change mid-pair, and a stale entry would " +
|
|
2096
|
+
"pin a file the instrument no longer reads [QK:RUNNER-SHIM-RUNTIME-GRAPH-EXACT]",
|
|
2097
|
+
);
|
|
2098
|
+
}
|
|
2099
|
+
// The path is read OUT OF THE RUNNER rather than restated here, so this
|
|
2100
|
+
// checks the file the runner actually arms. Restating it would pass happily
|
|
2101
|
+
// while the runner pointed somewhere else.
|
|
2102
|
+
const declared = /const PROBE_SHIM = join\(REPO_ROOT, "scripts", "fixtures", "([^"]+)"\);/.exec(RUNNER_SRC);
|
|
2103
|
+
const shimPath = declared ? join(REPO_ROOT, "scripts", "fixtures", declared[1]) : "";
|
|
2104
|
+
assert.ok(
|
|
2105
|
+
declared !== null &&
|
|
2106
|
+
existsSync(shimPath) &&
|
|
2107
|
+
statSync(shimPath).isFile() &&
|
|
2108
|
+
(statSync(shimPath).mode & 0o111) !== 0 &&
|
|
2109
|
+
SDK_SCRIPT_SUFFIXES.every((suffix) => !shimPath.endsWith(suffix)),
|
|
2110
|
+
"the shim the runner arms is present, executable and extensionless ON DISK — the pair is asserted onto the " +
|
|
2111
|
+
"direct-spawn branch, and a script suffix (or a path pointing at nothing) would silently move the " +
|
|
2112
|
+
"instrument to `node|bun <path>` or break the spawn outright [QK:RUNNER-SHIM-ON-DISK-NATIVE]",
|
|
2113
|
+
);
|
|
2114
|
+
}
|
|
2115
|
+
}
|
|
2116
|
+
|
|
2117
|
+
// --- 8e) shim events at the log door: judged payload is typed there ---------
|
|
2118
|
+
{
|
|
2119
|
+
const stampAt = (seq: number, tsMs: number) => ({ seq, pid: 9, ts: new Date(tsMs).toISOString(), tsMs, runId: "sh" });
|
|
2120
|
+
const doorPath = join(tmp, "payload-shim.ndjson");
|
|
2121
|
+
const lines = [
|
|
2122
|
+
{ ...stampAt(0, 3_000), event: PROBE_EVENTS.shimBoot, targetPath: "/x/claude" }, // sha missing
|
|
2123
|
+
{ ...stampAt(1, 3_001), event: PROBE_EVENTS.shimPromptForwarded, ordinal: 0 }, // ordinal < 1
|
|
2124
|
+
{ ...stampAt(2, 3_002), event: PROBE_EVENTS.shimInitSnapshot, tools: "nope", receivedAtMs: 1 },
|
|
2125
|
+
// interval inverted — receivedAtMs AFTER the event's own envelope stamp
|
|
2126
|
+
// (the downstream-write-callback moment, the interval's single-SSOT end)
|
|
2127
|
+
// would silently un-order the §11-7-c "after the wire" read → refused.
|
|
2128
|
+
{ ...stampAt(3, 3_003), event: PROBE_EVENTS.shimInitSnapshot, tools: ["a"], receivedAtMs: 3_500 },
|
|
2129
|
+
{ ...stampAt(4, 3_004), event: PROBE_EVENTS.shimBoot, targetPath: "/x/claude", targetSha256: "ab12" },
|
|
2130
|
+
{ ...stampAt(5, 3_005), event: PROBE_EVENTS.shimPromptForwarded, ordinal: 1 },
|
|
2131
|
+
{ ...stampAt(6, 3_006), event: PROBE_EVENTS.shimInitSnapshot, tools: ["a", "b"], receivedAtMs: 3_002 },
|
|
2132
|
+
];
|
|
2133
|
+
for (const l of lines) appendFileSync(doorPath, `${JSON.stringify(l)}\n`, "utf8");
|
|
2134
|
+
const door = readProbeEvents(doorPath);
|
|
2135
|
+
assert.ok(
|
|
2136
|
+
door.malformed.length === 4 && door.events.length === 3,
|
|
2137
|
+
"shim payload rules hold at the door: missing target hash, ordinal<1, non-array tools, and receivedAtMs AFTER the envelope stamp are MALFORMED; the well-formed trio passes [QK:PROBE-LOG-SNAPSHOT-PAYLOAD]",
|
|
2138
|
+
);
|
|
2139
|
+
}
|
|
2140
|
+
|
|
2141
|
+
// --- 8f) the B-name-snapshot verdict ladder over synthetic paired logs ------
|
|
2142
|
+
{
|
|
2143
|
+
let shimSeq = 5_000;
|
|
2144
|
+
const shimEv = (runId: string, event: string, tsMs: number, payload: Record<string, unknown>): ProbeEvent =>
|
|
2145
|
+
({ seq: shimSeq++, pid: 9, ts: new Date(tsMs).toISOString(), tsMs, runId, event, ...payload }) as ProbeEvent;
|
|
2146
|
+
|
|
2147
|
+
const EXPECTED_TARGET = { path: "/x/claude", sha256: "ab" } as const;
|
|
2148
|
+
|
|
2149
|
+
/** Shim channel for one run: boot → prompt frame → snapshot(s). The interval
|
|
2150
|
+
* END is the snapshot event's own envelope tsMs (`at`), the moment the shim
|
|
2151
|
+
* appends inside the downstream write callback; payload carries only
|
|
2152
|
+
* `receivedAtMs`. */
|
|
2153
|
+
const shimChannel = (
|
|
2154
|
+
runId: string,
|
|
2155
|
+
base: number,
|
|
2156
|
+
snapshots: Array<{ receivedAtMs: number; at: number; tools: string[] }>,
|
|
2157
|
+
opts: { boot?: boolean; promptForwardedAt?: number; bootTargetSha256?: string } = {},
|
|
2158
|
+
): ProbeEvent[] => {
|
|
2159
|
+
const out: ProbeEvent[] = [];
|
|
2160
|
+
if (opts.boot !== false) {
|
|
2161
|
+
out.push(
|
|
2162
|
+
shimEv(runId, PROBE_EVENTS.shimBoot, base + 150, {
|
|
2163
|
+
targetPath: EXPECTED_TARGET.path,
|
|
2164
|
+
targetSha256: opts.bootTargetSha256 ?? EXPECTED_TARGET.sha256,
|
|
2165
|
+
}),
|
|
2166
|
+
);
|
|
2167
|
+
}
|
|
2168
|
+
out.push(shimEv(runId, PROBE_EVENTS.shimPromptForwarded, opts.promptForwardedAt ?? base + 1_700, { ordinal: 1 }));
|
|
2169
|
+
for (const s of snapshots) {
|
|
2170
|
+
out.push(shimEv(runId, PROBE_EVENTS.shimInitSnapshot, s.at, { tools: s.tools, receivedAtMs: s.receivedAtMs }));
|
|
2171
|
+
}
|
|
2172
|
+
return out;
|
|
2173
|
+
};
|
|
2174
|
+
|
|
2175
|
+
// An armed roster record also CARRIES the expected target identity — the
|
|
2176
|
+
// classifier consumes it (condition 5), fail-closed when absent.
|
|
2177
|
+
const armed = (r: { record: ProbeRunRecord; events: ProbeEvent[] }, shim: ProbeEvent[]) => ({
|
|
2178
|
+
record: {
|
|
2179
|
+
...r.record,
|
|
2180
|
+
snapshotInstrumented: true,
|
|
2181
|
+
cliTargetPath: EXPECTED_TARGET.path,
|
|
2182
|
+
cliTargetSha256: EXPECTED_TARGET.sha256,
|
|
2183
|
+
},
|
|
2184
|
+
events: [...r.events, ...shim],
|
|
2185
|
+
});
|
|
2186
|
+
|
|
2187
|
+
// A calibrated, armed control: channel clean and the snapshot CONTAINS the
|
|
2188
|
+
// measured id (fixture call + nonce echo already hold in passingControl).
|
|
2189
|
+
const armedControl = (base = 0) =>
|
|
2190
|
+
armed(
|
|
2191
|
+
passingControl(base),
|
|
2192
|
+
shimChannel("ctl", base, [{ receivedAtMs: base + 600, at: base + 605, tools: [PROVIDER_ID_MEASURED, "other"] }], {
|
|
2193
|
+
promptForwardedAt: base + 500,
|
|
2194
|
+
}),
|
|
2195
|
+
);
|
|
2196
|
+
|
|
2197
|
+
// Intervention timing shape (base b): nsLatency 1500 → promptStart = b+1660;
|
|
2198
|
+
// wireAt 2000 → wire = b+2100 > promptStart (promptRanAhead).
|
|
2199
|
+
const ranAheadSpec = { wireAt: 2_000, nsLatency: 1_500, fixtureCall: false } as const;
|
|
2200
|
+
|
|
2201
|
+
// (1) full floor → B-name-snapshot, promotable.
|
|
2202
|
+
{
|
|
2203
|
+
const ctl = armedControl();
|
|
2204
|
+
const i1 = armed(
|
|
2205
|
+
intervention("d1", 2_000, 10_000, { ...ranAheadSpec }),
|
|
2206
|
+
shimChannel("d1", 10_000, [{ receivedAtMs: 12_190, at: 12_195, tools: ["unrelated_tool"] }]),
|
|
2207
|
+
);
|
|
2208
|
+
const res = classifyProbe([ctl.record, i1.record], [...ctl.events, ...i1.events]);
|
|
2209
|
+
assert.equal(
|
|
2210
|
+
res.verdict,
|
|
2211
|
+
"B-name-snapshot",
|
|
2212
|
+
"snapshot absence of the measured id + promptRanAhead + wire strictly before the interval + calibrated control → B-name-snapshot [QK:VERDICT-SNAPSHOT-PROMOTES]",
|
|
2213
|
+
);
|
|
2214
|
+
assert.ok(res.promotable && res.status.failureVerdict === "B-name-snapshot", "…and it is promotable on axis (b)");
|
|
2215
|
+
assert.notEqual(
|
|
2216
|
+
res.interventions[0].failure,
|
|
2217
|
+
"B",
|
|
2218
|
+
"…and it NEVER upgrades into runtime B — the report is not the failure",
|
|
2219
|
+
);
|
|
2220
|
+
}
|
|
2221
|
+
|
|
2222
|
+
// (2) armed control with NO shim events at all → named instrument absence.
|
|
2223
|
+
{
|
|
2224
|
+
const ctl = armed(passingControl(20_000), []);
|
|
2225
|
+
const i1 = armed(
|
|
2226
|
+
intervention("d1", 2_000, 30_000, { ...ranAheadSpec }),
|
|
2227
|
+
shimChannel("d1", 30_000, [{ receivedAtMs: 32_190, at: 32_195, tools: ["unrelated_tool"] }]),
|
|
2228
|
+
);
|
|
2229
|
+
const res = classifyProbe([ctl.record, i1.record], [...ctl.events, ...i1.events]);
|
|
2230
|
+
assert.equal(res.verdict, "P0", "an armed control whose shim never reported in cannot calibrate — P0");
|
|
2231
|
+
assert.equal(
|
|
2232
|
+
res.control.p0Reason,
|
|
2233
|
+
"snapshot-instrument-absent",
|
|
2234
|
+
"…and the reason NAMES the missing instrument (a hijacked/replaced override looks exactly like this) [QK:VERDICT-SNAPSHOT-INSTRUMENT-ABSENT]",
|
|
2235
|
+
);
|
|
2236
|
+
}
|
|
2237
|
+
|
|
2238
|
+
// (3) armed control, channel clean, but the snapshot LACKS the measured id.
|
|
2239
|
+
{
|
|
2240
|
+
const ctl = armed(
|
|
2241
|
+
passingControl(40_000),
|
|
2242
|
+
shimChannel("ctl", 40_000, [{ receivedAtMs: 40_600, at: 40_605, tools: ["only_this"] }], {
|
|
2243
|
+
promptForwardedAt: 40_500,
|
|
2244
|
+
}),
|
|
2245
|
+
);
|
|
2246
|
+
const i1 = armed(
|
|
2247
|
+
intervention("d1", 2_000, 50_000, { ...ranAheadSpec }),
|
|
2248
|
+
shimChannel("d1", 50_000, [{ receivedAtMs: 52_190, at: 52_195, tools: ["unrelated_tool"] }]),
|
|
2249
|
+
);
|
|
2250
|
+
const res = classifyProbe([ctl.record, i1.record], [...ctl.events, ...i1.events]);
|
|
2251
|
+
assert.ok(
|
|
2252
|
+
res.verdict === "P0" && res.control.p0Reason === "snapshot-calibration",
|
|
2253
|
+
"a control snapshot that cannot SEE the measured id fails calibration — absence readings need a baseline that shows presence [QK:VERDICT-SNAPSHOT-CALIBRATION]",
|
|
2254
|
+
);
|
|
2255
|
+
}
|
|
2256
|
+
|
|
2257
|
+
// (4) absence + wire before the interval, but the prompt did NOT run ahead.
|
|
2258
|
+
{
|
|
2259
|
+
const ctl = armedControl(60_000);
|
|
2260
|
+
// wireAt 200 → wire = 70_300, promptStart = 71_660 → NOT promptRanAhead;
|
|
2261
|
+
// snapshot received at 72_000 (after the wire) with the id absent.
|
|
2262
|
+
const i1 = armed(
|
|
2263
|
+
intervention("d1", 2_000, 70_000, { wireAt: 200, nsLatency: 1_500, fixtureCall: false }),
|
|
2264
|
+
shimChannel("d1", 70_000, [{ receivedAtMs: 72_000, at: 72_005, tools: ["unrelated_tool"] }]),
|
|
2265
|
+
);
|
|
2266
|
+
const res = classifyProbe([ctl.record, i1.record], [...ctl.events, ...i1.events]);
|
|
2267
|
+
assert.ok(
|
|
2268
|
+
res.verdict === "inconclusive" && !res.promotable,
|
|
2269
|
+
"snapshot absence WITHOUT promptRanAhead is not the delayed-window failure mode — never promoted [QK:VERDICT-SNAPSHOT-REQUIRES-RANAHEAD]",
|
|
2270
|
+
);
|
|
2271
|
+
}
|
|
2272
|
+
|
|
2273
|
+
// (5) the wire marker lands INSIDE the snapshot interval → unordered.
|
|
2274
|
+
{
|
|
2275
|
+
const ctl = armedControl(80_000);
|
|
2276
|
+
// wire = 92_100; interval [92_050, 92_150] straddles it.
|
|
2277
|
+
const i1 = armed(
|
|
2278
|
+
intervention("d1", 2_000, 90_000, { ...ranAheadSpec }),
|
|
2279
|
+
shimChannel("d1", 90_000, [{ receivedAtMs: 92_050, at: 92_150, tools: ["unrelated_tool"] }]),
|
|
2280
|
+
);
|
|
2281
|
+
const res = classifyProbe([ctl.record, i1.record], [...ctl.events, ...i1.events]);
|
|
2282
|
+
assert.ok(
|
|
2283
|
+
res.verdict === "inconclusive" &&
|
|
2284
|
+
!res.promotable &&
|
|
2285
|
+
res.interventions[0].evidence.includes("INSIDE the snapshot interval"),
|
|
2286
|
+
"a wire marker inside the received↔forwarded interval is UNORDERED — only wire strictly before the interval reads as after [QK:VERDICT-SNAPSHOT-INTERVAL-UNORDERED]",
|
|
2287
|
+
);
|
|
2288
|
+
}
|
|
2289
|
+
|
|
2290
|
+
// (6) snapshot received BEFORE the wire → a different claim, not promoted.
|
|
2291
|
+
{
|
|
2292
|
+
const ctl = armedControl(100_000);
|
|
2293
|
+
// wire = 112_100; interval [111_800, 111_805] fully before it.
|
|
2294
|
+
const i1 = armed(
|
|
2295
|
+
intervention("d1", 2_000, 110_000, { ...ranAheadSpec }),
|
|
2296
|
+
shimChannel("d1", 110_000, [{ receivedAtMs: 111_800, at: 111_805, tools: ["unrelated_tool"] }]),
|
|
2297
|
+
);
|
|
2298
|
+
const res = classifyProbe([ctl.record, i1.record], [...ctl.events, ...i1.events]);
|
|
2299
|
+
assert.ok(
|
|
2300
|
+
res.verdict === "inconclusive" && res.interventions[0].evidence.includes("snapshot-before-wire"),
|
|
2301
|
+
"a snapshot that predates wire-availability carries a different claim than the §11-7-c after-the-wire row",
|
|
2302
|
+
);
|
|
2303
|
+
}
|
|
2304
|
+
|
|
2305
|
+
// (7) runtime No-such-tool for the measured id + snapshot absence → the
|
|
2306
|
+
// runtime ladder OWNS the run: B, never B-name-snapshot.
|
|
2307
|
+
{
|
|
2308
|
+
const ctl = armedControl(120_000);
|
|
2309
|
+
const i1 = armed(
|
|
2310
|
+
intervention("d1", 2_000, 130_000, { ...ranAheadSpec, noSuchToolId: PROVIDER_ID_MEASURED }),
|
|
2311
|
+
shimChannel("d1", 130_000, [{ receivedAtMs: 132_190, at: 132_195, tools: ["unrelated_tool"] }]),
|
|
2312
|
+
);
|
|
2313
|
+
const res = classifyProbe([ctl.record, i1.record], [...ctl.events, ...i1.events]);
|
|
2314
|
+
assert.ok(
|
|
2315
|
+
res.verdict === "B" && res.interventions[0].failure === "B",
|
|
2316
|
+
"the model's own dispatch failing is the stronger runtime-B evidence and owns the combined run — the snapshot never substitutes for it",
|
|
2317
|
+
);
|
|
2318
|
+
}
|
|
2319
|
+
|
|
2320
|
+
// (8) TWO snapshots after the prompt frame → the exactly-one binding is a
|
|
2321
|
+
// named violation, never a pick-first.
|
|
2322
|
+
{
|
|
2323
|
+
const ctl = armedControl(140_000);
|
|
2324
|
+
const i1 = armed(
|
|
2325
|
+
intervention("d1", 2_000, 150_000, { ...ranAheadSpec }),
|
|
2326
|
+
shimChannel("d1", 150_000, [
|
|
2327
|
+
{ receivedAtMs: 152_190, at: 152_195, tools: ["unrelated_tool"] },
|
|
2328
|
+
{ receivedAtMs: 152_400, at: 152_405, tools: ["unrelated_tool", PROVIDER_ID_MEASURED] },
|
|
2329
|
+
]),
|
|
2330
|
+
);
|
|
2331
|
+
const res = classifyProbe([ctl.record, i1.record], [...ctl.events, ...i1.events]);
|
|
2332
|
+
assert.ok(
|
|
2333
|
+
res.verdict === "inconclusive" &&
|
|
2334
|
+
!res.promotable &&
|
|
2335
|
+
res.interventions[0].evidence.includes("snapshot-channel-violation"),
|
|
2336
|
+
"reinit/set-model re-emission making the binding ambiguous is a NAMED channel violation — the (b) reading is unavailable, never a pick-first promotion [QK:VERDICT-SNAPSHOT-ORDINAL-EXACTLY-ONE]",
|
|
2337
|
+
);
|
|
2338
|
+
assert.notEqual(res.status.orderingMeasurement, "unobserved", "…while axis (a) still carries its comparison");
|
|
2339
|
+
}
|
|
2340
|
+
|
|
2341
|
+
// (9) full promotion-shaped shim evidence under an UNARMED roster → ignored.
|
|
2342
|
+
// The roster is the authority on what was instrumented; found evidence never
|
|
2343
|
+
// promotes past the declaration.
|
|
2344
|
+
{
|
|
2345
|
+
const ctl = passingControl(160_000); // unarmed
|
|
2346
|
+
const i1raw = intervention("d1", 2_000, 170_000, { ...ranAheadSpec });
|
|
2347
|
+
const i1 = {
|
|
2348
|
+
record: i1raw.record, // snapshotInstrumented: false
|
|
2349
|
+
events: [
|
|
2350
|
+
...i1raw.events,
|
|
2351
|
+
...shimChannel("d1", 170_000, [{ receivedAtMs: 172_190, at: 172_195, tools: ["unrelated_tool"] }]),
|
|
2352
|
+
],
|
|
2353
|
+
};
|
|
2354
|
+
const res = classifyProbe([ctl.record, i1.record], [...ctl.events, ...i1.events]);
|
|
2355
|
+
assert.ok(
|
|
2356
|
+
res.verdict === "inconclusive" && !res.promotable && res.status.failureVerdict !== "B-name-snapshot",
|
|
2357
|
+
"shim-shaped evidence in the log of an UNARMED run is ignored — the roster declares the instrument, evidence alone never promotes [QK:VERDICT-SNAPSHOT-NEEDS-INSTRUMENT-FLAG]",
|
|
2358
|
+
);
|
|
2359
|
+
}
|
|
2360
|
+
|
|
2361
|
+
// (10) the binding is on the RECEIVE axis, not the append axis. Under stdout
|
|
2362
|
+
// backpressure a BOOT-time init (received before the prompt frame) can have
|
|
2363
|
+
// its downstream callback — and therefore its log append — land after the
|
|
2364
|
+
// prompt marker. A seq-only binding would promote that stale set as the turn
|
|
2365
|
+
// snapshot (GPT review 2026-07-29). Timeline: promptStart 191_660 < wire
|
|
2366
|
+
// 191_670 (ranAhead), shim prompt frame stamped 191_700; the ONLY snapshot
|
|
2367
|
+
// was received 191_680 (BEFORE the frame) but appended at 191_750.
|
|
2368
|
+
{
|
|
2369
|
+
const ctl = armedControl(180_000);
|
|
2370
|
+
const i1 = armed(
|
|
2371
|
+
intervention("d1", 2_000, 190_000, { wireAt: 1_570, nsLatency: 1_500, fixtureCall: false }),
|
|
2372
|
+
shimChannel("d1", 190_000, [{ receivedAtMs: 191_680, at: 191_750, tools: ["unrelated_tool"] }]),
|
|
2373
|
+
);
|
|
2374
|
+
const res = classifyProbe([ctl.record, i1.record], [...ctl.events, ...i1.events]);
|
|
2375
|
+
assert.ok(
|
|
2376
|
+
res.verdict === "inconclusive" &&
|
|
2377
|
+
!res.promotable &&
|
|
2378
|
+
res.interventions[0].evidence.includes("snapshot-channel-violation"),
|
|
2379
|
+
"an init RECEIVED before the prompt frame is not a candidate even when its append lands after — the receive axis, not the append/callback axis, binds; zero candidates is a named violation [QK:VERDICT-SNAPSHOT-BINDING-RECEIVE-AXIS]",
|
|
2380
|
+
);
|
|
2381
|
+
}
|
|
2382
|
+
|
|
2383
|
+
// (11) a broken shim IDENTITY on an armed intervention invalidates the RUN —
|
|
2384
|
+
// the shim intermediates the CLI spawn, so a run without the calibrated shim
|
|
2385
|
+
// did not share the pair's launch path, and it may NOT keep voting on axis
|
|
2386
|
+
// (a) (GPT review 2026-07-29: a missing-shim run could otherwise build A/B
|
|
2387
|
+
// causal windows out of a different stimulus).
|
|
2388
|
+
{
|
|
2389
|
+
const ctl = armedControl(200_000);
|
|
2390
|
+
const i1 = armed(intervention("d1", 2_000, 210_000, { ...ranAheadSpec }), []); // armed, NO shim events
|
|
2391
|
+
const res = classifyProbe([ctl.record, i1.record], [...ctl.events, ...i1.events]);
|
|
2392
|
+
assert.ok(
|
|
2393
|
+
res.verdict === "INVALIDATED" &&
|
|
2394
|
+
res.status.invalidRuns.some((r) => r.reason === "snapshot-topology") &&
|
|
2395
|
+
res.ordering.summary === "unobserved" &&
|
|
2396
|
+
res.status.orderingMeasurement !== "measured",
|
|
2397
|
+
"an armed intervention whose shim never reported in is INVALIDATED (snapshot-topology) and votes on NEITHER axis — not a (b)-only degradation [QK:VERDICT-SNAPSHOT-STRUCTURAL-INVALIDATES]",
|
|
2398
|
+
);
|
|
2399
|
+
}
|
|
2400
|
+
|
|
2401
|
+
// (12) the roster's expected target identity is CONSUMED: a shim boot
|
|
2402
|
+
// reporting a different content hash means this run did not execute the
|
|
2403
|
+
// pair's stimulus (env hijack, target swap) → INVALIDATED, never promoted.
|
|
2404
|
+
{
|
|
2405
|
+
const ctl = armedControl(220_000);
|
|
2406
|
+
const i1 = armed(
|
|
2407
|
+
intervention("d1", 2_000, 230_000, { ...ranAheadSpec }),
|
|
2408
|
+
shimChannel("d1", 230_000, [{ receivedAtMs: 232_190, at: 232_195, tools: ["unrelated_tool"] }], {
|
|
2409
|
+
bootTargetSha256: "zz-different",
|
|
2410
|
+
}),
|
|
2411
|
+
);
|
|
2412
|
+
const res = classifyProbe([ctl.record, i1.record], [...ctl.events, ...i1.events]);
|
|
2413
|
+
assert.ok(
|
|
2414
|
+
res.verdict === "INVALIDATED" && res.status.invalidRuns.some((r) => r.reason === "snapshot-topology"),
|
|
2415
|
+
"a shim boot whose target path+sha does not match the roster's expected identity INVALIDATES the run — identity is verified, not merely recorded [QK:VERDICT-SNAPSHOT-TARGET-IDENTITY]",
|
|
2416
|
+
);
|
|
2417
|
+
}
|
|
2418
|
+
|
|
2419
|
+
// (13) a snapshot CONTAINING the measured id blocks the ladder BEFORE the
|
|
2420
|
+
// promotion rung — the absence check is positional, so this rung is what
|
|
2421
|
+
// keeps a present-id snapshot from promoting as absence.
|
|
2422
|
+
{
|
|
2423
|
+
const ctl = armedControl(240_000);
|
|
2424
|
+
const i1 = armed(
|
|
2425
|
+
intervention("d1", 2_000, 250_000, { ...ranAheadSpec }),
|
|
2426
|
+
shimChannel("d1", 250_000, [
|
|
2427
|
+
{ receivedAtMs: 252_190, at: 252_195, tools: [PROVIDER_ID_MEASURED, "unrelated_tool"] },
|
|
2428
|
+
]),
|
|
2429
|
+
);
|
|
2430
|
+
const res = classifyProbe([ctl.record, i1.record], [...ctl.events, ...i1.events]);
|
|
2431
|
+
assert.ok(
|
|
2432
|
+
res.verdict === "inconclusive" &&
|
|
2433
|
+
!res.promotable &&
|
|
2434
|
+
res.interventions[0].evidence.includes("CONTAINS the measured id"),
|
|
2435
|
+
"a snapshot that CONTAINS the measured id reads model-compliance, never absence — the contains-id rung blocks promotion [QK:VERDICT-SNAPSHOT-CONTAINS-ID-BLOCKS]",
|
|
2436
|
+
);
|
|
2437
|
+
}
|
|
2438
|
+
}
|
|
2439
|
+
|
|
2440
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
2441
|
+
console.log("[check-probe-ordering] PASS — §11-7 probe seam: sameness pinned to backend.ts, phase attribution");
|
|
2442
|
+
console.log(" (set-model included), fixture wire markers + required probeRunId + legacy compat, the event-log");
|
|
2443
|
+
console.log(" door contract (reserved keys refused; unknown marker / broken axis / unjudgeable payload →");
|
|
2444
|
+
console.log(" MALFORMED, while a legitimately absent optional field stays an observation) plus the stream");
|
|
2445
|
+
console.log(" door (per-pid seq/clock judged on RAW append order), the observation-window protocol");
|
|
2446
|
+
console.log(" (censored ≠ candidate), runner-owned marker topology, the two reported axes, the");
|
|
2447
|
+
console.log(" paired-verdict truth table (P0/I0 outside the space, phase-qualified D, B promotion ladder, C,");
|
|
2448
|
+
console.log(" A's two-delay rule), and the §11-7-c consumer seam (CLI-target preconditions, upstream");
|
|
2449
|
+
console.log(" launch-semantics inspector, shim event doors, B-name-snapshot ladder with calibration and");
|
|
2450
|
+
console.log(" the roster-armed channel).");
|