@junghanacs/entwurf 0.12.9 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +135 -248
- package/BASELINE.md +46 -7
- package/CHANGELOG.md +35 -0
- package/CONTRIBUTING.md +4 -2
- package/DELIVERY.md +5 -4
- package/README.md +46 -18
- package/VERIFY.md +10 -6
- package/demo/demo.sh +7 -1
- package/docs/setup-clean-host.md +10 -4
- package/mcp/entwurf-bridge/dist/mcp/entwurf-bridge/src/index.js +159 -66
- package/mcp/entwurf-bridge/dist/pi-extensions/lib/acp/backend-adapter.js +148 -5
- package/mcp/entwurf-bridge/dist/pi-extensions/lib/acp/config.js +16 -4
- package/mcp/entwurf-bridge/dist/pi-extensions/lib/acp/models.js +66 -7
- package/mcp/entwurf-bridge/dist/pi-extensions/lib/acp/overlay.js +190 -3
- package/mcp/entwurf-bridge/dist/pi-extensions/lib/acp/tool-surface.js +12 -4
- package/mcp/entwurf-bridge/dist/pi-extensions/lib/entwurf-core.js +26 -9
- package/mcp/entwurf-bridge/dist/pi-extensions/lib/entwurf-deliverability.js +10 -6
- package/mcp/entwurf-bridge/dist/pi-extensions/lib/entwurf-fact-provider.js +15 -17
- package/mcp/entwurf-bridge/dist/pi-extensions/lib/entwurf-facts.js +13 -13
- package/mcp/entwurf-bridge/dist/pi-extensions/lib/entwurf-resume-args.js +31 -41
- package/mcp/entwurf-bridge/dist/pi-extensions/lib/entwurf-self-address.js +8 -9
- package/mcp/entwurf-bridge/dist/pi-extensions/lib/entwurf-v2-contract.js +32 -25
- package/mcp/entwurf-bridge/dist/pi-extensions/lib/entwurf-v2-decider.js +4 -4
- package/mcp/entwurf-bridge/dist/pi-extensions/lib/entwurf-v2-lock.js +5 -4
- package/mcp/entwurf-bridge/dist/pi-extensions/lib/entwurf-v2-production.js +9 -11
- package/mcp/entwurf-bridge/dist/pi-extensions/lib/entwurf-v2-spawn-production.js +3 -1
- package/mcp/entwurf-bridge/dist/pi-extensions/lib/meta-session.js +17 -20
- package/mcp/entwurf-bridge/src/index.ts +190 -82
- package/package.json +7 -3
- package/pi/settings.reference.json +1 -1
- package/pi-extensions/acp-provider.ts +20 -10
- package/pi-extensions/entwurf-control.ts +69 -27
- package/pi-extensions/lib/acp/augment.ts +60 -2
- package/pi-extensions/lib/acp/backend-adapter.ts +183 -8
- package/pi-extensions/lib/acp/backend.ts +5 -1
- package/pi-extensions/lib/acp/config.ts +19 -5
- package/pi-extensions/lib/acp/engraving.ts +3 -1
- package/pi-extensions/lib/acp/event-mapper.ts +10 -3
- package/pi-extensions/lib/acp/models.ts +69 -7
- package/pi-extensions/lib/acp/overlay.ts +234 -3
- package/pi-extensions/lib/acp/tool-surface.ts +12 -4
- package/pi-extensions/lib/entwurf-core.ts +26 -9
- package/pi-extensions/lib/entwurf-deliverability.ts +10 -6
- package/pi-extensions/lib/entwurf-fact-provider.ts +15 -17
- package/pi-extensions/lib/entwurf-facts.ts +13 -13
- package/pi-extensions/lib/entwurf-resume-args.ts +40 -46
- package/pi-extensions/lib/entwurf-self-address.ts +8 -9
- package/pi-extensions/lib/entwurf-v2-contract-schema.ts +7 -2
- package/pi-extensions/lib/entwurf-v2-contract.ts +32 -25
- package/pi-extensions/lib/entwurf-v2-decider.ts +8 -8
- package/pi-extensions/lib/entwurf-v2-lock.ts +5 -4
- package/pi-extensions/lib/entwurf-v2-production.ts +14 -11
- package/pi-extensions/lib/entwurf-v2-spawn-production.ts +3 -1
- package/pi-extensions/lib/meta-session.ts +17 -20
- package/run.sh +200 -62
- package/scripts/agy-bridge-config.py +249 -49
- package/scripts/agy-bridge.sh +59 -14
- package/scripts/check-acp-carrier-augment.ts +34 -2
- package/scripts/check-acp-cortex.ts +668 -0
- package/scripts/check-acp-provider-surface.ts +50 -6
- package/scripts/check-acp-sdk-surface.ts +22 -11
- package/scripts/check-acp-session-reuse.ts +64 -1
- package/scripts/check-agy-permission-matrix.py +655 -0
- package/scripts/check-entwurf-deliverability.ts +9 -6
- package/scripts/check-entwurf-fact-provider.ts +7 -7
- package/scripts/check-entwurf-facts.ts +12 -12
- package/scripts/check-entwurf-resume-args.ts +22 -60
- package/scripts/check-entwurf-self-address.ts +187 -4
- package/scripts/check-entwurf-v2-contract.ts +2 -2
- package/scripts/check-entwurf-v2-production.ts +5 -5
- package/scripts/check-entwurf-v2-spawn-production.ts +6 -2
- package/scripts/check-entwurf-v2-surface.ts +327 -14
- package/scripts/check-gate-qualification.ts +787 -0
- package/scripts/check-meta-identity-consumers.ts +1 -1
- package/scripts/check-probe-cli-shim.ts +879 -0
- package/scripts/check-probe-ordering.ts +2450 -0
- package/scripts/check-shell-quote.ts +5 -4
- package/scripts/fixtures/probe-cli-shim +20 -0
- package/scripts/fixtures/probe-mcp-server.ts +168 -12
- package/scripts/lib/mutation-qualify.ts +794 -0
- package/scripts/lib/probe-acp-turn.ts +207 -0
- package/scripts/lib/probe-cli-shim.ts +464 -0
- package/scripts/lib/probe-cli-target.ts +165 -0
- package/scripts/lib/probe-event-log.ts +383 -0
- package/scripts/lib/probe-verdict.ts +1213 -0
- package/scripts/mutants/acp-augment.json +30 -0
- package/scripts/mutants/acp-cortex.json +196 -0
- package/scripts/mutants/agy-permission.json +144 -0
- package/scripts/mutants/meta-identity.json +17 -0
- package/scripts/mutants/probe-ordering.json +1032 -0
- package/scripts/mutants/self-address.json +59 -0
- package/scripts/mutants/v2-surface.json +87 -0
- package/scripts/smoke-acp-cortex-live.ts +392 -0
- package/scripts/smoke-acp-ordering-probe-live.ts +848 -0
- package/scripts/smoke-acp-raw-turn-live.ts +1 -1
- package/scripts/smoke-acp-v2-send-live.ts +33 -18
- package/scripts/smoke-agy-install-state.sh +205 -20
- package/scripts/smoke-entwurf-v2-matrix-live.ts +3 -2
- package/pi-extensions/lib/entwurf-mailbox-guard.ts +0 -100
- package/scripts/check-entwurf-mailbox-guard.ts +0 -266
|
@@ -0,0 +1,1213 @@
|
|
|
1
|
+
// §11-7 ordering probe — the PURE verdict classifier (docs/acp-backend-rail.md).
|
|
2
|
+
//
|
|
3
|
+
// Input: the shared NDJSON events + the run roster (control + interventions).
|
|
4
|
+
// Output: per-run facts, per-intervention readings, and ONE paired verdict
|
|
5
|
+
// written as a delta against the control — never as an absolute.
|
|
6
|
+
//
|
|
7
|
+
// The §11-7 rules this encodes (each one is a gate-pinned claim):
|
|
8
|
+
// - P0 / INVALID BASELINE: the delay=0 control fails (including initialize
|
|
9
|
+
// there). Not a branch of the experiment — nothing else is judged.
|
|
10
|
+
// - I0 / INVALID RUN: control passed but an intervention failed at initialize.
|
|
11
|
+
// The injected delay cannot reach that phase (servers ride newSession), so
|
|
12
|
+
// it is environment drift, never a D.
|
|
13
|
+
// - D must name its phase (newSession | enforceModel | prompt) — initialize
|
|
14
|
+
// is structurally excluded, and the injected delay must sit well below the
|
|
15
|
+
// 30 s boundaries or the probe measures our own timeout.
|
|
16
|
+
// - A requires TWO nonzero delays whose newSession latency excess tracks D;
|
|
17
|
+
// one D yields at most the ordering observation with the wait verdict
|
|
18
|
+
// WITHHELD ("A-withheld"). B, C, D may be read off the first intervention.
|
|
19
|
+
// - Absence promotes to direct schema-absence evidence ONLY on the marker
|
|
20
|
+
// combination: wire-forwarded ∧ no fixture tools/call ∧ runtime
|
|
21
|
+
// `No such tool` naming EXACTLY the measured expectedProviderToolId.
|
|
22
|
+
// Alias mismatch and model prose never promote. Anything unlisted stays
|
|
23
|
+
// inconclusive by default.
|
|
24
|
+
//
|
|
25
|
+
// TWO AXES, not one (GPT review 2026-07-29). The first cut folded everything
|
|
26
|
+
// into a single verdict, and the first LIVE pair exposed what that costs: D1's
|
|
27
|
+
// `promptStart < wireForwarded` is a DIRECT, model-independent answer to the
|
|
28
|
+
// narrow question §11-7 exists to ask — does this server/path wait for delayed
|
|
29
|
+
// MCP wire availability before opening the turn? — yet the pair reported plain
|
|
30
|
+
// `inconclusive`, because the SEPARATE question (is that window sufficient to
|
|
31
|
+
// PRODUCE the failure) had no marker. One verdict let the missing (b) evidence
|
|
32
|
+
// hide the settled (a) observation. So every intervention now carries:
|
|
33
|
+
// (a) `ordering` — the ORDERING comparison, read off marker timestamps alone:
|
|
34
|
+
// wire-before-newSession-end | wire-before-prompt-request |
|
|
35
|
+
// prompt-request-ahead-of-wire | censored | unknown. Each value names the
|
|
36
|
+
// comparison, never a conclusion: `promptStart` is a CLIENT-side proxy, so
|
|
37
|
+
// `promptStart < wire` says WE issued the prompt request first and does NOT
|
|
38
|
+
// show the server failed to wait.
|
|
39
|
+
// (b) `failure` — the callability reading: callable | C | B |
|
|
40
|
+
// B-name-snapshot | candidate-handshake | inconclusive. B-name-snapshot
|
|
41
|
+
// (§11-7-c) is the CONTROLLED absence reading off the CLI's own per-turn
|
|
42
|
+
// name set — a REPORT, deliberately weaker than runtime B and never
|
|
43
|
+
// conflated with it; it needs the roster-armed shim channel, control
|
|
44
|
+
// calibration, promptRanAhead, and wire strictly before the snapshot
|
|
45
|
+
// interval.
|
|
46
|
+
// A (the WAIT verdict) still belongs to (a) and still needs `wire <
|
|
47
|
+
// newSessionEnd` plus latency scaling across two delays. B/C's causal window is
|
|
48
|
+
// `promptStart < wire` — used as a stated PREMISE (the prompt request is where
|
|
49
|
+
// we hand the turn over; that this is also when the model's tool set was fixed
|
|
50
|
+
// is what §11-7-c's snapshot would have to show) — and the newSession comparison
|
|
51
|
+
// is kept as a DIAGNOSTIC, not a verdict input. Hence three separately named
|
|
52
|
+
// flags rather than one overloaded `ranAhead`.
|
|
53
|
+
//
|
|
54
|
+
// Pure on purpose: no IO, no clocks — the deterministic gate replays synthetic
|
|
55
|
+
// logs through this exact module.
|
|
56
|
+
|
|
57
|
+
import { PROBE_EVENTS, PROBE_EXPECTED_TOOL, type ProbeEvent, type ProbeWindowReason } from "./probe-event-log.ts";
|
|
58
|
+
|
|
59
|
+
/** The tool whose call marks callability. One SSOT in probe-event-log.ts — the
|
|
60
|
+
* fixture serves it and the parser requires the wire marker to name it, so a
|
|
61
|
+
* private copy here could drift from either without a gate noticing. */
|
|
62
|
+
export const EXPECTED_SOURCE_TOOL = PROBE_EXPECTED_TOOL;
|
|
63
|
+
|
|
64
|
+
// "Well below BOTH the newSession and set-model 30 s boundaries" (§11-7): half.
|
|
65
|
+
export const DELAY_WELL_BELOW_MS = 15_000;
|
|
66
|
+
|
|
67
|
+
// A's discriminator is "newSession latency tracks D" — a BAND plus growth, not
|
|
68
|
+
// a floor alone (a floor-only check accepts excess=10s at D=2s, which is not
|
|
69
|
+
// tracking; GPT review 2026-07-28). Per intervention the excess over the
|
|
70
|
+
// control must absorb at least MIN_RATIO of the injected delay AND stay within
|
|
71
|
+
// SLACK above it; across delays the excess must GROW with D.
|
|
72
|
+
export const A_LATENCY_TRACK_MIN_RATIO = 0.8;
|
|
73
|
+
export const A_LATENCY_SLACK_MS = 2_000;
|
|
74
|
+
export const A_GROWTH_MIN_RATIO = 0.6;
|
|
75
|
+
|
|
76
|
+
/** Axis (a) — the ORDERING observation, from marker timestamps alone.
|
|
77
|
+
*
|
|
78
|
+
* Every value is named for the comparison it IS, not for a conclusion drawn
|
|
79
|
+
* from it. That is deliberate and was corrected under review (2026-07-29): the
|
|
80
|
+
* first cut called `promptStart < wire` "no-wait", which claims something these
|
|
81
|
+
* stamps cannot show. `promptStart` is a CLIENT-side proxy — the moment we
|
|
82
|
+
* issued the ACP prompt request — and a server may perfectly well accept that
|
|
83
|
+
* request and then wait internally for the MCP install before serving the
|
|
84
|
+
* model. Likewise `wire < newSessionEnd` is not "the server waited": on its own
|
|
85
|
+
* it is one ordering, and only A's latency scaling over two delays turns it
|
|
86
|
+
* into wait evidence. So:
|
|
87
|
+
* - `wire-before-newSession-end` wire < newSessionEnd. A's input.
|
|
88
|
+
* - `wire-before-prompt-request` newSessionEnd < wire < promptStart.
|
|
89
|
+
* - `prompt-request-ahead-of-wire` promptStart < wire. Model-independent, and
|
|
90
|
+
* the ONLY thing it establishes is that WE issued the prompt request first.
|
|
91
|
+
* - `censored` our observation window closed before the marker could land —
|
|
92
|
+
* a fact about the probe, never about the server.
|
|
93
|
+
* - `unknown` no marker under a sufficient window, or a same-ms
|
|
94
|
+
* cross-process tie, which is unordered at this resolution. */
|
|
95
|
+
export type OrderingObservation =
|
|
96
|
+
| "wire-before-newSession-end"
|
|
97
|
+
| "wire-before-prompt-request"
|
|
98
|
+
| "prompt-request-ahead-of-wire"
|
|
99
|
+
| "censored"
|
|
100
|
+
| "unknown";
|
|
101
|
+
|
|
102
|
+
/** Axis (b) — the callability reading.
|
|
103
|
+
* - `callable` the expected tool's call reached the fixture.
|
|
104
|
+
* - `C` callable, but the prompt request had been issued
|
|
105
|
+
* ahead of the wire: late / dynamic readiness with no client fence.
|
|
106
|
+
* - `B` the marker-grade absence combination (see below).
|
|
107
|
+
* - `B-name-snapshot` the §11-7-c CONTROLLED absence reading — the CLI's
|
|
108
|
+
* own per-turn tool-NAME set, captured by the shim, lacks the measured id
|
|
109
|
+
* under the full promotion floor. DELIBERATELY a separate value: it is a
|
|
110
|
+
* REPORT of what the CLI said it had, not the model's dispatch failing, so
|
|
111
|
+
* it may never inherit `B`'s strength and `B` may never be read off it.
|
|
112
|
+
* Only reachable when the run's roster entry declares the snapshot channel
|
|
113
|
+
* armed (`snapshotInstrumented`) — evidence without the declared instrument
|
|
114
|
+
* never promotes.
|
|
115
|
+
* - `candidate-handshake` no wire marker under a SUFFICIENT window: an MCP
|
|
116
|
+
* handshake / fixture / config candidate. Only reachable when the window
|
|
117
|
+
* closed on its deadline — a censored run never lands here, which is exactly
|
|
118
|
+
* the misattribution the first LIVE pair produced.
|
|
119
|
+
* - `inconclusive` everything else, by default. */
|
|
120
|
+
export type FailureReading = "callable" | "C" | "B" | "B-name-snapshot" | "candidate-handshake" | "inconclusive";
|
|
121
|
+
|
|
122
|
+
/** Why a run is INVALIDATED — outside the verdict space entirely, like P0/I0.
|
|
123
|
+
* `snapshot-topology` (§11-7-c): the shim's IDENTITY is broken on an armed run
|
|
124
|
+
* — no/duplicate boot, several shim pids, or a boot target that does not match
|
|
125
|
+
* the roster's expected path+sha. The shim intermediates the CLI spawn, i.e.
|
|
126
|
+
* it sits on the TIMING path, so a run that did not execute the calibrated
|
|
127
|
+
* shim did not share the pair's launch path and its (a) axis is polluted too —
|
|
128
|
+
* unlike a READING failure of the channel, which stays (b)-only. */
|
|
129
|
+
export type RunInvalidReason = "observation-window-closed" | "topology" | "snapshot-topology";
|
|
130
|
+
|
|
131
|
+
export interface ProbeRunRecord {
|
|
132
|
+
runId: string;
|
|
133
|
+
role: "control" | "intervention";
|
|
134
|
+
delayMs: number;
|
|
135
|
+
probeRunId: string;
|
|
136
|
+
/** §11-7-c: whether the CLI-shim snapshot channel was ARMED for this run.
|
|
137
|
+
* The roster is the authority on what was instrumented: shim-shaped events
|
|
138
|
+
* in a log whose roster says `false` are ignored (never promoted), and a
|
|
139
|
+
* `true` here obliges the channel to be present and coherent — the control
|
|
140
|
+
* P0s on a missing/incoherent instrument instead of quietly reading like an
|
|
141
|
+
* uninstrumented pair. The LIVE runner pins `false` until the shim (the
|
|
142
|
+
* producer half) lands. */
|
|
143
|
+
snapshotInstrumented: boolean;
|
|
144
|
+
/** §11-7-c condition 5: the pair's EXPECTED CLI target identity, stamped by
|
|
145
|
+
* the runner from its pre-run resolution. The classifier CONSUMES this —
|
|
146
|
+
* an armed run whose shim boot reports a different path or content hash
|
|
147
|
+
* did not execute the pair's stimulus (env hijack, target swap) and is
|
|
148
|
+
* INVALIDATED as `snapshot-topology`; an armed roster that omits the
|
|
149
|
+
* identity is the same finding (nothing to verify against is fail-closed,
|
|
150
|
+
* never fail-open). Unarmed runs carry them as forensics only. */
|
|
151
|
+
cliTargetPath?: string;
|
|
152
|
+
cliTargetSha256?: string;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
interface PhaseFact {
|
|
156
|
+
started: boolean;
|
|
157
|
+
ok: boolean;
|
|
158
|
+
timedOut: boolean;
|
|
159
|
+
error?: string;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
export interface RunFacts {
|
|
163
|
+
runId: string;
|
|
164
|
+
role: "control" | "intervention";
|
|
165
|
+
delayMs: number;
|
|
166
|
+
probeRunId: string;
|
|
167
|
+
initialize: PhaseFact;
|
|
168
|
+
newSession: PhaseFact;
|
|
169
|
+
enforceModel: PhaseFact;
|
|
170
|
+
prompt: PhaseFact;
|
|
171
|
+
newSessionStartMs?: number;
|
|
172
|
+
newSessionEndMs?: number;
|
|
173
|
+
newSessionLatencyMs?: number;
|
|
174
|
+
promptStartMs?: number;
|
|
175
|
+
promptEndMs?: number;
|
|
176
|
+
/** ts of tools_list_response_forwarded — THE wire-availability marker. */
|
|
177
|
+
wireForwardedMs?: number;
|
|
178
|
+
/** How the runner's observation window closed. Absent = the run predates the
|
|
179
|
+
* window protocol, which is a topology violation, not a permissive default. */
|
|
180
|
+
windowReason?: ProbeWindowReason;
|
|
181
|
+
windowMarkerSeen?: boolean;
|
|
182
|
+
/** Runner-owned marker topology. Violations invalidate the run: these markers
|
|
183
|
+
* are OURS, emitted exactly once by construction, so a duplicate, an
|
|
184
|
+
* end-without-start, or an out-of-order pair means the log describing the run
|
|
185
|
+
* is not the run. */
|
|
186
|
+
topologyViolations: string[];
|
|
187
|
+
/** Topology PLUS window-close coherence — the full "is this log a description
|
|
188
|
+
* of one run" answer, computed once here so the control and the interventions
|
|
189
|
+
* are held to the SAME bar. The first cut checked coherence inside
|
|
190
|
+
* classifyIntervention only, which left a control free to claim a wire marker
|
|
191
|
+
* it never logged (GPT review round 2, 2026-07-29). */
|
|
192
|
+
validityViolations: string[];
|
|
193
|
+
/** A fixture tools/call for the expected tool with THIS run's probeRunId —
|
|
194
|
+
* §11-7's only callability marker. */
|
|
195
|
+
fixtureToolCall: boolean;
|
|
196
|
+
fixtureToolCallMs?: number;
|
|
197
|
+
/** §11-7-c snapshot channel (populated ONLY when the roster armed it). */
|
|
198
|
+
snapshotInstrumented: boolean;
|
|
199
|
+
shimBootCount: number;
|
|
200
|
+
shimTargetPath?: string;
|
|
201
|
+
shimTargetSha256?: string;
|
|
202
|
+
/** STRUCTURAL findings — the shim's identity is broken (no/dup boot, several
|
|
203
|
+
* shim pids, boot target ≠ roster expectation, armed roster without an
|
|
204
|
+
* expected identity). The shim sits on the timing path, so these INVALIDATE
|
|
205
|
+
* the run (`snapshot-topology`) — both axes, not just (b). */
|
|
206
|
+
snapshotStructuralViolations: string[];
|
|
207
|
+
/** READING findings — the instrument ran but its exactly-one binding failed
|
|
208
|
+
* (prompt-frame cardinality/ordinal, candidate count ≠ 1). Deliberately NOT
|
|
209
|
+
* run-invalidating: the channel is one instrument on axis (b), and axis (a)
|
|
210
|
+
* plus the runner/fixture markers are untouched by a binding that stuttered
|
|
211
|
+
* — widening one instrument's reading failure over both axes would re-create
|
|
212
|
+
* the very axis-conflation §11-7-0 undid. The CONTROL is stricter: any entry
|
|
213
|
+
* here is a calibration failure and P0s the pair. */
|
|
214
|
+
snapshotChannelViolations: string[];
|
|
215
|
+
/** The ONE bound snapshot (§11-7-c: exactly one init RECEIVED after the
|
|
216
|
+
* prompt frame), present only when the channel is clean. The interval end
|
|
217
|
+
* is the event's envelope tsMs (stamped in the downstream write callback). */
|
|
218
|
+
snapshotTools?: string[];
|
|
219
|
+
snapshotReceivedAtMs?: number;
|
|
220
|
+
snapshotForwardedAtMs?: number;
|
|
221
|
+
/** ACP-side tool_call observed with matching probeRunId (visibility + the
|
|
222
|
+
* provider-bound id measurement). */
|
|
223
|
+
acpProviderToolId?: string;
|
|
224
|
+
/** Runtime `No such tool available: <id>` observed during the turn. */
|
|
225
|
+
noSuchToolId?: string;
|
|
226
|
+
nonceEchoed: boolean;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
export type InterventionReadingKind =
|
|
230
|
+
| "I0"
|
|
231
|
+
| "INVALIDATED"
|
|
232
|
+
| "D-newSession"
|
|
233
|
+
| "D-enforceModel"
|
|
234
|
+
| "D-prompt"
|
|
235
|
+
| "B"
|
|
236
|
+
| "B-name-snapshot"
|
|
237
|
+
| "C"
|
|
238
|
+
| "ordering-kept"
|
|
239
|
+
| "inconclusive";
|
|
240
|
+
|
|
241
|
+
/** The three ordering comparisons, kept SEPARATE on purpose. Collapsing them
|
|
242
|
+
* into one `ranAhead` is what let a single flag serve A (whose axis is
|
|
243
|
+
* `wire < newSessionEnd`) and B/C (whose causal window is `promptStart < wire`);
|
|
244
|
+
* replacing that one flag with the prompt axis would have silently broken A. */
|
|
245
|
+
export interface OrderingDeltas {
|
|
246
|
+
/** wire < newSessionEnd — session creation absorbed the delay (A's axis). */
|
|
247
|
+
newSessionOrderingKept: boolean;
|
|
248
|
+
/** newSessionEnd < wire — DIAGNOSTIC only; never a B/C verdict input. */
|
|
249
|
+
newSessionRanAhead: boolean;
|
|
250
|
+
/** promptStart < wire — the causal window for B/C. */
|
|
251
|
+
promptRanAhead: boolean;
|
|
252
|
+
/** newSessionEnd − wire (positive = session ended that many ms early). */
|
|
253
|
+
newSessionAheadOfWireMs?: number;
|
|
254
|
+
/** promptStart − wire (positive = we issued the prompt request that many ms
|
|
255
|
+
* before the wire marker landed — a client-side fact, not a server one). */
|
|
256
|
+
promptAheadOfWireMs?: number;
|
|
257
|
+
/** How much turn was LEFT after the wire marker landed. A large value with no
|
|
258
|
+
* tool call says the model had time and did not use it — a model-compliance
|
|
259
|
+
* reading, not a server reading, and the artifact must show it. */
|
|
260
|
+
postWireTurnMs?: number;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
export interface InterventionReading {
|
|
264
|
+
runId: string;
|
|
265
|
+
delayMs: number;
|
|
266
|
+
kind: InterventionReadingKind;
|
|
267
|
+
/** Axis (a) — server-wait observation. Independent of the model's behavior. */
|
|
268
|
+
ordering: OrderingObservation;
|
|
269
|
+
/** Axis (b) — callability reading. */
|
|
270
|
+
failure: FailureReading;
|
|
271
|
+
deltas: OrderingDeltas;
|
|
272
|
+
/** Only a B backed by the exact §11-7 marker combination may enter the
|
|
273
|
+
* readiness ledger. */
|
|
274
|
+
promotable: boolean;
|
|
275
|
+
evidence: string;
|
|
276
|
+
invalidReason?: RunInvalidReason;
|
|
277
|
+
newSessionExcessMs?: number;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
export interface ProbeClassification {
|
|
281
|
+
control: { runId: string; pass: boolean; p0Reason?: string };
|
|
282
|
+
/** Measured off the control's ACP tool_call — never hardcoded (§11-7). */
|
|
283
|
+
expectedProviderToolId?: string;
|
|
284
|
+
interventions: InterventionReading[];
|
|
285
|
+
verdict:
|
|
286
|
+
| "P0"
|
|
287
|
+
| "I0"
|
|
288
|
+
| "INVALIDATED"
|
|
289
|
+
| "A"
|
|
290
|
+
| "A-withheld"
|
|
291
|
+
| "B"
|
|
292
|
+
| "B-name-snapshot"
|
|
293
|
+
| "C"
|
|
294
|
+
| "D-newSession"
|
|
295
|
+
| "D-enforceModel"
|
|
296
|
+
| "D-prompt"
|
|
297
|
+
| "inconclusive";
|
|
298
|
+
/** Axis (a) reported on its OWN terms, so a settled ordering observation is
|
|
299
|
+
* never hidden by the (b) axis lacking a marker. The summary values are named
|
|
300
|
+
* for the comparison, never for a conclusion: neither of them asserts that the
|
|
301
|
+
* server did or did not wait. `wire-before-newSession-end` is A's input and
|
|
302
|
+
* becomes wait evidence only with A's latency scaling on top;
|
|
303
|
+
* `prompt-request-ahead-of-wire` says WE issued the prompt request first and
|
|
304
|
+
* nothing about what the server then did internally. */
|
|
305
|
+
ordering: {
|
|
306
|
+
summary: "wire-before-newSession-end" | "prompt-request-ahead-of-wire" | "mixed" | "unobserved";
|
|
307
|
+
perRun: { runId: string; delayMs: number; observation: OrderingObservation }[];
|
|
308
|
+
};
|
|
309
|
+
/** The run's outcome split into the three things a caller actually decides on.
|
|
310
|
+
* Folding them into one verdict is what made the LIVE runner treat a pair that
|
|
311
|
+
* MEASURED its ordering axis as a failed run, purely because the callability
|
|
312
|
+
* axis had no marker (GPT review 2026-07-29).
|
|
313
|
+
* - `validity` fatal only. Under P0/I0/INVALIDATED nothing was measured.
|
|
314
|
+
* - `orderingMeasurement` did axis (a) produce a comparison? `measured` says
|
|
315
|
+
* the comparison exists — it does NOT say the server waited or did not.
|
|
316
|
+
* - `failureVerdict` axis (b), or `not-judged` when validity is fatal. */
|
|
317
|
+
status: {
|
|
318
|
+
/** `partial` = the pair was read, but at least one intervention was thrown
|
|
319
|
+
* out. Calling that plain `valid` hides a missing delay point behind a
|
|
320
|
+
* healthy-looking label (GPT review round 2, 2026-07-29). */
|
|
321
|
+
validity: "valid" | "partial" | "P0" | "I0" | "INVALIDATED";
|
|
322
|
+
orderingMeasurement: "measured" | "censored" | "unobserved";
|
|
323
|
+
failureVerdict: FailureReading | "not-judged";
|
|
324
|
+
/** Which runs were thrown out, and why — never summarized away. */
|
|
325
|
+
invalidRuns: { runId: string; reason: RunInvalidReason }[];
|
|
326
|
+
};
|
|
327
|
+
promotable: boolean;
|
|
328
|
+
detail: string;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
function phaseFact(events: ProbeEvent[], startName: string, endName: string): PhaseFact {
|
|
332
|
+
const started = events.some((e) => e.event === startName);
|
|
333
|
+
const end = events.find((e) => e.event === endName);
|
|
334
|
+
return {
|
|
335
|
+
started,
|
|
336
|
+
ok: end?.ok === true,
|
|
337
|
+
timedOut: end?.timedOut === true,
|
|
338
|
+
error: typeof end?.error === "string" ? end.error : undefined,
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
function eventTs(events: ProbeEvent[], name: string): number | undefined {
|
|
343
|
+
const hit = events.find((e) => e.event === name);
|
|
344
|
+
return hit?.tsMs;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
/** The runner-owned phase pairs, in production order. */
|
|
348
|
+
const PHASE_PAIRS: ReadonlyArray<readonly [string, string, string]> = [
|
|
349
|
+
["initialize", PROBE_EVENTS.initializeStart, PROBE_EVENTS.initializeEnd],
|
|
350
|
+
["newSession", PROBE_EVENTS.newSessionStart, PROBE_EVENTS.newSessionEnd],
|
|
351
|
+
["enforceModel", PROBE_EVENTS.setModelStart, PROBE_EVENTS.setModelEnd],
|
|
352
|
+
["prompt", PROBE_EVENTS.promptStart, PROBE_EVENTS.promptEnd],
|
|
353
|
+
];
|
|
354
|
+
|
|
355
|
+
/** Markers the RUNNER emits exactly once per run, by construction. Exported so
|
|
356
|
+
* the gate can pin it against a HAND-WRITTEN literal: dropping a member here
|
|
357
|
+
* would quietly retire a topology rule, and reading the set off the module under
|
|
358
|
+
* test would make the gate agree with whatever it was handed. */
|
|
359
|
+
export const RUNNER_EXACTLY_ONCE: ReadonlyArray<string> = [
|
|
360
|
+
PROBE_EVENTS.runStart,
|
|
361
|
+
PROBE_EVENTS.observationWindowEnd,
|
|
362
|
+
PROBE_EVENTS.runEnd,
|
|
363
|
+
];
|
|
364
|
+
|
|
365
|
+
/** Window-close coherence. The close is SELF-REPORTED by the runner and
|
|
366
|
+
* everything downstream turns on it, so it is checked against the log rather
|
|
367
|
+
* than believed: a close claiming the marker was seen, in a run whose log has
|
|
368
|
+
* none, would walk a censored run straight into the candidate branch. Applied
|
|
369
|
+
* to EVERY run — control included, since the baseline is what the pair is a
|
|
370
|
+
* delta against. */
|
|
371
|
+
function windowCoherenceViolations(windowEnd: ProbeEvent | undefined, wireInLog: boolean): string[] {
|
|
372
|
+
if (windowEnd === undefined) {
|
|
373
|
+
return [
|
|
374
|
+
`no ${PROBE_EVENTS.observationWindowEnd} for this run — absence of the wire marker cannot be told apart from our own teardown`,
|
|
375
|
+
];
|
|
376
|
+
}
|
|
377
|
+
const reason = windowEnd.reason as ProbeWindowReason;
|
|
378
|
+
const markerSeen = windowEnd.markerSeen;
|
|
379
|
+
const out: string[] = [];
|
|
380
|
+
if (markerSeen !== wireInLog) {
|
|
381
|
+
out.push(
|
|
382
|
+
`window close reports markerSeen=${markerSeen} but the run's log ${wireInLog ? "DOES" : "does NOT"} carry ${PROBE_EVENTS.toolsListResponseForwarded} — the self-reported window contradicts its own evidence`,
|
|
383
|
+
);
|
|
384
|
+
}
|
|
385
|
+
if (reason === "wire-marker" && markerSeen !== true) {
|
|
386
|
+
out.push("window closed on 'wire-marker' with markerSeen=false — incoherent close");
|
|
387
|
+
}
|
|
388
|
+
if ((reason === "deadline" || reason === "child-exit") && markerSeen === true) {
|
|
389
|
+
out.push(`window closed on '${reason}' with markerSeen=true — those reasons mean the marker did NOT arrive`);
|
|
390
|
+
}
|
|
391
|
+
return out;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
/** Runner-owned marker topology for ONE run.
|
|
395
|
+
*
|
|
396
|
+
* This is deliberately narrow: it judges only markers the runner itself emits,
|
|
397
|
+
* where cardinality is a property of our own code rather than of the model or
|
|
398
|
+
* the transport. `acp_tool_call_raw` repeats by design, a model may produce
|
|
399
|
+
* several tool calls, and a client may re-request tools/list — those are
|
|
400
|
+
* repeatable markers governed by an earliest-wins read, not by this check
|
|
401
|
+
* (GPT review 2026-07-29 rejected the wider "everything but tools/list is
|
|
402
|
+
* exactly-once" rule as over-broad, and it was).
|
|
403
|
+
*
|
|
404
|
+
* Order is compared on `seq`, not timestamps: all runner markers come from one
|
|
405
|
+
* process, so `seq` is an exact in-process order, while two markers in the same
|
|
406
|
+
* millisecond are unordered on the shared wall-clock axis. A runner marker
|
|
407
|
+
* arriving from a SECOND pid is itself a violation — the run would be describing
|
|
408
|
+
* two writers as one. */
|
|
409
|
+
function runnerTopologyViolations(events: ProbeEvent[]): string[] {
|
|
410
|
+
const violations: string[] = [];
|
|
411
|
+
const runnerOwned = new Set<string>([
|
|
412
|
+
...RUNNER_EXACTLY_ONCE,
|
|
413
|
+
...PHASE_PAIRS.flatMap(([, s, e]) => [s, e]),
|
|
414
|
+
PROBE_EVENTS.promptReply,
|
|
415
|
+
]);
|
|
416
|
+
const owned = events.filter((e) => runnerOwned.has(e.event));
|
|
417
|
+
const pids = new Set(owned.map((e) => e.pid));
|
|
418
|
+
if (pids.size > 1) {
|
|
419
|
+
violations.push(
|
|
420
|
+
`runner-owned markers came from ${pids.size} pids (${[...pids].join(", ")}) — one run has one runner`,
|
|
421
|
+
);
|
|
422
|
+
}
|
|
423
|
+
const seqOf = (name: string): number | undefined => owned.find((e) => e.event === name)?.seq;
|
|
424
|
+
const countOf = (name: string): number => owned.filter((e) => e.event === name).length;
|
|
425
|
+
|
|
426
|
+
for (const name of RUNNER_EXACTLY_ONCE) {
|
|
427
|
+
const n = countOf(name);
|
|
428
|
+
if (n !== 1) violations.push(`${name} appears ${n} times — the runner emits it exactly once per run`);
|
|
429
|
+
}
|
|
430
|
+
// A reply is judged payload (nonceEchoed), so a second one would silently
|
|
431
|
+
// change the reading depending on which is read first.
|
|
432
|
+
const replies = countOf(PROBE_EVENTS.promptReply);
|
|
433
|
+
if (replies > 1) violations.push(`${PROBE_EVENTS.promptReply} appears ${replies} times — at most one per run`);
|
|
434
|
+
|
|
435
|
+
for (const [phase, startName, endName] of PHASE_PAIRS) {
|
|
436
|
+
const starts = countOf(startName);
|
|
437
|
+
const ends = countOf(endName);
|
|
438
|
+
if (starts > 1) violations.push(`${phase}: ${startName} appears ${starts} times — a phase starts at most once`);
|
|
439
|
+
if (ends > 1) violations.push(`${phase}: ${endName} appears ${ends} times — a phase ends at most once`);
|
|
440
|
+
if (ends > 0 && starts === 0) violations.push(`${phase}: ${endName} without ${startName} — an end with no start`);
|
|
441
|
+
// runPhase logs the end on BOTH paths (ok / error), so a started phase that
|
|
442
|
+
// never ends means the log lost a line, not that the phase is still running.
|
|
443
|
+
if (starts > 0 && ends === 0) violations.push(`${phase}: ${startName} without ${endName} — the phase never closed`);
|
|
444
|
+
const s = seqOf(startName);
|
|
445
|
+
const e = seqOf(endName);
|
|
446
|
+
if (s !== undefined && e !== undefined && !(s < e)) {
|
|
447
|
+
violations.push(`${phase}: ${endName} (seq ${e}) does not follow ${startName} (seq ${s})`);
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
// PHASE-TO-PHASE production order, not just start<end within a phase. Checking
|
|
451
|
+
// only the pairs would pass a log whose phases are wholly transposed —
|
|
452
|
+
// prompt before newSession, say — while every pair looked fine (GPT review
|
|
453
|
+
// 2026-07-29). The driver's real order is initialize → newSession →
|
|
454
|
+
// enforceModel → prompt, and a failed phase is a PREFIX of it: everything
|
|
455
|
+
// before it ran, nothing after it started.
|
|
456
|
+
const reached = PHASE_PAIRS.filter(([, startName]) => seqOf(startName) !== undefined);
|
|
457
|
+
const reachedNames = reached.map(([phase]) => phase);
|
|
458
|
+
const expectedPrefix = PHASE_PAIRS.slice(0, reached.length).map(([phase]) => phase);
|
|
459
|
+
if (reachedNames.join(">") !== expectedPrefix.join(">")) {
|
|
460
|
+
violations.push(
|
|
461
|
+
`phases reached [${reachedNames.join(", ")}] are not a prefix of the production order [${PHASE_PAIRS.map(([p]) => p).join(", ")}]`,
|
|
462
|
+
);
|
|
463
|
+
}
|
|
464
|
+
for (let i = 1; i < reached.length; i++) {
|
|
465
|
+
const prevEnd = seqOf(reached[i - 1][2]);
|
|
466
|
+
const thisStart = seqOf(reached[i][1]);
|
|
467
|
+
if (prevEnd !== undefined && thisStart !== undefined && !(prevEnd < thisStart)) {
|
|
468
|
+
violations.push(
|
|
469
|
+
`${reached[i][0]} starts (seq ${thisStart}) before ${reached[i - 1][0]} ends (seq ${prevEnd}) — phases are sequential`,
|
|
470
|
+
);
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
// A FAILED phase must be the LAST reached one. Being a prefix of the phase
|
|
474
|
+
// list is not enough on its own: `initialize_end ok=false` followed by a tidy
|
|
475
|
+
// newSession → enforceModel → prompt is a perfect prefix and still describes a
|
|
476
|
+
// driver that kept going after a phase failed, which ours never does.
|
|
477
|
+
const okOf = (endName: string): boolean | undefined => {
|
|
478
|
+
const e = owned.find((x) => x.event === endName);
|
|
479
|
+
return e === undefined ? undefined : e.ok === true;
|
|
480
|
+
};
|
|
481
|
+
const failedIdx = reached.findIndex(([, , endName]) => okOf(endName) === false);
|
|
482
|
+
if (failedIdx !== -1 && failedIdx !== reached.length - 1) {
|
|
483
|
+
violations.push(
|
|
484
|
+
`${reached[failedIdx][0]} failed but ${reached
|
|
485
|
+
.slice(failedIdx + 1)
|
|
486
|
+
.map(([p]) => p)
|
|
487
|
+
.join(", ")} started after it — the driver stops at the first failing phase`,
|
|
488
|
+
);
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
// The reply is logged by the runner AFTER driveProbeTurn returns, so the real
|
|
492
|
+
// writer order is prompt_end → prompt_reply → window close. (The first cut
|
|
493
|
+
// said "inside the prompt phase" and only checked prompt_start < reply, which
|
|
494
|
+
// admitted a reply stamped before the phase had even ended.)
|
|
495
|
+
const replySeq = seqOf(PROBE_EVENTS.promptReply);
|
|
496
|
+
const promptEndSeq = seqOf(PROBE_EVENTS.promptEnd);
|
|
497
|
+
const promptOk = okOf(PROBE_EVENTS.promptEnd);
|
|
498
|
+
if (replySeq !== undefined && promptEndSeq !== undefined && !(promptEndSeq < replySeq)) {
|
|
499
|
+
violations.push(`prompt_reply (seq ${replySeq}) does not follow prompt_end (seq ${promptEndSeq})`);
|
|
500
|
+
}
|
|
501
|
+
if (promptOk === true && countOf(PROBE_EVENTS.promptReply) !== 1) {
|
|
502
|
+
violations.push(
|
|
503
|
+
`prompt ended ok but prompt_reply appears ${countOf(PROBE_EVENTS.promptReply)} times — expected exactly one`,
|
|
504
|
+
);
|
|
505
|
+
}
|
|
506
|
+
if (promptOk === false && countOf(PROBE_EVENTS.promptReply) !== 0) {
|
|
507
|
+
violations.push("prompt FAILED yet a prompt_reply was stamped — there is no reply to record");
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
// The window closes BEFORE the run is declared over — the whole point is that
|
|
511
|
+
// teardown may not happen while the marker could still land — and AFTER every
|
|
512
|
+
// other runner marker, since it is the last thing stamped before teardown.
|
|
513
|
+
const runStartSeq = seqOf(PROBE_EVENTS.runStart);
|
|
514
|
+
const windowSeq = seqOf(PROBE_EVENTS.observationWindowEnd);
|
|
515
|
+
const runEndSeq = seqOf(PROBE_EVENTS.runEnd);
|
|
516
|
+
// run_start opens the run: nothing the runner does may precede it.
|
|
517
|
+
const firstPhaseStartSeq = reached.length > 0 ? seqOf(reached[0][1]) : undefined;
|
|
518
|
+
if (runStartSeq !== undefined && firstPhaseStartSeq !== undefined && !(runStartSeq < firstPhaseStartSeq)) {
|
|
519
|
+
violations.push(
|
|
520
|
+
`${reached[0][0]} starts (seq ${firstPhaseStartSeq}) before run_start (seq ${runStartSeq}) — run_start opens the run`,
|
|
521
|
+
);
|
|
522
|
+
}
|
|
523
|
+
if (windowSeq !== undefined) {
|
|
524
|
+
const laterThanWindow = owned.filter((e) => e.event !== PROBE_EVENTS.runEnd && e.seq > windowSeq);
|
|
525
|
+
if (laterThanWindow.length > 0) {
|
|
526
|
+
violations.push(
|
|
527
|
+
`runner marker(s) ${laterThanWindow.map((e) => e.event).join(", ")} were stamped AFTER the observation window closed`,
|
|
528
|
+
);
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
if (runStartSeq !== undefined && windowSeq !== undefined && !(runStartSeq < windowSeq)) {
|
|
532
|
+
violations.push(
|
|
533
|
+
`${PROBE_EVENTS.observationWindowEnd} (seq ${windowSeq}) does not follow run_start (seq ${runStartSeq})`,
|
|
534
|
+
);
|
|
535
|
+
}
|
|
536
|
+
if (windowSeq !== undefined && runEndSeq !== undefined && !(windowSeq < runEndSeq)) {
|
|
537
|
+
violations.push(
|
|
538
|
+
`run_end (seq ${runEndSeq}) does not follow ${PROBE_EVENTS.observationWindowEnd} (seq ${windowSeq}) — the window must close before the run does`,
|
|
539
|
+
);
|
|
540
|
+
}
|
|
541
|
+
return violations;
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
/** §11-7-c snapshot channel for ONE run. Computed ONLY when the roster armed
|
|
545
|
+
* the channel — shim-shaped events under an unarmed roster are ignored, so
|
|
546
|
+
* found-in-the-wild evidence can never promote past the instrument declaration.
|
|
547
|
+
*
|
|
548
|
+
* Findings split into TWO severities (GPT review 2026-07-29):
|
|
549
|
+
* - STRUCTURAL — the shim's identity is broken (no/dup boot, several pids,
|
|
550
|
+
* boot target ≠ the roster's expected path+sha, armed roster without an
|
|
551
|
+
* expected identity). These invalidate the RUN: the shim intermediates the
|
|
552
|
+
* CLI spawn, so a run without the calibrated shim did not share the pair's
|
|
553
|
+
* launch path and its timing axis is polluted too.
|
|
554
|
+
* - READING — the instrument ran but the exactly-one binding failed. (b)-only.
|
|
555
|
+
*
|
|
556
|
+
* The ordinal binding is the NARROW single-prompt contract, on purpose: this
|
|
557
|
+
* probe drives exactly one serialized prompt per run. A candidate must be
|
|
558
|
+
* RECEIVED after the prompt frame passed — `receivedAtMs` strictly greater
|
|
559
|
+
* than the prompt-forward stamp — not merely appended after it: under stdout
|
|
560
|
+
* backpressure a BOOT-time init can have its downstream callback (and log
|
|
561
|
+
* append) land after the prompt marker, and a seq-only binding would promote
|
|
562
|
+
* that stale set as the turn snapshot (GPT review 2026-07-29). Same-writer
|
|
563
|
+
* `seq` still orders the append as a sanity floor; a same-ms receive tie is
|
|
564
|
+
* fail-closed (not a candidate). Zero candidates (the CLI did not re-emit) or
|
|
565
|
+
* several (reinitialize / set-model re-emission) is a NAMED reading violation,
|
|
566
|
+
* never a pick-first. Multi-prompt generalization is out of §11-7-c scope. */
|
|
567
|
+
function snapshotChannelFacts(
|
|
568
|
+
run: ProbeRunRecord,
|
|
569
|
+
events: ProbeEvent[],
|
|
570
|
+
): Pick<
|
|
571
|
+
RunFacts,
|
|
572
|
+
| "snapshotInstrumented"
|
|
573
|
+
| "shimBootCount"
|
|
574
|
+
| "shimTargetPath"
|
|
575
|
+
| "shimTargetSha256"
|
|
576
|
+
| "snapshotStructuralViolations"
|
|
577
|
+
| "snapshotChannelViolations"
|
|
578
|
+
| "snapshotTools"
|
|
579
|
+
| "snapshotReceivedAtMs"
|
|
580
|
+
| "snapshotForwardedAtMs"
|
|
581
|
+
> {
|
|
582
|
+
if (run.snapshotInstrumented !== true) {
|
|
583
|
+
return {
|
|
584
|
+
snapshotInstrumented: false,
|
|
585
|
+
shimBootCount: 0,
|
|
586
|
+
snapshotStructuralViolations: [],
|
|
587
|
+
snapshotChannelViolations: [],
|
|
588
|
+
};
|
|
589
|
+
}
|
|
590
|
+
const shimNames: ReadonlySet<string> = new Set([
|
|
591
|
+
PROBE_EVENTS.shimBoot,
|
|
592
|
+
PROBE_EVENTS.shimPromptForwarded,
|
|
593
|
+
PROBE_EVENTS.shimInitSnapshot,
|
|
594
|
+
]);
|
|
595
|
+
const shim = events.filter((e) => shimNames.has(e.event));
|
|
596
|
+
const structural: string[] = [];
|
|
597
|
+
const reading: string[] = [];
|
|
598
|
+
const boots = shim.filter((e) => e.event === PROBE_EVENTS.shimBoot);
|
|
599
|
+
const pids = new Set(shim.map((e) => e.pid));
|
|
600
|
+
if (pids.size > 1) {
|
|
601
|
+
structural.push(`shim events came from ${pids.size} pids (${[...pids].join(", ")}) — one run has one shim`);
|
|
602
|
+
}
|
|
603
|
+
if (boots.length === 0) {
|
|
604
|
+
structural.push(
|
|
605
|
+
`no ${PROBE_EVENTS.shimBoot} — the roster armed the snapshot channel but the shim never reported in ` +
|
|
606
|
+
"(an ambient/managed-policy override replacing the shim looks exactly like this; that hijack must be " +
|
|
607
|
+
"a NAMED absence, not an anonymous no-snapshot)",
|
|
608
|
+
);
|
|
609
|
+
} else if (boots.length > 1) {
|
|
610
|
+
structural.push(`${PROBE_EVENTS.shimBoot} appears ${boots.length} times — one shim boots once per run`);
|
|
611
|
+
}
|
|
612
|
+
const boot = boots[0];
|
|
613
|
+
// Condition 5: the roster's expected identity is CONSUMED, not just recorded.
|
|
614
|
+
if (typeof run.cliTargetPath !== "string" || typeof run.cliTargetSha256 !== "string") {
|
|
615
|
+
structural.push(
|
|
616
|
+
"armed roster carries no expected CLI target identity (cliTargetPath/cliTargetSha256) — nothing to verify " +
|
|
617
|
+
"the shim's boot report against; fail-closed",
|
|
618
|
+
);
|
|
619
|
+
} else if (boots.length === 1) {
|
|
620
|
+
if (boot.targetPath !== run.cliTargetPath || boot.targetSha256 !== run.cliTargetSha256) {
|
|
621
|
+
structural.push(
|
|
622
|
+
`shim boot reports target ${boot.targetPath} (sha256 ${boot.targetSha256}) but the roster expected ` +
|
|
623
|
+
`${run.cliTargetPath} (sha256 ${run.cliTargetSha256}) — this run did not execute the pair's stimulus`,
|
|
624
|
+
);
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
const prompts = shim.filter((e) => e.event === PROBE_EVENTS.shimPromptForwarded);
|
|
628
|
+
let bound: ProbeEvent | undefined;
|
|
629
|
+
if (prompts.length !== 1) {
|
|
630
|
+
reading.push(
|
|
631
|
+
`${PROBE_EVENTS.shimPromptForwarded} appears ${prompts.length} times — the single-prompt binding needs exactly one`,
|
|
632
|
+
);
|
|
633
|
+
} else if (prompts[0].ordinal !== 1) {
|
|
634
|
+
reading.push(
|
|
635
|
+
`${PROBE_EVENTS.shimPromptForwarded} carries ordinal ${prompts[0].ordinal} — this probe serializes exactly one prompt (ordinal 1)`,
|
|
636
|
+
);
|
|
637
|
+
} else {
|
|
638
|
+
// Candidates must satisfy BOTH orders: append after the anchor (same
|
|
639
|
+
// writer, exact seq) AND received after the anchor's stamp on the
|
|
640
|
+
// receive-time axis — the backpressure counter-example above is exactly
|
|
641
|
+
// a candidate that passes the first and fails the second.
|
|
642
|
+
const anchor = prompts[0];
|
|
643
|
+
const candidates = shim.filter(
|
|
644
|
+
(e) =>
|
|
645
|
+
e.event === PROBE_EVENTS.shimInitSnapshot &&
|
|
646
|
+
e.pid === anchor.pid &&
|
|
647
|
+
e.seq > anchor.seq &&
|
|
648
|
+
(e.receivedAtMs as number) > anchor.tsMs,
|
|
649
|
+
);
|
|
650
|
+
if (candidates.length !== 1) {
|
|
651
|
+
reading.push(
|
|
652
|
+
`${candidates.length} init snapshots RECEIVED after the prompt frame — the exactly-one binding is violated ` +
|
|
653
|
+
"(0 = the CLI did not re-emit init for this turn, or the only post-append snapshot was received " +
|
|
654
|
+
"before the frame; >1 = reinitialize/set-model re-emission made the binding ambiguous). Never pick-first",
|
|
655
|
+
);
|
|
656
|
+
} else {
|
|
657
|
+
bound = candidates[0];
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
const clean = structural.length === 0 && reading.length === 0;
|
|
661
|
+
return {
|
|
662
|
+
snapshotInstrumented: true,
|
|
663
|
+
shimBootCount: boots.length,
|
|
664
|
+
shimTargetPath: typeof boot?.targetPath === "string" ? boot.targetPath : undefined,
|
|
665
|
+
shimTargetSha256: typeof boot?.targetSha256 === "string" ? boot.targetSha256 : undefined,
|
|
666
|
+
snapshotStructuralViolations: structural,
|
|
667
|
+
snapshotChannelViolations: reading,
|
|
668
|
+
snapshotTools: clean && bound !== undefined ? (bound.tools as string[]) : undefined,
|
|
669
|
+
snapshotReceivedAtMs: clean && bound !== undefined ? (bound.receivedAtMs as number) : undefined,
|
|
670
|
+
// The interval END is the snapshot event's own envelope stamp — the shim
|
|
671
|
+
// appends inside the downstream write callback, so one clock read IS the
|
|
672
|
+
// callback moment (no second SSOT to drift).
|
|
673
|
+
snapshotForwardedAtMs: clean && bound !== undefined ? bound.tsMs : undefined,
|
|
674
|
+
};
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
/** Derive one run's facts from the shared log (events already parsed+sorted). */
|
|
678
|
+
export function deriveRunFacts(run: ProbeRunRecord, allEvents: ProbeEvent[]): RunFacts {
|
|
679
|
+
const events = allEvents.filter((e) => e.runId === run.runId);
|
|
680
|
+
const newSessionStartMs = eventTs(events, PROBE_EVENTS.newSessionStart);
|
|
681
|
+
const newSessionEndEvent = events.find((e) => e.event === PROBE_EVENTS.newSessionEnd);
|
|
682
|
+
const newSessionEndMs = newSessionEndEvent?.tsMs;
|
|
683
|
+
const fixtureCall = events.find(
|
|
684
|
+
(e) =>
|
|
685
|
+
e.event === PROBE_EVENTS.fixtureToolsCallReceived &&
|
|
686
|
+
e.tool === EXPECTED_SOURCE_TOOL &&
|
|
687
|
+
e.probeRunId === run.probeRunId,
|
|
688
|
+
);
|
|
689
|
+
const acpCall = events.find((e) => e.event === PROBE_EVENTS.acpToolCallObserved && e.probeRunId === run.probeRunId);
|
|
690
|
+
const noSuchTool = events.find((e) => e.event === PROBE_EVENTS.acpNoSuchTool);
|
|
691
|
+
const reply = events.find((e) => e.event === PROBE_EVENTS.promptReply);
|
|
692
|
+
const windowEnd = events.find((e) => e.event === PROBE_EVENTS.observationWindowEnd);
|
|
693
|
+
const topology = runnerTopologyViolations(events);
|
|
694
|
+
const wireForwardedMs = eventTs(events, PROBE_EVENTS.toolsListResponseForwarded);
|
|
695
|
+
return {
|
|
696
|
+
runId: run.runId,
|
|
697
|
+
role: run.role,
|
|
698
|
+
delayMs: run.delayMs,
|
|
699
|
+
probeRunId: run.probeRunId,
|
|
700
|
+
initialize: phaseFact(events, PROBE_EVENTS.initializeStart, PROBE_EVENTS.initializeEnd),
|
|
701
|
+
newSession: phaseFact(events, PROBE_EVENTS.newSessionStart, PROBE_EVENTS.newSessionEnd),
|
|
702
|
+
enforceModel: phaseFact(events, PROBE_EVENTS.setModelStart, PROBE_EVENTS.setModelEnd),
|
|
703
|
+
prompt: phaseFact(events, PROBE_EVENTS.promptStart, PROBE_EVENTS.promptEnd),
|
|
704
|
+
newSessionStartMs,
|
|
705
|
+
newSessionEndMs,
|
|
706
|
+
newSessionLatencyMs:
|
|
707
|
+
newSessionStartMs !== undefined && newSessionEndMs !== undefined && newSessionEndEvent?.ok === true
|
|
708
|
+
? newSessionEndMs - newSessionStartMs
|
|
709
|
+
: undefined,
|
|
710
|
+
promptStartMs: eventTs(events, PROBE_EVENTS.promptStart),
|
|
711
|
+
promptEndMs: eventTs(events, PROBE_EVENTS.promptEnd),
|
|
712
|
+
wireForwardedMs,
|
|
713
|
+
fixtureToolCall: fixtureCall !== undefined,
|
|
714
|
+
fixtureToolCallMs: fixtureCall?.tsMs,
|
|
715
|
+
...snapshotChannelFacts(run, events),
|
|
716
|
+
acpProviderToolId: typeof acpCall?.providerToolId === "string" ? acpCall.providerToolId : undefined,
|
|
717
|
+
noSuchToolId: typeof noSuchTool?.toolId === "string" ? noSuchTool.toolId : undefined,
|
|
718
|
+
nonceEchoed: reply?.carriesNonce === true,
|
|
719
|
+
windowReason: windowEnd?.reason as ProbeWindowReason | undefined,
|
|
720
|
+
windowMarkerSeen: typeof windowEnd?.markerSeen === "boolean" ? windowEnd.markerSeen : undefined,
|
|
721
|
+
topologyViolations: topology,
|
|
722
|
+
validityViolations: [...topology, ...windowCoherenceViolations(windowEnd, wireForwardedMs !== undefined)],
|
|
723
|
+
};
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
function controlP0Reason(facts: RunFacts): string | undefined {
|
|
727
|
+
if (!facts.initialize.ok) return "initialize";
|
|
728
|
+
if (!facts.newSession.ok) return "newSession";
|
|
729
|
+
if (!facts.enforceModel.ok) return "enforceModel";
|
|
730
|
+
if (!facts.prompt.ok) return "prompt";
|
|
731
|
+
// visible ∧ callable: the ACP-side tool_call (visibility + id measurement),
|
|
732
|
+
// the fixture-side call marker (callability), and the end-to-end nonce echo.
|
|
733
|
+
if (!facts.fixtureToolCall) return "tool-unavailable";
|
|
734
|
+
if (facts.acpProviderToolId === undefined) return "tool-unavailable";
|
|
735
|
+
if (!facts.nonceEchoed) return "nonce-missing";
|
|
736
|
+
// §11-7-c condition 7 calibration — only when the roster ARMED the channel.
|
|
737
|
+
// The control must prove the instrument works before any intervention absence
|
|
738
|
+
// may be read: shim reported in with the EXPECTED target, channel coherent,
|
|
739
|
+
// and the snapshot CONTAINS the measured id while the fixture call + nonce
|
|
740
|
+
// echo succeeded (both already held above). A control that cannot show all of
|
|
741
|
+
// that is not a baseline for an absence claim — and each failure class keeps
|
|
742
|
+
// its NAME (absent instrument / broken shim identity / broken reading) so a
|
|
743
|
+
// hijacked override never reads like an uninstrumented pair.
|
|
744
|
+
if (facts.snapshotInstrumented) {
|
|
745
|
+
if (facts.shimBootCount === 0) return "snapshot-instrument-absent";
|
|
746
|
+
if (facts.snapshotStructuralViolations.length > 0) return "snapshot-topology";
|
|
747
|
+
if (facts.snapshotChannelViolations.length > 0) return "snapshot-calibration";
|
|
748
|
+
if (facts.snapshotTools === undefined || !facts.snapshotTools.includes(facts.acpProviderToolId)) {
|
|
749
|
+
return "snapshot-calibration";
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
return undefined;
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
/** The three ordering comparisons. STRICT inequalities on both sides: the stamps
|
|
756
|
+
* come from different processes and the shared axis has millisecond resolution,
|
|
757
|
+
* so an equal-ms pair is unordered (probe-event-log sorts such ties by pid for
|
|
758
|
+
* stability only). Reading meaning into a tie would manufacture evidence. */
|
|
759
|
+
function orderingDeltas(facts: RunFacts): OrderingDeltas {
|
|
760
|
+
const w = facts.wireForwardedMs;
|
|
761
|
+
const n = facts.newSessionEndMs;
|
|
762
|
+
const p = facts.promptStartMs;
|
|
763
|
+
return {
|
|
764
|
+
newSessionOrderingKept: w !== undefined && n !== undefined && w < n,
|
|
765
|
+
newSessionRanAhead: w !== undefined && n !== undefined && n < w,
|
|
766
|
+
promptRanAhead: w !== undefined && p !== undefined && p < w,
|
|
767
|
+
newSessionAheadOfWireMs: w !== undefined && n !== undefined ? w - n : undefined,
|
|
768
|
+
promptAheadOfWireMs: w !== undefined && p !== undefined ? w - p : undefined,
|
|
769
|
+
postWireTurnMs: w !== undefined && facts.promptEndMs !== undefined ? facts.promptEndMs - w : undefined,
|
|
770
|
+
};
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
/** Render the deltas so the ARTIFACT carries the observation, not just the
|
|
774
|
+
* classification. §11-7-b's first pair classified D1 correctly and still left a
|
|
775
|
+
* reader unable to see the 1.9 s ran-ahead or the 2.3 s of turn that remained
|
|
776
|
+
* after the wire marker: a diagnosability defect, separate from the verdict. */
|
|
777
|
+
function deltaEvidence(d: OrderingDeltas): string {
|
|
778
|
+
const parts: string[] = [];
|
|
779
|
+
parts.push(
|
|
780
|
+
d.promptAheadOfWireMs === undefined
|
|
781
|
+
? "promptStart↔wire unmeasured"
|
|
782
|
+
: `promptStart ${d.promptAheadOfWireMs > 0 ? `${d.promptAheadOfWireMs}ms BEFORE` : `${-d.promptAheadOfWireMs}ms after`} wire`,
|
|
783
|
+
);
|
|
784
|
+
parts.push(
|
|
785
|
+
d.newSessionAheadOfWireMs === undefined
|
|
786
|
+
? "newSessionEnd↔wire unmeasured"
|
|
787
|
+
: `newSessionEnd ${d.newSessionAheadOfWireMs > 0 ? `${d.newSessionAheadOfWireMs}ms BEFORE` : `${-d.newSessionAheadOfWireMs}ms after`} wire`,
|
|
788
|
+
);
|
|
789
|
+
if (d.postWireTurnMs !== undefined) parts.push(`${d.postWireTurnMs}ms of turn remained after wire`);
|
|
790
|
+
return parts.join("; ");
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
function classifyIntervention(facts: RunFacts, expectedProviderToolId: string | undefined): InterventionReading {
|
|
794
|
+
const deltas = orderingDeltas(facts);
|
|
795
|
+
const base = { runId: facts.runId, delayMs: facts.delayMs, deltas };
|
|
796
|
+
const invalid = (reason: RunInvalidReason, evidence: string): InterventionReading => ({
|
|
797
|
+
...base,
|
|
798
|
+
kind: "INVALIDATED",
|
|
799
|
+
ordering: "censored",
|
|
800
|
+
failure: "inconclusive",
|
|
801
|
+
promotable: false,
|
|
802
|
+
invalidReason: reason,
|
|
803
|
+
evidence,
|
|
804
|
+
});
|
|
805
|
+
|
|
806
|
+
// Validity first: if the runner's own markers do not describe one coherent run
|
|
807
|
+
// — topology OR a window close that contradicts the log — nothing read off
|
|
808
|
+
// them is a measurement of anything. Same list the control is held to.
|
|
809
|
+
if (facts.validityViolations.length > 0) {
|
|
810
|
+
return invalid("topology", `run validity violated — ${facts.validityViolations.join(" | ")}`);
|
|
811
|
+
}
|
|
812
|
+
// §11-7-c: a broken shim IDENTITY on an armed run invalidates the RUN, not
|
|
813
|
+
// just the (b) reading — the shim intermediates the CLI spawn, so a run that
|
|
814
|
+
// did not execute the calibrated shim (missing/dup boot, second pid, wrong
|
|
815
|
+
// target) did not share the pair's launch path, and letting it keep voting on
|
|
816
|
+
// axis (a) would build A/B causal windows out of a different stimulus.
|
|
817
|
+
if (facts.snapshotStructuralViolations.length > 0) {
|
|
818
|
+
return invalid(
|
|
819
|
+
"snapshot-topology",
|
|
820
|
+
`shim identity violated on an armed run — ${facts.snapshotStructuralViolations.join(" | ")}`,
|
|
821
|
+
);
|
|
822
|
+
}
|
|
823
|
+
if (facts.delayMs <= 0 || facts.delayMs >= DELAY_WELL_BELOW_MS) {
|
|
824
|
+
return {
|
|
825
|
+
...base,
|
|
826
|
+
kind: "inconclusive",
|
|
827
|
+
ordering: "unknown",
|
|
828
|
+
failure: "inconclusive",
|
|
829
|
+
promotable: false,
|
|
830
|
+
evidence: `delay ${facts.delayMs}ms is not 0 < D < ${DELAY_WELL_BELOW_MS}ms — D readings would measure our own timeouts`,
|
|
831
|
+
};
|
|
832
|
+
}
|
|
833
|
+
// I0: the injected delay cannot reach initialize (servers ride newSession) —
|
|
834
|
+
// an initialize failure here is environment drift, never a D.
|
|
835
|
+
if (!facts.initialize.ok) {
|
|
836
|
+
return {
|
|
837
|
+
...base,
|
|
838
|
+
kind: "I0",
|
|
839
|
+
ordering: "unknown",
|
|
840
|
+
failure: "inconclusive",
|
|
841
|
+
promotable: false,
|
|
842
|
+
evidence: facts.initialize.error ?? "initialize failed",
|
|
843
|
+
};
|
|
844
|
+
}
|
|
845
|
+
// Phase-qualified fail-loud observations.
|
|
846
|
+
for (const [kind, phase] of [
|
|
847
|
+
["D-newSession", facts.newSession],
|
|
848
|
+
["D-enforceModel", facts.enforceModel],
|
|
849
|
+
["D-prompt", facts.prompt],
|
|
850
|
+
] as const) {
|
|
851
|
+
if (!phase.ok) {
|
|
852
|
+
return {
|
|
853
|
+
...base,
|
|
854
|
+
kind,
|
|
855
|
+
ordering: "unknown",
|
|
856
|
+
failure: "inconclusive",
|
|
857
|
+
promotable: false,
|
|
858
|
+
evidence: phase.error ?? "failed",
|
|
859
|
+
};
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
// CENSORED — the window closed for a reason outside the marker, so a missing
|
|
864
|
+
// marker says nothing about the server. This is the exact misattribution the
|
|
865
|
+
// first LIVE pair produced: D2's child was torn down at turn end while the
|
|
866
|
+
// fixture was still inside its injected delay, and a wire-marker-less run was
|
|
867
|
+
// then filed as an MCP handshake / fixture / config candidate.
|
|
868
|
+
if (!facts.windowMarkerSeen && facts.windowReason !== "deadline") {
|
|
869
|
+
return invalid(
|
|
870
|
+
"observation-window-closed",
|
|
871
|
+
`the observation window closed on '${facts.windowReason}' with the wire marker unseen — right-censored, NOT a handshake/fixture/config candidate (${deltaEvidence(deltas)})`,
|
|
872
|
+
);
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
// ---- axis (a): the server-wait observation ------------------------------
|
|
876
|
+
const ordering: OrderingObservation =
|
|
877
|
+
facts.wireForwardedMs === undefined
|
|
878
|
+
? "unknown"
|
|
879
|
+
: deltas.promptRanAhead
|
|
880
|
+
? "prompt-request-ahead-of-wire"
|
|
881
|
+
: deltas.newSessionOrderingKept
|
|
882
|
+
? "wire-before-newSession-end"
|
|
883
|
+
: deltas.newSessionRanAhead
|
|
884
|
+
? "wire-before-prompt-request"
|
|
885
|
+
: "unknown";
|
|
886
|
+
|
|
887
|
+
// ---- axis (b): the callability reading ----------------------------------
|
|
888
|
+
let failure: FailureReading;
|
|
889
|
+
let promotable = false;
|
|
890
|
+
let evidence: string;
|
|
891
|
+
if (facts.fixtureToolCall) {
|
|
892
|
+
// Callable in the end. Prompt-request-ahead + late success = C (late /
|
|
893
|
+
// dynamic readiness with no client fence). The causal window is the PROMPT
|
|
894
|
+
// comparison — used as a stated PREMISE, not as proof of when the model's
|
|
895
|
+
// tool set was fixed; only §11-7-c's snapshot could show that.
|
|
896
|
+
if (deltas.promptRanAhead) {
|
|
897
|
+
failure = "C";
|
|
898
|
+
evidence = `the prompt request was issued ahead of wire-availability and a later tools/call succeeded — ${deltaEvidence(deltas)}`;
|
|
899
|
+
} else {
|
|
900
|
+
failure = "callable";
|
|
901
|
+
evidence =
|
|
902
|
+
ordering === "wire-before-newSession-end"
|
|
903
|
+
? `tools/list forwarded before newSession end — ${deltaEvidence(deltas)}`
|
|
904
|
+
: `tools/call arrived and the prompt request was NOT issued ahead of the wire — ${deltaEvidence(deltas)}`;
|
|
905
|
+
}
|
|
906
|
+
} else if (facts.wireForwardedMs === undefined) {
|
|
907
|
+
// Reachable ONLY under a sufficient window (the censored branch returned
|
|
908
|
+
// above), so this really is the handshake/fixture/config candidate that
|
|
909
|
+
// the §11-7 ladder describes rather than a self-inflicted absence.
|
|
910
|
+
failure = "candidate-handshake";
|
|
911
|
+
evidence = `no tools_list_response_forwarded although the window stayed open to its deadline — MCP handshake / fixture / config candidate, not server-behavior evidence`;
|
|
912
|
+
} else if (facts.noSuchToolId !== undefined) {
|
|
913
|
+
// The delta-table B is "the delayed run puts the turn AHEAD of
|
|
914
|
+
// wire-availability and yields absence" — the ran-ahead half is part of the
|
|
915
|
+
// verdict, not decoration (GPT review round 2, 2026-07-28), and its axis is
|
|
916
|
+
// promptStart — the point at which WE issued the request. That it is also
|
|
917
|
+
// when the model's tool set was fixed is a PREMISE this probe cannot yet
|
|
918
|
+
// show; §11-7-c's snapshot is what would.
|
|
919
|
+
if (
|
|
920
|
+
deltas.promptRanAhead &&
|
|
921
|
+
expectedProviderToolId !== undefined &&
|
|
922
|
+
facts.noSuchToolId === expectedProviderToolId
|
|
923
|
+
) {
|
|
924
|
+
failure = "B";
|
|
925
|
+
promotable = true;
|
|
926
|
+
evidence = `prompt request issued ahead of wire-availability ∧ no fixture tools/call ∧ runtime No-such-tool for measured id ${expectedProviderToolId} — the delta-B failure mode with direct schema-absence evidence (${deltaEvidence(deltas)})`;
|
|
927
|
+
} else if (expectedProviderToolId !== undefined && facts.noSuchToolId === expectedProviderToolId) {
|
|
928
|
+
failure = "inconclusive";
|
|
929
|
+
evidence = `direct No-such-tool for the measured id but the prompt request was NOT issued ahead of wire-availability (wire first, or a same-ms tie) — not the delayed-window failure mode; unlisted combination stays inconclusive (${deltaEvidence(deltas)})`;
|
|
930
|
+
} else {
|
|
931
|
+
// §11-7 ladder — a No-such-tool naming a bare/alias id proves nothing: the real
|
|
932
|
+
// provider-bound id may have been in the schema all along.
|
|
933
|
+
failure = "inconclusive";
|
|
934
|
+
evidence = `No-such-tool named ${facts.noSuchToolId} but the measured provider-bound id is ${expectedProviderToolId ?? "unmeasured"} — model/alias mismatch, not absence evidence`;
|
|
935
|
+
}
|
|
936
|
+
} else if (facts.snapshotInstrumented) {
|
|
937
|
+
// §11-7-c B-name-snapshot ladder — reachable ONLY in the branch with no
|
|
938
|
+
// fixture call and no runtime No-such-tool. That placement is the
|
|
939
|
+
// anti-conflation rule in structure: a runtime No-such-tool run stays on
|
|
940
|
+
// the runtime ladder above (only the exact measured id reads B there),
|
|
941
|
+
// and a snapshot absence can never upgrade into B because this branch
|
|
942
|
+
// cannot assign it. The promotion floor (condition 7): control
|
|
943
|
+
// calibration already held (P0 gated), snapshot absence of the MEASURED
|
|
944
|
+
// id, promptRanAhead, wire strictly before the snapshot interval, valid
|
|
945
|
+
// topology/window (gated above).
|
|
946
|
+
const wire = facts.wireForwardedMs;
|
|
947
|
+
if (facts.snapshotChannelViolations.length > 0) {
|
|
948
|
+
failure = "inconclusive";
|
|
949
|
+
evidence = `snapshot-channel-violation: ${facts.snapshotChannelViolations.join(" | ")} — the (b) snapshot reading is unavailable for this run; axis (a) and the runner/fixture markers are unaffected (${deltaEvidence(deltas)})`;
|
|
950
|
+
} else if (
|
|
951
|
+
expectedProviderToolId === undefined ||
|
|
952
|
+
facts.snapshotTools === undefined ||
|
|
953
|
+
facts.snapshotReceivedAtMs === undefined ||
|
|
954
|
+
facts.snapshotForwardedAtMs === undefined
|
|
955
|
+
) {
|
|
956
|
+
failure = "inconclusive";
|
|
957
|
+
evidence = `snapshot channel clean but the comparison inputs are incomplete (measured id ${expectedProviderToolId ?? "unmeasured"}) — unlisted combination stays inconclusive (${deltaEvidence(deltas)})`;
|
|
958
|
+
} else if (facts.snapshotTools.includes(expectedProviderToolId)) {
|
|
959
|
+
failure = "inconclusive";
|
|
960
|
+
evidence = `the CLI's turn-init name set CONTAINS the measured id ${expectedProviderToolId} yet no tools/call arrived — model-compliance reading, not absence (${deltaEvidence(deltas)})`;
|
|
961
|
+
} else if (!deltas.promptRanAhead) {
|
|
962
|
+
failure = "inconclusive";
|
|
963
|
+
evidence = `snapshot absence of ${expectedProviderToolId} but the prompt request was NOT issued ahead of wire-availability — not the delayed-window failure mode (${deltaEvidence(deltas)})`;
|
|
964
|
+
} else if (wire < facts.snapshotReceivedAtMs) {
|
|
965
|
+
// The interval rule (condition 6): only wire STRICTLY before the
|
|
966
|
+
// interval's start reads as "snapshot after wire". This orders the
|
|
967
|
+
// REPORT, not the set's assembly — the CLI assembled the name set at
|
|
968
|
+
// some unobservable earlier moment — so the promoted claim stays "the
|
|
969
|
+
// CLI's per-turn account, received after wire-availability, lacked the
|
|
970
|
+
// id", never "the schema was fixed at T".
|
|
971
|
+
failure = "B-name-snapshot";
|
|
972
|
+
promotable = true;
|
|
973
|
+
evidence = `prompt request issued ahead of wire-availability ∧ no fixture tools/call ∧ CLI turn-init name set (received ${facts.snapshotReceivedAtMs - wire}ms after the wire marker) lacks the measured id ${expectedProviderToolId} — the §11-7-c controlled absence reading, WEAKER than runtime-B by design (${deltaEvidence(deltas)})`;
|
|
974
|
+
} else if (wire <= facts.snapshotForwardedAtMs) {
|
|
975
|
+
failure = "inconclusive";
|
|
976
|
+
evidence = `the wire marker lands INSIDE the snapshot interval [received ${facts.snapshotReceivedAtMs}, forwarded ${facts.snapshotForwardedAtMs}] — unordered at this resolution, not promotable (${deltaEvidence(deltas)})`;
|
|
977
|
+
} else {
|
|
978
|
+
failure = "inconclusive";
|
|
979
|
+
evidence = `snapshot-before-wire: the CLI reported its name set before the tools reached the wire — absence there is a different claim than §11-7-c's "after the wire" row (${deltaEvidence(deltas)})`;
|
|
980
|
+
}
|
|
981
|
+
} else {
|
|
982
|
+
// No call marker, no direct runtime error. Model prose alone never promotes
|
|
983
|
+
// — but the ORDERING observation above is unaffected by that, which is the
|
|
984
|
+
// whole reason the two axes are reported separately now.
|
|
985
|
+
failure = "inconclusive";
|
|
986
|
+
evidence = `turn completed without tools/call and without a direct No-such-tool error — model-compliance / insufficient evidence for (b); the (a) comparison is '${ordering}' (${deltaEvidence(deltas)})`;
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
// Composite label, kept for the paired verdict and the A tally.
|
|
990
|
+
const kind: InterventionReadingKind =
|
|
991
|
+
failure === "B"
|
|
992
|
+
? "B"
|
|
993
|
+
: failure === "B-name-snapshot"
|
|
994
|
+
? "B-name-snapshot"
|
|
995
|
+
: failure === "C"
|
|
996
|
+
? "C"
|
|
997
|
+
: failure === "callable" && ordering === "wire-before-newSession-end"
|
|
998
|
+
? "ordering-kept"
|
|
999
|
+
: "inconclusive";
|
|
1000
|
+
|
|
1001
|
+
return { ...base, kind, ordering, failure, promotable, evidence };
|
|
1002
|
+
}
|
|
1003
|
+
|
|
1004
|
+
/** Axis (a), summarized over the pair. Only JUDGEABLE interventions vote: an
|
|
1005
|
+
* INVALIDATED (censored / topology) run has no ordering observation to give, and
|
|
1006
|
+
* a phase failure never reached the comparison. Reported separately from the
|
|
1007
|
+
* verdict so a settled ordering fact survives an unsettled (b) axis. */
|
|
1008
|
+
/** Fatal validity + the two axes, derived once so no return site can disagree. */
|
|
1009
|
+
function runStatus(
|
|
1010
|
+
validity: ProbeClassification["status"]["validity"],
|
|
1011
|
+
ordering: ProbeClassification["ordering"],
|
|
1012
|
+
failureVerdict: ProbeClassification["status"]["failureVerdict"],
|
|
1013
|
+
readings: InterventionReading[] = [],
|
|
1014
|
+
): ProbeClassification["status"] {
|
|
1015
|
+
const orderingMeasurement: ProbeClassification["status"]["orderingMeasurement"] =
|
|
1016
|
+
ordering.summary === "unobserved"
|
|
1017
|
+
? ordering.perRun.some((r) => r.observation === "censored")
|
|
1018
|
+
? "censored"
|
|
1019
|
+
: "unobserved"
|
|
1020
|
+
: "measured";
|
|
1021
|
+
const invalidRuns = readings
|
|
1022
|
+
.filter((r) => r.kind === "INVALIDATED")
|
|
1023
|
+
.map((r) => ({ runId: r.runId, reason: r.invalidReason ?? "topology" }));
|
|
1024
|
+
// A pair that lost a run is not simply "valid" — the label has to carry it.
|
|
1025
|
+
const effective = validity === "valid" && invalidRuns.length > 0 ? "partial" : validity;
|
|
1026
|
+
const judged = effective === "valid" || effective === "partial";
|
|
1027
|
+
return {
|
|
1028
|
+
validity: effective,
|
|
1029
|
+
orderingMeasurement,
|
|
1030
|
+
failureVerdict: judged ? failureVerdict : "not-judged",
|
|
1031
|
+
invalidRuns,
|
|
1032
|
+
};
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
function summarizeOrdering(readings: InterventionReading[]): ProbeClassification["ordering"] {
|
|
1036
|
+
const perRun = readings.map((r) => ({ runId: r.runId, delayMs: r.delayMs, observation: r.ordering }));
|
|
1037
|
+
const voting = readings.filter(
|
|
1038
|
+
(r) =>
|
|
1039
|
+
r.ordering === "wire-before-newSession-end" ||
|
|
1040
|
+
r.ordering === "wire-before-prompt-request" ||
|
|
1041
|
+
r.ordering === "prompt-request-ahead-of-wire",
|
|
1042
|
+
);
|
|
1043
|
+
if (voting.length === 0) return { summary: "unobserved", perRun };
|
|
1044
|
+
if (voting.every((r) => r.ordering === "prompt-request-ahead-of-wire"))
|
|
1045
|
+
return { summary: "prompt-request-ahead-of-wire", perRun };
|
|
1046
|
+
if (voting.every((r) => r.ordering === "wire-before-newSession-end")) {
|
|
1047
|
+
return { summary: "wire-before-newSession-end", perRun };
|
|
1048
|
+
}
|
|
1049
|
+
return { summary: "mixed", perRun };
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
/** Classify one paired probe (control + interventions) into the §11-7 verdict. */
|
|
1053
|
+
export function classifyProbe(runs: ProbeRunRecord[], events: ProbeEvent[]): ProbeClassification {
|
|
1054
|
+
const control = runs.find((r) => r.role === "control");
|
|
1055
|
+
if (!control) throw new Error("probe roster has no control run — a paired probe is control + interventions");
|
|
1056
|
+
const controlFacts = deriveRunFacts(control, events);
|
|
1057
|
+
const noOrdering: ProbeClassification["ordering"] = { summary: "unobserved", perRun: [] };
|
|
1058
|
+
|
|
1059
|
+
// The control's own log must describe one coherent run before anything is read
|
|
1060
|
+
// off it — including the baseline the whole pair is a delta against.
|
|
1061
|
+
if (controlFacts.validityViolations.length > 0) {
|
|
1062
|
+
const why = controlFacts.validityViolations.join(" | ");
|
|
1063
|
+
return {
|
|
1064
|
+
control: { runId: control.runId, pass: false, p0Reason: "topology" },
|
|
1065
|
+
expectedProviderToolId: controlFacts.acpProviderToolId,
|
|
1066
|
+
interventions: [],
|
|
1067
|
+
verdict: "INVALIDATED",
|
|
1068
|
+
status: runStatus("INVALIDATED", noOrdering, "not-judged"),
|
|
1069
|
+
ordering: noOrdering,
|
|
1070
|
+
promotable: false,
|
|
1071
|
+
detail: `INVALIDATED (control validity): ${why} — the baseline log is not a description of one run, so nothing is judged.`,
|
|
1072
|
+
};
|
|
1073
|
+
}
|
|
1074
|
+
const p0Reason = controlP0Reason(controlFacts);
|
|
1075
|
+
if (p0Reason !== undefined) {
|
|
1076
|
+
return {
|
|
1077
|
+
control: { runId: control.runId, pass: false, p0Reason },
|
|
1078
|
+
expectedProviderToolId: controlFacts.acpProviderToolId,
|
|
1079
|
+
interventions: [],
|
|
1080
|
+
verdict: "P0",
|
|
1081
|
+
status: runStatus("P0", noOrdering, "not-judged"),
|
|
1082
|
+
ordering: noOrdering,
|
|
1083
|
+
promotable: false,
|
|
1084
|
+
detail:
|
|
1085
|
+
`P0 / INVALID BASELINE (reason=${p0Reason}): the delay=0 control is not a judgeable baseline — ` +
|
|
1086
|
+
"no intervention is judged; preserve the artifact and classify setup/pin/config/fixture/model-compliance first.",
|
|
1087
|
+
};
|
|
1088
|
+
}
|
|
1089
|
+
const expectedProviderToolId = controlFacts.acpProviderToolId;
|
|
1090
|
+
|
|
1091
|
+
const interventionRuns = runs.filter((r) => r.role === "intervention");
|
|
1092
|
+
const readings = interventionRuns.map((r) => classifyIntervention(deriveRunFacts(r, events), expectedProviderToolId));
|
|
1093
|
+
const ordering = summarizeOrdering(readings);
|
|
1094
|
+
|
|
1095
|
+
const controlResult = { control: { runId: control.runId, pass: true }, expectedProviderToolId, ordering };
|
|
1096
|
+
|
|
1097
|
+
const i0 = readings.find((r) => r.kind === "I0");
|
|
1098
|
+
if (i0) {
|
|
1099
|
+
return {
|
|
1100
|
+
...controlResult,
|
|
1101
|
+
interventions: readings,
|
|
1102
|
+
verdict: "I0",
|
|
1103
|
+
status: runStatus("I0", ordering, "not-judged", readings),
|
|
1104
|
+
promotable: false,
|
|
1105
|
+
detail:
|
|
1106
|
+
`I0 / INVALID RUN (${i0.runId}): control passed but the intervention failed at initialize, which the ` +
|
|
1107
|
+
"injected delay cannot reach — environment drift. Re-run the same pair once; if it recurs, stop and root-cause.",
|
|
1108
|
+
};
|
|
1109
|
+
}
|
|
1110
|
+
|
|
1111
|
+
// Every intervention invalidated → the pair measured nothing. A PARTIAL
|
|
1112
|
+
// invalidation does not stop the rest from being read (the (a) summary above
|
|
1113
|
+
// already excluded the censored runs), but it does block A below, which needs
|
|
1114
|
+
// every delay point it declares.
|
|
1115
|
+
const invalidated = readings.filter((r) => r.kind === "INVALIDATED");
|
|
1116
|
+
if (readings.length > 0 && invalidated.length === readings.length) {
|
|
1117
|
+
return {
|
|
1118
|
+
...controlResult,
|
|
1119
|
+
interventions: readings,
|
|
1120
|
+
verdict: "INVALIDATED",
|
|
1121
|
+
status: runStatus("INVALIDATED", ordering, "not-judged", readings),
|
|
1122
|
+
promotable: false,
|
|
1123
|
+
detail: `INVALIDATED: every intervention is outside the verdict space (${invalidated
|
|
1124
|
+
.map((r) => `${r.runId}=${r.invalidReason}`)
|
|
1125
|
+
.join(", ")}). Preserve the artifact; re-run the pair with the observation window honored.`,
|
|
1126
|
+
};
|
|
1127
|
+
}
|
|
1128
|
+
|
|
1129
|
+
const decisive = readings.find(
|
|
1130
|
+
(r) => r.kind === "B" || r.kind === "B-name-snapshot" || r.kind === "C" || r.kind.startsWith("D-"),
|
|
1131
|
+
);
|
|
1132
|
+
if (decisive) {
|
|
1133
|
+
return {
|
|
1134
|
+
...controlResult,
|
|
1135
|
+
interventions: readings,
|
|
1136
|
+
verdict: decisive.kind as ProbeClassification["verdict"],
|
|
1137
|
+
status: runStatus("valid", ordering, decisive.failure, readings),
|
|
1138
|
+
promotable: decisive.promotable,
|
|
1139
|
+
detail: `${decisive.kind} on ${decisive.runId} (D=${decisive.delayMs}ms): ${decisive.evidence}`,
|
|
1140
|
+
};
|
|
1141
|
+
}
|
|
1142
|
+
|
|
1143
|
+
// A path: every intervention kept ordering; the wait verdict additionally
|
|
1144
|
+
// needs ≥2 distinct nonzero delays whose newSession excess tracks D. An
|
|
1145
|
+
// invalidated run in the pair is disqualifying here — A is a claim about the
|
|
1146
|
+
// whole delay series, and a censored point is a missing point, not a passing one.
|
|
1147
|
+
const orderingKept = readings.filter((r) => r.kind === "ordering-kept");
|
|
1148
|
+
if (orderingKept.length !== readings.length || readings.length === 0) {
|
|
1149
|
+
return {
|
|
1150
|
+
...controlResult,
|
|
1151
|
+
interventions: readings,
|
|
1152
|
+
verdict: "inconclusive",
|
|
1153
|
+
status: runStatus("valid", ordering, "inconclusive", readings),
|
|
1154
|
+
promotable: false,
|
|
1155
|
+
detail:
|
|
1156
|
+
`no decisive (b) reading and not every intervention kept ordering — unlisted combination stays inconclusive for (b). ` +
|
|
1157
|
+
`The (a) ordering axis reads '${ordering.summary}': ${ordering.perRun
|
|
1158
|
+
.map((r) => `${r.runId}(D=${r.delayMs}ms)=${r.observation}`)
|
|
1159
|
+
.join(", ")}.`,
|
|
1160
|
+
};
|
|
1161
|
+
}
|
|
1162
|
+
const controlLatency = controlFacts.newSessionLatencyMs;
|
|
1163
|
+
const withExcess = orderingKept.map((r) => {
|
|
1164
|
+
const facts = deriveRunFacts(interventionRuns.find((run) => run.runId === r.runId) as ProbeRunRecord, events);
|
|
1165
|
+
const excess =
|
|
1166
|
+
controlLatency !== undefined && facts.newSessionLatencyMs !== undefined
|
|
1167
|
+
? facts.newSessionLatencyMs - controlLatency
|
|
1168
|
+
: undefined;
|
|
1169
|
+
return { ...r, newSessionExcessMs: excess };
|
|
1170
|
+
});
|
|
1171
|
+
const distinctDelays = new Set(withExcess.map((r) => r.delayMs));
|
|
1172
|
+
// Band per point: MIN_RATIO·D ≤ excess ≤ D + SLACK. A floor alone would call
|
|
1173
|
+
// a 10 s excess at D=2 s "tracking" — an overshoot is NOT wait-for-delay
|
|
1174
|
+
// evidence, it is some other stall.
|
|
1175
|
+
const inBand = withExcess.every(
|
|
1176
|
+
(r) =>
|
|
1177
|
+
r.newSessionExcessMs !== undefined &&
|
|
1178
|
+
r.newSessionExcessMs >= r.delayMs * A_LATENCY_TRACK_MIN_RATIO &&
|
|
1179
|
+
r.newSessionExcessMs <= r.delayMs + A_LATENCY_SLACK_MS,
|
|
1180
|
+
);
|
|
1181
|
+
// Growth across delays: excess must move WITH D between adjacent delays —
|
|
1182
|
+
// "latency shifts with D" is the whole discriminator (§11-7).
|
|
1183
|
+
const byDelay = [...withExcess].sort((a, b) => a.delayMs - b.delayMs);
|
|
1184
|
+
let growsWithD = true;
|
|
1185
|
+
for (let i = 1; i < byDelay.length; i++) {
|
|
1186
|
+
const dDelay = byDelay[i].delayMs - byDelay[i - 1].delayMs;
|
|
1187
|
+
const dExcess = (byDelay[i].newSessionExcessMs ?? 0) - (byDelay[i - 1].newSessionExcessMs ?? 0);
|
|
1188
|
+
if (dDelay > 0 && dExcess < dDelay * A_GROWTH_MIN_RATIO) growsWithD = false;
|
|
1189
|
+
}
|
|
1190
|
+
if (distinctDelays.size >= 2 && inBand && growsWithD) {
|
|
1191
|
+
return {
|
|
1192
|
+
...controlResult,
|
|
1193
|
+
interventions: withExcess,
|
|
1194
|
+
verdict: "A",
|
|
1195
|
+
status: runStatus("valid", ordering, "callable", readings),
|
|
1196
|
+
promotable: false,
|
|
1197
|
+
detail:
|
|
1198
|
+
"A: ordering kept and newSession latency tracks D across ≥2 delays — wait evidence ON THIS SERVER AND PATH, " +
|
|
1199
|
+
"not a general guarantee.",
|
|
1200
|
+
};
|
|
1201
|
+
}
|
|
1202
|
+
return {
|
|
1203
|
+
...controlResult,
|
|
1204
|
+
interventions: withExcess,
|
|
1205
|
+
verdict: "A-withheld",
|
|
1206
|
+
status: runStatus("valid", ordering, "callable", readings),
|
|
1207
|
+
promotable: false,
|
|
1208
|
+
detail:
|
|
1209
|
+
distinctDelays.size < 2
|
|
1210
|
+
? "ordering kept, but only one nonzero delay — scaling is unobservable, wait verdict WITHHELD (§11-7 requires D1 and D2)"
|
|
1211
|
+
: "ordering kept, but newSession latency does not track D (outside the [0.8·D, D+slack] band, or excess not growing with D) — wait verdict WITHHELD",
|
|
1212
|
+
};
|
|
1213
|
+
}
|