agent-control-plane-core 0.5.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -6
- package/bin/codex-hook.mjs +3 -2
- package/package.json +1 -1
- package/src/adapters/amp.mjs +89 -17
- package/src/adapters/claude.mjs +75 -36
- package/src/adapters/codex.mjs +234 -61
- package/src/adapters/gemini.mjs +196 -78
- package/src/conformance.mjs +614 -1
- package/src/control-plane.mjs +173 -10
- package/src/fixtures/amp.json +8 -3
- package/src/fixtures/codex.json +277 -0
- package/src/fixtures/gemini.json +99 -2
- package/src/index.mjs +5 -1
- package/types/adapters/amp.d.mts +48 -6
- package/types/adapters/claude.d.mts +34 -3
- package/types/adapters/codex.d.mts +63 -11
- package/types/adapters/gemini.d.mts +58 -8
- package/types/conformance.d.mts +16 -1
- package/types/control-plane.d.mts +178 -75
- package/types/index.d.mts +1 -1
package/src/conformance.mjs
CHANGED
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
coverageAllowsVeto,
|
|
10
10
|
isCoverageStatus,
|
|
11
11
|
lookup,
|
|
12
|
+
VERDICT_CONTENT_FIELDS,
|
|
12
13
|
} from "./control-plane.mjs";
|
|
13
14
|
|
|
14
15
|
/**
|
|
@@ -36,6 +37,577 @@ const DRIFT_PROBE_NATIVE = Object.freeze({
|
|
|
36
37
|
version: "9999.0.0",
|
|
37
38
|
});
|
|
38
39
|
|
|
40
|
+
/**
|
|
41
|
+
* The value rule ⑩ puts in each content field it probes, and the shapes each
|
|
42
|
+
* field's contract allows it to take (`mutated_input` must be an object). It is
|
|
43
|
+
* a sentinel rather than realistic content because the rule asks whether the
|
|
44
|
+
* VALUE reached the wire, which is answered by searching the serialized render
|
|
45
|
+
* for it — a check that survives an adapter wrapping, renaming or annotating the
|
|
46
|
+
* channel, where comparing two whole renders does not.
|
|
47
|
+
*/
|
|
48
|
+
const CONTENT_PROBE_SENTINEL = "acpc-conformance-content-probe-b6f1";
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* `meta.native_event` on a synthesized probe event — a name no host uses, so an
|
|
52
|
+
* adapter that selects its output schema from that field takes its
|
|
53
|
+
* unrecognized-event branch rather than a fallback naming a real one.
|
|
54
|
+
*/
|
|
55
|
+
const PROBE_NATIVE_EVENT = "acpc-conformance-synthesized-event";
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* The `ReadonlySet` members an `UNRENDERED_FIELDS` row must carry. A consumer
|
|
59
|
+
* reads a row every way a Set can be read, so a row that answers only `has` is
|
|
60
|
+
* a crash waiting for the first caller that iterates it.
|
|
61
|
+
*/
|
|
62
|
+
const SET_SURFACE = Object.freeze([
|
|
63
|
+
"has",
|
|
64
|
+
"forEach",
|
|
65
|
+
"keys",
|
|
66
|
+
"values",
|
|
67
|
+
"entries",
|
|
68
|
+
Symbol.iterator,
|
|
69
|
+
]);
|
|
70
|
+
/** One field's sentinel — distinct per field, so a probe carrying several can
|
|
71
|
+
* say WHICH of them reached the wire. */
|
|
72
|
+
const sentinelFor = (/** @type {string} */ field) =>
|
|
73
|
+
`${CONTENT_PROBE_SENTINEL}-${field}`;
|
|
74
|
+
|
|
75
|
+
/** JSON-shaped deep equality, for comparing one render against another. */
|
|
76
|
+
const same = (/** @type {unknown} */ a, /** @type {unknown} */ b) =>
|
|
77
|
+
JSON.stringify(a) === JSON.stringify(b);
|
|
78
|
+
|
|
79
|
+
const isBranch = (/** @type {unknown} */ v) =>
|
|
80
|
+
v !== null && typeof v === "object";
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Every path at which RENDERS do not all agree, outermost first.
|
|
84
|
+
*
|
|
85
|
+
* A path that varies with the value is a position the field could be reaching;
|
|
86
|
+
* one that does not vary cannot be, which is what drops a warning keyed on the
|
|
87
|
+
* field's presence out of the comparison.
|
|
88
|
+
*
|
|
89
|
+
* ANCESTORS are included, not only the innermost disagreements: a structured
|
|
90
|
+
* value sits at the path whose CHILDREN differ, so returning the children alone
|
|
91
|
+
* would never offer the position holding the whole value.
|
|
92
|
+
*/
|
|
93
|
+
/**
|
|
94
|
+
* @param {unknown[]} renders
|
|
95
|
+
* @param {string[]} path
|
|
96
|
+
* @returns {string[][]}
|
|
97
|
+
*/
|
|
98
|
+
const divergences = (renders, path = []) => {
|
|
99
|
+
const [first, ...rest] = renders;
|
|
100
|
+
if (rest.every((r) => same(r, first))) return [];
|
|
101
|
+
const alike =
|
|
102
|
+
renders.every(isBranch) &&
|
|
103
|
+
renders.every((r) => Array.isArray(r) === Array.isArray(first));
|
|
104
|
+
if (!alike) return [path];
|
|
105
|
+
const keys = [
|
|
106
|
+
...new Set(renders.flatMap((r) => Object.keys(/** @type {any} */ (r)))),
|
|
107
|
+
];
|
|
108
|
+
return [
|
|
109
|
+
path,
|
|
110
|
+
...keys.flatMap((key) =>
|
|
111
|
+
divergences(
|
|
112
|
+
renders.map((r) => /** @type {any} */ (r)[key]),
|
|
113
|
+
[...path, key],
|
|
114
|
+
),
|
|
115
|
+
),
|
|
116
|
+
];
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
/** What RENDER holds at PATH. */
|
|
120
|
+
const at = (/** @type {unknown} */ render, /** @type {string[]} */ path) =>
|
|
121
|
+
path.reduce((node, key) => /** @type {any} */ (node)?.[key], render);
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* The values each content field is probed with — every shape its contract
|
|
125
|
+
* allows, and at least TWO per field, because the rule reads a field's channel
|
|
126
|
+
* off the positions that VARY as the value changes.
|
|
127
|
+
*
|
|
128
|
+
* A sentinel rides in the VALUE, never a key: an adapter forwarding an input's
|
|
129
|
+
* key set while dropping the values would serialize a sentinel KEY and pass,
|
|
130
|
+
* which is the self-consistent broken render this rule exists to catch
|
|
131
|
+
* independently of the golden fixtures. The values that hold no sentinel at all
|
|
132
|
+
* are carried by the same structural comparison.
|
|
133
|
+
*/
|
|
134
|
+
// Fields the contract carries VERBATIM, so a native path must hold the value
|
|
135
|
+
// itself. `additional_context` is not one: it is prose for the model, and a
|
|
136
|
+
// host with one message channel may legitimately COMPOSE it with its own text —
|
|
137
|
+
// gemini joins it with the post-tool warning. That field only has to reach the
|
|
138
|
+
// wire intact somewhere inside a varying path.
|
|
139
|
+
const CARRIED_VERBATIM = Object.freeze(["mutated_input", "mutated_output"]);
|
|
140
|
+
|
|
141
|
+
const CONTENT_PROBE_VALUES = Object.freeze({
|
|
142
|
+
// A `Verdict`'s `mutated_input` is any `Record<string, unknown>`, not a Bash
|
|
143
|
+
// call: a Read/Edit/Write replacement carries `file_path` and no `command`,
|
|
144
|
+
// and an EMPTY record is a legitimate replacement too. An adapter that
|
|
145
|
+
// forwards only what looks like a shell invocation drops the rest.
|
|
146
|
+
mutated_input: Object.freeze([
|
|
147
|
+
Object.freeze({ command: sentinelFor("mutated_input") }),
|
|
148
|
+
Object.freeze({ command: `${sentinelFor("mutated_input")}-2`, argv: [] }),
|
|
149
|
+
Object.freeze({ file_path: sentinelFor("mutated_input") }),
|
|
150
|
+
Object.freeze({}),
|
|
151
|
+
// FALSY members. A render that keeps only what `Boolean(value)` admits
|
|
152
|
+
// turns this into `{}`, dropping `enabled: false` and `retries: 0` from a
|
|
153
|
+
// replacement the caller meant.
|
|
154
|
+
Object.freeze({
|
|
155
|
+
command: sentinelFor("mutated_input"),
|
|
156
|
+
enabled: false,
|
|
157
|
+
retries: 0,
|
|
158
|
+
note: "",
|
|
159
|
+
parent: null,
|
|
160
|
+
}),
|
|
161
|
+
// POPULATED NESTED members. Every other probe's members are primitives or
|
|
162
|
+
// empty, so a render that replaces a nested value with a placeholder — or
|
|
163
|
+
// keeps only the members it can flatten — carries them all intact.
|
|
164
|
+
Object.freeze({
|
|
165
|
+
command: sentinelFor("mutated_input"),
|
|
166
|
+
options: Object.freeze({
|
|
167
|
+
cwd: "/tmp",
|
|
168
|
+
retries: 2,
|
|
169
|
+
env: Object.freeze({ PATH: "/usr/bin" }),
|
|
170
|
+
}),
|
|
171
|
+
argv: Object.freeze(["--flag", `${sentinelFor("mutated_input")}-arg`]),
|
|
172
|
+
steps: Object.freeze([Object.freeze({ id: 1 })]),
|
|
173
|
+
}),
|
|
174
|
+
]),
|
|
175
|
+
// `mutated_output` is a tool's output verbatim, so every JSON shape a tool
|
|
176
|
+
// can return is one an adapter must forward. A render that type-switches
|
|
177
|
+
// passes the shapes it happens to handle and drops the rest, and the EMPTY
|
|
178
|
+
// ones are what an `if (verdict.mutated_output)` render loses — an empty
|
|
179
|
+
// output being how a redaction removes all content.
|
|
180
|
+
mutated_output: Object.freeze([
|
|
181
|
+
sentinelFor("mutated_output"),
|
|
182
|
+
Object.freeze({ content: sentinelFor("mutated_output") }),
|
|
183
|
+
Object.freeze([Object.freeze({ text: sentinelFor("mutated_output") })]),
|
|
184
|
+
null,
|
|
185
|
+
true,
|
|
186
|
+
false,
|
|
187
|
+
"",
|
|
188
|
+
0,
|
|
189
|
+
Object.freeze({}),
|
|
190
|
+
Object.freeze([]),
|
|
191
|
+
908172635441,
|
|
192
|
+
]),
|
|
193
|
+
// A context string is prose, so it carries newlines and punctuation a render
|
|
194
|
+
// may strip, escape or truncate on its way to a native field. An EMPTY string
|
|
195
|
+
// is deliberately absent: this field is checked by containment, and every
|
|
196
|
+
// string contains the empty one, so the probe could not tell a dropped empty
|
|
197
|
+
// context from a carried one.
|
|
198
|
+
additional_context: Object.freeze([
|
|
199
|
+
sentinelFor("additional_context"),
|
|
200
|
+
`${sentinelFor("additional_context")}-2`,
|
|
201
|
+
`${sentinelFor("additional_context")}\nsecond line\twith <&>"' punctuation`,
|
|
202
|
+
]),
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
// A field added to VERDICT_CONTENT_FIELDS with no probe value would be dropped
|
|
206
|
+
// by normalizeVerdict and then reported as "reaches no native channel" against
|
|
207
|
+
// every adapter, naming the adapter for the harness's own omission.
|
|
208
|
+
if (
|
|
209
|
+
VERDICT_CONTENT_FIELDS.some(
|
|
210
|
+
(field) => (lookup(CONTENT_PROBE_VALUES, field) ?? []).length < 2,
|
|
211
|
+
)
|
|
212
|
+
)
|
|
213
|
+
throw new Error(
|
|
214
|
+
`conformance: CONTENT_PROBE_VALUES needs two or more values for each of ${VERDICT_CONTENT_FIELDS.join(", ")}`,
|
|
215
|
+
);
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* Rule ⑩'s first half: the adapter declares a row for EVERY {@link EventKind}.
|
|
219
|
+
*
|
|
220
|
+
* Iterated over the SSOT rather than over the fixtures, because a kind an
|
|
221
|
+
* adapter's `parse` never emits is exactly the one whose row goes missing — and
|
|
222
|
+
* an absent row reads as "every content field reaches a channel here", which is
|
|
223
|
+
* the reverse of the truth on a transport that carries none. Gemini's
|
|
224
|
+
* `session_start` and Amp's `unknown` are the live cases: no fixture produces
|
|
225
|
+
* either, so a fixture-driven check certifies neither.
|
|
226
|
+
* @param {import("./control-plane.mjs").Adapter} adapter
|
|
227
|
+
* @param {any} assert
|
|
228
|
+
*/
|
|
229
|
+
function assertEveryKindHasARow(adapter, assert) {
|
|
230
|
+
// An adapter written against the earlier contract has no map at all, and
|
|
231
|
+
// `lookup` would throw a bare TypeError on it. Name the missing member and its
|
|
232
|
+
// one-line migration instead — a third-party adapter's author is who reads
|
|
233
|
+
// this, and a stack trace into the harness tells them nothing.
|
|
234
|
+
assert.ok(
|
|
235
|
+
adapter.UNRENDERED_FIELDS !== null &&
|
|
236
|
+
typeof adapter.UNRENDERED_FIELDS === "object",
|
|
237
|
+
`${adapter.AGENT}: adapter declares no UNRENDERED_FIELDS — every adapter needs one row per EventKind naming the VERDICT_CONTENT_FIELDS this host has no channel for (UNRENDERED_ON_UNKNOWN for a kind that carries none, readonlySet([]) for one that carries all three)`,
|
|
238
|
+
);
|
|
239
|
+
// The MAP, not only its rows: a consumer that can swap the row for `pre_tool`
|
|
240
|
+
// rewrites the declaration after conformance certifies it, which is the same
|
|
241
|
+
// corruption immutable rows exist to prevent, one level out.
|
|
242
|
+
assert.ok(
|
|
243
|
+
Object.isFrozen(adapter.UNRENDERED_FIELDS),
|
|
244
|
+
`${adapter.AGENT}: UNRENDERED_FIELDS is not frozen — a consumer can replace a whole row after conformance certifies it; wrap the map in Object.freeze`,
|
|
245
|
+
);
|
|
246
|
+
for (const kind of Object.values(EventKind)) {
|
|
247
|
+
const row = lookup(adapter.UNRENDERED_FIELDS, kind);
|
|
248
|
+
assert.ok(
|
|
249
|
+
row !== undefined,
|
|
250
|
+
`${adapter.AGENT}: UNRENDERED_FIELDS has no row for '${kind}' — every EventKind needs one, so an omission cannot read as full support`,
|
|
251
|
+
);
|
|
252
|
+
// A row is only a declaration if it answers like the ReadonlySet the
|
|
253
|
+
// contract promises. `null` reads as "declares nothing" at every `has` call,
|
|
254
|
+
// so the adapter claims every channel; `{ has: () => true }` passes a
|
|
255
|
+
// has-only check while a consumer that iterates it, or reads `size`, fails
|
|
256
|
+
// at runtime.
|
|
257
|
+
assert.ok(
|
|
258
|
+
SET_SURFACE.every(
|
|
259
|
+
(member) => typeof (/** @type {any} */ (row)?.[member]) === "function",
|
|
260
|
+
) && typeof (/** @type {any} */ (row)?.size) === "number",
|
|
261
|
+
`${adapter.AGENT}: UNRENDERED_FIELDS row for '${kind}' is not a ReadonlySet — use readonlySet([...]), which carries ${SET_SURFACE.length} members plus size`,
|
|
262
|
+
);
|
|
263
|
+
// The members are not the contract, the SEMANTICS are: a Map carries every
|
|
264
|
+
// one of them and hands a consumer `[key, value]` pairs. Iterating has to
|
|
265
|
+
// yield the declared field names themselves, agree with `has`, and count
|
|
266
|
+
// `size` — and each name has to be a field a Verdict actually carries, or a
|
|
267
|
+
// typo declares nothing while reading as a declaration.
|
|
268
|
+
assertRowReads(adapter.AGENT, kind, row, assert);
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/**
|
|
273
|
+
* One `UNRENDERED_FIELDS` row, read every way a `ReadonlySet` can be read.
|
|
274
|
+
*
|
|
275
|
+
* The members are not the contract, the semantics are: a `Map` carries all of
|
|
276
|
+
* them and hands a consumer `[key, value]` pairs, and a row whose `forEach`
|
|
277
|
+
* throws passes any check that only asks whether it is callable. Every reader
|
|
278
|
+
* has to answer the same field names, each name has to be a field a `Verdict`
|
|
279
|
+
* carries, and `size` has to count them — a typo like `mutatedOutput` otherwise
|
|
280
|
+
* declares nothing while reading as a declaration.
|
|
281
|
+
* @param {string} agent
|
|
282
|
+
* @param {string} kind
|
|
283
|
+
* @param {any} row
|
|
284
|
+
* @param {any} assert
|
|
285
|
+
*/
|
|
286
|
+
function assertRowReads(agent, kind, row, assert) {
|
|
287
|
+
const where = `${agent}: UNRENDERED_FIELDS row for '${kind}'`;
|
|
288
|
+
const declared = [...row];
|
|
289
|
+
assert.equal(
|
|
290
|
+
new Set(declared).size,
|
|
291
|
+
declared.length,
|
|
292
|
+
`${where} iterates a value twice — a set has each of its members once`,
|
|
293
|
+
);
|
|
294
|
+
// A row must be immutable, not merely read-only-looking. `Object.freeze` does
|
|
295
|
+
// NOT stop `new Set([...]).clear()`, so a certified row could be emptied
|
|
296
|
+
// afterwards and would then report channels `render` still drops — the stale
|
|
297
|
+
// declaration this whole rule exists to prevent. `readonlySet` exposes no
|
|
298
|
+
// mutator at all, which is what makes every call below throw.
|
|
299
|
+
//
|
|
300
|
+
// Throwing is not enough on its own: a mutator that inserts and THEN throws
|
|
301
|
+
// has already changed the row, and a caller who catches it observes the new
|
|
302
|
+
// contents. So the members are compared across every attempt.
|
|
303
|
+
for (const [mutator, argument] of [
|
|
304
|
+
["add", "mutated_input"],
|
|
305
|
+
["delete", declared[0]],
|
|
306
|
+
["clear", undefined],
|
|
307
|
+
]) {
|
|
308
|
+
assert.throws(
|
|
309
|
+
() => /** @type {any} */ (row)[mutator](argument),
|
|
310
|
+
`${where} exposes a working '${mutator}' — a consumer can rewrite the declaration after conformance certifies it; build the row with readonlySet`,
|
|
311
|
+
);
|
|
312
|
+
assert.deepEqual(
|
|
313
|
+
[...row],
|
|
314
|
+
declared,
|
|
315
|
+
`${where}: '${mutator}' changed the row before it threw — a caller that catches it reads the new declaration`,
|
|
316
|
+
);
|
|
317
|
+
}
|
|
318
|
+
assert.deepEqual(
|
|
319
|
+
declared.filter((field) => VERDICT_CONTENT_FIELDS.includes(field)),
|
|
320
|
+
declared,
|
|
321
|
+
`${where} does not iterate as a set of ${VERDICT_CONTENT_FIELDS.join("/")} — got ${JSON.stringify(declared)}`,
|
|
322
|
+
);
|
|
323
|
+
// `has` has to answer for EVERY content field, not only the iterated ones:
|
|
324
|
+
// a row that iterates one field while `has` says yes to all three tells rule
|
|
325
|
+
// ⑩ one declaration and a consumer that iterates it another.
|
|
326
|
+
assert.deepEqual(
|
|
327
|
+
VERDICT_CONTENT_FIELDS.filter((field) => row.has(field)),
|
|
328
|
+
VERDICT_CONTENT_FIELDS.filter((field) => declared.includes(field)),
|
|
329
|
+
`${where}: has() and iteration disagree about ${VERDICT_CONTENT_FIELDS.join("/")}`,
|
|
330
|
+
);
|
|
331
|
+
assert.equal(declared.length, row.size, `${where} miscounts its own size`);
|
|
332
|
+
assert.deepEqual([...row.keys()], declared, `${where}: keys() disagrees`);
|
|
333
|
+
assert.deepEqual([...row.values()], declared, `${where}: values() disagrees`);
|
|
334
|
+
assert.deepEqual(
|
|
335
|
+
[...row.entries()],
|
|
336
|
+
declared.map((field) => [field, field]),
|
|
337
|
+
`${where}: entries() is not a set's [value, value]`,
|
|
338
|
+
);
|
|
339
|
+
/** @type {string[][]} */
|
|
340
|
+
const walked = [];
|
|
341
|
+
/** @type {unknown[]} */
|
|
342
|
+
const handed = [];
|
|
343
|
+
row.forEach(
|
|
344
|
+
(
|
|
345
|
+
/** @type {string} */ value,
|
|
346
|
+
/** @type {string} */ key,
|
|
347
|
+
/** @type {any} */ set,
|
|
348
|
+
) => {
|
|
349
|
+
walked.push([value, key]);
|
|
350
|
+
handed.push(set);
|
|
351
|
+
},
|
|
352
|
+
);
|
|
353
|
+
assert.deepEqual(
|
|
354
|
+
walked,
|
|
355
|
+
declared.map((field) => [field, field]),
|
|
356
|
+
`${where}: forEach does not pass (value, value)`,
|
|
357
|
+
);
|
|
358
|
+
// The third argument is the row ITSELF, by identity. A row that hands the
|
|
359
|
+
// callback its private inner collection gives a consumer a handle the frozen
|
|
360
|
+
// facade exists to withhold, and a look-alike carrying the same readers
|
|
361
|
+
// compares equal while hiding its own mutable state — so `===`, not equality.
|
|
362
|
+
assert.equal(
|
|
363
|
+
handed.length,
|
|
364
|
+
declared.length,
|
|
365
|
+
`${where}: forEach called its callback ${handed.length} times for ${declared.length} members`,
|
|
366
|
+
);
|
|
367
|
+
assert.ok(
|
|
368
|
+
handed.every((set) => set === row),
|
|
369
|
+
`${where}: forEach does not pass the row itself as its third argument`,
|
|
370
|
+
);
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/**
|
|
374
|
+
* Rule ⑩'s other half: every declared row agrees with what `render` emits.
|
|
375
|
+
*
|
|
376
|
+
* Existence is not agreement. A row for a kind no fixture produces could claim
|
|
377
|
+
* a channel the render drops, and the per-fixture probes would never ask.
|
|
378
|
+
* @param {import("./control-plane.mjs").Adapter} adapter
|
|
379
|
+
* @param {Record<string, import("./control-plane.mjs").ToolCallEvent>} byKind one parsed event per kind a fixture produced
|
|
380
|
+
* @param {any} assert
|
|
381
|
+
* @param {Set<string>} seen fields observed to REACH a wire
|
|
382
|
+
*/
|
|
383
|
+
function assertEveryKindRenders(adapter, byKind, assert, seen) {
|
|
384
|
+
for (const kind of Object.values(EventKind))
|
|
385
|
+
assertContentChannels(
|
|
386
|
+
adapter,
|
|
387
|
+
lookup(byKind, kind) ?? coherentEvent(adapter, byKind, kind),
|
|
388
|
+
`every-kind probe '${kind}'`,
|
|
389
|
+
assert,
|
|
390
|
+
seen,
|
|
391
|
+
);
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
/**
|
|
395
|
+
* A representative event OF `kind`, preferring one a fixture actually produced.
|
|
396
|
+
*
|
|
397
|
+
* Re-labelling a parsed event is not enough: a `pre_tool` event relabelled
|
|
398
|
+
* `prompt_submit` still carries a `tool` the contract says is null there, and
|
|
399
|
+
* still names `PreToolUse` on `meta.native_event` — so an adapter that picks its
|
|
400
|
+
* native schema from that field would be probed about the wrong channel. The
|
|
401
|
+
* synthesized event drops what the kind cannot carry, and names
|
|
402
|
+
* {@link PROBE_NATIVE_EVENT} rather than a real native event it is not.
|
|
403
|
+
*
|
|
404
|
+
* `NATIVE_EVENT_FOR` is the adapter's own answer for that field, so a render
|
|
405
|
+
* that selects its output schema from the native name takes the branch this
|
|
406
|
+
* kind really takes. {@link PROBE_NATIVE_EVENT} is the fallback for a kind the
|
|
407
|
+
* adapter names no native event for — `unknown` by definition, and any kind its
|
|
408
|
+
* transport does not carry. The marker is a value rather than an absence
|
|
409
|
+
* because `parse` always stamps this field: reading `undefined` sends a
|
|
410
|
+
* renderer down a fallback no real event reaches, and Codex's names
|
|
411
|
+
* `PreToolUse`.
|
|
412
|
+
* @param {import("./control-plane.mjs").Adapter} adapter
|
|
413
|
+
* @param {Record<string, import("./control-plane.mjs").ToolCallEvent>} byKind
|
|
414
|
+
* @param {string} kind
|
|
415
|
+
* @returns {import("./control-plane.mjs").ToolCallEvent}
|
|
416
|
+
*/
|
|
417
|
+
function coherentEvent(adapter, byKind, kind) {
|
|
418
|
+
const carriesTool =
|
|
419
|
+
kind === EventKind.PRE_TOOL || kind === EventKind.POST_TOOL;
|
|
420
|
+
// A tool kind seeded from a prompt/session event inherits `tool: null` and
|
|
421
|
+
// that event's input, which is an event no `parse` produces. Prefer a
|
|
422
|
+
// tool-bearing one; the first parsed event is the fallback.
|
|
423
|
+
const preferred = carriesTool
|
|
424
|
+
? (lookup(byKind, EventKind.PRE_TOOL) ??
|
|
425
|
+
lookup(byKind, EventKind.POST_TOOL))
|
|
426
|
+
: undefined;
|
|
427
|
+
const seed = preferred ?? Object.values(byKind)[0];
|
|
428
|
+
// `native_tool` names the tool this event is about, so a kind that carries no
|
|
429
|
+
// tool must not carry it either — an event `parse` never produces.
|
|
430
|
+
const { native_tool, ...seedMeta } = seed.meta ?? {};
|
|
431
|
+
const meta = {
|
|
432
|
+
...seedMeta,
|
|
433
|
+
...(carriesTool && native_tool !== undefined ? { native_tool } : {}),
|
|
434
|
+
native_event:
|
|
435
|
+
lookup(adapter.NATIVE_EVENT_FOR ?? {}, kind) ?? PROBE_NATIVE_EVENT,
|
|
436
|
+
};
|
|
437
|
+
const { response, ...rest } = seed;
|
|
438
|
+
void response;
|
|
439
|
+
return {
|
|
440
|
+
...rest,
|
|
441
|
+
event: /** @type {any} */ (kind),
|
|
442
|
+
// A synthesized event is one no fixture produced, so nothing here says this
|
|
443
|
+
// adapter gates this kind — Amp seeds every other kind from its vetoable
|
|
444
|
+
// `pre_tool` fixture and can gate none of them. Claiming enforcement would
|
|
445
|
+
// report a block the host never performs, which `makeEvent` refuses outright
|
|
446
|
+
// for UNKNOWN. Rule ⑧ probes the non-vetoable deny path on its own.
|
|
447
|
+
this_call_vetoable: false,
|
|
448
|
+
tool: carriesTool ? seed.tool : null,
|
|
449
|
+
input: carriesTool ? seed.input : {},
|
|
450
|
+
...(kind === EventKind.POST_TOOL && "response" in seed
|
|
451
|
+
? { response: seed.response }
|
|
452
|
+
: {}),
|
|
453
|
+
meta: /** @type {any} */ (meta),
|
|
454
|
+
};
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
/**
|
|
458
|
+
* Rule ⑩, for one parsed event: every {@link VERDICT_CONTENT_FIELDS} entry
|
|
459
|
+
* either reaches this host's wire or is DECLARED unreachable in the adapter's
|
|
460
|
+
* `UNRENDERED_FIELDS`, and the render agrees with the declaration both ways.
|
|
461
|
+
*
|
|
462
|
+
* The gap this closes is silent loss. Three adapters carry a `Verdict` field no
|
|
463
|
+
* native channel accepts — Gemini and Codex have no output-rewrite field, Amp
|
|
464
|
+
* has no stdout body at all — and each simply ignored it, so a redaction verdict
|
|
465
|
+
* rendered a bare allow and the unredacted output reached the model with nothing
|
|
466
|
+
* anywhere saying so. Declaring the drop is what makes it reviewable; checking
|
|
467
|
+
* the declaration against the render is what stops it going stale, in the
|
|
468
|
+
* direction that matters most (a channel that quietly stops carrying a field
|
|
469
|
+
* reads as a working redaction).
|
|
470
|
+
*
|
|
471
|
+
* Probed on a synthesized verdict rather than a fixture, for rule ⑧'s reason: an
|
|
472
|
+
* adapter with no channel for a field has no fixture to write for it, so a
|
|
473
|
+
* fixture-driven check would sit vacuous on exactly the adapters the rule is
|
|
474
|
+
* for.
|
|
475
|
+
*
|
|
476
|
+
* SCOPE — the probe carries an abstaining `allow`, so the declaration this rule
|
|
477
|
+
* checks is about the ALLOW path. An enforced deny may legitimately drop more:
|
|
478
|
+
* Gemini's exit-2 System Block returns no stdout body at all, so a deny there
|
|
479
|
+
* carries no context whatever `UNRENDERED_FIELDS` says. Widening the map to
|
|
480
|
+
* (kind, decision) would state that, and is not worth its weight while the only
|
|
481
|
+
* field a blocked call can still want is `additional_context`.
|
|
482
|
+
*
|
|
483
|
+
* Reaching the wire is checked as SERIALIZATION, so the rule proves a field is
|
|
484
|
+
* carried, not that the key it is carried in is one the host reads. Nothing
|
|
485
|
+
* machine-readable describes a host's schema, so that half stays with the
|
|
486
|
+
* declaration's prose and its review.
|
|
487
|
+
* @param {import("./control-plane.mjs").Adapter} adapter
|
|
488
|
+
* @param {import("./control-plane.mjs").ToolCallEvent} event
|
|
489
|
+
* @param {string} caseName
|
|
490
|
+
* @param {any} assert
|
|
491
|
+
* @param {Set<string>} seen fields observed to REACH a wire, for the non-vacuity check
|
|
492
|
+
*/
|
|
493
|
+
function assertContentChannels(adapter, event, caseName, assert, seen) {
|
|
494
|
+
const declared = lookup(adapter.UNRENDERED_FIELDS, event.event);
|
|
495
|
+
|
|
496
|
+
// Every value of a field is rendered and the renders compared to EACH OTHER.
|
|
497
|
+
// A path that varies with the value is a position the field reaches, and ONE
|
|
498
|
+
// such path must hold every value VERBATIM — same type, same structure, not
|
|
499
|
+
// merely a marker somewhere inside it.
|
|
500
|
+
//
|
|
501
|
+
// ONE path, not all of them: a native schema may annotate the output it
|
|
502
|
+
// carries with a type discriminator or a length, and that metadata varies
|
|
503
|
+
// with the value while equalling none of them. Requiring every varying path
|
|
504
|
+
// to hold the value would reject the adapter for describing what it carried.
|
|
505
|
+
//
|
|
506
|
+
// Comparing values to each other is what drops behaviour keyed on the field's
|
|
507
|
+
// PRESENCE — gemini's post-tool warning — out of the comparison: it is
|
|
508
|
+
// identical at every value, so no path varies with it. A diff against the
|
|
509
|
+
// field-absent render would score that declared drop as "carried".
|
|
510
|
+
//
|
|
511
|
+
// HELD carries the other content fields, constant across the renders, so an
|
|
512
|
+
// adapter that loses this one only when another shares the channel has
|
|
513
|
+
// nowhere to hide.
|
|
514
|
+
/** @param {string} field @param {Record<string, unknown>} held @param {string} shape */
|
|
515
|
+
const carries = (field, held, shape) => {
|
|
516
|
+
const values = lookup(CONTENT_PROBE_VALUES, field) ?? [];
|
|
517
|
+
if (values.length < 2) return;
|
|
518
|
+
const renders = values.map((value) =>
|
|
519
|
+
adapter.render(
|
|
520
|
+
{ decision: Decision.ALLOW, ...held, [field]: value },
|
|
521
|
+
event,
|
|
522
|
+
),
|
|
523
|
+
);
|
|
524
|
+
const paths = divergences(renders);
|
|
525
|
+
const where = `'${caseName}' (${adapter.AGENT}), ${field} ${shape}`;
|
|
526
|
+
if (declared?.has(field)) {
|
|
527
|
+
assert.deepEqual(
|
|
528
|
+
paths,
|
|
529
|
+
[],
|
|
530
|
+
`${where}: UNRENDERED_FIELDS declares ${field} has no channel on ${event.event}, but the render changes with its value`,
|
|
531
|
+
);
|
|
532
|
+
return;
|
|
533
|
+
}
|
|
534
|
+
assert.ok(
|
|
535
|
+
paths.length,
|
|
536
|
+
`${where}: ${field} reaches no native channel on ${event.event}, though the row declares one — the verdict is silently lost`,
|
|
537
|
+
);
|
|
538
|
+
const verbatim = CARRIED_VERBATIM.includes(field);
|
|
539
|
+
/** Does PATH hold the value the verdict set for probe INDEX? */
|
|
540
|
+
const holdsAt = (
|
|
541
|
+
/** @type {string[]} */ path,
|
|
542
|
+
/** @type {number} */ index,
|
|
543
|
+
) => {
|
|
544
|
+
const found = at(renders[index], path);
|
|
545
|
+
const value = values[index];
|
|
546
|
+
if (verbatim) return same(found, value);
|
|
547
|
+
return (
|
|
548
|
+
typeof found === "string" &&
|
|
549
|
+
typeof value === "string" &&
|
|
550
|
+
found.includes(value)
|
|
551
|
+
);
|
|
552
|
+
};
|
|
553
|
+
// A host may split its channels by SHAPE — text at one native key,
|
|
554
|
+
// structured results at another — so the values need not share one path.
|
|
555
|
+
// Each path must carry at least TWO of them, which is what stops an
|
|
556
|
+
// unrelated varying key from claiming a single value by coincidence: a
|
|
557
|
+
// render that drops the output while emitting `has_output: false` holds the
|
|
558
|
+
// `false` probe there and nothing else.
|
|
559
|
+
const channels = paths
|
|
560
|
+
.map((path) => values.map((_, index) => holdsAt(path, index)))
|
|
561
|
+
.filter(
|
|
562
|
+
(held) => held.filter(Boolean).length >= Math.min(2, values.length),
|
|
563
|
+
);
|
|
564
|
+
const missing = values.findIndex(
|
|
565
|
+
(_, index) => !channels.some((held) => held[index]),
|
|
566
|
+
);
|
|
567
|
+
if (missing === -1) {
|
|
568
|
+
seen.add(field);
|
|
569
|
+
return;
|
|
570
|
+
}
|
|
571
|
+
// One value reached no channel. Name it at the INNERMOST varying position,
|
|
572
|
+
// which is the one a reader can act on.
|
|
573
|
+
const path = paths[paths.length - 1];
|
|
574
|
+
const index = missing;
|
|
575
|
+
assert.fail(
|
|
576
|
+
`${where}: no native path carries ${field} ${verbatim ? "verbatim" : "intact"} on ${event.event}. At ${path.join(".") || "the render root"} the render holds ${JSON.stringify(at(renders[index], path))} where the verdict set ${JSON.stringify(values[index])}`,
|
|
577
|
+
);
|
|
578
|
+
};
|
|
579
|
+
|
|
580
|
+
for (const field of VERDICT_CONTENT_FIELDS) {
|
|
581
|
+
carries(field, {}, "alone");
|
|
582
|
+
// The HELD fields take every COMBINATION of their shapes, because an
|
|
583
|
+
// adapter can lose one channel only for a particular pairing of the other
|
|
584
|
+
// two — dropping the context exactly when the input is a record and the
|
|
585
|
+
// output an object. Zipping the held lists by index reaches each held
|
|
586
|
+
// field's every shape but only one pairing of them.
|
|
587
|
+
//
|
|
588
|
+
// The cross-product is 594 renders per case over three fields today, and
|
|
589
|
+
// `render` is a pure call. A fourth content
|
|
590
|
+
// field would multiply that, and is the point to reconsider.
|
|
591
|
+
const others = VERDICT_CONTENT_FIELDS.filter((name) => name !== field);
|
|
592
|
+
const held = others.reduce(
|
|
593
|
+
(combinations, name) =>
|
|
594
|
+
combinations.flatMap((carried) =>
|
|
595
|
+
(lookup(CONTENT_PROBE_VALUES, name) ?? []).map((value) => ({
|
|
596
|
+
...carried,
|
|
597
|
+
[name]: value,
|
|
598
|
+
})),
|
|
599
|
+
),
|
|
600
|
+
/** @type {Record<string, unknown>[]} */ ([{}]),
|
|
601
|
+
);
|
|
602
|
+
for (const [index, carried] of held.entries())
|
|
603
|
+
carries(
|
|
604
|
+
field,
|
|
605
|
+
carried,
|
|
606
|
+
`with the other fields, combination ${index + 1}`,
|
|
607
|
+
);
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
|
|
39
611
|
/**
|
|
40
612
|
* Assert an adapter's {@link import("./control-plane.mjs").Adapter.COVERAGE}
|
|
41
613
|
* hook-coverage matrix is well-formed: it classifies EXACTLY the canonical
|
|
@@ -234,13 +806,27 @@ function assertAliasedInput(agent, caseName, event, assert) {
|
|
|
234
806
|
* throwing away the objection entirely. The strongest honest signal a
|
|
235
807
|
* transport has left — an ask, an advisory body — must survive. An
|
|
236
808
|
* OBSERVE_ONLY render is exempt: it has no pre-emption channel to differ in.
|
|
809
|
+
* 9. drift honesty: `parse` must ANSWER an event name no host emits with
|
|
810
|
+
* EventKind.UNKNOWN rather than throw, and a deny rendered on that answer
|
|
811
|
+
* must not claim a block. Probed on a synthesized payload, since an adapter
|
|
812
|
+
* whose parse can never emit UNKNOWN has no such fixture to write.
|
|
813
|
+
* 10. content-channel honesty: for every parsed event, each Verdict content
|
|
814
|
+
* field (mutated_input, mutated_output, additional_context) either reaches
|
|
815
|
+
* the host's wire or is DECLARED unreachable in the adapter's
|
|
816
|
+
* UNRENDERED_FIELDS — and the render must agree with the declaration in
|
|
817
|
+
* both directions. Without it an adapter with no channel for a field just
|
|
818
|
+
* ignored it, so a redaction verdict rendered a bare allow and the
|
|
819
|
+
* unredacted output reached the model with nothing saying so. The fields
|
|
820
|
+
* that DID reach a wire come back as `contentFieldsSeen`; a suite covering
|
|
821
|
+
* several adapters asserts their union to keep the positive half of the
|
|
822
|
+
* rule non-vacuous, since no single host has a channel for all three.
|
|
237
823
|
*
|
|
238
824
|
* `assert` is injected (node:assert/strict) so the harness stays test-framework
|
|
239
825
|
* neutral; it throws on the first mismatch. Returns a summary the caller can
|
|
240
826
|
* assert further on.
|
|
241
827
|
*
|
|
242
828
|
* @param {{ adapter: import("./control-plane.mjs").Adapter, fixtures: any, assert: any }} args
|
|
243
|
-
* @returns {{ cases: number, renders: number, decisionsSeen: Set<string>, mutationSeen: boolean, enforcedDenySeen: boolean, vetoableDenySeen: boolean, unknownKindSeen: boolean, coverageClassesChecked: Set<string>, unenforceableDenyChecks: number }}
|
|
829
|
+
* @returns {{ cases: number, renders: number, decisionsSeen: Set<string>, mutationSeen: boolean, contentFieldsSeen: Set<string>, enforcedDenySeen: boolean, vetoableDenySeen: boolean, unknownKindSeen: boolean, coverageClassesChecked: Set<string>, unenforceableDenyChecks: number }}
|
|
244
830
|
*/
|
|
245
831
|
export function runAdapterConformance({ adapter, fixtures, assert }) {
|
|
246
832
|
assert.equal(
|
|
@@ -250,15 +836,20 @@ export function runAdapterConformance({ adapter, fixtures, assert }) {
|
|
|
250
836
|
);
|
|
251
837
|
|
|
252
838
|
assertCoverageWellFormed(adapter, assert);
|
|
839
|
+
assertEveryKindHasARow(adapter, assert);
|
|
253
840
|
|
|
254
841
|
/** @type {Set<string>} */
|
|
255
842
|
const decisionsSeen = new Set();
|
|
256
843
|
/** @type {Set<string>} call classes exercised by a tagged fixture case */
|
|
257
844
|
const coverageClassesChecked = new Set();
|
|
258
845
|
let mutationSeen = false;
|
|
846
|
+
/** @type {Set<string>} content fields rule ⑩ saw reach this host's wire */
|
|
847
|
+
const contentFieldsSeen = new Set();
|
|
259
848
|
let enforcedDenySeen = false;
|
|
260
849
|
let vetoableDenySeen = false;
|
|
261
850
|
let unknownKindSeen = false;
|
|
851
|
+
/** @type {Record<string, import("./control-plane.mjs").ToolCallEvent>} */
|
|
852
|
+
const parsedByKind = {};
|
|
262
853
|
let preToolCases = 0;
|
|
263
854
|
let unenforceableDenyChecks = 0;
|
|
264
855
|
let renders = 0;
|
|
@@ -366,6 +957,15 @@ export function runAdapterConformance({ adapter, fixtures, assert }) {
|
|
|
366
957
|
renders += 1;
|
|
367
958
|
}
|
|
368
959
|
|
|
960
|
+
parsedByKind[parsed.event] ??= parsed;
|
|
961
|
+
assertContentChannels(
|
|
962
|
+
adapter,
|
|
963
|
+
parsed,
|
|
964
|
+
testCase.name,
|
|
965
|
+
assert,
|
|
966
|
+
contentFieldsSeen,
|
|
967
|
+
);
|
|
968
|
+
|
|
369
969
|
// Rule ⑧, probed on a synthesized non-vetoable variant rather than on a
|
|
370
970
|
// fixture flag: an adapter whose COVERAGE marks every reachable call class
|
|
371
971
|
// as gated can never PARSE a non-vetoable event, so a fixture-driven check
|
|
@@ -431,9 +1031,21 @@ export function runAdapterConformance({ adapter, fixtures, assert }) {
|
|
|
431
1031
|
false,
|
|
432
1032
|
"a deny on an unmodelled event rendered as an enforced block",
|
|
433
1033
|
);
|
|
1034
|
+
assertContentChannels(
|
|
1035
|
+
adapter,
|
|
1036
|
+
drifted,
|
|
1037
|
+
"rule ⑨ drift probe",
|
|
1038
|
+
assert,
|
|
1039
|
+
contentFieldsSeen,
|
|
1040
|
+
);
|
|
434
1041
|
unknownKindSeen = true;
|
|
435
1042
|
}
|
|
436
1043
|
assert.ok(renders > 0, "conformance fixtures render nothing");
|
|
1044
|
+
assert.ok(
|
|
1045
|
+
Object.keys(parsedByKind).length > 0,
|
|
1046
|
+
"the every-EventKind declaration sweep has no parsed event to build from",
|
|
1047
|
+
);
|
|
1048
|
+
assertEveryKindRenders(adapter, parsedByKind, assert, contentFieldsSeen);
|
|
437
1049
|
// EVERY pre-tool case, not merely one: the count is what catches rule ⑧ being
|
|
438
1050
|
// skipped by a stray `continue` rather than merely present somewhere. A suite
|
|
439
1051
|
// with no pre-tool case has no veto surface to lose, and rule ⑤'s own
|
|
@@ -449,6 +1061,7 @@ export function runAdapterConformance({ adapter, fixtures, assert }) {
|
|
|
449
1061
|
renders,
|
|
450
1062
|
decisionsSeen,
|
|
451
1063
|
mutationSeen,
|
|
1064
|
+
contentFieldsSeen,
|
|
452
1065
|
enforcedDenySeen,
|
|
453
1066
|
vetoableDenySeen,
|
|
454
1067
|
unknownKindSeen,
|