@openwop/openwop-conformance 2.40.2 → 2.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +1 -1
- package/dist/lib/certification-bundle-verify.js +24 -2
- package/dist/spec-artifacts.lock.json +2 -2
- package/fixtures/conformance-context-budget-live.json +44 -0
- package/fixtures/conformance-context-budget-multiturn.json +1 -1
- package/fixtures.md +2 -1
- package/package.json +2 -2
- package/requirements.json +85 -56
- package/scenario-majors.json +5 -3
- package/schemas/CORPUS-STAMP.json +11 -11
- package/src/lib/certification-bundle-verify.ts +23 -1
- package/src/lib/context-budget.ts +198 -0
- package/src/lib/host-callback.ts +8 -0
- package/src/scenarios/certification-bundle-redaction.test.ts +30 -7
- package/src/scenarios/context-budget-transcript-bound.test.ts +85 -183
- package/src/scenarios/context-summarization-replay.test.ts +81 -81
- package/src/scenarios/v2-bound-id-kinds.test.ts +2 -0
- package/src/scenarios/v2-durability-recovery.test.ts +2 -0
- package/src/scenarios/v2-revocation-honored.test.ts +11 -4
- package/src/scenarios/v2-terminal-event-once.test.ts +2 -0
- package/src/scenarios/v2-webhook-delivery-isolation.test.ts +2 -0
- package/src/scenarios/v2-webhook-delivery-shape.test.ts +2 -0
- package/src/scenarios/v2-webhook-durable-delivery.test.ts +2 -0
- package/src/scenarios/v2-webhook-unregister-stops-delivery.test.ts +2 -0
- package/src/scenarios/webhook-signed-delivery.test.ts +2 -0
|
@@ -16,18 +16,32 @@
|
|
|
16
16
|
* summarizedRanges }`. The seam is OPTIONAL — the scenario soft-skips on
|
|
17
17
|
* `404`/`405` (the RFC defers reference-host implementation).
|
|
18
18
|
*
|
|
19
|
-
* Asserts, for each iteration the host reports
|
|
19
|
+
* Asserts, for each iteration the host reports (the rules live in the pure
|
|
20
|
+
* checker `lib/context-budget.ts`, whose sabotage cases are self-tests):
|
|
20
21
|
* 1. `tokenCounter` equals the advertised `contextBudget.tokenCounter`.
|
|
21
22
|
* 2. `tokenCount ≤ transcriptTokenBudget` (the per-turn bound).
|
|
22
|
-
* 3.
|
|
23
|
-
* `
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
* 4.
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
* `
|
|
23
|
+
* 3. RECOUNT (RFC 0111 (b), corrected 2026-09-26) — for `tokenCounter:
|
|
24
|
+
* "chars"` the seam MUST return `entries[] { eventId, rendered }` and the
|
|
25
|
+
* suite sums the rendered text's code points itself; the sum MUST equal
|
|
26
|
+
* `tokenCount`. Other units are advertise-and-attest.
|
|
27
|
+
* 4. REAL EVENTS — every `eventIds` / `entries[]` id is an event of the run;
|
|
28
|
+
* the verbatim entries are `eventIds` in order.
|
|
29
|
+
* 5. RECENT TAIL — fed events are in log order, none at or after the
|
|
30
|
+
* iteration's own `runOrchestrator.decided`, and no eligible event (same
|
|
31
|
+
* `(type, nodeId)`) newer than the oldest fed one was dropped.
|
|
32
|
+
* 6. SUMMARIZED-RANGE — every `summaryRef` has a `context.summarized` event.
|
|
33
|
+
* 7. PRESSURE — at least one iteration shows the budget acting (an older
|
|
34
|
+
* eligible event evicted, or a summarized range). Without it the row
|
|
35
|
+
* records `partial-witness:` — a budget of 10^9 is not a witness.
|
|
36
|
+
*
|
|
37
|
+
* Both majors (2.41.0). Major 1 gates through `behaviorGate`; major 2 reads the
|
|
38
|
+
* `multiAgent` record's presence and records `inapplicable` when
|
|
39
|
+
* `contextBudget` is absent — never a strict-mode failure for a host that does
|
|
40
|
+
* not claim the capability. The seam is in the seams profile at major 2.
|
|
41
|
+
* Gated on the LIVE fixture `conformance-context-budget-live` (no mock
|
|
42
|
+
* decisions): the scripted `conformance-context-budget-multiturn` drives the
|
|
43
|
+
* supervisor through `mockDecisions`, which is exactly the mock RFC 0111
|
|
44
|
+
* §Scope forbids from advertising `contextBudget`.
|
|
31
45
|
*
|
|
32
46
|
* Honest non-vacuity ceiling (RFC 0111 §"Conformance seam"): the model-facing
|
|
33
47
|
* prompt is genuinely host-internal, so this proves the host's DECLARED
|
|
@@ -47,209 +61,97 @@ import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
|
47
61
|
import { readCapabilityFamily } from '../lib/discovery-capabilities.js';
|
|
48
62
|
import { queryTestEvents } from '../lib/event-log-query.js';
|
|
49
63
|
import { req } from '../lib/requirement-ids.js';
|
|
50
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
64
|
+
import { softSkip, seamAbsent } from '../lib/soft-skip.js';
|
|
65
|
+
import { seamsProfileAdvertised, targetMajor } from '../lib/seams.js';
|
|
66
|
+
import { familyAdvertised, v2Discovery } from '../lib/v2.js';
|
|
67
|
+
import { runsPath } from '../lib/memoryAttribution.js';
|
|
68
|
+
import { checkTranscriptWindow, parseTranscriptWindow, type LogEvent, type TranscriptWindow } from '../lib/context-budget.js';
|
|
51
69
|
|
|
52
|
-
const FIXTURE = 'conformance-context-budget-
|
|
70
|
+
const FIXTURE = 'conformance-context-budget-live';
|
|
53
71
|
const PROFILE = 'openwop-context-budget';
|
|
54
72
|
const MAX_ITERATIONS_PROBED = 16;
|
|
73
|
+
const ID = 'openwop.it.context-budget-transcript-bound.bounds-the-per-turn-transcript-to-transcripttokenbudget-with-an-internally-consi';
|
|
55
74
|
|
|
56
|
-
interface SummarizationCap {
|
|
57
|
-
readonly supported?: boolean;
|
|
58
|
-
readonly strategy?: string;
|
|
59
|
-
readonly keepLastTurns?: number;
|
|
60
|
-
}
|
|
61
|
-
interface ContextBudgetCap {
|
|
62
|
-
readonly transcriptTokenBudget?: number;
|
|
63
|
-
readonly tokenCounter?: string;
|
|
64
|
-
readonly summarization?: SummarizationCap;
|
|
65
|
-
}
|
|
66
|
-
interface ExecutionModelCap {
|
|
67
|
-
readonly contextBudget?: ContextBudgetCap;
|
|
68
|
-
}
|
|
69
|
-
interface MultiAgentCap {
|
|
70
|
-
readonly executionModel?: ExecutionModelCap;
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
// ── cast-free typed accessors (no `as`) ──────────────────────────────────
|
|
74
75
|
function isRecord(v: unknown): v is Record<string, unknown> {
|
|
75
76
|
return typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
76
77
|
}
|
|
77
|
-
function
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
return typeof v === 'number';
|
|
82
|
-
}
|
|
83
|
-
function stringOf(v: unknown): string | undefined {
|
|
84
|
-
return isString(v) ? v : undefined;
|
|
85
|
-
}
|
|
86
|
-
function numberOf(v: unknown): number | undefined {
|
|
87
|
-
return isNumber(v) ? v : undefined;
|
|
88
|
-
}
|
|
89
|
-
function stringArrayOf(v: unknown): string[] | undefined {
|
|
90
|
-
return Array.isArray(v) && v.every(isString) ? v : undefined;
|
|
78
|
+
function recordAt(v: unknown, ...keys: string[]): Record<string, unknown> | undefined {
|
|
79
|
+
let cur: unknown = v;
|
|
80
|
+
for (const k of keys) cur = isRecord(cur) ? cur[k] : undefined;
|
|
81
|
+
return isRecord(cur) ? cur : undefined;
|
|
91
82
|
}
|
|
92
83
|
function runIdOf(v: unknown): string | undefined {
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
interface SummarizedRange {
|
|
97
|
-
readonly summaryRef: string;
|
|
98
|
-
readonly replacedTurns: string[];
|
|
99
|
-
}
|
|
100
|
-
interface TranscriptWindow {
|
|
101
|
-
readonly tokenCounter: string;
|
|
102
|
-
readonly tokenCount: number;
|
|
103
|
-
readonly eventIds: string[];
|
|
104
|
-
readonly summarizedRanges: SummarizedRange[];
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
function summarizedRangeOf(v: unknown): SummarizedRange | undefined {
|
|
108
|
-
if (!isRecord(v)) return undefined;
|
|
109
|
-
const summaryRef = stringOf(v['summaryRef']);
|
|
110
|
-
const replacedTurns = stringArrayOf(v['replacedTurns']);
|
|
111
|
-
if (summaryRef === undefined || replacedTurns === undefined) return undefined;
|
|
112
|
-
return { summaryRef, replacedTurns };
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
/** Parse the seam response into a typed window — undefined if the shape is wrong. */
|
|
116
|
-
function transcriptWindowOf(v: unknown): TranscriptWindow | undefined {
|
|
117
|
-
if (!isRecord(v)) return undefined;
|
|
118
|
-
const tokenCounter = stringOf(v['tokenCounter']);
|
|
119
|
-
const tokenCount = numberOf(v['tokenCount']);
|
|
120
|
-
const eventIds = stringArrayOf(v['eventIds']);
|
|
121
|
-
if (tokenCounter === undefined || tokenCount === undefined || eventIds === undefined) return undefined;
|
|
122
|
-
const rawRanges = v['summarizedRanges'];
|
|
123
|
-
const summarizedRanges: SummarizedRange[] = [];
|
|
124
|
-
if (Array.isArray(rawRanges)) {
|
|
125
|
-
for (const r of rawRanges) {
|
|
126
|
-
const parsed = summarizedRangeOf(r);
|
|
127
|
-
if (parsed === undefined) return undefined; // malformed range → fail loudly via caller
|
|
128
|
-
summarizedRanges.push(parsed);
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
return { tokenCounter, tokenCount, eventIds, summarizedRanges };
|
|
84
|
+
const r = isRecord(v) ? v['runId'] : undefined;
|
|
85
|
+
return typeof r === 'string' ? r : undefined;
|
|
132
86
|
}
|
|
133
87
|
|
|
134
88
|
describe('context-budget-transcript-bound (RFC 0111 §"Context economy")', () => {
|
|
135
89
|
it('bounds the per-turn transcript to transcriptTokenBudget with an internally-consistent, recent-tail accounting', async () => {
|
|
136
|
-
const
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
const
|
|
150
|
-
|
|
90
|
+
const major = targetMajor();
|
|
91
|
+
let cb: Record<string, unknown> | undefined;
|
|
92
|
+
if (major === 2) {
|
|
93
|
+
const doc = await v2Discovery();
|
|
94
|
+
if (!doc) return softSkip('blocked', 'v2 discovery unreachable');
|
|
95
|
+
cb = recordAt(await familyAdvertised('multiAgent'), 'executionModel', 'contextBudget');
|
|
96
|
+
if (typeof cb?.['transcriptTokenBudget'] !== 'number') return softSkip('inapplicable', 'multiAgent.executionModel.contextBudget.transcriptTokenBudget is not advertised at major 2');
|
|
97
|
+
if (!seamsProfileAdvertised(doc)) return softSkip('inapplicable', 'the transcript-window seam is in the seams profile — conformance.seamsProfile is not openwop-conformance-seams-v2');
|
|
98
|
+
} else {
|
|
99
|
+
cb = recordAt(await readCapabilityFamily<Record<string, unknown>>('multiAgent'), 'executionModel', 'contextBudget');
|
|
100
|
+
if (!behaviorGate(PROFILE, typeof cb?.['transcriptTokenBudget'] === 'number')) return;
|
|
101
|
+
}
|
|
102
|
+
const budget = cb?.['transcriptTokenBudget'];
|
|
103
|
+
const advertisedCounter = cb?.['tokenCounter'];
|
|
104
|
+
if (typeof budget !== 'number') return softSkip('inapplicable', 'transcriptTokenBudget not advertised');
|
|
105
|
+
if (!isFixtureAdvertised(FIXTURE)) return softSkip('inapplicable', `the live fixture ${FIXTURE} is not advertised — the scripted multiturn fixture drives a mock supervisor, which RFC 0111 §Scope forbids from advertising contextBudget`);
|
|
106
|
+
expect(typeof advertisedCounter === 'string', req(ID, 'RFC 0111', 'tokenCounter MUST be advertised when transcriptTokenBudget is present (schema if/then)')).toBe(true);
|
|
107
|
+
if (typeof advertisedCounter !== 'string') return softSkip('blocked', 'contextBudget.tokenCounter is not advertised (the assertion above records the failure)');
|
|
108
|
+
|
|
109
|
+
const create = await driver.post(runsPath(), { workflowId: FIXTURE });
|
|
110
|
+
expect(create.status, req(ID, 'RFC 0111', `POST ${runsPath()} MUST create the live-fixture run`)).toBe(201);
|
|
151
111
|
const runId = runIdOf(create.json);
|
|
152
|
-
expect(runId, req(
|
|
153
|
-
if (runId === undefined) return softSkip('blocked', '
|
|
112
|
+
expect(runId, req(ID, 'RFC 0111', 'the create response MUST carry a runId')).toBeDefined();
|
|
113
|
+
if (runId === undefined) return softSkip('blocked', 'no runId');
|
|
154
114
|
await pollUntilTerminal(runId);
|
|
155
115
|
|
|
156
|
-
// Probe the per-iteration transcript-window seam (OPTIONAL).
|
|
157
116
|
const windows: Array<{ iteration: number; window: TranscriptWindow }> = [];
|
|
158
117
|
for (let iteration = 1; iteration <= MAX_ITERATIONS_PROBED; iteration += 1) {
|
|
159
|
-
const res = await driver.get(
|
|
160
|
-
`/v1/host/sample/agent/transcript-window?runId=${encodeURIComponent(runId)}&iteration=${iteration}`,
|
|
161
|
-
);
|
|
118
|
+
const res = await driver.get(`/v1/host/sample/agent/transcript-window?runId=${encodeURIComponent(runId)}&iteration=${iteration}`);
|
|
162
119
|
if (res.status === 404 || res.status === 405) {
|
|
163
|
-
if (iteration === 1) return
|
|
164
|
-
break;
|
|
120
|
+
if (iteration === 1) return major === 2 ? seamAbsent(`contextBudget is advertised but the transcript-window seam answered ${res.status} (host-sample-test-seams.md §14)`) : softSkip('blocked', `the transcript-window seam answered ${res.status} (host-sample-test-seams.md §14)`);
|
|
121
|
+
break;
|
|
165
122
|
}
|
|
166
|
-
if (res.status === 400 || res.status === 422) break;
|
|
167
|
-
expect(
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
)
|
|
171
|
-
const window = transcriptWindowOf(res.json);
|
|
172
|
-
expect(
|
|
173
|
-
window,
|
|
174
|
-
req('openwop.it.context-budget-transcript-bound.bounds-the-per-turn-transcript-to-transcripttokenbudget-with-an-internally-consi', 'host-sample-test-seams.md §14', 'the seam MUST return { tokenCounter, tokenCount, eventIds, summarizedRanges }'),
|
|
175
|
-
).toBeDefined();
|
|
176
|
-
if (window === undefined) return softSkip('blocked', 'precondition not met — `window === undefined` returned early (seam, prior step, or fixture unavailable)');
|
|
123
|
+
if (res.status === 400 || res.status === 422) break;
|
|
124
|
+
expect(res.status, req(ID, 'host-sample-test-seams.md §14', `iteration ${iteration}: the transcript-window seam MUST return 200 for a valid iteration`)).toBe(200);
|
|
125
|
+
const window = parseTranscriptWindow(res.json);
|
|
126
|
+
expect(window, req(ID, 'host-sample-test-seams.md §14', `iteration ${iteration}: the seam MUST return { tokenCounter, tokenCount, eventIds, summarizedRanges, entries? } with a well-formed entries[] when present`)).toBeDefined();
|
|
127
|
+
if (window === undefined) return softSkip('blocked', `iteration ${iteration}: the seam answer was malformed (the assertion above records the failure)`);
|
|
177
128
|
windows.push({ iteration, window });
|
|
178
129
|
}
|
|
130
|
+
expect(windows.length, req(ID, 'host-sample-test-seams.md §14', 'a wired transcript-window seam MUST report at least one orchestrator iteration')).toBeGreaterThan(0);
|
|
179
131
|
|
|
180
|
-
// Non-vacuity: a wired seam MUST report at least one iteration.
|
|
181
|
-
expect(windows.length, req('openwop.it.context-budget-transcript-bound.bounds-the-per-turn-transcript-to-transcripttokenbudget-with-an-internally-consi', 'host-sample-test-seams.md §14', 'a wired transcript-window seam MUST report at least one orchestrator iteration')).toBeGreaterThan(0);
|
|
182
|
-
|
|
183
|
-
// Independent event-log read for the cross-check (OPTIONAL seam).
|
|
184
132
|
const q = await queryTestEvents(runId);
|
|
185
|
-
const
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
if (e.type === 'context.summarized') {
|
|
191
|
-
const ref = stringOf(e.payload['summaryRef']);
|
|
192
|
-
if (ref !== undefined) summarizedRefs.add(ref);
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
}
|
|
196
|
-
|
|
133
|
+
const log: LogEvent[] | null = q.ok
|
|
134
|
+
? q.events.map((e) => ({ eventId: e.eventId, type: e.type, sequence: e.sequence, payload: e.payload, ...(e.nodeId !== undefined ? { nodeId: e.nodeId } : {}) }))
|
|
135
|
+
: null;
|
|
136
|
+
const cap = { transcriptTokenBudget: budget, tokenCounter: advertisedCounter };
|
|
137
|
+
let pressure = false;
|
|
197
138
|
for (const { iteration, window } of windows) {
|
|
198
|
-
|
|
199
|
-
expect(
|
|
200
|
-
|
|
201
|
-
req('openwop.it.context-budget-transcript-bound.bounds-the-per-turn-transcript-to-transcripttokenbudget-with-an-internally-consi', 'RFC 0111', `iteration ${iteration}: seam tokenCounter MUST equal the advertised contextBudget.tokenCounter`),
|
|
202
|
-
).toBe(advertisedCounter);
|
|
203
|
-
|
|
204
|
-
// 2 — the per-turn token bound.
|
|
205
|
-
if (budget !== undefined) {
|
|
206
|
-
expect(
|
|
207
|
-
window.tokenCount,
|
|
208
|
-
req('openwop.it.context-budget-transcript-bound.bounds-the-per-turn-transcript-to-transcripttokenbudget-with-an-internally-consi', 'RFC 0111', `iteration ${iteration}: tokenCount MUST NOT exceed transcriptTokenBudget`),
|
|
209
|
-
).toBeLessThanOrEqual(budget);
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
// 3 — internal consistency: every named id is a real persisted event.
|
|
213
|
-
if (q.ok) {
|
|
214
|
-
for (const id of window.eventIds) {
|
|
215
|
-
expect(
|
|
216
|
-
logEventIds.has(id),
|
|
217
|
-
req('openwop.it.context-budget-transcript-bound.bounds-the-per-turn-transcript-to-transcripttokenbudget-with-an-internally-consi', 'RFC 0111 §"Conformance seam"', `iteration ${iteration}: eventId "${id}" in the seam accounting MUST be a real persisted run event`),
|
|
218
|
-
).toBe(true);
|
|
219
|
-
}
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
// 4 — recent-tail: ids are unique (no double-count inflating the window).
|
|
223
|
-
const uniqueIds = new Set(window.eventIds);
|
|
224
|
-
expect(
|
|
225
|
-
uniqueIds.size,
|
|
226
|
-
req('openwop.it.context-budget-transcript-bound.bounds-the-per-turn-transcript-to-transcripttokenbudget-with-an-internally-consi', 'RFC 0111 §"Conformance seam"', `iteration ${iteration}: eventIds MUST be a tail with no repeated entry`),
|
|
227
|
-
).toBe(window.eventIds.length);
|
|
228
|
-
|
|
229
|
-
// 5 — every summarized range references a recorded context.summarized event.
|
|
230
|
-
if (q.ok) {
|
|
231
|
-
for (const range of window.summarizedRanges) {
|
|
232
|
-
expect(
|
|
233
|
-
summarizedRefs.has(range.summaryRef),
|
|
234
|
-
req('openwop.it.context-budget-transcript-bound.bounds-the-per-turn-transcript-to-transcripttokenbudget-with-an-internally-consi', 'RFC 0111', `iteration ${iteration}: summarizedRanges summaryRef "${range.summaryRef}" MUST have a matching context.summarized event`),
|
|
235
|
-
).toBe(true);
|
|
236
|
-
}
|
|
237
|
-
}
|
|
139
|
+
const v = checkTranscriptWindow(iteration, window, cap, log);
|
|
140
|
+
expect(v.violations, req(ID, 'RFC 0111 §"Conformance seam" (b)/(c), host-sample-test-seams.md §14', `iteration ${iteration}: the host's transcript accounting MUST be within budget, recountable, real, and a recent tail`)).toEqual([]);
|
|
141
|
+
pressure ||= v.pressure;
|
|
238
142
|
}
|
|
239
143
|
|
|
240
144
|
// keepLastTurns verbatim — a kept turn is fed verbatim, never inside a summarized range.
|
|
241
|
-
const keepLastTurns =
|
|
242
|
-
if (keepLastTurns
|
|
145
|
+
const keepLastTurns = recordAt(cb, 'summarization')?.['keepLastTurns'];
|
|
146
|
+
if (typeof keepLastTurns === 'number' && keepLastTurns > 0) {
|
|
243
147
|
const last = windows[windows.length - 1].window;
|
|
244
|
-
const summarizedIds = new Set
|
|
245
|
-
for (const
|
|
246
|
-
|
|
247
|
-
for (const id of verbatimTail) {
|
|
248
|
-
expect(
|
|
249
|
-
summarizedIds.has(id),
|
|
250
|
-
req('openwop.it.context-budget-transcript-bound.bounds-the-per-turn-transcript-to-transcripttokenbudget-with-an-internally-consi', 'RFC 0111', `a kept (verbatim) turn "${id}" MUST NOT appear inside a summarized range`),
|
|
251
|
-
).toBe(false);
|
|
148
|
+
const summarizedIds = new Set(last.summarizedRanges.flatMap((r) => r.replacedTurns));
|
|
149
|
+
for (const id of last.eventIds.slice(Math.max(0, last.eventIds.length - keepLastTurns))) {
|
|
150
|
+
expect(summarizedIds.has(id), req(ID, 'RFC 0111', `a kept (verbatim) turn "${id}" MUST NOT appear inside a summarized range`)).toBe(false);
|
|
252
151
|
}
|
|
253
152
|
}
|
|
153
|
+
|
|
154
|
+
if (log === null) return softSkip('blocked', 'the run event-log seam is unavailable, so the real-event, recent-tail and pressure rules were not measured');
|
|
155
|
+
if (!pressure) softSkip('inapplicable', `no iteration shows budget pressure — every eligible event fit under transcriptTokenBudget ${budget}, so the bound was never exercised (a budget the run never reaches is not a witness)`);
|
|
254
156
|
});
|
|
255
157
|
});
|
|
@@ -23,6 +23,15 @@
|
|
|
23
23
|
* The RFC defers reference-host implementation; the witness comes from a host
|
|
24
24
|
* that runs real orchestrator turns and summarizes.
|
|
25
25
|
*
|
|
26
|
+
* STRENGTHENED 2.41.0 (the RFC 0111 correction on record, 2026-09-26). Equal
|
|
27
|
+
* `summaryRef`s prove the fork cited the same artifact; they do not prove the
|
|
28
|
+
* MODEL saw the same summary. When the transcript-window seam serves
|
|
29
|
+
* `entries[]` for both runs, the summary texts the host fed on the fork MUST be
|
|
30
|
+
* the source's, byte for byte — the replay MUST NOT re-summarize, on the text.
|
|
31
|
+
* Both majors: major 2 reads `summarization` by presence (the v2 schema has no
|
|
32
|
+
* `supported` seat) and records `inapplicable`, never a strict-mode failure,
|
|
33
|
+
* when it is absent. Gated on the live fixture `conformance-context-budget-live`.
|
|
34
|
+
*
|
|
26
35
|
* @see RFCS/0111-context-economy.md
|
|
27
36
|
* @see spec/v1/multi-agent-execution.md §"Context economy (RFC 0111)"
|
|
28
37
|
*/
|
|
@@ -36,122 +45,113 @@ import { readCapabilityFamily } from '../lib/discovery-capabilities.js';
|
|
|
36
45
|
import { queryTestEvents, type TestEvent } from '../lib/event-log-query.js';
|
|
37
46
|
import { req } from '../lib/requirement-ids.js';
|
|
38
47
|
import { softSkip } from '../lib/soft-skip.js';
|
|
48
|
+
import { seamsProfileAdvertised, targetMajor } from '../lib/seams.js';
|
|
49
|
+
import { familyAdvertised, v2Discovery } from '../lib/v2.js';
|
|
50
|
+
import { runsPath } from '../lib/memoryAttribution.js';
|
|
51
|
+
import { parseTranscriptWindow, summarizationDeclared, summaryTexts, type LogEvent } from '../lib/context-budget.js';
|
|
39
52
|
|
|
40
|
-
const FIXTURE = 'conformance-context-budget-
|
|
53
|
+
const FIXTURE = 'conformance-context-budget-live';
|
|
41
54
|
const PROFILE = 'openwop-context-summarization';
|
|
55
|
+
const ID = 'openwop.it.context-summarization-replay.replay-reuses-the-recorded-context-summarized-summaryref-never-re-summarizes';
|
|
56
|
+
const MAX_ITERATIONS_PROBED = 16;
|
|
42
57
|
|
|
43
|
-
interface SummarizationCap {
|
|
44
|
-
readonly supported?: boolean;
|
|
45
|
-
}
|
|
46
|
-
interface ContextBudgetCap {
|
|
47
|
-
readonly summarization?: SummarizationCap;
|
|
48
|
-
}
|
|
49
|
-
interface ExecutionModelCap {
|
|
50
|
-
readonly contextBudget?: ContextBudgetCap;
|
|
51
|
-
}
|
|
52
|
-
interface MultiAgentCap {
|
|
53
|
-
readonly executionModel?: ExecutionModelCap;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
// ── cast-free typed accessors (no `as`) ──────────────────────────────────
|
|
57
58
|
function isRecord(v: unknown): v is Record<string, unknown> {
|
|
58
59
|
return typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
59
60
|
}
|
|
60
|
-
function
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
return isString(v) ? v : undefined;
|
|
61
|
+
function recordAt(v: unknown, ...keys: string[]): Record<string, unknown> | undefined {
|
|
62
|
+
let cur: unknown = v;
|
|
63
|
+
for (const k of keys) cur = isRecord(cur) ? cur[k] : undefined;
|
|
64
|
+
return isRecord(cur) ? cur : undefined;
|
|
65
65
|
}
|
|
66
66
|
function stringArrayOf(v: unknown): string[] | undefined {
|
|
67
|
-
return Array.isArray(v) && v.every(
|
|
67
|
+
return Array.isArray(v) && v.every((x) => typeof x === 'string') ? v : undefined;
|
|
68
68
|
}
|
|
69
69
|
function runIdOf(v: unknown): string | undefined {
|
|
70
|
-
|
|
70
|
+
const r = isRecord(v) ? v['runId'] : undefined;
|
|
71
|
+
return typeof r === 'string' ? r : undefined;
|
|
71
72
|
}
|
|
72
73
|
function replayModesOf(v: unknown): string[] {
|
|
73
|
-
|
|
74
|
-
const replay = v['replay'];
|
|
75
|
-
if (!isRecord(replay)) return [];
|
|
76
|
-
return stringArrayOf(replay['modes']) ?? [];
|
|
74
|
+
return stringArrayOf(recordAt(v, 'replay')?.['modes']) ?? [];
|
|
77
75
|
}
|
|
78
|
-
|
|
79
76
|
/** A summary fingerprint: summaryRef plus the (ordered) replaced-turn ids. */
|
|
80
|
-
function summaryFingerprint(e: TestEvent): string | undefined {
|
|
81
|
-
const ref = stringOf(e.payload['summaryRef']);
|
|
82
|
-
const replaced = stringArrayOf(e.payload['replacedTurns']);
|
|
83
|
-
if (ref === undefined || replaced === undefined) return undefined;
|
|
84
|
-
return `${ref}::${replaced.join(',')}`;
|
|
85
|
-
}
|
|
86
|
-
|
|
87
77
|
function summaryFingerprints(events: readonly TestEvent[]): string[] {
|
|
88
78
|
const out: string[] = [];
|
|
89
79
|
for (const e of events) {
|
|
90
80
|
if (e.type !== 'context.summarized') continue;
|
|
91
|
-
const
|
|
92
|
-
|
|
93
|
-
|
|
81
|
+
const ref = e.payload['summaryRef'];
|
|
82
|
+
const replaced = stringArrayOf(e.payload['replacedTurns']);
|
|
83
|
+
expect(typeof ref === 'string' && replaced !== undefined, req(ID, 'RFC 0111', 'a context.summarized event MUST carry summaryRef + replacedTurns')).toBe(true);
|
|
84
|
+
if (typeof ref === 'string' && replaced !== undefined) out.push(`${ref}::${replaced.join(',')}`);
|
|
94
85
|
}
|
|
95
86
|
return out.sort();
|
|
96
87
|
}
|
|
88
|
+
const toLog = (events: readonly TestEvent[]): LogEvent[] => events.map((e) => ({ eventId: e.eventId, type: e.type, sequence: e.sequence, payload: e.payload }));
|
|
89
|
+
|
|
90
|
+
/** The summary texts the host fed across a run's iterations, via the seam; null when the seam or entries[] are not served. */
|
|
91
|
+
async function fedSummaryTexts(runId: string, log: readonly LogEvent[]): Promise<string[] | null> {
|
|
92
|
+
const out: string[] = [];
|
|
93
|
+
let served = false;
|
|
94
|
+
for (let iteration = 1; iteration <= MAX_ITERATIONS_PROBED; iteration += 1) {
|
|
95
|
+
const res = await driver.get(`/v1/host/sample/agent/transcript-window?runId=${encodeURIComponent(runId)}&iteration=${iteration}`);
|
|
96
|
+
if (res.status !== 200) break;
|
|
97
|
+
const w = parseTranscriptWindow(res.json);
|
|
98
|
+
if (w?.entries === undefined) return null;
|
|
99
|
+
served = true;
|
|
100
|
+
out.push(...summaryTexts(w, log));
|
|
101
|
+
}
|
|
102
|
+
return served ? out : null;
|
|
103
|
+
}
|
|
97
104
|
|
|
98
105
|
describe('context-summarization-replay (RFC 0111 §"Replay determinism")', () => {
|
|
99
106
|
it('replay reuses the recorded context.summarized summaryRef — never re-summarizes', async () => {
|
|
100
|
-
const
|
|
101
|
-
|
|
102
|
-
if (
|
|
103
|
-
|
|
107
|
+
const major = targetMajor();
|
|
108
|
+
let wellKnown: unknown;
|
|
109
|
+
if (major === 2) {
|
|
110
|
+
const doc = await v2Discovery();
|
|
111
|
+
if (!doc) return softSkip('blocked', 'v2 discovery unreachable');
|
|
112
|
+
const cb = recordAt(await familyAdvertised('multiAgent'), 'executionModel', 'contextBudget');
|
|
113
|
+
if (!summarizationDeclared(cb, 2)) return softSkip('inapplicable', 'multiAgent.executionModel.contextBudget.summarization is not advertised at major 2');
|
|
114
|
+
if (!seamsProfileAdvertised(doc)) return softSkip('inapplicable', 'the run event-log is read through the seams profile — conformance.seamsProfile is not openwop-conformance-seams-v2');
|
|
115
|
+
wellKnown = doc;
|
|
116
|
+
} else {
|
|
117
|
+
const cb = recordAt(await readCapabilityFamily<Record<string, unknown>>('multiAgent'), 'executionModel', 'contextBudget');
|
|
118
|
+
if (!behaviorGate(PROFILE, summarizationDeclared(cb, 1))) return;
|
|
119
|
+
wellKnown = (await driver.get('/.well-known/openwop')).json;
|
|
120
|
+
}
|
|
121
|
+
if (!isFixtureAdvertised(FIXTURE)) return softSkip('inapplicable', `the live fixture ${FIXTURE} is not advertised — RFC 0111 §Scope forbids a mock supervisor from advertising contextBudget`);
|
|
104
122
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
expect(create.status).toBe(201);
|
|
123
|
+
const create = await driver.post(runsPath(), { workflowId: FIXTURE });
|
|
124
|
+
expect(create.status, req(ID, 'RFC 0111', `POST ${runsPath()} MUST create the live-fixture run`)).toBe(201);
|
|
108
125
|
const sourceRunId = runIdOf(create.json);
|
|
109
|
-
expect(sourceRunId, req(
|
|
110
|
-
if (sourceRunId === undefined) return softSkip('blocked', '
|
|
126
|
+
expect(sourceRunId, req(ID, 'rest-endpoints.md POST /v1/runs', 'the create response MUST carry a runId')).toBeDefined();
|
|
127
|
+
if (sourceRunId === undefined) return softSkip('blocked', 'no runId');
|
|
111
128
|
await pollUntilTerminal(sourceRunId);
|
|
112
129
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
if (!sourceQ.ok) return softSkip('blocked', 'precondition not met — `!sourceQ.ok` returned early (event-log seam unwired — soft-skip) (seam, prior step, or fixture unavailable)'); // event-log seam unwired — soft-skip
|
|
130
|
+
const sourceQ = await queryTestEvents(sourceRunId);
|
|
131
|
+
if (!sourceQ.ok) return softSkip('blocked', 'the run event-log seam is unavailable');
|
|
116
132
|
const sourceFingerprints = summaryFingerprints(sourceQ.events);
|
|
117
133
|
if (sourceFingerprints.length === 0) {
|
|
118
|
-
|
|
119
|
-
// to prove about reuse. Honest soft-skip; not a vacuous pass of the MUST.
|
|
120
|
-
// eslint-disable-next-line no-console
|
|
121
|
-
console.warn(`[${PROFILE}] run produced no context.summarized events; replay-reuse leg soft-skipped`);
|
|
122
|
-
return softSkip('blocked', 'precondition not met — `sourceFingerprints.length === 0` returned early ([…] run produced no context.summarized events; replay-reuse leg soft-skipped) (seam, prior step, or fixture unavailable)');
|
|
134
|
+
return softSkip('blocked', 'the live run produced no context.summarized event — a host advertising summarization must summarize this fixture for reuse to be observable');
|
|
123
135
|
}
|
|
136
|
+
if (!replayModesOf(wellKnown).includes('replay')) return softSkip('inapplicable', 'the host advertises no replay fork mode');
|
|
124
137
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
const fork = await driver.post(
|
|
130
|
-
`/v1/runs/${encodeURIComponent(sourceRunId)}:fork`,
|
|
131
|
-
{ fromSeq: 0, mode: 'replay' },
|
|
132
|
-
);
|
|
133
|
-
if (fork.status === 501 || fork.status === 404) return softSkip('blocked', 'precondition not met — `fork.status === 501 || fork.status === 404` returned early (replay not implemented for this run — soft-skip) (seam, prior step, or fixture unavailable)'); // replay not implemented for this run — soft-skip
|
|
134
|
-
expect(
|
|
135
|
-
fork.status,
|
|
136
|
-
req('openwop.it.context-summarization-replay.replay-reuses-the-recorded-context-summarized-summaryref-never-re-summarizes', 'rest-endpoints.md POST /v1/runs/{runId}:fork', 'replay fork MUST return 201'),
|
|
137
|
-
).toBe(201);
|
|
138
|
+
const fork = await driver.post(`${runsPath()}/${encodeURIComponent(sourceRunId)}:fork`, { fromSeq: 0, mode: 'replay' });
|
|
139
|
+
if (fork.status === 501 || fork.status === 404) return softSkip('blocked', `replay fork answered ${fork.status}`);
|
|
140
|
+
expect(fork.status, req(ID, 'rest-endpoints.md POST /v1/runs/{runId}:fork', 'replay fork MUST return 201')).toBe(201);
|
|
138
141
|
const forkRunId = runIdOf(fork.json);
|
|
139
|
-
expect(forkRunId, req(
|
|
140
|
-
if (forkRunId === undefined) return softSkip('blocked', '
|
|
142
|
+
expect(forkRunId, req(ID, 'rest-endpoints.md POST /v1/runs/{runId}:fork', 'replay fork MUST return a runId')).toBeDefined();
|
|
143
|
+
if (forkRunId === undefined) return softSkip('blocked', 'no fork runId');
|
|
141
144
|
await pollUntilTerminal(forkRunId);
|
|
142
145
|
|
|
143
|
-
const forkQ = await queryTestEvents(forkRunId
|
|
144
|
-
if (!forkQ.ok) return softSkip('blocked', '
|
|
145
|
-
|
|
146
|
+
const forkQ = await queryTestEvents(forkRunId);
|
|
147
|
+
if (!forkQ.ok) return softSkip('blocked', 'the event-log seam is unavailable for the fork');
|
|
148
|
+
expect(summaryFingerprints(forkQ.events), req(ID, 'RFC 0111 §"Replay determinism"', 'a replay fork MUST reuse the recorded context.summarized summaryRef (never re-summarize to a different transcript)')).toEqual(sourceFingerprints);
|
|
146
149
|
|
|
147
|
-
// The
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
'a replay fork MUST reuse the recorded context.summarized summaryRef (never re-summarize to a different transcript)',
|
|
154
|
-
),
|
|
155
|
-
).toEqual(sourceFingerprints);
|
|
150
|
+
// The model-facing half: the summary TEXT the host fed on the fork.
|
|
151
|
+
const sourceTexts = await fedSummaryTexts(sourceRunId, toLog(sourceQ.events));
|
|
152
|
+
const forkTexts = await fedSummaryTexts(forkRunId, toLog(forkQ.events));
|
|
153
|
+
if (sourceTexts === null || forkTexts === null) return softSkip('inapplicable', 'the transcript-window seam serves no entries[] for these runs, so the model-facing summary text was not compared (summaryRef reuse was)');
|
|
154
|
+
expect(sourceTexts.length, req(ID, 'RFC 0111 §"Replay determinism"', 'the source run summarized, so its transcript windows MUST carry the summary text it fed')).toBeGreaterThan(0);
|
|
155
|
+
expect(forkTexts, req(ID, 'RFC 0111 §"Replay determinism"', 'the replay MUST feed the model the recorded summary text, byte for byte — never a re-summarization')).toEqual(sourceTexts);
|
|
156
156
|
});
|
|
157
157
|
});
|
|
@@ -36,6 +36,8 @@ import { scaledTimeoutMs } from '../lib/polling.js';
|
|
|
36
36
|
import { req } from '../lib/requirement-ids.js';
|
|
37
37
|
import { projectBoundId, BOUND_ID } from '../lib/bound-id.js';
|
|
38
38
|
|
|
39
|
+
export const REQUIRES_HOST_CALLBACK = 'the host POSTs a webhook delivery to the suite-owned scoped receiver behind OPENWOP_WEBHOOK_RECEIVER_URL';
|
|
40
|
+
|
|
39
41
|
const DOC = 'spec/v2/core/identity.md §5 (RFC 0187 §A)';
|
|
40
42
|
const WEBHOOK_ID = 'openwop.requirement.0187.bound-id-kinds.webhook';
|
|
41
43
|
const PER_KIND = 'openwop.requirement.0187.bound-id-kinds.per-kind';
|
|
@@ -75,6 +75,8 @@ import { startEffectReceiver, waitForFirstArrival } from '../lib/effect-receiver
|
|
|
75
75
|
import { watchForResumption, type Observation, type Watch } from '../lib/durability-watch.js';
|
|
76
76
|
import { noteEvidence, parseRecoveryBounds, EVIDENCE_NAME_PATTERN } from '../lib/durability-evidence.js';
|
|
77
77
|
|
|
78
|
+
export const REQUIRES_HOST_CALLBACK = 'the host makes an outbound effect call to the suite-owned effect receiver (OPENWOP_WEBHOOK_RECEIVER_PORT)';
|
|
79
|
+
|
|
78
80
|
const FIXTURE = 'conformance-noop';
|
|
79
81
|
const FAILURE_FIXTURE = 'conformance-failure';
|
|
80
82
|
|
|
@@ -11,7 +11,9 @@
|
|
|
11
11
|
* drives `POST /conformance/seams/sample/auth/credential/{mint,revoke}` (the
|
|
12
12
|
* v1-shaped address `/v1/host/sample/auth/credential/…` through `seamPath()`)
|
|
13
13
|
* with `{ lane }` / `{ lane, credential }` and records `blocked` when the host
|
|
14
|
-
* does not mount it.
|
|
14
|
+
* does not mount it. A mint answer carrying `presentation: { kind: "cookie", name }`
|
|
15
|
+
* (2.40.3) is presented as `Cookie: <name>=<credential>`, so a cookie-borne
|
|
16
|
+
* `session` lane is witnessed as its production credential travels; bearer otherwise. Windowed lanes (`exp-and-recheck`, `short-lived`, `rebind`)
|
|
15
17
|
* have no wire-observable "next request" and record `inapplicable`.
|
|
16
18
|
*/
|
|
17
19
|
|
|
@@ -48,17 +50,22 @@ describe('v2 revocation-honored (RFC 0170 §B.3 — seam-gated)', () => {
|
|
|
48
50
|
const minted = await http(() => driver.post(MINT, { lane }));
|
|
49
51
|
if (minted === null) return softSkip('blocked', `${MINT} unreachable (fetch failed)`);
|
|
50
52
|
if (minted.status === 404 || minted.status === 403) return seamAbsent(`${MINT} not mounted (${minted.status}) — the host advertises the seams profile but does not serve the RFC 0170 §B.3 credential mint/revoke seams (api/seams-v2.yaml mintLaneCredential / revokeLaneCredential; host-sample-test-seams.md)`);
|
|
51
|
-
const
|
|
53
|
+
const body = minted.json as { credential?: unknown; presentation?: { kind?: unknown; name?: unknown } } | undefined;
|
|
54
|
+
const credential = body?.credential;
|
|
55
|
+
// 2.40.3 — a cookie-borne lane (a `session` lane) is presented the way its
|
|
56
|
+
// production credential is, `Cookie: <name>=<credential>`. Bearer otherwise.
|
|
57
|
+
const cookieName = body?.presentation?.kind === 'cookie' && typeof body.presentation.name === 'string' ? body.presentation.name : null;
|
|
58
|
+
const present = (c: unknown): Record<string, string> => (cookieName ? { Cookie: `${cookieName}=${String(c)}` } : { Authorization: `Bearer ${String(c)}` });
|
|
52
59
|
expect(typeof credential, req('openwop.requirement.0170.revocation-honored', DOC, `the mint seam MUST answer { credential } for lane ${lane}`)).toBe('string');
|
|
53
60
|
|
|
54
61
|
// Positive control — the minted credential authenticates before revocation.
|
|
55
|
-
const before = await http(() => driver.get('/runs/openwop-conformance-tenant/revocationprobe0123456789', { authenticated: false, headers:
|
|
62
|
+
const before = await http(() => driver.get('/runs/openwop-conformance-tenant/revocationprobe0123456789', { authenticated: false, headers: present(credential) }));
|
|
56
63
|
expect(before !== null && before.status !== 401, req('openwop.requirement.0170.revocation-honored', DOC, `a freshly minted ${lane} credential MUST authenticate before revocation (got ${before?.status ?? 'no response'})`)).toBe(true);
|
|
57
64
|
|
|
58
65
|
const revoked = await http(() => driver.post(REVOKE, { lane, credential }));
|
|
59
66
|
expect(revoked !== null && revoked.status < 400, req('openwop.requirement.0170.revocation-honored', DOC, `the revoke seam MUST accept the credential it minted for lane ${lane} (got ${revoked?.status ?? 'no response'})`)).toBe(true);
|
|
60
67
|
|
|
61
|
-
const after = await http(() => driver.get('/runs/openwop-conformance-tenant/revocationprobe0123456789', { authenticated: false, headers:
|
|
68
|
+
const after = await http(() => driver.get('/runs/openwop-conformance-tenant/revocationprobe0123456789', { authenticated: false, headers: present(credential) }));
|
|
62
69
|
expect(after?.status, req('openwop.requirement.0170.revocation-honored', DOC, `lane ${lane} MUST refuse a revoked credential on the next request with 401`)).toBe(401);
|
|
63
70
|
expect(readErrorCode(after?.json), req('openwop.requirement.0170.revocation-honored', DOC, `the refusal code MUST be credential_revoked (lane ${lane})`)).toBe('credential_revoked');
|
|
64
71
|
}
|
|
@@ -39,6 +39,8 @@ import { req } from '../lib/requirement-ids.js';
|
|
|
39
39
|
import { startEffectReceiver, waitForFirstArrival } from '../lib/effect-receiver.js';
|
|
40
40
|
import { terminalShapeViolation, TERMINAL_RUN_EVENTS } from '../lib/terminal-shape.js';
|
|
41
41
|
|
|
42
|
+
export const REQUIRES_HOST_CALLBACK = 'the host makes an outbound effect call to the suite-owned effect receiver (OPENWOP_WEBHOOK_RECEIVER_PORT)';
|
|
43
|
+
|
|
42
44
|
const ONCE = 'openwop.requirement.0194.terminal-once';
|
|
43
45
|
const DUP = 'openwop.requirement.0194.terminal-once.duplicate-delivery';
|
|
44
46
|
const DOC = 'spec/v2/core/events.md §The terminal event (RFC 0194 §A)';
|
|
@@ -55,6 +55,8 @@ import { blockedDespiteAssertions, softSkip } from '../lib/soft-skip.js';
|
|
|
55
55
|
import { req } from '../lib/requirement-ids.js';
|
|
56
56
|
import { RETRY_WAIT_FLOOR_MS } from '../lib/webhook-retry-window.js';
|
|
57
57
|
|
|
58
|
+
export const REQUIRES_HOST_CALLBACK = 'the host POSTs webhook deliveries to nine suite-owned subscriptions on the scoped receiver behind OPENWOP_WEBHOOK_RECEIVER_URL, eight of which the suite holds open';
|
|
59
|
+
|
|
58
60
|
const ID = 'openwop.requirement.0215.no-head-of-line';
|
|
59
61
|
const DOC = 'webhooks.md §Durability (RFC 0215 §A.1, §A.2)';
|
|
60
62
|
const FIXTURE = 'conformance-delay';
|