@openwop/openwop-conformance 2.46.44 → 2.48.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,244 @@
1
+ /**
2
+ * v2 — compensation behavior, witnessed through a failing fixture (RFC 0265;
3
+ * the rules are `spec/v2/core/security-defaults.md` §Compensation, the
4
+ * `compensation.*` events in `api/v2/asyncapi.yaml` and
5
+ * `schemas/v2/compensation-projection.schema.json`).
6
+ *
7
+ * `v2-compensation-read-projection` reads the projection on a run with nothing
8
+ * to unwind. This scenario makes a host unwind. The `conformance-compensate-unwind`
9
+ * fixture runs three compensable effects (`e1 → e2 → e3`, the conformance-RESERVED
10
+ * `core.conformance.compensable-effect`) and then a node that fails, under a
11
+ * `reverse-completion` policy triggered on `node-failure`. The suite causes that
12
+ * condition unaided and reads the result through normative paths only — the
13
+ * run's event log, `GET /runs/{runId}/compensation` and `GET /runs/{runId}/effects`.
14
+ * No seam is involved: the v1 legs (`compensation-behavior`, `compensation-recovery`)
15
+ * asked a host seam how many effects re-fired, and a host that never compensates
16
+ * can answer 0.
17
+ *
18
+ * One run is shared by every leg (a module-level lazy promise); judges are in
19
+ * `lib/compensation-unwind.ts`.
20
+ * 1. `0265.plan-before-inverse` — `compensation.requested` precedes the first
21
+ * `compensation.started` in the log (asyncapi: "MUST follow").
22
+ * 2. `0265.reverse-completion` — the projection's `attempts[]` names `e3, e2, e1`,
23
+ * the reverse of the forward `node.completed` order.
24
+ * 3. `0265.inverse-once` — the effect ledger holds exactly three inverse
25
+ * records, one per forward effect, each performed once.
26
+ * 4. `0265.projection-agrees` — the projection's `status` is `completed` exactly
27
+ * when the log holds `compensation.completed`, and `plan[]` names the three
28
+ * effect nodes.
29
+ * 5. `0265.replay-no-refire` — a `mode: "replay"` fork adds no newly performed
30
+ * effect to the fork's ledger (invariant `compensation-replay-no-refire`).
31
+ *
32
+ * Dispositions: `compensation` not advertised, or the fixture not listed ⇒
33
+ * `inapplicable` (every leg). Legs 3 and 5 also need the effect ledger, which
34
+ * idempotency.md §Witness owes only under `idempotency`; leg 5 needs
35
+ * `replay.modes ∋ replay` — absent ⇒ `inapplicable`. A run that cannot be
36
+ * created or finished, a fixture whose effects did not all complete or whose
37
+ * run did not fail, an unreadable projection or ledger, or a ledger with no
38
+ * forward record for an effect node (RFC 0265 R1: the host did not map the
39
+ * reserved node to a ledgered effect) ⇒ `blocked`.
40
+ *
41
+ * @see RFCS/0265-compensation-unwind-witness.md
42
+ * @see spec/v2/core/security-defaults.md §Compensation
43
+ * @see spec/v2/core/replay.md §Suppression
44
+ * @see conformance/fixtures.md `conformance-compensate-unwind`
45
+ */
46
+
47
+ import { describe, it, expect } from 'vitest';
48
+ import { driver, type OpenWOPResponse } from '../lib/driver.js';
49
+ import { v2Discovery, gateFamily, familyAdvertised } from '../lib/v2.js';
50
+ import { isFixtureAdvertised } from '../lib/fixtures.js';
51
+ import { projectBoundId } from '../lib/bound-id.js';
52
+ import { readErrorCode } from '../lib/error-envelope.js';
53
+ import { softSkip } from '../lib/soft-skip.js';
54
+ import { req } from '../lib/requirement-ids.js';
55
+ import { scaledTimeoutMs } from '../lib/polling.js';
56
+ import {
57
+ EFFECT_NODES, COMPENSATION_TERMINAL,
58
+ forwardCompletionOrder, judgePlanBeforeInverse, judgeReverseCompletion, splitLedger,
59
+ judgeInverseOnce, judgeProjectionAgrees, refiredOnFork,
60
+ type Attempt, type Ev, type Finding, type LedgerRow, type Projection,
61
+ } from '../lib/compensation-unwind.js';
62
+
63
+ const FIXTURE = 'conformance-compensate-unwind';
64
+ const DOC = 'security-defaults.md §Compensation; asyncapi compensation.*; compensation-projection.schema.json (RFC 0265)';
65
+ const ID_PLAN = 'openwop.requirement.0265.plan-before-inverse';
66
+ const ID_ORDER = 'openwop.requirement.0265.reverse-completion';
67
+ const ID_ONCE = 'openwop.requirement.0265.inverse-once';
68
+ const ID_AGREES = 'openwop.requirement.0265.projection-agrees';
69
+ const ID_REPLAY = 'openwop.requirement.0265.replay-no-refire';
70
+ const RUN_TERMINAL = /^run\.(completed|failed|cancelled)$/;
71
+
72
+ type Skip = readonly ['inapplicable' | 'blocked', string];
73
+ type Rec = Record<string, unknown>;
74
+ const isRec = (v: unknown): v is Rec => typeof v === 'object' && v !== null && !Array.isArray(v);
75
+ async function http(fn: () => Promise<OpenWOPResponse>): Promise<OpenWOPResponse | null> { try { return await fn(); } catch { return null; } }
76
+ const assertAll = (id: string, fs: readonly Finding[]): void => { for (const x of fs) expect(x.ok, req(id, DOC, x.message)).toBe(true); };
77
+ const seqOf = (e: Ev): number => (typeof e.sequence === 'number' ? e.sequence : Number.NaN);
78
+
79
+ async function logOf(runId: string): Promise<Ev[]> {
80
+ const res = await http(() => driver.get(`/runs/${projectBoundId(runId)}/events/poll?timeout=1`));
81
+ const body = res?.json;
82
+ const events = isRec(body) ? body['events'] : undefined;
83
+ return Array.isArray(events) ? (events as Ev[]).filter((e) => Number.isFinite(seqOf(e))).sort((a, b) => seqOf(a) - seqOf(b)) : [];
84
+ }
85
+
86
+ /**
87
+ * Poll until the run has a terminal run event and, if an unwind began (or, for
88
+ * the source run, is owed), a compensation terminal event. The compensation
89
+ * tail MAY follow the run's terminal event (replay.md §Refusals), so the run
90
+ * event alone is not the end. On the deadline the log is returned as it stands:
91
+ * an unwind that never finishes is for the legs to judge, not a reason to skip.
92
+ */
93
+ async function settle(runId: string, unwindOwed: boolean): Promise<{ log: Ev[] } | { reason: string }> {
94
+ const deadline = Date.now() + scaledTimeoutMs(30_000);
95
+ for (;;) {
96
+ const log = await logOf(runId);
97
+ const ended = log.some((e) => typeof e.type === 'string' && RUN_TERMINAL.test(e.type));
98
+ const begun = unwindOwed || log.some((e) => e.type === 'compensation.requested' || e.type === 'compensation.started');
99
+ const unwound = log.some((e) => typeof e.type === 'string' && COMPENSATION_TERMINAL.has(e.type));
100
+ if (ended && (!begun || unwound)) return { log };
101
+ if (Date.now() > deadline) return ended ? { log } : { reason: `run ${runId} logged no terminal run event within the bound (${log.length} event(s))` };
102
+ await new Promise((r) => setTimeout(r, 250));
103
+ }
104
+ }
105
+
106
+ let disc: Promise<Rec | null> | undefined;
107
+ const discovery = (): Promise<Rec | null> => (disc ??= v2Discovery().catch(() => null));
108
+
109
+ /** The two gates every leg shares: the `compensation` record and the fixture. */
110
+ async function gate(): Promise<Skip | null> {
111
+ if (!(await discovery())) return ['blocked', 'v2 discovery unreachable — /.well-known/openwop did not answer 200 JSON under OpenWOP-Version: 2.0'];
112
+ if (!(await gateFamily('compensation'))) return ['inapplicable', 'compensation family not advertised — no obligation (gate recorded under openwop.family.compensation)'];
113
+ if (!isFixtureAdvertised(FIXTURE)) return ['inapplicable', `the host does not advertise fixture ${FIXTURE} (serving it is OPTIONAL, RFC 0265 §A)`];
114
+ return null;
115
+ }
116
+ async function ledgerGate(): Promise<Skip | null> {
117
+ if (!(await familyAdvertised('idempotency'))) return ['inapplicable', 'the host does not advertise idempotency, so it owes no effect ledger (idempotency.md §Witness) and the inverse records cannot be read'];
118
+ return null;
119
+ }
120
+
121
+ interface Unwind { readonly runId: string; readonly log: Ev[]; readonly projection: OpenWOPResponse | null }
122
+
123
+ /** The shared source run: start the fixture, wait out the run and its unwind, read the projection. Asserts nothing. */
124
+ async function drive(): Promise<{ skip: Skip } | { u: Unwind }> {
125
+ const created = await http(() => driver.post('/runs', { workflowId: FIXTURE }));
126
+ const createdBody = created?.json;
127
+ const runId = isRec(createdBody) ? createdBody['runId'] : undefined;
128
+ if (created === null || created.status !== 201 || typeof runId !== 'string') return { skip: ['blocked', `POST /runs {workflowId: ${FIXTURE}} answered ${created?.status ?? 'nothing'} ${readErrorCode(created?.json) ?? ''}`.trim()] };
129
+ const s = await settle(runId, true);
130
+ if ('reason' in s) return { skip: ['blocked', s.reason] };
131
+ if (!s.log.some((e) => e.type === 'run.failed')) {
132
+ return { skip: ['blocked', `the ${FIXTURE} run did not end run.failed (its terminal node is core.fail), so the condition an unwind exists for was not caused — the host does not serve the fixture as fixtures.md specifies`] };
133
+ }
134
+ const projection = await http(() => driver.get(`/runs/${projectBoundId(runId)}/compensation`));
135
+ return { u: { runId, log: s.log, projection } };
136
+ }
137
+ let source: Promise<{ skip: Skip } | { u: Unwind }> | undefined;
138
+ const once = (): Promise<{ skip: Skip } | { u: Unwind }> => (source ??= drive());
139
+
140
+ /** The shared source run's effect ledger, read once after the unwind. */
141
+ let ledger: Promise<{ skip: Skip } | { rows: LedgerRow[] }> | undefined;
142
+ const sourceLedger = (runId: string): Promise<{ skip: Skip } | { rows: LedgerRow[] }> => (ledger ??= (async () => {
143
+ const res = await http(() => driver.get(`/runs/${projectBoundId(runId)}/effects`));
144
+ const body = res?.json;
145
+ const effects = isRec(body) ? body['effects'] : undefined;
146
+ if (res === null || res.status !== 200 || !Array.isArray(effects)) {
147
+ return { skip: ['blocked', `GET /runs/{runId}/effects answered ${res?.status ?? 'nothing'} ${readErrorCode(res?.json) ?? ''} — idempotency.md §Witness owns that contract`.trim()] as const };
148
+ }
149
+ return { rows: effects as LedgerRow[] };
150
+ })());
151
+
152
+ function projectionOf(u: Unwind): { body: Projection } | { skip: Skip } {
153
+ const p = u.projection;
154
+ const body = p?.json;
155
+ if (p === null || p.status !== 200 || !isRec(body)) {
156
+ return { skip: ['blocked', `GET /runs/{runId}/compensation answered ${p?.status ?? 'nothing'} ${readErrorCode(body) ?? ''} on the unwound run — v2-compensation-read-projection owns that obligation`.trim()] };
157
+ }
158
+ return { body };
159
+ }
160
+
161
+ describe('v2 RFC 0265 — compensation unwind through a failing fixture (host)', () => {
162
+ it('compensation.requested precedes the first compensation.started', async () => {
163
+ const g = await gate();
164
+ if (g) return softSkip(...g);
165
+ const s = await once();
166
+ if ('skip' in s) return softSkip(...s.skip);
167
+ assertAll(ID_PLAN, judgePlanBeforeInverse(s.u.log));
168
+ }, 120_000);
169
+
170
+ it('the projection attempts the effect nodes in the reverse of their forward completion order', async () => {
171
+ const g = await gate();
172
+ if (g) return softSkip(...g);
173
+ const s = await once();
174
+ if ('skip' in s) return softSkip(...s.skip);
175
+ const forward = forwardCompletionOrder(s.u.log);
176
+ if (forward.length !== EFFECT_NODES.length) {
177
+ return softSkip('blocked', `the log holds a forward node.completed for ${forward.join(', ') || 'none'} of ${EFFECT_NODES.join(', ')}, so there is no forward completion order to reverse`);
178
+ }
179
+ const p = projectionOf(s.u);
180
+ if ('skip' in p) return softSkip(...p.skip);
181
+ const attempts = Array.isArray(p.body.attempts) ? (p.body.attempts as Attempt[]) : [];
182
+ assertAll(ID_ORDER, judgeReverseCompletion(forward, attempts));
183
+ }, 120_000);
184
+
185
+ it('the effect ledger holds exactly one inverse record per forward effect', async () => {
186
+ const g = (await gate()) ?? (await ledgerGate());
187
+ if (g) return softSkip(...g);
188
+ const s = await once();
189
+ if ('skip' in s) return softSkip(...s.skip);
190
+ const l = await sourceLedger(s.u.runId);
191
+ if ('skip' in l) return softSkip(...l.skip);
192
+ const split = splitLedger(l.rows);
193
+ if (split.missingForward.length > 0) {
194
+ return softSkip('blocked', `the effect ledger has no record for ${split.missingForward.join(', ')}: the forward effects are absent too, so the host did not map core.conformance.compensable-effect to a ledgered effect (RFC 0265 §A, risk R1) and the inverse count cannot be measured`);
195
+ }
196
+ assertAll(ID_ONCE, judgeInverseOnce(split));
197
+ }, 120_000);
198
+
199
+ it('the projection status and plan agree with the event log', async () => {
200
+ const g = await gate();
201
+ if (g) return softSkip(...g);
202
+ const s = await once();
203
+ if ('skip' in s) return softSkip(...s.skip);
204
+ const p = projectionOf(s.u);
205
+ if ('skip' in p) return softSkip(...p.skip);
206
+ assertAll(ID_AGREES, judgeProjectionAgrees(p.body, s.u.log));
207
+ }, 120_000);
208
+
209
+ it('a replay fork of the unwound run performs no inverse again', async () => {
210
+ const g = (await gate()) ?? (await ledgerGate());
211
+ if (g) return softSkip(...g);
212
+ const replay = await familyAdvertised('replay');
213
+ if (!replay || !(Array.isArray(replay['modes']) && replay['modes'].includes('replay'))) return softSkip('inapplicable', 'replay.modes does not list replay, so no replay fork can be requested');
214
+ const s = await once();
215
+ if ('skip' in s) return softSkip(...s.skip);
216
+ const l = await sourceLedger(s.u.runId);
217
+ if ('skip' in l) return softSkip(...l.skip);
218
+ const split = splitLedger(l.rows);
219
+ if (split.missingForward.length > 0 || split.inverse.length === 0) {
220
+ return softSkip('blocked', `the source run's ledger records ${split.missingForward.length > 0 ? `no forward effect for ${split.missingForward.join(', ')}` : 'no inverse effect'}, so a fork that also records none would compare nothing against nothing (RFC 0265 R1)`);
221
+ }
222
+ const first = s.u.log[0];
223
+ if (first === undefined) return softSkip('blocked', 'the source run logged no event to fork from');
224
+ const forked = await http(() => driver.post(`/runs/${projectBoundId(s.u.runId)}:fork`, { mode: 'replay', fromSeq: first.sequence }));
225
+ const forkedBody = forked?.json;
226
+ const forkId = isRec(forkedBody) ? forkedBody['runId'] : undefined;
227
+ if (forked === null || forked.status !== 201 || typeof forkId !== 'string') {
228
+ return softSkip('blocked', `POST /runs/{runId}:fork {mode: replay} answered ${forked?.status ?? 'nothing'} ${readErrorCode(forked?.json) ?? ''} on the unwound run`.trim());
229
+ }
230
+ const settled = await settle(forkId, false);
231
+ if ('reason' in settled) return softSkip('blocked', settled.reason);
232
+ const forkLedger = await http(() => driver.get(`/runs/${projectBoundId(forkId)}/effects`));
233
+ const forkLedgerBody = forkLedger?.json;
234
+ const forkRows = isRec(forkLedgerBody) ? forkLedgerBody['effects'] : undefined;
235
+ if (forkLedger === null || forkLedger.status !== 200 || !Array.isArray(forkRows)) {
236
+ return softSkip('blocked', `GET /runs/{runId}/effects answered ${forkLedger?.status ?? 'nothing'} on the replay fork`);
237
+ }
238
+ const refired = refiredOnFork(l.rows, forkRows as LedgerRow[]);
239
+ expect(
240
+ refired.length,
241
+ req(ID_REPLAY, 'replay.md §Suppression; invariant compensation-replay-no-refire (RFC 0265 §B.5)', `a mode: "replay" fork MUST NOT perform an inverse action again — the fork's effect ledger records ${refired.length} effect(s) the source run never made: ${JSON.stringify(refired).slice(0, 300)}`),
242
+ ).toBe(0);
243
+ }, 180_000);
244
+ });
@@ -0,0 +1,160 @@
1
+ /**
2
+ * v2 — LLM cache key recipe 2 (RFC 0260; the rules are the
3
+ * `replay.llmCacheKeyRecipe` facet description in `spec/v2/facets/replay.schema.json`
4
+ * and the `cacheKey` description in `schemas/v2/effect-ledger-projection.schema.json`).
5
+ *
6
+ * Server-free (always on):
7
+ * 1. `0260.shape` — the effect-ledger record admits `cacheKey: {recipe, digest}`
8
+ * with a closed recipe vocabulary and a 64-hex digest, and refuses anything else.
9
+ *
10
+ * Host legs, unaided through the `conformance-llm-cache-key` fixture: its node makes
11
+ * one `ctx.callAI` call through the host's conformance mock with the run's
12
+ * `inputs.request` (prompt and params) unchanged. The suite reads the call's record
13
+ * on `GET /runs/{runId}/effects` (idempotency.md §Witness), which is a normative
14
+ * observation path, so no seam is needed:
15
+ * 2. `0260.key-recorded` — the record carries `cacheKey` with recipe `openwop-llm-key-2`
16
+ * 3. `0260.params-in-key` — runs differing only in `maxTokens` have different digests
17
+ * 4. `0260.key-stable` — two identical runs have the same digest
18
+ * 5. `0260.key-portable` — the digest equals recipe 2 recomputed by the suite
19
+ * (lib/llm-cache-key-2.ts) over the inputs the fixture pins
20
+ *
21
+ * Dispositions: `replay.llmCacheKeyRecipe` not advertised, `idempotency` absent, or the
22
+ * fixture not listed ⇒ `inapplicable`. A run that cannot be created, read or finished,
23
+ * or a ledger with no record for the node ⇒ `blocked`.
24
+ *
25
+ * @see RFCS/0260-llm-cache-key-recipe-2.md
26
+ * @see spec/v2/core/replay.md §Suppression (the cache key for LLM-calling nodes)
27
+ * @see conformance/fixtures.md §The LLM cache-key fixture
28
+ */
29
+
30
+ import { describe, it, expect } from 'vitest';
31
+ import { driver, type OpenWOPResponse } from '../lib/driver.js';
32
+ import { v2Discovery, v2RefValidator } from '../lib/v2.js';
33
+ import { isFixtureAdvertised } from '../lib/fixtures.js';
34
+ import { projectBoundId } from '../lib/bound-id.js';
35
+ import { readErrorCode } from '../lib/error-envelope.js';
36
+ import { softSkip } from '../lib/soft-skip.js';
37
+ import { req } from '../lib/requirement-ids.js';
38
+ import { scaledTimeoutMs } from '../lib/polling.js';
39
+ import {
40
+ RECIPE_2, FIXTURE, NODE_ID, fixtureInputs, readLedgerKey,
41
+ judgeRecorded, judgeParamsInKey, judgeStable, judgePortable, type Finding, type LedgerKey, type NamedParam,
42
+ } from '../lib/llm-cache-key-2.js';
43
+
44
+ const DOC = 'replay facet llmCacheKeyRecipe + effect-ledger-projection cacheKey (RFC 0260)';
45
+ const R = (slug: string): string => `openwop.requirement.0260.${slug}`;
46
+ const TERMINAL = new Set(['completed', 'failed', 'cancelled']);
47
+ const PROMPT = 'Say ok.';
48
+ const PARAMS_A: Partial<Record<NamedParam, unknown>> = { maxTokens: 16, temperature: 0 };
49
+ const PARAMS_B: Partial<Record<NamedParam, unknown>> = { maxTokens: 32, temperature: 0 };
50
+
51
+ type Skip = readonly ['inapplicable' | 'blocked', string];
52
+ const isRec = (v: unknown): v is Record<string, unknown> => typeof v === 'object' && v !== null && !Array.isArray(v);
53
+ async function http(fn: () => Promise<OpenWOPResponse>): Promise<OpenWOPResponse | null> { try { return await fn(); } catch { return null; } }
54
+ const assertAll = (id: string, fs: readonly Finding[]): void => { for (const x of fs) expect(x.ok, req(id, DOC, x.message)).toBe(true); };
55
+
56
+ let disc: Promise<Record<string, unknown> | null> | undefined;
57
+ const discovery = (): Promise<Record<string, unknown> | null> => (disc ??= v2Discovery().catch(() => null));
58
+
59
+ async function gate(): Promise<Skip | null> {
60
+ const doc = await discovery();
61
+ if (!doc) return ['blocked', 'v2 discovery unreachable — /.well-known/openwop did not answer 200 JSON under OpenWOP-Version: 2.0'];
62
+ const replay = doc['replay'];
63
+ if (!isRec(replay) || replay['llmCacheKeyRecipe'] !== RECIPE_2) return ['inapplicable', `the host does not advertise replay.llmCacheKeyRecipe "${RECIPE_2}" (absent ⇒ the RFC 0041 recipe, opt-in per RFC 0260)`];
64
+ if (!isRec(doc['idempotency'])) return ['inapplicable', 'the host does not advertise idempotency, so it serves no effect ledger to read the key from'];
65
+ if (!isFixtureAdvertised(FIXTURE)) return ['inapplicable', `the host does not advertise fixture ${FIXTURE}`];
66
+ return null;
67
+ }
68
+
69
+ async function terminal(runId: string): Promise<string | null> {
70
+ const deadline = Date.now() + scaledTimeoutMs(30_000);
71
+ for (;;) {
72
+ const snap = await http(() => driver.get(`/runs/${projectBoundId(runId)}`));
73
+ const status = String((snap?.json as { status?: unknown } | undefined)?.status ?? '');
74
+ if (TERMINAL.has(status)) return status;
75
+ if (Date.now() > deadline) return null;
76
+ await new Promise((r) => setTimeout(r, 250));
77
+ }
78
+ }
79
+
80
+ /** One fixture run with these params, and the key its provider-call record carries. Asserts nothing. */
81
+ async function keyed(params: Partial<Record<NamedParam, unknown>>): Promise<{ skip: Skip } | { key: LedgerKey | undefined }> {
82
+ const created = await http(() => driver.post('/runs', { workflowId: FIXTURE, inputs: { request: { prompt: PROMPT, params } } }));
83
+ const runId = (created?.json as { runId?: unknown } | undefined)?.runId;
84
+ if (created === null || created.status !== 201 || typeof runId !== 'string') return { skip: ['blocked', `POST /runs {workflowId: ${FIXTURE}} answered ${created?.status ?? 'nothing'} ${readErrorCode(created?.json) ?? ''}`.trim()] };
85
+ const status = await terminal(runId);
86
+ if (status !== 'completed') return { skip: ['blocked', `the ${FIXTURE} run ended ${status ?? 'not at all within the timeout'}, so its provider call is not on record`] };
87
+ const ledger = await http(() => driver.get(`/runs/${projectBoundId(runId)}/effects`));
88
+ if (ledger === null || ledger.status !== 200) return { skip: ['blocked', `GET /runs/{runId}/effects answered ${ledger?.status ?? 'nothing'} — idempotency.md §Witness owns that contract`] };
89
+ const key = readLedgerKey(ledger.json, NODE_ID);
90
+ if (key === 'no-record') return { skip: ['blocked', `the effect ledger has no record for node ${NODE_ID}: the host did not record the provider call as an effect (idempotency.md §Composition)`] };
91
+ return { key };
92
+ }
93
+
94
+ const memo = new Map<string, ReturnType<typeof keyed>>();
95
+ const once = (name: 'a' | 'a2' | 'b'): ReturnType<typeof keyed> => {
96
+ let p = memo.get(name);
97
+ if (!p) { p = keyed(name === 'b' ? PARAMS_B : PARAMS_A); memo.set(name, p); }
98
+ return p;
99
+ };
100
+
101
+ describe('v2 RFC 0260 — cacheKey wire shape (server-free)', () => {
102
+ it('the effect-ledger record admits cacheKey {recipe, digest} and refuses anything else', () => {
103
+ const v = v2RefValidator('effect-ledger-projection.schema.json');
104
+ const base = { effectId: 'acme/0123456789abcdef', nodeId: 'llm-call', attempt: 1, keying: 'activity-recipe', state: 'completed', at: '2026-11-13T00:00:00Z' };
105
+ const doc = (rec: Record<string, unknown>) => ({ runId: 'acme/0123456789abcdef', effects: [rec] });
106
+ const digest = 'f1714d7aa2abf5994dd375f66aac14514b1c2439f203a6171dc6053c59f54402';
107
+ expect(v(doc(base)).ok, req(R('shape'), DOC, 'a record without cacheKey MUST validate (the facet is opt-in)')).toBe(true);
108
+ expect(v(doc({ ...base, cacheKey: { recipe: RECIPE_2, digest } })).ok, req(R('shape'), DOC, 'cacheKey {recipe: openwop-llm-key-2, digest: 64 hex} MUST validate')).toBe(true);
109
+ expect(v(doc({ ...base, cacheKey: { recipe: 'spec-rfc-0041', digest } })).ok, req(R('shape'), DOC, 'an unregistered recipe MUST fail')).toBe(false);
110
+ expect(v(doc({ ...base, cacheKey: { recipe: RECIPE_2, digest: digest.toUpperCase() } })).ok, req(R('shape'), DOC, 'an uppercase or short digest MUST fail')).toBe(false);
111
+ expect(v(doc({ ...base, cacheKey: { recipe: RECIPE_2 } })).ok, req(R('shape'), DOC, 'digest is REQUIRED')).toBe(false);
112
+ expect(v(doc({ ...base, cacheKey: { recipe: RECIPE_2, digest, preimage: '{}' } })).ok, req(R('shape'), DOC, 'cacheKey is closed: a preimage member MUST fail')).toBe(false);
113
+ });
114
+ });
115
+
116
+ describe('v2 RFC 0260 — recipe 2 on the effect ledger (host)', () => {
117
+ it('a host advertising replay.llmCacheKeyRecipe also advertises idempotency', async () => {
118
+ const doc = await discovery();
119
+ if (!doc) return softSkip('blocked', 'v2 discovery unreachable — /.well-known/openwop did not answer 200 JSON under OpenWOP-Version: 2.0');
120
+ const replay = doc['replay'];
121
+ if (!isRec(replay) || replay['llmCacheKeyRecipe'] === undefined) return softSkip('inapplicable', 'the host does not advertise replay.llmCacheKeyRecipe');
122
+ expect(isRec(doc['idempotency']), req(R('implies-idempotency'), DOC, 'a host advertising replay.llmCacheKeyRecipe MUST advertise idempotency: the effect ledger is where the key is recorded')).toBe(true);
123
+ });
124
+
125
+ it('the provider-call record carries cacheKey with recipe openwop-llm-key-2', async () => {
126
+ const g = await gate();
127
+ if (g) return softSkip(...g);
128
+ const a = await once('a');
129
+ if ('skip' in a) return softSkip(...a.skip);
130
+ assertAll(R('key-recorded'), judgeRecorded(a.key));
131
+ }, 120_000);
132
+
133
+ it('two runs that differ only in maxTokens have different keys', async () => {
134
+ const g = await gate();
135
+ if (g) return softSkip(...g);
136
+ const a = await once('a');
137
+ if ('skip' in a) return softSkip(...a.skip);
138
+ const b = await once('b');
139
+ if ('skip' in b) return softSkip(...b.skip);
140
+ assertAll(R('params-in-key'), judgeParamsInKey(a.key, b.key));
141
+ }, 120_000);
142
+
143
+ it('two identical runs have the same key', async () => {
144
+ const g = await gate();
145
+ if (g) return softSkip(...g);
146
+ const a = await once('a');
147
+ if ('skip' in a) return softSkip(...a.skip);
148
+ const a2 = await once('a2');
149
+ if ('skip' in a2) return softSkip(...a2.skip);
150
+ assertAll(R('key-stable'), judgeStable(a.key, a2.key));
151
+ }, 120_000);
152
+
153
+ it('the key equals recipe 2 recomputed over the inputs the fixture pins', async () => {
154
+ const g = await gate();
155
+ if (g) return softSkip(...g);
156
+ const a = await once('a');
157
+ if ('skip' in a) return softSkip(...a.skip);
158
+ assertAll(R('key-portable'), judgePortable(a.key, fixtureInputs(PROMPT, PARAMS_A)));
159
+ }, 120_000);
160
+ });
@@ -0,0 +1,181 @@
1
+ /**
2
+ * v2 — the `telemetry` family: a host exports OTLP traces (RFC 0262; the rules
3
+ * are `spec/v2/core/telemetry.md`, the facets `spec/v2/facets/telemetry.schema.json`).
4
+ *
5
+ * Server-free (always on):
6
+ * 1. `0262.shape` — the generated `telemetry` record requires `otlp` with a
7
+ * non-empty `protocols` from the closed transport vocabulary; the optional
8
+ * `genAiSemconv` is a semver string.
9
+ *
10
+ * Host legs — gated on the `telemetry` family record AND on the suite's OTLP
11
+ * collector (`OPENWOP_OTEL_COLLECTOR=true`, the host's exporter pointed at it
12
+ * with `OTEL_EXPORTER_OTLP_ENDPOINT`). The host calls the suite, so a host in
13
+ * another network namespace records `blocked`, not a failure.
14
+ * 2. `0262.run-span` — a `conformance-noop` run exports an `openwop.run` span
15
+ * carrying its `openwop.run_id` and `openwop.workflow_id`.
16
+ * 3. `0262.node-span` — the same run exports an `openwop.node.<typeId>` span
17
+ * carrying `openwop.node_id`, `openwop.node_type` and a non-negative
18
+ * `openwop.node_attempt`.
19
+ * 4. `0262.traceparent-continued` — a run created with a suite-minted
20
+ * `traceparent` exports its spans under that `traceId`.
21
+ * 5. `0262.no-secret-in-spans` — after an `openwop-smoke-byok-roundtrip` run,
22
+ * no captured span name, attribute or resource attribute carries the store
23
+ * canary (digest detector, plus the plaintext when the operator supplies
24
+ * `OPENWOP_CANARY_SECRET_VALUE`; `lib/secret-scan.ts`).
25
+ * 6. `0262.gen-ai-attributes` — gated on `telemetry.genAiSemconv` and the
26
+ * `conformance-callai-attempt-accounting` fixture: the model-call span
27
+ * carries `gen_ai.operation.name`, a provider attribute, `gen_ai.request.model`
28
+ * and both usage-token attributes.
29
+ *
30
+ * A host leg whose run exported NO span records `blocked`: an empty export is
31
+ * not a pass, so a host that advertises the family and exports nothing cannot
32
+ * certify (RFC 0262 R2).
33
+ *
34
+ * @see RFCS/0262-telemetry-family.md
35
+ * @see spec/v2/core/telemetry.md
36
+ */
37
+
38
+ import { describe, it, expect } from 'vitest';
39
+ import { driver, type OpenWOPResponse } from '../lib/driver.js';
40
+ import { familyAdvertised, v2Discovery, v2RefValidator } from '../lib/v2.js';
41
+ import { isFixtureAdvertised } from '../lib/fixtures.js';
42
+ import { readErrorCode } from '../lib/error-envelope.js';
43
+ import { pollUntilTerminal } from '../lib/polling.js';
44
+ import { softSkip } from '../lib/soft-skip.js';
45
+ import { req } from '../lib/requirement-ids.js';
46
+ import { getCollector, waitForRunSpans, type CapturedSpan } from '../lib/otel-collector.js';
47
+ import { digestDetector, findSecretDigest, leaks, valueDetector, type SecretDetector, type Surface } from '../lib/secret-scan.js';
48
+
49
+ /**
50
+ * Callback-shaped: the host exports OTLP to the suite's collector.
51
+ * Unwitnessable when the host is in a separate network namespace — see
52
+ * `../lib/host-callback.ts`. Not host non-conformance.
53
+ */
54
+ export const REQUIRES_HOST_CALLBACK = "the host exports OTLP traces to the suite's collector";
55
+
56
+ const DOC = 'spec/v2/core/telemetry.md (RFC 0262)';
57
+ const R = (slug: string): string => `openwop.requirement.0262.${slug}`;
58
+ const NOOP = 'conformance-noop';
59
+ const BYOK = 'openwop-smoke-byok-roundtrip';
60
+ const CALLAI = 'conformance-callai-attempt-accounting';
61
+
62
+ type Skip = { skip: ['inapplicable' | 'blocked', string] };
63
+ const skipped = (s: Skip): undefined => softSkip(s.skip[0], s.skip[1]);
64
+ const isRec = (v: unknown): v is Record<string, unknown> => typeof v === 'object' && v !== null && !Array.isArray(v);
65
+ async function http(fn: () => Promise<OpenWOPResponse>): Promise<OpenWOPResponse | null> { try { return await fn(); } catch { return null; } }
66
+
67
+ /** The family record, or why the host legs do not bind. */
68
+ async function gate(fixture: string): Promise<Skip | { record: Record<string, unknown> }> {
69
+ if (!(await v2Discovery())) return { skip: ['blocked', 'v2 discovery unreachable — /.well-known/openwop did not answer 200 under OpenWOP-Version: 2.0'] };
70
+ const record = await familyAdvertised('telemetry');
71
+ if (record === null) return { skip: ['inapplicable', 'the telemetry family is not advertised — no export claim'] };
72
+ if (!getCollector()) return { skip: ['blocked', 'the suite OTLP collector is not running (set OPENWOP_OTEL_COLLECTOR=true and point the host exporter at it)'] };
73
+ if (!isFixtureAdvertised(fixture)) return { skip: ['inapplicable', `fixture ${fixture} is not advertised`] };
74
+ return { record };
75
+ }
76
+
77
+ /** Start a run, wait for it to settle, then wait for its exported spans. */
78
+ async function exportedRun(body: Record<string, unknown>, headers?: Record<string, string>): Promise<Skip | { runId: string; created: OpenWOPResponse; spans: readonly CapturedSpan[] }> {
79
+ const created = await http(() => driver.post('/runs', body, headers ? { headers } : {}));
80
+ const runId = isRec(created?.json) ? created.json['runId'] : undefined;
81
+ if (created === null || created.status !== 201 || typeof runId !== 'string') return { skip: ['blocked', `POST /runs (${String(body['workflowId'])}) answered ${created?.status ?? 'nothing'} ${readErrorCode(created?.json) ?? ''}`.trim()] };
82
+ try { await pollUntilTerminal(runId, { timeoutMs: 20_000 }); } catch { return { skip: ['blocked', `run ${runId} did not settle within 20 s`] }; }
83
+ const spans = await waitForRunSpans(runId, { timeoutMs: 8_000, minCount: 2 });
84
+ if (spans.length === 0) return { skip: ['blocked', `the collector captured no span carrying openwop.run_id ${runId} — the host advertised telemetry but nothing arrived (an empty export is not a pass)`] };
85
+ return { runId, created, spans };
86
+ }
87
+
88
+ let noopRun: ReturnType<typeof exportedRun> | undefined;
89
+ const noop = (): ReturnType<typeof exportedRun> => {
90
+ if (!noopRun) { getCollector()?.reset(); noopRun = exportedRun({ workflowId: NOOP }); }
91
+ return noopRun;
92
+ };
93
+
94
+ function spanSurfaces(spans: readonly CapturedSpan[]): Surface[] {
95
+ return spans.map((s) => ({
96
+ name: `span ${s.name}`,
97
+ text: JSON.stringify({ name: s.name, attributes: Object.fromEntries(s.attributes), resource: Object.fromEntries(s.resourceAttributes) }),
98
+ }));
99
+ }
100
+
101
+ describe('v2 RFC 0262 — telemetry facet shape (server-free)', () => {
102
+ it('the telemetry record requires otlp.protocols from the closed transports; genAiSemconv is semver', () => {
103
+ const v = v2RefValidator('capabilities.schema.json#/properties/telemetry');
104
+ const base = { status: 'experimental', since: '2.0', witness: 'witnessable-gated' };
105
+ expect(v({ ...base, otlp: { protocols: ['http/protobuf'] } }).ok, req(R('shape'), DOC, 'a record with otlp.protocols MUST validate')).toBe(true);
106
+ expect(v({ ...base, otlp: { protocols: ['grpc', 'http/json'] }, genAiSemconv: '1.37.0' }).ok, req(R('shape'), DOC, 'genAiSemconv as semver MUST validate')).toBe(true);
107
+ expect(v({ ...base }).ok, req(R('shape'), DOC, 'otlp is REQUIRED')).toBe(false);
108
+ expect(v({ ...base, otlp: { protocols: [] } }).ok, req(R('shape'), DOC, 'protocols MUST be non-empty')).toBe(false);
109
+ expect(v({ ...base, otlp: { protocols: ['zipkin'] } }).ok, req(R('shape'), DOC, 'an unknown transport MUST fail')).toBe(false);
110
+ expect(v({ ...base, otlp: { protocols: ['grpc'] }, genAiSemconv: 'latest' }).ok, req(R('shape'), DOC, 'a non-semver genAiSemconv MUST fail')).toBe(false);
111
+ });
112
+ });
113
+
114
+ describe('v2 RFC 0262 — OTLP export (host, telemetry-gated)', () => {
115
+ it('a run exports an openwop.run span with its run and workflow ids', async () => {
116
+ const g = await gate(NOOP);
117
+ if ('skip' in g) return skipped(g);
118
+ const r = await noop();
119
+ if ('skip' in r) return skipped(r);
120
+ const run = r.spans.find((s) => s.name === 'openwop.run' || s.name.startsWith('openwop.run.'));
121
+ expect(run, req(R('run-span'), DOC, 'every run MUST produce a span named openwop.run carrying openwop.run_id')).toBeDefined();
122
+ expect(run?.attributes.get('openwop.workflow_id'), req(R('run-span'), DOC, 'the run span MUST carry openwop.workflow_id')).toBe(NOOP);
123
+ });
124
+
125
+ it('each node execution exports an openwop.node.<typeId> span with its node attributes', async () => {
126
+ const g = await gate(NOOP);
127
+ if ('skip' in g) return skipped(g);
128
+ const r = await noop();
129
+ if ('skip' in r) return skipped(r);
130
+ const node = r.spans.find((s) => s.name.startsWith('openwop.node.'));
131
+ expect(node, req(R('node-span'), DOC, 'every node execution MUST produce a span named openwop.node.<typeId>')).toBeDefined();
132
+ expect(typeof node?.attributes.get('openwop.node_id'), req(R('node-span'), DOC, 'the node span MUST carry openwop.node_id')).toBe('string');
133
+ const type = node?.attributes.get('openwop.node_type');
134
+ expect(typeof type === 'string' && node?.name === `openwop.node.${type}`, req(R('node-span'), DOC, 'the node span MUST carry openwop.node_type, and its name MUST be openwop.node.<typeId>')).toBe(true);
135
+ const attempt = node?.attributes.get('openwop.node_attempt');
136
+ expect(typeof attempt === 'number' && Number.isInteger(attempt) && attempt >= 0, req(R('node-span'), DOC, 'openwop.node_attempt MUST be a non-negative integer')).toBe(true);
137
+ });
138
+
139
+ it('a run created with a traceparent exports its spans in that trace', async () => {
140
+ const g = await gate(NOOP);
141
+ if ('skip' in g) return skipped(g);
142
+ const traceId = '0af7651916cd43dd8448eb211c80319c';
143
+ const r = await exportedRun({ workflowId: NOOP }, { traceparent: `00-${traceId}-b7ad6b7169203331-01` });
144
+ if ('skip' in r) return skipped(r);
145
+ const inTrace = r.spans.filter((s) => s.traceId.toLowerCase() === traceId);
146
+ expect(inTrace.length, req(R('traceparent-continued'), DOC, 'a run created with a valid W3C traceparent MUST emit its spans in that trace')).toBe(r.spans.length);
147
+ });
148
+
149
+ it('no exported span carries the resolved store secret', async () => {
150
+ const g = await gate(BYOK);
151
+ if ('skip' in g) return skipped(g);
152
+ if (!(await familyAdvertised('secrets'))) return softSkip('inapplicable', 'the secrets family is not advertised, so the store-canary fixture binds nothing');
153
+ const r = await exportedRun({ workflowId: BYOK });
154
+ if ('skip' in r) return skipped(r);
155
+ const snap = await http(() => driver.get(`/runs/${encodeURIComponent(r.runId)}`));
156
+ const poll = await http(() => driver.get(`/runs/${encodeURIComponent(r.runId)}/events/poll?timeout=1&streamMode=debug`));
157
+ const detectors: SecretDetector[] = [];
158
+ const digest = findSecretDigest([r.created.json, snap?.json, poll?.json]);
159
+ if (digest !== null) detectors.push(digestDetector(digest.sha256, digest.length));
160
+ const operator = process.env['OPENWOP_CANARY_SECRET_VALUE'];
161
+ if (operator && operator.length > 0) detectors.push(valueDetector(operator));
162
+ if (detectors.length === 0) return softSkip('inapplicable', 'the run surfaced no {secretSha256, secretLength} and OPENWOP_CANARY_SECRET_VALUE is unset — nothing can be recognised as the canary');
163
+ const found = leaks(spanSurfaces(r.spans), detectors);
164
+ expect(found, req(R('no-secret-in-spans'), DOC, 'a span name, attribute, event or resource attribute MUST NOT carry a resolved secret')).toEqual([]);
165
+ });
166
+
167
+ it('a host advertising genAiSemconv puts the gen_ai attributes on the model-call span', async () => {
168
+ const g = await gate(CALLAI);
169
+ if ('skip' in g) return skipped(g);
170
+ const version = g.record['genAiSemconv'];
171
+ if (typeof version !== 'string') return softSkip('inapplicable', 'telemetry.genAiSemconv is not advertised — no GenAI mapping claim');
172
+ const r = await exportedRun({ workflowId: CALLAI, inputs: { request: { prompt: 'conformance', maxTokens: 64 }, script: [{ kind: 'valid', outputTokens: 5 }] } });
173
+ if ('skip' in r) return skipped(r);
174
+ const call = r.spans.find((s) => s.attributes.has('gen_ai.operation.name'));
175
+ expect(call, req(R('gen-ai-attributes'), DOC, `with genAiSemconv ${version}, each model-call span MUST carry gen_ai.operation.name`)).toBeDefined();
176
+ const has = (k: string): boolean => call?.attributes.get(k) !== undefined && call?.attributes.get(k) !== null;
177
+ expect(has('gen_ai.provider.name') || has('gen_ai.system'), req(R('gen-ai-attributes'), DOC, 'the model-call span MUST carry the provider attribute its version names')).toBe(true);
178
+ expect(has('gen_ai.request.model'), req(R('gen-ai-attributes'), DOC, 'the model-call span MUST carry gen_ai.request.model')).toBe(true);
179
+ expect(has('gen_ai.usage.input_tokens') && has('gen_ai.usage.output_tokens'), req(R('gen-ai-attributes'), DOC, 'the model-call span MUST carry input and output token usage')).toBe(true);
180
+ });
181
+ });