@openwop/openwop-conformance 2.45.13 → 2.45.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/CHANGELOG.md +24 -0
  2. package/README.md +2 -2
  3. package/dist/spec-artifacts.lock.json +2 -2
  4. package/package.json +2 -2
  5. package/requirements.json +1446 -4
  6. package/scenario-majors.json +65 -2
  7. package/schemas/CORPUS-STAMP.json +14 -12
  8. package/src/lib/backpressure-witness.ts +2 -2
  9. package/src/lib/budget-witness.ts +3 -3
  10. package/src/lib/envelope-schema-static.ts +166 -0
  11. package/src/lib/exhaustion-facet-witness.ts +1 -1
  12. package/src/lib/family-advert-witness.ts +212 -0
  13. package/src/lib/i18n-double.ts +99 -0
  14. package/src/lib/i18n-negotiation-witness.ts +173 -0
  15. package/src/lib/localized-content-witness.ts +49 -0
  16. package/src/lib/model-capability-insufficient-witness.ts +121 -0
  17. package/src/lib/prompt-double.ts +217 -0
  18. package/src/lib/prompt-events-witness.ts +211 -0
  19. package/src/lib/prompt-library-witness.ts +406 -0
  20. package/src/lib/run-bounds-witness.ts +111 -0
  21. package/src/lib/scratch-host.ts +4 -1
  22. package/src/lib/table-schema-witness.ts +3 -1
  23. package/src/scenarios/v2-ai-envelope-shape-static.test.ts +93 -0
  24. package/src/scenarios/v2-ai-providers-advertisement.test.ts +94 -0
  25. package/src/scenarios/v2-envelope-reasoning-shape.test.ts +81 -0
  26. package/src/scenarios/v2-envelope-tier-one-subset-static.test.ts +77 -0
  27. package/src/scenarios/v2-envelope-variant-discriminator-static.test.ts +57 -0
  28. package/src/scenarios/v2-envelopes-advertisement.test.ts +85 -0
  29. package/src/scenarios/v2-i18n-negotiation.test.ts +91 -0
  30. package/src/scenarios/v2-limits-envelope-caps-advertisement.test.ts +61 -0
  31. package/src/scenarios/v2-localized-content-delivery.test.ts +136 -0
  32. package/src/scenarios/v2-media-inline-cap-static.test.ts +59 -0
  33. package/src/scenarios/v2-model-capabilities-advertisement.test.ts +61 -0
  34. package/src/scenarios/v2-model-capability-insufficient.test.ts +63 -0
  35. package/src/scenarios/v2-prompt-all-four-kinds-events.test.ts +69 -0
  36. package/src/scenarios/v2-prompt-end-to-end-events.test.ts +71 -0
  37. package/src/scenarios/v2-prompt-list-and-fetch.test.ts +103 -0
  38. package/src/scenarios/v2-prompt-mutable-lifecycle.test.ts +130 -0
  39. package/src/scenarios/v2-prompt-pack-install.test.ts +78 -0
  40. package/src/scenarios/v2-prompt-render-deterministic.test.ts +87 -0
  41. package/src/scenarios/v2-prompt-resolution-chain-event.test.ts +46 -0
  42. package/src/scenarios/v2-prompt-template-shape.test.ts +112 -0
  43. package/src/scenarios/v2-run-execution-bounds.test.ts +67 -0
@@ -0,0 +1,217 @@
1
+ /**
2
+ * A scratch double for the prompt surface at major 2: the `/prompts*`
3
+ * operations (`host-services.md` §`prompts` → §Library) and a run surface that
4
+ * plays the two prompt fixtures' event logs (`agent.prompt-resolved`,
5
+ * `prompt.composed`). Each self-test turns on ONE defect to show
6
+ * `prompt-library-witness.ts` or `prompt-events-witness.ts` convicts it; a
7
+ * standalone instance lets the scenario files run against it.
8
+ *
9
+ * A test double, not a host: it executes nothing, and nothing it does is
10
+ * evidence about any implementation.
11
+ */
12
+
13
+ import { createHash } from 'node:crypto';
14
+ import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http';
15
+ import type { AddressInfo } from 'node:net';
16
+ import { unprojectBoundId } from './bound-id.js';
17
+ import { ALL_FOUR_FIXTURE, ALL_FOUR_REFS, END_TO_END_FIXTURE, END_TO_END_NODE, END_TO_END_REF } from './prompt-events-witness.js';
18
+ import { REFERENCE_PACK } from './prompt-library-witness.js';
19
+
20
+ export type PromptDefect =
21
+ // library
22
+ | 'none' | 'list-invalid-item' | 'filter-ignored' | 'no-etag' | 'etag-no-304' | 'unknown-200' | 'unknown-nested-envelope' | 'gate-off-serves'
23
+ | 'create-no-location' | 'source-not-user' | 'dup-accepted' | 'update-not-stored' | 'non-monotonic-accepted' | 'delete-not-gone'
24
+ | 'builtin-deletable' | 'writes-unauthenticated' | 'mutable-gate-off-serves'
25
+ | 'render-nondeterministic' | 'render-ignores-vars' | 'render-bad-hash' | 'render-leaks-body' | 'render-unbound-ok'
26
+ | 'pack-unstamped' | 'reference-pack-404'
27
+ // events
28
+ | 'composed-before-resolved' | 'missing-few-shot-2' | 'chain-two-applied' | 'chain-source-mismatch' | 'v1-event-name'
29
+ | 'body-under-hashed' | 'bad-hash' | 'wrong-layer';
30
+
31
+ export const API_KEY = 'k';
32
+ type Tpl = Record<string, unknown> & { templateId: string; version: string; kind: string; text: string; meta?: Record<string, unknown> };
33
+
34
+ const sha = (s: string): string => `sha256:${createHash('sha256').update(s).digest('hex')}`;
35
+ const input = (name: string, required = true): Record<string, unknown> => ({ name, type: 'string', required, source: 'input' });
36
+
37
+ function seed(defect: PromptDefect): Tpl[] {
38
+ const pack = (templateId: string): Tpl => ({ templateId, version: '1.0.0', kind: 'system', text: `${templateId} body`, meta: { source: 'pack', packName: REFERENCE_PACK, ...(defect === 'pack-unstamped' ? {} : { packVersion: '1.0.0' }) } });
39
+ return [
40
+ { templateId: 'conformance.prompt.writer-system', version: '1.0.0', kind: 'system', text: 'You are a writer. Topic: {{topic}}', variables: [input('topic'), input('tone', false)], meta: { source: 'host' } },
41
+ { templateId: 'conformance.prompt.few-shot', version: '1.0.0', kind: 'few-shot', text: 'Example about {{topic}}', variables: [input('topic')], meta: { source: 'host' } },
42
+ { templateId: 'conformance.prompt.secret-only', version: '1.0.0', kind: 'user', text: 'Key {{key}}', variables: [{ name: 'key', type: 'string', required: true, source: 'secret' }], meta: { source: 'host' } },
43
+ pack('writer-system'),
44
+ pack('critic-system'),
45
+ ];
46
+ }
47
+
48
+ export class PromptDouble {
49
+ defect: PromptDefect = 'none';
50
+ url = '';
51
+ private server: Server | undefined;
52
+ private store: Tpl[] = seed('none');
53
+ private renders = 0;
54
+ private runs = new Map<string, string>();
55
+ private seq = 0;
56
+ discovery: Record<string, unknown> = PromptDouble.adverts();
57
+
58
+ static adverts(facets: Record<string, unknown> = {}): Record<string, unknown> {
59
+ return { prompts: { status: 'stable', since: '2.0', witness: 'witnessable-gated', endpointsSupported: true, mutableLibrary: true, packsSupported: true, observability: 'hashed', ...facets } };
60
+ }
61
+
62
+ async start(): Promise<string> {
63
+ this.server = createServer((req, res) => { void this.handle(req, res); });
64
+ await new Promise<void>((r) => this.server!.listen(0, '127.0.0.1', r));
65
+ this.url = `http://127.0.0.1:${(this.server.address() as AddressInfo).port}`;
66
+ return this.url;
67
+ }
68
+ async stop(): Promise<void> { const s = this.server; if (s) { s.closeAllConnections(); await new Promise<void>((r) => s.close(() => r())); } }
69
+ reset(defect: PromptDefect, discovery: Record<string, unknown> = PromptDouble.adverts()): void {
70
+ this.defect = defect; this.discovery = discovery; this.store = seed(defect); this.renders = 0; this.runs.clear();
71
+ }
72
+
73
+ private facet(k: string): unknown { return (this.discovery['prompts'] as Record<string, unknown> | undefined)?.[k]; }
74
+ private send(res: ServerResponse, status: number, body: unknown, headers: Record<string, string> = {}): void {
75
+ res.writeHead(status, { 'content-type': 'application/json', ...headers });
76
+ res.end(body === undefined ? '' : JSON.stringify(body));
77
+ }
78
+ private notFound(res: ServerResponse, what: string): void {
79
+ if (this.defect === 'unknown-nested-envelope') return this.send(res, 404, { error: { code: 'not_found', message: what } });
80
+ this.send(res, 404, { error: 'not_found', message: what });
81
+ }
82
+ private async body(req: IncomingMessage): Promise<Record<string, unknown>> {
83
+ const c: Buffer[] = []; for await (const x of req) c.push(x as Buffer);
84
+ try { const v: unknown = JSON.parse(Buffer.concat(c).toString('utf8') || '{}'); return v !== null && typeof v === 'object' && !Array.isArray(v) ? v as Record<string, unknown> : {}; } catch { return {}; }
85
+ }
86
+ private latest(id: string, library?: string): Tpl | undefined {
87
+ return this.store.filter((t) => t.templateId === id && (library === undefined || t.meta?.['packName'] === library))
88
+ .sort((x, y) => y.version.localeCompare(x.version, undefined, { numeric: true }))[0];
89
+ }
90
+
91
+ private async handle(req: IncomingMessage, res: ServerResponse): Promise<void> {
92
+ const u = new URL(req.url ?? '/', 'http://x');
93
+ const path = u.pathname;
94
+ const authed = req.headers['authorization'] === `Bearer ${API_KEY}`;
95
+ if (req.method === 'GET' && path === '/.well-known/openwop') return this.send(res, 200, this.discovery);
96
+
97
+ if (path.startsWith('/runs')) return this.handleRuns(req, res, path);
98
+
99
+ if (!path.startsWith('/prompts')) return this.notFound(res, `no route ${req.method} ${path}`);
100
+ const endpoints = this.facet('endpointsSupported') === true;
101
+ const mutable = endpoints && this.facet('mutableLibrary') === true;
102
+ const write = req.method !== 'GET' && path !== '/prompts:render';
103
+ if (!endpoints && this.defect !== 'gate-off-serves') return this.send(res, 404, { error: 'not_found', message: 'prompts endpoints are not served' });
104
+ if (write && !mutable && this.defect !== 'mutable-gate-off-serves') return this.send(res, 404, { error: 'not_found', message: 'the prompt library is not mutable' });
105
+ if (write && !authed && this.defect !== 'writes-unauthenticated') return this.send(res, 401, { error: 'unauthenticated', message: 'credential required' });
106
+
107
+ if (path === '/prompts:render' && req.method === 'POST') return this.render(res, await this.body(req));
108
+
109
+ if (path === '/prompts') {
110
+ if (req.method === 'GET') {
111
+ const kind = u.searchParams.get('kind'); const source = u.searchParams.get('source');
112
+ const limit = Number(u.searchParams.get('limit') ?? '50');
113
+ const filter = this.defect !== 'filter-ignored';
114
+ let items: Record<string, unknown>[] = this.store.filter((t) => !filter || ((kind === null || t.kind === kind) && (source === null || t.meta?.['source'] === source)));
115
+ if (this.defect === 'list-invalid-item' && kind === null && source === null) items = [...items, { templateId: 'conformance.broken', version: '1.0.0', kind: 'user', meta: { source: 'user' } }];
116
+ return this.send(res, 200, { items: items.slice(0, limit) });
117
+ }
118
+ if (req.method === 'POST') {
119
+ const b = await this.body(req) as Tpl;
120
+ if (this.store.some((t) => t.templateId === b.templateId && t.version === b.version)) {
121
+ if (this.defect === 'dup-accepted') return this.send(res, 201, undefined, { location: `/prompts/${b.templateId}` });
122
+ return this.send(res, 409, { error: 'conflict', message: 'duplicate (templateId, version)' });
123
+ }
124
+ this.store.push({ ...b, meta: { source: this.defect === 'source-not-user' ? 'host' : 'user' } });
125
+ return this.send(res, 201, undefined, this.defect === 'create-no-location' ? {} : { location: `/prompts/${b.templateId}` });
126
+ }
127
+ }
128
+
129
+ const m = /^\/prompts\/([^/]+)$/.exec(path);
130
+ if (!m) return this.notFound(res, `no route ${req.method} ${path}`);
131
+ const id = unprojectBoundId(m[1]!);
132
+ const t = this.latest(id, u.searchParams.get('libraryId') ?? undefined);
133
+ if (req.method === 'GET') {
134
+ if (t === undefined) return this.defect === 'unknown-200' ? this.send(res, 200, { templateId: id }) : this.notFound(res, `no template ${id}`);
135
+ if (this.defect === 'reference-pack-404' && t.meta?.['source'] === 'pack') return this.notFound(res, `no template ${id}`);
136
+ const etag = `"${sha(JSON.stringify(t)).slice(7, 39)}"`;
137
+ if (this.defect !== 'etag-no-304' && req.headers['if-none-match'] === etag) return this.send(res, 304, undefined);
138
+ return this.send(res, 200, t, this.defect === 'no-etag' ? {} : { etag, 'cache-control': 'max-age=60' });
139
+ }
140
+ if (t === undefined) return this.notFound(res, `no template ${id}`);
141
+ // Read-only by origin (seeded built-in or pack), not by the stamp a defect may have changed.
142
+ const builtin = seed('none').some((x) => x.templateId === t.templateId && x.version === t.version);
143
+ if (builtin && this.defect !== 'builtin-deletable') return this.send(res, 403, { error: 'forbidden', message: 'built-in and pack templates are read-only' });
144
+ if (req.method === 'PUT') {
145
+ const b = await this.body(req) as Tpl;
146
+ if (b.version.localeCompare(t.version, undefined, { numeric: true }) <= 0 && this.defect !== 'non-monotonic-accepted') return this.send(res, 409, { error: 'conflict', message: 'version must increase' });
147
+ if (this.defect !== 'update-not-stored') this.store.push({ ...b, meta: { source: 'user' } });
148
+ return this.send(res, 200, b);
149
+ }
150
+ if (req.method === 'DELETE') {
151
+ if (this.defect !== 'delete-not-gone') this.store = this.store.filter((x) => x.templateId !== id);
152
+ return this.send(res, 204, undefined);
153
+ }
154
+ return this.notFound(res, `no route ${req.method} ${path}`);
155
+ }
156
+
157
+ private render(res: ServerResponse, b: Record<string, unknown>): void {
158
+ const m = /^prompt:([^@]+)(?:@(.+))?$/.exec(String(b['ref']));
159
+ const t = m ? this.store.find((x) => x.templateId === m[1] && (m[2] === undefined || x.version === m[2])) : undefined;
160
+ if (t === undefined) return this.send(res, 400, { error: 'validation_error', message: 'unresolvable ref' });
161
+ const vars = (b['variables'] ?? {}) as Record<string, unknown>;
162
+ const declared = (t['variables'] as Array<{ name: string; required: boolean }> | undefined) ?? [];
163
+ const unbound = declared.find((v) => v.required && vars[v.name] === undefined);
164
+ if (unbound && this.defect !== 'render-unbound-ok') return this.send(res, 400, { error: 'validation_error', message: `required variable ${unbound.name} unbound` });
165
+ const composed = t.text.replace(/\{\{(\w+)\}\}/g, (_m, n: string) => (this.defect === 'render-ignores-vars' ? '' : String(vars[n] ?? '')));
166
+ const variableHashes = Object.fromEntries(Object.entries(vars).map(([k, v]) => [k, sha(this.defect === 'render-ignores-vars' ? '' : String(v))]));
167
+ let hash = sha(this.defect === 'render-nondeterministic' ? `${composed}#${++this.renders}` : composed);
168
+ if (this.defect === 'render-bad-hash') hash = hash.toUpperCase();
169
+ const full = this.facet('observability') === 'full' || this.defect === 'render-leaks-body';
170
+ return this.send(res, 200, { hash, refs: [String(b['ref'])], variableHashes, contentTrust: 'trusted', ...(full ? { composed } : {}) });
171
+ }
172
+
173
+ // ── run surface: the two prompt fixtures ─────────────────────────────────
174
+ private events(fixture: string): Array<Record<string, unknown>> {
175
+ const d = this.defect;
176
+ const resolvedType = d === 'v1-event-name' ? 'agent.promptResolved' : 'agent.prompt-resolved';
177
+ const obs = this.facet('observability');
178
+ const out: Array<Record<string, unknown>> = [];
179
+ const pair = (nodeId: string, kind: string, ref: string, compKind: string): void => {
180
+ const chain: Array<Record<string, unknown>> = [
181
+ { layer: d === 'wrong-layer' ? 'host-defaults' : 'node', source: d === 'chain-source-mismatch' ? 'prompt:other@9.9.9' : ref, applied: true },
182
+ { layer: 'workflow-defaults', applied: d === 'chain-two-applied', reason: 'superseded by node-config layer' },
183
+ { layer: 'host-defaults', applied: false, reason: 'no candidate at this layer' },
184
+ ];
185
+ const resolved = { type: resolvedType, nodeId, payload: { nodeId, kind, chain, resolved: ref } };
186
+ const body = `composed ${ref}`;
187
+ const showBody = obs === 'full' || d === 'body-under-hashed';
188
+ const composed = { type: 'prompt.composed', nodeId, payload: { nodeId, refs: [ref], kind: compKind, hash: d === 'bad-hash' ? 'sha256:abc' : sha(body), variableHashes: {}, ...(showBody ? { composed: body, ...(compKind === 'system-only' ? { systemPrompt: body } : {}) } : {}) } };
189
+ const emitComposed = obs !== 'off' || d === 'body-under-hashed';
190
+ out.push(...(d === 'composed-before-resolved' ? [...(emitComposed ? [composed] : []), resolved] : [resolved, ...(emitComposed ? [composed] : [])]));
191
+ };
192
+ if (fixture === END_TO_END_FIXTURE) pair(END_TO_END_NODE, 'system', END_TO_END_REF, 'system-only');
193
+ if (fixture === ALL_FOUR_FIXTURE) {
194
+ const kinds = ['system', 'user', 'schema-hint', 'few-shot', 'few-shot'];
195
+ ALL_FOUR_REFS.forEach((ref, i) => pair('all-kinds', kinds[i]!, d === 'missing-few-shot-2' && i === 4 ? ALL_FOUR_REFS[3] : ref, 'system+user'));
196
+ }
197
+ return [{ type: 'run.started', payload: {} }, ...out, { type: 'run.completed', payload: { outputs: {} } }]
198
+ .map((e, i) => ({ eventId: `evt${String(i + 1).padStart(16, '0')}`, sequence: i + 1, schemaVersion: 2, timestamp: new Date(0).toISOString(), ...e }));
199
+ }
200
+
201
+ private async handleRuns(req: IncomingMessage, res: ServerResponse, path: string): Promise<void> {
202
+ if (req.method === 'POST' && path === '/runs') {
203
+ const b = await this.body(req);
204
+ const wf = String(b['workflowId']);
205
+ if (wf !== ALL_FOUR_FIXTURE && wf !== END_TO_END_FIXTURE) return this.send(res, 404, { error: 'not_found', message: `no workflow ${wf}` });
206
+ const runId = `acme/run-${String(++this.seq).padStart(16, '0')}`;
207
+ this.runs.set(runId, wf);
208
+ return this.send(res, 201, { runId, status: 'running' });
209
+ }
210
+ const m = /^\/runs\/([^/]+)(\/events\/poll)?$/.exec(path);
211
+ const runId = m ? unprojectBoundId(m[1]!) : '';
212
+ const wf = this.runs.get(runId);
213
+ if (!m || wf === undefined) return this.send(res, 404, { error: 'not_found', message: 'no such run' });
214
+ if (m[2]) return this.send(res, 200, { events: this.events(wf).map((e) => ({ ...e, runId })) });
215
+ return this.send(res, 200, { runId, workflowId: wf, status: 'completed' });
216
+ }
217
+ }
@@ -0,0 +1,211 @@
1
+ /**
2
+ * The prompt resolution + composition events witness (`host-services.md`
3
+ * §`prompts` → §Resolution, §Composition; RFCs 0027, 0029), shared by every
4
+ * major. A fixture run is created on the normative run surface and its durable
5
+ * log read through `events/poll`; no seam. What differs between majors (the
6
+ * run path, the id segment, the event names — `agent.promptResolved` is
7
+ * `agent.prompt-resolved` at major 2 — and the payload schemas) is the profile
8
+ * row plus the validators the scenario hands in.
9
+ *
10
+ * {@link driveFixture} observes and asserts nothing; the judges are pure:
11
+ * {@link judgeAllFourKinds} `conformance-prompt-all-four-kinds`: one resolution
12
+ * per configured ref, every ref composed;
13
+ * {@link judgeOrdering} every composition follows a resolution for its node;
14
+ * {@link judgeEndToEnd} `conformance-prompt-end-to-end`: node layer wins,
15
+ * the composed hash, and bodies only under `full`;
16
+ * {@link judgeChain} every resolution carries a well-formed chain whose
17
+ * one applied layer is mirrored by `resolved`.
18
+ */
19
+
20
+ import { driver } from './driver.js';
21
+ import { isFixtureAdvertised } from './fixtures.js';
22
+ import type { MajorProfile } from './major-profile.js';
23
+
24
+ export interface EventFinding { readonly ok: boolean; readonly doc: string; readonly message: string }
25
+ export type Skip = { readonly kind: 'skip'; readonly disposition: 'inapplicable' | 'blocked'; readonly reason: string };
26
+ export type Validate = (doc: unknown) => { ok: boolean; errors: string };
27
+
28
+ export const ALL_FOUR_FIXTURE = 'conformance-prompt-all-four-kinds';
29
+ export const END_TO_END_FIXTURE = 'conformance-prompt-end-to-end';
30
+ /** The five refs `conformance-prompt-all-four-kinds` configures, few-shot twice. */
31
+ export const ALL_FOUR_REFS = [
32
+ 'prompt:conformance.prompt.writer-system@1.0.0',
33
+ 'prompt:conformance.prompt.writer-user@1.0.0',
34
+ 'prompt:conformance.prompt.schema-hint@1.0.0',
35
+ 'prompt:conformance.prompt.few-shot@1.0.0',
36
+ 'prompt:conformance.prompt.few-shot-2@1.0.0',
37
+ ] as const;
38
+ export const END_TO_END_REF = 'prompt:conformance.prompt.writer-system@1.0.0';
39
+ export const END_TO_END_NODE = 'writer';
40
+
41
+ const isRecord = (v: unknown): v is Record<string, unknown> => v !== null && typeof v === 'object' && !Array.isArray(v);
42
+ const f = (ok: boolean, doc: string, message: string): EventFinding => ({ ok, doc, message });
43
+ const SHA = /^sha256:[0-9a-f]{64}$/;
44
+ const DOC_RES = 'host-services.md §prompts → §Resolution';
45
+ const DOC_COMP = 'host-services.md §prompts → §Composition';
46
+
47
+ export interface Ev { readonly type: string; readonly sequence: number; readonly nodeId?: unknown; readonly payload: Record<string, unknown> }
48
+ export interface FixtureObservation {
49
+ readonly fixture: string;
50
+ readonly createStatus: number;
51
+ readonly terminalStatus: string | undefined;
52
+ /** Sorted by `sequence`. */
53
+ readonly events: readonly Ev[];
54
+ /** This major's names for the two events. */
55
+ readonly resolvedType: string;
56
+ readonly composedType: string;
57
+ readonly observability: 'off' | 'hashed' | 'full';
58
+ }
59
+
60
+ /** The gate both fixture scenarios share: the `prompts` family, the fixture, and the event names. */
61
+ export function eventsGate(profile: MajorProfile, doc: unknown, fixture: string): Skip | { resolvedType: string; composedType: string; observability: 'off' | 'hashed' | 'full' } {
62
+ const fam = profile.family(doc, 'prompts');
63
+ if (fam === null) return { kind: 'skip', disposition: 'inapplicable', reason: 'the host does not advertise prompts' };
64
+ if (!isFixtureAdvertised(fixture)) return { kind: 'skip', disposition: 'inapplicable', reason: `fixture ${fixture} is not advertised — the fixture is the opt-in` };
65
+ const resolvedType = profile.eventType('agent.promptResolved');
66
+ const composedType = profile.eventType('prompt.composed');
67
+ if (resolvedType === undefined || composedType === undefined) return { kind: 'skip', disposition: 'blocked', reason: 'the event codemap is not on disk in this layout — the suite will not guess an event name' };
68
+ const obs = fam['observability'];
69
+ return { resolvedType, composedType, observability: obs === 'off' || obs === 'full' ? obs : 'hashed' };
70
+ }
71
+
72
+ /** Create a fixture run, wait for a terminal status, read the log. Asserts nothing. */
73
+ export async function driveFixture(profile: MajorProfile, doc: unknown, fixture: string, opts: { timeoutMs?: number; pollMs?: number } = {}): Promise<Skip | FixtureObservation> {
74
+ const g = eventsGate(profile, doc, fixture);
75
+ if ('kind' in g) return g;
76
+ const created = await driver.post(profile.runsPath, { workflowId: fixture });
77
+ const runId = isRecord(created.json) ? created.json['runId'] : undefined;
78
+ const base = { fixture, createStatus: created.status, ...g };
79
+ if ((created.status !== 201 && created.status !== 202) || typeof runId !== 'string') return { ...base, terminalStatus: undefined, events: [] };
80
+ const at = `${profile.runsPath}/${profile.idSegment(runId)}`;
81
+ const deadline = Date.now() + (opts.timeoutMs ?? 30_000);
82
+ let status: string | undefined;
83
+ for (;;) {
84
+ const r = await driver.get(at);
85
+ status = isRecord(r.json) && typeof r.json['status'] === 'string' ? (r.json['status'] as string) : undefined;
86
+ if ((status !== undefined && ['completed', 'failed', 'cancelled'].includes(status)) || Date.now() > deadline) break;
87
+ await new Promise((res) => setTimeout(res, opts.pollMs ?? 250));
88
+ }
89
+ const poll = await driver.get(`${at}/events/poll`);
90
+ const raw = isRecord(poll.json) && Array.isArray(poll.json['events']) ? (poll.json['events'] as unknown[]) : [];
91
+ const events = raw.filter(isRecord).map((e): Ev => ({
92
+ type: String(e['type']),
93
+ sequence: typeof e['sequence'] === 'number' ? e['sequence'] : Number.MAX_SAFE_INTEGER,
94
+ nodeId: e['nodeId'],
95
+ payload: isRecord(e['payload']) ? e['payload'] : {},
96
+ })).sort((x, y) => x.sequence - y.sequence);
97
+ return { ...base, terminalStatus: status, events };
98
+ }
99
+
100
+ const of = (o: FixtureObservation, type: string): Ev[] => o.events.filter((e) => e.type === type);
101
+ const nodeOf = (e: Ev): unknown => e.payload['nodeId'] ?? e.nodeId;
102
+
103
+ function runFindings(o: FixtureObservation): EventFinding[] {
104
+ return [
105
+ f(o.createStatus === 201 || o.createStatus === 202, 'runs.md createRun', `createRun of ${o.fixture} MUST be accepted (got ${o.createStatus})`),
106
+ f(o.terminalStatus === 'completed', `fixtures.md ${o.fixture}`, `the fixture run MUST end completed (got ${String(o.terminalStatus)})`),
107
+ ];
108
+ }
109
+ const payloadFindings = (events: readonly Ev[], validate: Validate, name: string): EventFinding[] =>
110
+ events.map((e) => { const v = validate(e.payload); return f(v.ok, `schemas run-event-payloads ${name}`, `every ${e.type} payload MUST validate (seq ${e.sequence}): ${v.errors}`); });
111
+
112
+ /** `conformance-prompt-all-four-kinds`: every kind resolved, every configured ref resolved and composed. Pure. */
113
+ export function judgeAllFourKinds(o: FixtureObservation, v: { resolved: Validate; composed: Validate }): EventFinding[] {
114
+ const out = runFindings(o);
115
+ const resolved = of(o, o.resolvedType);
116
+ const composed = of(o, o.composedType);
117
+ const kinds = new Set(resolved.map((e) => e.payload['kind']));
118
+ const resolvedRefs = new Set(resolved.map((e) => e.payload['resolved']));
119
+ const composedRefs = composed.flatMap((e) => (Array.isArray(e.payload['refs']) ? e.payload['refs'].filter((r): r is string => typeof r === 'string') : []));
120
+ for (const k of ['system', 'user', 'schema-hint', 'few-shot']) {
121
+ out.push(f(kinds.has(k), DOC_RES, `the host MUST emit ${o.resolvedType} with kind ${k} when the node carries the matching ref`));
122
+ }
123
+ for (const ref of ALL_FOUR_REFS) {
124
+ out.push(f(resolvedRefs.has(ref), DOC_RES, `${o.resolvedType}.resolved MUST surface ${ref} — the node configures it (fewShotPromptRefs[1] included)`));
125
+ if (o.observability !== 'off') out.push(f(composedRefs.includes(ref), DOC_COMP, `${o.composedType}.refs MUST contain ${ref} — every resolved ref is composed`));
126
+ }
127
+ if (o.observability === 'off') out.push(f(composed.length === 0, DOC_COMP, `under observability off the host MUST NOT emit ${o.composedType} (got ${composed.length})`));
128
+ out.push(...payloadFindings(resolved, v.resolved, 'agentPromptResolved'), ...payloadFindings(composed, v.composed, 'promptComposed'));
129
+ return out;
130
+ }
131
+
132
+ /** Every composition follows a resolution for its node. Pure. */
133
+ export function judgeOrdering(o: FixtureObservation): EventFinding[] {
134
+ const out = runFindings(o);
135
+ const resolved = of(o, o.resolvedType);
136
+ const composed = of(o, o.composedType);
137
+ out.push(f(resolved.length > 0, DOC_RES, `the log MUST carry ${o.resolvedType}`));
138
+ if (o.observability !== 'off') out.push(f(composed.length > 0, DOC_COMP, `the log MUST carry ${o.composedType} (observability ${o.observability})`));
139
+ for (const c of composed) {
140
+ const before = resolved.some((r) => r.sequence < c.sequence && nodeOf(r) === nodeOf(c));
141
+ out.push(f(before, DOC_RES, `${o.resolvedType} MUST precede ${o.composedType} for node ${String(nodeOf(c))} (composed at seq ${c.sequence})`));
142
+ }
143
+ return out;
144
+ }
145
+
146
+ /** `conformance-prompt-end-to-end`: the node layer wins, the hash, and bodies only under `full`. Pure. */
147
+ export function judgeEndToEnd(o: FixtureObservation, v: { resolved: Validate; composed: Validate }): EventFinding[] {
148
+ const out = runFindings(o);
149
+ const r = of(o, o.resolvedType).find((e) => e.payload['kind'] === 'system');
150
+ out.push(f(r !== undefined, DOC_RES, `the host MUST emit ${o.resolvedType} (kind system) for a node carrying systemPromptRef`));
151
+ if (r !== undefined) {
152
+ const chain = Array.isArray(r.payload['chain']) ? r.payload['chain'].filter(isRecord) : [];
153
+ const applied = chain.find((c) => c['applied'] === true);
154
+ out.push(
155
+ f(nodeOf(r) === END_TO_END_NODE, DOC_RES, `${o.resolvedType}.nodeId MUST be ${END_TO_END_NODE} (got ${String(nodeOf(r))})`),
156
+ f(applied?.['layer'] === 'node', `${DOC_RES} (layer 1)`, `the node config ref MUST win: the applied chain entry is layer node (got ${String(applied?.['layer'])})`),
157
+ f(r.payload['resolved'] === END_TO_END_REF, DOC_RES, `${o.resolvedType}.resolved MUST be ${END_TO_END_REF} (got ${String(r.payload['resolved'])})`),
158
+ ...payloadFindings([r], v.resolved, 'agentPromptResolved'),
159
+ );
160
+ }
161
+ const composed = of(o, o.composedType);
162
+ if (o.observability === 'off') {
163
+ out.push(f(composed.length === 0, DOC_COMP, `under observability off the host MUST NOT emit ${o.composedType} (got ${composed.length})`));
164
+ return out;
165
+ }
166
+ const c = composed[0];
167
+ out.push(f(c !== undefined, DOC_COMP, `unless observability is off, the host MUST emit ${o.composedType} for the composition`));
168
+ if (c === undefined) return out;
169
+ const p = c.payload;
170
+ out.push(
171
+ f(typeof p['hash'] === 'string' && SHA.test(p['hash']), 'schemas run-event-payloads promptComposed.hash', `hash MUST match sha256:<hex64> (got ${String(p['hash'])})`),
172
+ f(p['kind'] === 'system-only', 'schemas run-event-payloads promptComposed.kind', `a system-only composition MUST say kind system-only (got ${String(p['kind'])})`),
173
+ ...payloadFindings([c], v.composed, 'promptComposed'),
174
+ );
175
+ if (o.observability === 'full') {
176
+ out.push(
177
+ f(typeof p['composed'] === 'string' && p['composed'].length > 0, DOC_COMP, 'under observability full, composed MUST carry the body'),
178
+ f(p['systemPrompt'] === p['composed'], 'schemas run-event-payloads promptComposed.systemPrompt', 'for a system template, systemPrompt MUST equal composed'),
179
+ );
180
+ } else {
181
+ const leaked = ['composed', 'systemPrompt', 'userPrompt', 'variableBindings'].filter((k) => p[k] !== undefined);
182
+ out.push(f(leaked.length === 0, `${DOC_COMP} ("carrying bodies only under full")`, `under observability hashed, ${o.composedType} MUST carry no body (got ${JSON.stringify(leaked)})`));
183
+ }
184
+ return out;
185
+ }
186
+
187
+ const LAYERS = new Set(['run-configurable', 'node', 'agent-intrinsic', 'agent-overrides', 'agent-library-default', 'workflow-defaults', 'host-defaults']);
188
+
189
+ /** Every resolution carries a well-formed chain with at most one applied layer, mirrored by `resolved`. Pure. */
190
+ export function judgeChain(o: FixtureObservation): EventFinding[] {
191
+ const out = runFindings(o);
192
+ const resolved = of(o, o.resolvedType);
193
+ out.push(f(resolved.length > 0, DOC_RES, `a host advertising prompts MUST emit ${o.resolvedType} when a node carries a PromptRef`));
194
+ for (const e of resolved) {
195
+ const chain = Array.isArray(e.payload['chain']) ? e.payload['chain'] : [];
196
+ out.push(f(chain.length > 0, DOC_RES, `${o.resolvedType} MUST carry one chain[] entry per layer tried (seq ${e.sequence})`));
197
+ for (const c of chain) {
198
+ const rec = isRecord(c) ? c : {};
199
+ out.push(
200
+ f(typeof rec['layer'] === 'string' && LAYERS.has(rec['layer']), DOC_RES, `each chain entry MUST name a layer (got ${String(rec['layer'])})`),
201
+ f(typeof rec['applied'] === 'boolean', DOC_RES, 'each chain entry MUST carry a boolean applied'),
202
+ );
203
+ }
204
+ const applied = chain.filter((c) => isRecord(c) && c['applied'] === true) as Array<Record<string, unknown>>;
205
+ out.push(f(applied.length <= 1, DOC_RES, `at most one chain entry is applied (got ${applied.length})`));
206
+ out.push(applied.length === 1
207
+ ? f(e.payload['resolved'] === applied[0]!['source'], 'schemas run-event-payloads agentPromptResolved.resolved', `resolved MUST mirror the applied entry's source (got ${String(e.payload['resolved'])} vs ${String(applied[0]!['source'])})`)
208
+ : f(e.payload['resolved'] === null, 'schemas run-event-payloads agentPromptResolved.resolved', `with no applied layer, resolved MUST be null (got ${String(e.payload['resolved'])})`));
209
+ }
210
+ return out;
211
+ }