@openwop/openwop-conformance 2.46.34 → 2.46.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/README.md +3 -3
- package/coverage.md +4 -5
- package/dist/spec-artifacts.lock.json +2 -2
- package/fixtures/conformance-interrupt-form-refused.json +108 -0
- package/fixtures/conformance-interrupt-form-unbound.json +108 -0
- package/fixtures/conformance-interrupt-form.json +107 -0
- package/fixtures/conformance-stream-text.json +1 -2
- package/fixtures.md +44 -4
- package/package.json +2 -2
- package/requirement-aliases.json +98 -1
- package/requirements.json +663 -804
- package/scenario-majors.json +16 -19
- package/schemas/CORPUS-STAMP.json +72 -85
- package/src/lib/callai-attempts-witness.ts +13 -1
- package/src/lib/deployment-lifecycle-witness.ts +308 -0
- package/src/lib/family-advert-witness.ts +37 -50
- package/src/scenarios/stream-text-fixture.test.ts +13 -5
- package/src/scenarios/v2-deployment-lifecycle.test.ts +526 -0
- package/src/scenarios/v2-envelope-surfaces-absent.test.ts +55 -0
- package/src/scenarios/v2-interrupt-form-rules.test.ts +143 -0
- package/src/scenarios/v2-limits-clarification-rounds-advertisement.test.ts +42 -0
- package/src/scenarios/v2-peer-dependency-declared.test.ts +1 -1
- package/src/scenarios/v2-provider-output-reliability.test.ts +94 -0
- package/src/scenarios/v2-structured-output-advertisement.test.ts +82 -0
- package/src/lib/envelope-schema-static.ts +0 -167
- package/src/scenarios/v2-a2ui-v09-surface.test.ts +0 -323
- package/src/scenarios/v2-ai-envelope-advertisement.test.ts +0 -43
- package/src/scenarios/v2-envelope-contracts-advertisement.test.ts +0 -48
- package/src/scenarios/v2-envelope-reasoning-shape.test.ts +0 -81
- package/src/scenarios/v2-envelope-tier-one-subset-static.test.ts +0 -64
- package/src/scenarios/v2-envelopes-advertisement.test.ts +0 -85
- package/src/scenarios/v2-limits-envelope-caps-advertisement.test.ts +0 -61
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RFC 0082 at major 2: the deployment lifecycle. Pure judges for
|
|
3
|
+
* `v2-deployment-lifecycle.test.ts`, over answers from `listAgentDeployments`
|
|
4
|
+
* and `transitionAgentDeployment` and over the event logs of the
|
|
5
|
+
* `conformance-agent-channel-dispatch` fixture's runs.
|
|
6
|
+
*
|
|
7
|
+
* The rules are the `agents.deployment` facet description in
|
|
8
|
+
* `spec/v2/facets/agents.schema.json`, `schemas/v2/agent-deployment.schema.json`
|
|
9
|
+
* (the record and its legal transitions), `agent-deployment-transition.schema.json`
|
|
10
|
+
* and the two operations in `api/v2/openapi.yaml`, with the transition rules
|
|
11
|
+
* RFC 0254 adds to that facet text. Each judge returns findings and asserts
|
|
12
|
+
* nothing; the scenario maps them to `req()` ids.
|
|
13
|
+
*
|
|
14
|
+
* @see RFCS/0082-agent-deployment-lifecycle.md §B, §C, §E, §F
|
|
15
|
+
* @see RFCS/0254-deployment-transitions-at-v2.md
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { readErrorCode } from './error-envelope.js';
|
|
19
|
+
|
|
20
|
+
export interface DeployFinding { readonly ok: boolean; readonly message: string }
|
|
21
|
+
export interface Answer {
|
|
22
|
+
readonly status: number;
|
|
23
|
+
readonly json: unknown;
|
|
24
|
+
/** The `WWW-Authenticate` header, when the answer carried one. */
|
|
25
|
+
readonly wwwAuthenticate?: string | undefined;
|
|
26
|
+
}
|
|
27
|
+
export type Validate = (doc: unknown) => { ok: boolean; errors: string };
|
|
28
|
+
type Rec = Record<string, unknown>;
|
|
29
|
+
const isRec = (v: unknown): v is Rec => typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
30
|
+
const f = (ok: boolean, message: string): DeployFinding => ({ ok, message });
|
|
31
|
+
const shown = (a: Answer | null): string => (a === null ? 'no answer' : `${a.status} ${readErrorCode(a.json) ?? ''}`.trim());
|
|
32
|
+
|
|
33
|
+
/** The forward promotion path `agent-deployment.schema.json` names. */
|
|
34
|
+
export const PROMOTION_PATH = ['draft', 'test', 'staged', 'active'] as const;
|
|
35
|
+
export const ALL_STATES = ['draft', 'test', 'staged', 'active', 'paused', 'deprecated', 'rolled-back'] as const;
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* The promotion path over the advertised states: absent `states` means all
|
|
39
|
+
* seven, and over a subset `promote` moves to the next advertised state in
|
|
40
|
+
* draft→test→staged→active order (RFC 0254).
|
|
41
|
+
*/
|
|
42
|
+
export function promotionPath(states: readonly string[] | null): string[] {
|
|
43
|
+
return PROMOTION_PATH.filter((s) => states === null || states.includes(s));
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** `details` of an error body: the canonical root `details`, else the legacy nested `error.details`. */
|
|
47
|
+
export function errorDetails(body: unknown): Rec {
|
|
48
|
+
if (!isRec(body)) return {};
|
|
49
|
+
if (isRec(body['details'])) return body['details'];
|
|
50
|
+
return isRec(body['error']) && isRec(body['error']['details']) ? body['error']['details'] : {};
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** The record for `version` in a list body, or undefined. */
|
|
54
|
+
export function recordOf(list: unknown, version: string): Rec | undefined {
|
|
55
|
+
return Array.isArray(list) ? list.find((r): r is Rec => isRec(r) && r['version'] === version) : undefined;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** The versions listed `active` with `channel` in `channels`. */
|
|
59
|
+
export function activeOn(list: unknown, channel: string): string[] {
|
|
60
|
+
if (!Array.isArray(list)) return [];
|
|
61
|
+
return list.filter((r): r is Rec => isRec(r) && r['state'] === 'active' && Array.isArray(r['channels']) && (r['channels'] as unknown[]).includes(channel) && typeof r['version'] === 'string')
|
|
62
|
+
.map((r) => r['version'] as string);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** The first `draft` record's version, or undefined. */
|
|
66
|
+
export function pickDraft(list: unknown): string | undefined {
|
|
67
|
+
if (!Array.isArray(list)) return undefined;
|
|
68
|
+
const r = list.find((x): x is Rec => isRec(x) && x['state'] === 'draft' && typeof x['version'] === 'string');
|
|
69
|
+
return r === undefined ? undefined : (r['version'] as string);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** `listAgentDeployments` answers 200 with an array of valid records of the agent. */
|
|
73
|
+
export function judgeList(a: Answer | null, agentId: string, validate: Validate): DeployFinding[] {
|
|
74
|
+
if (a === null || a.status !== 200) return [f(false, `GET /agents/{agentId}/deployments MUST answer 200 for an advertised facet (got ${shown(a)})`)];
|
|
75
|
+
if (!Array.isArray(a.json)) return [f(false, 'the 200 body MUST be an array of deployment records')];
|
|
76
|
+
const out: DeployFinding[] = [f(a.json.length > 0, 'the fixture agent MUST have at least one listed deployment record')];
|
|
77
|
+
a.json.forEach((r, i) => {
|
|
78
|
+
const v = validate(r);
|
|
79
|
+
out.push(f(v.ok, `record [${i}] MUST validate against agent-deployment.schema.json (${v.errors})`));
|
|
80
|
+
out.push(f(isRec(r) && r['agentId'] === agentId, `record [${i}] MUST belong to ${agentId} (got ${JSON.stringify(isRec(r) ? r['agentId'] : r)})`));
|
|
81
|
+
});
|
|
82
|
+
return out;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const KEPT = ['state', 'channels', 'canaryPercent', 'rollbackPointer'] as const;
|
|
86
|
+
function unchanged(before: Rec, after: unknown, why: string): DeployFinding[] {
|
|
87
|
+
const version = String(before['version']);
|
|
88
|
+
const now = recordOf(after, version);
|
|
89
|
+
if (now === undefined) return [f(false, `the record for ${version} MUST still be listed after ${why}`)];
|
|
90
|
+
return KEPT.map((k) => f(JSON.stringify(before[k]) === JSON.stringify(now[k]), `${why} MUST NOT change ${k} (was ${JSON.stringify(before[k])}, now ${JSON.stringify(now[k])})`));
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* A transition the record's state allows no edge for, or a `promote` past the
|
|
95
|
+
* next state: `409 deployment_state_conflict` naming the current state and the
|
|
96
|
+
* transition, and the record does not move (RFC 0254).
|
|
97
|
+
*/
|
|
98
|
+
export function judgeStateConflict(label: string, transition: string, a: Answer | null, before: Rec, after: unknown): DeployFinding[] {
|
|
99
|
+
const d = errorDetails(a?.json);
|
|
100
|
+
return [
|
|
101
|
+
f(a?.status === 409, `${label} is not a legal edge from ${JSON.stringify(before['state'])} and MUST be refused 409 (got ${shown(a)})`),
|
|
102
|
+
f(readErrorCode(a?.json) === 'deployment_state_conflict', `${label} MUST carry deployment_state_conflict (got ${JSON.stringify(readErrorCode(a?.json))})`),
|
|
103
|
+
f(d['state'] === before['state'], `details.state MUST be the record's current state ${JSON.stringify(before['state'])} (got ${JSON.stringify(d['state'])})`),
|
|
104
|
+
f(d['transition'] === transition, `details.transition MUST name the refused transition ${JSON.stringify(transition)} (got ${JSON.stringify(d['transition'])})`),
|
|
105
|
+
...unchanged(before, after, `the refused ${label}`),
|
|
106
|
+
];
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** The scopes a `WWW-Authenticate` challenge names in `scope="…"`, or undefined when it names none. */
|
|
110
|
+
export function challengeScopes(header: string): string[] | undefined {
|
|
111
|
+
const m = /\bscope="([^"]*)"/i.exec(header);
|
|
112
|
+
return m === null ? undefined : (m[1] ?? '').split(/\s+/).filter((x) => x.length > 0);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* A transition by a principal without `deploy:*` is refused `403 forbidden`
|
|
117
|
+
* and moves nothing. `deploy:*` is one literal scope: a `WWW-Authenticate`
|
|
118
|
+
* challenge, when present, names it in `scope=` (RFC 0254).
|
|
119
|
+
*/
|
|
120
|
+
export function judgeScopeRefused(a: Answer | null, before: Rec, after: unknown): DeployFinding[] {
|
|
121
|
+
const out = [
|
|
122
|
+
f(a?.status === 403, `a transition by a principal without the deploy:* scope MUST be refused 403, fail-closed (got ${shown(a)})`),
|
|
123
|
+
f(readErrorCode(a?.json) === 'forbidden', `the 403 MUST carry forbidden (got ${JSON.stringify(readErrorCode(a?.json))})`),
|
|
124
|
+
];
|
|
125
|
+
const h = a?.wwwAuthenticate;
|
|
126
|
+
if (typeof h === 'string' && h.length > 0) {
|
|
127
|
+
const scopes = challengeScopes(h);
|
|
128
|
+
out.push(f(scopes !== undefined && scopes.includes('deploy:*'), `a WWW-Authenticate challenge on the 403 MUST name scope="deploy:*" (got ${JSON.stringify(h)})`));
|
|
129
|
+
}
|
|
130
|
+
return [...out, ...unchanged(before, after, 'the unauthorized transition')];
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** `400 validation_error`, the record unchanged: a transition the advertised `states` or `canary` cannot satisfy. */
|
|
134
|
+
export function judgeValidationRefused(label: string, a: Answer | null, before: Rec, after: unknown): DeployFinding[] {
|
|
135
|
+
return [
|
|
136
|
+
f(a?.status === 400, `${label} MUST be refused 400 (got ${shown(a)})`),
|
|
137
|
+
f(readErrorCode(a?.json) === 'validation_error', `${label} MUST carry validation_error (got ${JSON.stringify(readErrorCode(a?.json))})`),
|
|
138
|
+
...unchanged(before, after, `the refused ${label}`),
|
|
139
|
+
];
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** One legal step answers 200 with the valid record in the target state. */
|
|
143
|
+
export function judgeStep(a: Answer | null, version: string, toState: string, validate: Validate): DeployFinding[] {
|
|
144
|
+
if (a === null || a.status !== 200) return [f(false, `promote ${version} to ${toState} is a legal transition and MUST answer 200 (got ${shown(a)})`)];
|
|
145
|
+
const v = validate(a.json);
|
|
146
|
+
const r = isRec(a.json) ? a.json : {};
|
|
147
|
+
return [
|
|
148
|
+
f(v.ok, `the record returned for ${toState} MUST validate against agent-deployment.schema.json (${v.errors})`),
|
|
149
|
+
f(r['version'] === version, `the returned record MUST be version ${version} (got ${JSON.stringify(r['version'])})`),
|
|
150
|
+
f(r['state'] === toState, `the returned record MUST be in ${toState} (got ${JSON.stringify(r['state'])})`),
|
|
151
|
+
];
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** After the walk the list shows the version `active` on the channel the last promote named. */
|
|
155
|
+
export function judgeBound(after: unknown, version: string, channel: string): DeployFinding[] {
|
|
156
|
+
const r = recordOf(after, version);
|
|
157
|
+
return [
|
|
158
|
+
f(r?.['state'] === 'active', `after the walk ${version} MUST be listed active (got ${JSON.stringify(r?.['state'])})`),
|
|
159
|
+
f(Array.isArray(r?.['channels']) && (r['channels'] as unknown[]).includes(channel), `a promote naming channel ${channel} MUST bind it (channels ${JSON.stringify(r?.['channels'])})`),
|
|
160
|
+
];
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** The payload of the first `agent.invocation-started`, or undefined. */
|
|
164
|
+
export function firstInvocation(events: ReadonlyArray<{ type: string; payload: Rec }>, type: string): Rec | undefined {
|
|
165
|
+
return events.find((e) => e.type === type)?.payload;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** A channel-bound run records the channel and a version active on it when the run started. */
|
|
169
|
+
export function judgePinned(payload: Rec | undefined, channel: string, activeVersions: readonly string[]): DeployFinding[] {
|
|
170
|
+
if (payload === undefined) return [f(false, 'a channel-bound agent run MUST emit agent.invocation-started')];
|
|
171
|
+
const v = payload['resolvedAgentVersion'];
|
|
172
|
+
return [
|
|
173
|
+
f(payload['resolvedChannel'] === channel, `agent.invocation-started MUST carry resolvedChannel ${JSON.stringify(channel)} (got ${JSON.stringify(payload['resolvedChannel'])})`),
|
|
174
|
+
f(typeof v === 'string' && activeVersions.includes(v), `resolvedAgentVersion MUST be a version active on ${channel} at run start, ${JSON.stringify(activeVersions)} (got ${JSON.stringify(v)})`),
|
|
175
|
+
];
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** A replay of the source run re-reads its recorded version, not the moved channel. */
|
|
179
|
+
export function judgeReplayPinned(source: string, moved: string, payload: Rec | undefined): DeployFinding[] {
|
|
180
|
+
if (payload === undefined) return [f(false, 'the replay MUST emit agent.invocation-started')];
|
|
181
|
+
const v = payload['resolvedAgentVersion'];
|
|
182
|
+
return [f(v === source, `a replay MUST re-read the recorded resolvedAgentVersion ${source}, not re-resolve the channel now serving ${moved} (got ${JSON.stringify(v)})`)];
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** `adjust-canary` answers 200 with the record at the requested share, and the list agrees. */
|
|
186
|
+
export function judgeCanaryAdjusted(a: Answer | null, version: string, percent: number, after: unknown, validate: Validate): DeployFinding[] {
|
|
187
|
+
if (a === null || a.status !== 200) return [f(false, `adjust-canary of active ${version} to ${percent} on a canary host MUST answer 200 (got ${shown(a)})`)];
|
|
188
|
+
const v = validate(a.json);
|
|
189
|
+
const r = isRec(a.json) ? a.json : {};
|
|
190
|
+
return [
|
|
191
|
+
f(v.ok, `the adjusted record MUST validate against agent-deployment.schema.json (${v.errors})`),
|
|
192
|
+
f(r['canaryPercent'] === percent, `the returned record MUST carry canaryPercent ${percent} (got ${JSON.stringify(r['canaryPercent'])})`),
|
|
193
|
+
f(recordOf(after, version)?.['canaryPercent'] === percent, `the list MUST then show canaryPercent ${percent} (got ${JSON.stringify(recordOf(after, version)?.['canaryPercent'])})`),
|
|
194
|
+
];
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* A rollback leaves the version `rolled-back`, and restores the version that
|
|
199
|
+
* was `active` on the channel immediately before it: the incumbent, which
|
|
200
|
+
* `rollbackPointer` names and which is listed `active` afterwards (RFC 0254).
|
|
201
|
+
*/
|
|
202
|
+
export function judgeRolledBack(a: Answer | null, version: string, incumbent: readonly string[], after: unknown, validate: Validate): DeployFinding[] {
|
|
203
|
+
if (a === null || a.status !== 200) return [f(false, `rollback of active ${version} on a rollback host MUST answer 200 (got ${shown(a)})`)];
|
|
204
|
+
const v = validate(a.json);
|
|
205
|
+
const r = isRec(a.json) ? a.json : {};
|
|
206
|
+
const ptr = r['rollbackPointer'];
|
|
207
|
+
const restored = typeof ptr === 'string' ? recordOf(after, ptr) : undefined;
|
|
208
|
+
return [
|
|
209
|
+
f(v.ok, `the rolled-back record MUST validate against agent-deployment.schema.json (${v.errors})`),
|
|
210
|
+
f(r['state'] === 'rolled-back', `the returned record MUST be rolled-back (got ${JSON.stringify(r['state'])})`),
|
|
211
|
+
f(typeof ptr === 'string' && incumbent.includes(ptr), `rollbackPointer MUST name the version active on the channel immediately before ${version}, ${JSON.stringify(incumbent)} (got ${JSON.stringify(ptr)})`),
|
|
212
|
+
f(restored?.['state'] === 'active', `the version rollbackPointer names MUST be listed active (got ${JSON.stringify(restored?.['state'])})`),
|
|
213
|
+
f(recordOf(after, version)?.['state'] === 'rolled-back', `the list MUST then show ${version} rolled-back (got ${JSON.stringify(recordOf(after, version)?.['state'])})`),
|
|
214
|
+
];
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/** A `pause` then a `promote` of a paused version: each 200, ending `paused` then `active` (RFC 0254). */
|
|
218
|
+
export function judgeResumed(pause: Answer | null, resume: Answer | null, version: string, after: unknown, validate: Validate): DeployFinding[] {
|
|
219
|
+
if (pause === null || pause.status !== 200) return [f(false, `pause of active ${version} is a legal edge and MUST answer 200 (got ${shown(pause)})`)];
|
|
220
|
+
const p = isRec(pause.json) ? pause.json : {};
|
|
221
|
+
const vp = validate(pause.json);
|
|
222
|
+
const out = [
|
|
223
|
+
f(vp.ok, `the paused record MUST validate against agent-deployment.schema.json (${vp.errors})`),
|
|
224
|
+
f(p['state'] === 'paused', `the pause MUST return the record paused (got ${JSON.stringify(p['state'])})`),
|
|
225
|
+
];
|
|
226
|
+
if (resume === null || resume.status !== 200) return [...out, f(false, `promote of paused ${version} is the paused→active edge and MUST answer 200 (got ${shown(resume)})`)];
|
|
227
|
+
const r = isRec(resume.json) ? resume.json : {};
|
|
228
|
+
const vr = validate(resume.json);
|
|
229
|
+
return [
|
|
230
|
+
...out,
|
|
231
|
+
f(vr.ok, `the resumed record MUST validate against agent-deployment.schema.json (${vr.errors})`),
|
|
232
|
+
f(r['state'] === 'active', `promote of a paused version MUST return it active (got ${JSON.stringify(r['state'])})`),
|
|
233
|
+
f(recordOf(after, version)?.['state'] === 'active', `the list MUST then show ${version} active (got ${JSON.stringify(recordOf(after, version)?.['state'])})`),
|
|
234
|
+
];
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/** One transition the host accepted with 200: what was asked, and the record it returned. */
|
|
238
|
+
export interface Accepted {
|
|
239
|
+
readonly label: string;
|
|
240
|
+
readonly transition: string;
|
|
241
|
+
readonly version: string;
|
|
242
|
+
/** The record's state before the transition. */
|
|
243
|
+
readonly fromState: string;
|
|
244
|
+
readonly answer: Answer;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/** Every 200 transition answer names its deployment-management run in `transitionRunId` (RFC 0254). */
|
|
248
|
+
export function judgeRunLinked(accepted: readonly Accepted[]): DeployFinding[] {
|
|
249
|
+
return accepted.map((x) => {
|
|
250
|
+
const id = isRec(x.answer.json) ? x.answer.json['transitionRunId'] : undefined;
|
|
251
|
+
return f(typeof id === 'string' && id.length > 0, `the record ${x.label} returned MUST carry transitionRunId naming its deployment-management run (got ${JSON.stringify(id)})`);
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/** The `transitionRunId` of an accepted answer, or undefined. */
|
|
256
|
+
export function runIdOf(x: Accepted): string | undefined {
|
|
257
|
+
const id = isRec(x.answer.json) ? x.answer.json['transitionRunId'] : undefined;
|
|
258
|
+
return typeof id === 'string' && id.length > 0 ? id : undefined;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/** A `deployment.*` event the transition must have recorded: its v1 name (the codemap key), its payload $def, and how it names this transition. */
|
|
262
|
+
export interface Expected { readonly v1: string; readonly def: string; readonly matches: (p: Rec) => boolean }
|
|
263
|
+
|
|
264
|
+
/**
|
|
265
|
+
* The `deployment.*` event a transition must record. A resume (`promote` of a
|
|
266
|
+
* paused version) records `deployment.state.changed`, as pause and deprecate
|
|
267
|
+
* do; `deployment.promoted` records only a forward step (RFC 0254 §A).
|
|
268
|
+
*/
|
|
269
|
+
export function expectedEvents(x: Accepted): Expected[] {
|
|
270
|
+
const r = isRec(x.answer.json) ? x.answer.json : {};
|
|
271
|
+
const state = r['state'];
|
|
272
|
+
const promoted: Expected = { v1: 'deployment.promoted', def: 'deploymentPromoted', matches: (p) => p['toVersion'] === x.version && p['toState'] === state };
|
|
273
|
+
const changed: Expected = { v1: 'deployment.state.changed', def: 'deploymentStateChanged', matches: (p) => p['version'] === x.version && p['fromState'] === x.fromState && p['toState'] === state };
|
|
274
|
+
switch (x.transition) {
|
|
275
|
+
case 'promote': return x.fromState === 'paused' ? [changed] : [promoted];
|
|
276
|
+
case 'pause':
|
|
277
|
+
case 'deprecate': return [changed];
|
|
278
|
+
case 'adjust-canary': return [{ v1: 'deployment.canary.adjusted', def: 'deploymentCanaryAdjusted', matches: (p) => p['version'] === x.version && p['toPercent'] === r['canaryPercent'] }];
|
|
279
|
+
case 'rollback': return [{ v1: 'deployment.rolled-back', def: 'deploymentRolledBack', matches: (p) => p['fromVersion'] === x.version && p['toVersion'] === r['rollbackPointer'] }];
|
|
280
|
+
default: return [];
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* The management run's events include the `deployment.*` event matching the
|
|
286
|
+
* transition, and its payload validates against its `run-event-payloads` $def.
|
|
287
|
+
* `typeOf` maps a v1 name to the major's wire name; `validators` are keyed by $def.
|
|
288
|
+
*/
|
|
289
|
+
export function judgeEventRecorded(
|
|
290
|
+
x: Accepted,
|
|
291
|
+
events: ReadonlyArray<{ type: string; payload: Rec }> | undefined,
|
|
292
|
+
typeOf: (v1: string) => string,
|
|
293
|
+
validators: Readonly<Record<string, Validate>>,
|
|
294
|
+
): DeployFinding[] {
|
|
295
|
+
const runId = runIdOf(x);
|
|
296
|
+
if (runId === undefined) return [f(false, `${x.label}: no transitionRunId, so its deployment.* event cannot be read`)];
|
|
297
|
+
if (events === undefined) return [f(false, `${x.label}: the events of management run ${runId} MUST be readable by a caller with runs:read in the same tenant`)];
|
|
298
|
+
const want = expectedEvents(x);
|
|
299
|
+
const names = want.map((w) => typeOf(w.v1));
|
|
300
|
+
const hit = want.flatMap((w) => events.filter((e) => e.type === typeOf(w.v1) && w.matches(e.payload)).map((e) => ({ w, e })))[0];
|
|
301
|
+
if (hit === undefined) {
|
|
302
|
+
const seen = events.filter((e) => e.type.startsWith('deployment.')).map((e) => `${e.type} ${JSON.stringify(e.payload)}`);
|
|
303
|
+
return [f(false, `${x.label}: management run ${runId} MUST record ${names.join(' or ')} for this transition (deployment.* seen: ${seen.length === 0 ? 'none' : seen.join('; ')})`)];
|
|
304
|
+
}
|
|
305
|
+
const validate = validators[hit.w.def];
|
|
306
|
+
const v = validate === undefined ? { ok: false, errors: `no validator for ${hit.w.def}` } : validate(hit.e.payload);
|
|
307
|
+
return [f(v.ok, `${x.label}: the ${hit.e.type} payload MUST validate against run-event-payloads#/$defs/${hit.w.def}, content-free (${v.errors})`)];
|
|
308
|
+
}
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The advertisement-shape witness at major 2. Wave 1: the claims-check AI
|
|
3
|
-
* families `
|
|
4
|
-
*
|
|
3
|
+
* families `limits` (`clarificationRounds`), `modelCapabilities` and
|
|
4
|
+
* `aiProviders`, whose structured-output facets came from the `envelopes`
|
|
5
|
+
* family RFC 0255 deleted. Wave 2: the record and facet shapes of `selfHostedRunner`,
|
|
5
6
|
* `credentials`, `authorization`, `providerUsage`, `subWorkflow`, `scheduling`,
|
|
6
|
-
* `artifactTypes`, `
|
|
7
|
+
* `artifactTypes`, `deadLetter` and `portability`. Wave 3: the
|
|
7
8
|
* families no host serves at v2 — `dataResidency`, `multiPartyConversation`,
|
|
8
9
|
* `nodePackRuntimes`, `uiPlugins`, `purposePropagation` and `agentRuntime` get
|
|
9
10
|
* facet legs; `conversationTurnModelProvenance`, `channelPresence`,
|
|
10
|
-
* `nondeterminismPolicy
|
|
11
|
+
* `nondeterminismPolicy` and `promptLibrary` carry no facet
|
|
11
12
|
* a host-binding v2 rule gives meaning to, so they get the record leg only.
|
|
12
13
|
* Wave 4 (suite 2.46.3): the `multiAgent.executionModel` sub-blocks a version
|
|
13
14
|
* level requires and the escalation kind it may name, and `auth.subjectLinkKey`
|
|
@@ -71,12 +72,12 @@ function onFamily(profile: MajorProfile, discovery: unknown, family: string, leg
|
|
|
71
72
|
|
|
72
73
|
/** The record as a whole against its seat in the capabilities schema. Catches every v1 shape v2 retired (`supported`, `selfHosted[]`, the per-provider `authModes` map, …). */
|
|
73
74
|
export type AdvertFamily =
|
|
74
|
-
| '
|
|
75
|
+
| 'modelCapabilities' | 'aiProviders'
|
|
75
76
|
| 'selfHostedRunner' | 'credentials' | 'authorization' | 'providerUsage' | 'subWorkflow'
|
|
76
|
-
| 'scheduling' | 'artifactTypes' | '
|
|
77
|
+
| 'scheduling' | 'artifactTypes' | 'deadLetter' | 'portability'
|
|
77
78
|
| 'dataResidency' | 'conversationTurnModelProvenance' | 'multiPartyConversation' | 'channelPresence'
|
|
78
79
|
| 'nodePackRuntimes' | 'uiPlugins' | 'purposePropagation' | 'nondeterminismPolicy'
|
|
79
|
-
| 'agentRuntime' | 'promptLibrary'
|
|
80
|
+
| 'agentRuntime' | 'promptLibrary'
|
|
80
81
|
| 'multiAgent' | 'auth' | 'agents';
|
|
81
82
|
|
|
82
83
|
export function recordSchemaLeg(profile: MajorProfile, discovery: unknown, family: AdvertFamily, doc: string): AdvertOutcome {
|
|
@@ -84,86 +85,72 @@ export function recordSchemaLeg(profile: MajorProfile, discovery: unknown, famil
|
|
|
84
85
|
}
|
|
85
86
|
|
|
86
87
|
// ---------------------------------------------------------------------------
|
|
87
|
-
//
|
|
88
|
+
// aiProviders structured output — spec/v2/core/host-services.md §Structured output
|
|
88
89
|
// ---------------------------------------------------------------------------
|
|
89
90
|
|
|
90
|
-
const
|
|
91
|
-
const
|
|
91
|
+
const SO_DOC = 'host-services.md §Structured output';
|
|
92
|
+
const SO = '/properties/aiProviders/properties';
|
|
92
93
|
|
|
93
|
-
/** `
|
|
94
|
-
export const
|
|
94
|
+
/** `aiProviders.reasoning`: an object whose `promptDirective` is one of three values. */
|
|
95
|
+
export const structuredOutputReasoningLeg = (p: MajorProfile, d: unknown): AdvertOutcome => onFamily(p, d, 'aiProviders', (rec) => {
|
|
95
96
|
const r = rec['reasoning'];
|
|
96
|
-
if (r === undefined) return inapplicable('the host does not advertise
|
|
97
|
-
const out = [bySchema(`${
|
|
97
|
+
if (r === undefined) return inapplicable('the host does not advertise aiProviders.reasoning (optional)');
|
|
98
|
+
const out = [bySchema(`${SO}/reasoning`, r, SO_DOC, 'aiProviders.reasoning')];
|
|
98
99
|
if (isRecord(r) && r['promptDirective'] !== undefined) {
|
|
99
|
-
out.push({ ok: ['mandatory', 'advisory', 'off'].includes(String(r['promptDirective'])), doc:
|
|
100
|
+
out.push({ ok: ['mandatory', 'advisory', 'off'].includes(String(r['promptDirective'])), doc: SO_DOC, message: `aiProviders.reasoning.promptDirective MUST be mandatory, advisory or off (got ${JSON.stringify(r['promptDirective'])})` });
|
|
100
101
|
}
|
|
101
102
|
return observed(out);
|
|
102
103
|
});
|
|
103
104
|
|
|
104
|
-
/** `
|
|
105
|
-
export const
|
|
105
|
+
/** `aiProviders.tierOneSubsetCompliance`: one of strict, warn, off. */
|
|
106
|
+
export const structuredOutputTierOneLeg = (p: MajorProfile, d: unknown): AdvertOutcome => onFamily(p, d, 'aiProviders', (rec) => {
|
|
106
107
|
const v = rec['tierOneSubsetCompliance'];
|
|
107
|
-
if (v === undefined) return inapplicable('the host does not advertise
|
|
108
|
-
return observed([bySchema(`${
|
|
108
|
+
if (v === undefined) return inapplicable('the host does not advertise aiProviders.tierOneSubsetCompliance (optional)');
|
|
109
|
+
return observed([bySchema(`${SO}/tierOneSubsetCompliance`, v, SO_DOC, 'aiProviders.tierOneSubsetCompliance')]);
|
|
109
110
|
});
|
|
110
111
|
|
|
111
|
-
/** The six reliability events by their v2 names
|
|
112
|
+
/** The six reliability events, by their v2 names. */
|
|
112
113
|
export const RELIABILITY_EVENTS = ['envelope.retry-attempted', 'envelope.retry-exhausted', 'envelope.refusal', 'envelope.truncated', 'envelope.nl-to-format-engaged', 'envelope.recovery-applied'] as const;
|
|
113
|
-
export const RELIABILITY_ALIASES: Readonly<Record<string, string>> = {
|
|
114
|
-
'envelope.retry.attempted': 'envelope.retry-attempted',
|
|
115
|
-
'envelope.retry.exhausted': 'envelope.retry-exhausted',
|
|
116
|
-
'envelope.nlToFormat.engaged': 'envelope.nl-to-format-engaged',
|
|
117
|
-
'envelope.recovery.applied': 'envelope.recovery-applied',
|
|
118
|
-
};
|
|
119
|
-
/** A listed reliability event by its v2 name: an alias folds to its hyphenated equivalent; anything else is returned unchanged. */
|
|
120
|
-
export const canonicalReliabilityEvent = (e: string): string => RELIABILITY_ALIASES[e] ?? e;
|
|
121
114
|
|
|
122
|
-
/**
|
|
123
|
-
|
|
124
|
-
* or v1 alias), and — after folding aliases — includes the two MUST-tier ones;
|
|
125
|
-
* `maxRetryAttempts` is an integer in [1, 16].
|
|
126
|
-
*/
|
|
127
|
-
export const envelopesReliabilityLeg = (p: MajorProfile, d: unknown): AdvertOutcome => onFamily(p, d, 'envelopes', (rec) => {
|
|
115
|
+
/** `aiProviders.reliability`: `events[]` names only reliability events and includes the two MUST-tier ones. */
|
|
116
|
+
export const structuredOutputReliabilityLeg = (p: MajorProfile, d: unknown): AdvertOutcome => onFamily(p, d, 'aiProviders', (rec) => {
|
|
128
117
|
const r = rec['reliability'];
|
|
129
|
-
if (r === undefined) return inapplicable('the host does not advertise
|
|
130
|
-
const out = [bySchema(`${
|
|
118
|
+
if (r === undefined) return inapplicable('the host does not advertise aiProviders.reliability (optional)');
|
|
119
|
+
const out = [bySchema(`${SO}/reliability`, r, SO_DOC, 'aiProviders.reliability')];
|
|
131
120
|
const events = isRecord(r) ? r['events'] : undefined;
|
|
132
121
|
if (!Array.isArray(events)) {
|
|
133
|
-
out.push({ ok: false, doc:
|
|
122
|
+
out.push({ ok: false, doc: SO_DOC, message: `a host advertising aiProviders.reliability MUST list the events it emits in reliability.events[] (got ${JSON.stringify(events)})` });
|
|
134
123
|
return observed(out);
|
|
135
124
|
}
|
|
136
|
-
const canonical = new Set(events.filter((e): e is string => typeof e === 'string').map(canonicalReliabilityEvent));
|
|
137
125
|
for (const e of events) {
|
|
138
|
-
|
|
139
|
-
out.push({ ok: name !== undefined && (RELIABILITY_EVENTS as readonly string[]).includes(name), doc: ENV_DOC, message: `reliability.events[] MUST name only reliability events, by v2 name or v1 alias (got ${JSON.stringify(e)})` });
|
|
126
|
+
out.push({ ok: typeof e === 'string' && (RELIABILITY_EVENTS as readonly string[]).includes(e), doc: SO_DOC, message: `reliability.events[] MUST name only reliability events (got ${JSON.stringify(e)})` });
|
|
140
127
|
}
|
|
141
128
|
for (const must of ['envelope.retry-exhausted', 'envelope.refusal']) {
|
|
142
|
-
out.push({ ok:
|
|
129
|
+
out.push({ ok: events.includes(must), doc: SO_DOC, message: `a host advertising aiProviders.reliability MUST emit and list ${must} (listed: ${JSON.stringify(events)})` });
|
|
143
130
|
}
|
|
144
131
|
return observed(out);
|
|
145
132
|
});
|
|
146
133
|
|
|
147
|
-
/** `
|
|
148
|
-
export const
|
|
134
|
+
/** `aiProviders.reliability.completion`: `distinguishesTruncation` is a boolean; `truncationBudgetMultiplier` a number in [1, 8]. */
|
|
135
|
+
export const structuredOutputCompletionLeg = (p: MajorProfile, d: unknown): AdvertOutcome => onFamily(p, d, 'aiProviders', (rec) => {
|
|
149
136
|
const r = rec['reliability'];
|
|
150
137
|
const c = isRecord(r) ? r['completion'] : undefined;
|
|
151
|
-
if (c === undefined) return inapplicable('the host does not advertise
|
|
152
|
-
const out = [bySchema(`${
|
|
138
|
+
if (c === undefined) return inapplicable('the host does not advertise aiProviders.reliability.completion (optional)');
|
|
139
|
+
const out = [bySchema(`${SO}/reliability/properties/completion`, c, SO_DOC, 'aiProviders.reliability.completion')];
|
|
153
140
|
const dt = isRecord(c) ? c['distinguishesTruncation'] : undefined;
|
|
154
|
-
out.push({ ok: typeof dt === 'boolean', doc:
|
|
141
|
+
out.push({ ok: typeof dt === 'boolean', doc: SO_DOC, message: `reliability.completion.distinguishesTruncation MUST be a boolean (got ${JSON.stringify(dt)})` });
|
|
155
142
|
return observed(out);
|
|
156
143
|
});
|
|
157
144
|
|
|
158
145
|
// ---------------------------------------------------------------------------
|
|
159
|
-
// limits — spec/v2/core/runs.md §Limits: the
|
|
146
|
+
// limits — spec/v2/core/runs.md §Limits: the cap every record carries
|
|
160
147
|
// ---------------------------------------------------------------------------
|
|
161
148
|
|
|
162
|
-
export const
|
|
163
|
-
export type
|
|
149
|
+
export const ALWAYS_CARRIED_CAPS = ['clarificationRounds'] as const;
|
|
150
|
+
export type AlwaysCarriedCap = (typeof ALWAYS_CARRIED_CAPS)[number];
|
|
164
151
|
|
|
165
152
|
/** `limits.<cap>`: present (runs.md: "`limits` always carries" it) and a non-negative integer. */
|
|
166
|
-
export const limitsCapLeg = (p: MajorProfile, d: unknown, cap:
|
|
153
|
+
export const limitsCapLeg = (p: MajorProfile, d: unknown, cap: AlwaysCarriedCap): AdvertOutcome => onFamily(p, d, 'limits', (rec) => {
|
|
167
154
|
const v = rec[cap];
|
|
168
155
|
const doc = 'runs.md §Limits';
|
|
169
156
|
if (v === undefined) return observed([{ ok: false, doc, message: `limits MUST always carry ${cap} (absent)` }]);
|
|
@@ -87,13 +87,21 @@ async function isStreamTextMockAdvertised(): Promise<boolean> {
|
|
|
87
87
|
return Array.isArray(testing?.mockProviders) && testing.mockProviders.includes('stream-text');
|
|
88
88
|
}
|
|
89
89
|
|
|
90
|
-
/**
|
|
91
|
-
|
|
92
|
-
|
|
90
|
+
/**
|
|
91
|
+
* Why the fixture gate holds, as RFC 0148 §A names it (openwop#1686: a
|
|
92
|
+
* describe-level skip recorded no disposition). The mock is what makes the
|
|
93
|
+
* fixture runnable, so a host serving the mock without the fixture is blocked;
|
|
94
|
+
* a host serving neither has nothing to stream through and is inapplicable.
|
|
95
|
+
*/
|
|
96
|
+
async function gateWhy(): Promise<readonly [SoftSkipKind, string]> {
|
|
97
|
+
return (await isStreamTextMockAdvertised())
|
|
98
|
+
? ['blocked', `the stream-text mock provider is advertised but the \`${FIXTURE_ID}\` fixture is not`] as const
|
|
99
|
+
: ['inapplicable', `neither the \`${FIXTURE_ID}\` fixture nor the stream-text mock provider is advertised`] as const;
|
|
100
|
+
}
|
|
93
101
|
|
|
94
102
|
describe('stream-text-fixture: messages-mode fold through the stream-text mock (F1)', () => {
|
|
95
103
|
it('delivers the mocked tokens in order, terminates with isLast + meta, and server-closes', async () => {
|
|
96
|
-
if (SKIP_NO_FIXTURE) return softSkip(...
|
|
104
|
+
if (SKIP_NO_FIXTURE) return softSkip(...(await gateWhy()));
|
|
97
105
|
if (!(await isStreamTextMockAdvertised())) {
|
|
98
106
|
// eslint-disable-next-line no-console
|
|
99
107
|
console.warn(
|
|
@@ -196,7 +204,7 @@ describe('stream-text-fixture: messages-mode fold through the stream-text mock (
|
|
|
196
204
|
});
|
|
197
205
|
|
|
198
206
|
it('an unknown mockProvider.id is rejected with 400 unsupported_mock_provider', async () => {
|
|
199
|
-
if (SKIP_NO_FIXTURE) return softSkip(...
|
|
207
|
+
if (SKIP_NO_FIXTURE) return softSkip(...(await gateWhy()));
|
|
200
208
|
if (!(await isStreamTextMockAdvertised())) {
|
|
201
209
|
// eslint-disable-next-line no-console
|
|
202
210
|
console.warn('[stream-text-fixture] stream-text mock not advertised; skipping negative leg');
|