@openwop/openwop-conformance 2.45.13 → 2.45.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/README.md +2 -2
- package/dist/spec-artifacts.lock.json +2 -2
- package/package.json +2 -2
- package/requirements.json +1446 -4
- package/scenario-majors.json +65 -2
- package/schemas/CORPUS-STAMP.json +14 -12
- package/src/lib/backpressure-witness.ts +2 -2
- package/src/lib/budget-witness.ts +3 -3
- package/src/lib/envelope-schema-static.ts +166 -0
- package/src/lib/exhaustion-facet-witness.ts +1 -1
- package/src/lib/family-advert-witness.ts +212 -0
- package/src/lib/i18n-double.ts +99 -0
- package/src/lib/i18n-negotiation-witness.ts +173 -0
- package/src/lib/localized-content-witness.ts +49 -0
- package/src/lib/model-capability-insufficient-witness.ts +121 -0
- package/src/lib/prompt-double.ts +217 -0
- package/src/lib/prompt-events-witness.ts +211 -0
- package/src/lib/prompt-library-witness.ts +406 -0
- package/src/lib/run-bounds-witness.ts +111 -0
- package/src/lib/scratch-host.ts +4 -1
- package/src/lib/table-schema-witness.ts +3 -1
- package/src/scenarios/v2-ai-envelope-shape-static.test.ts +93 -0
- package/src/scenarios/v2-ai-providers-advertisement.test.ts +94 -0
- package/src/scenarios/v2-envelope-reasoning-shape.test.ts +81 -0
- package/src/scenarios/v2-envelope-tier-one-subset-static.test.ts +77 -0
- package/src/scenarios/v2-envelope-variant-discriminator-static.test.ts +57 -0
- package/src/scenarios/v2-envelopes-advertisement.test.ts +85 -0
- package/src/scenarios/v2-i18n-negotiation.test.ts +91 -0
- package/src/scenarios/v2-limits-envelope-caps-advertisement.test.ts +61 -0
- package/src/scenarios/v2-localized-content-delivery.test.ts +136 -0
- package/src/scenarios/v2-media-inline-cap-static.test.ts +59 -0
- package/src/scenarios/v2-model-capabilities-advertisement.test.ts +61 -0
- package/src/scenarios/v2-model-capability-insufficient.test.ts +63 -0
- package/src/scenarios/v2-prompt-all-four-kinds-events.test.ts +69 -0
- package/src/scenarios/v2-prompt-end-to-end-events.test.ts +71 -0
- package/src/scenarios/v2-prompt-list-and-fetch.test.ts +103 -0
- package/src/scenarios/v2-prompt-mutable-lifecycle.test.ts +130 -0
- package/src/scenarios/v2-prompt-pack-install.test.ts +78 -0
- package/src/scenarios/v2-prompt-render-deterministic.test.ts +87 -0
- package/src/scenarios/v2-prompt-resolution-chain-event.test.ts +46 -0
- package/src/scenarios/v2-prompt-template-shape.test.ts +112 -0
- package/src/scenarios/v2-run-execution-bounds.test.ts +67 -0
|
@@ -0,0 +1,406 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The prompt-library witness on the normative `/prompts*` surface
|
|
3
|
+
* (`host-services.md` §`prompts` + §Library; RFC 0028), shared by every major
|
|
4
|
+
* that serves it. What differs between majors (the path prefix, how an id
|
|
5
|
+
* becomes a path segment, how a family is advertised, the schemas) is the
|
|
6
|
+
* profile row plus the validators the scenario hands in.
|
|
7
|
+
*
|
|
8
|
+
* Legs, grouped by the scenario that owns them:
|
|
9
|
+
* list-and-fetch list (envelope + every item validates), filters
|
|
10
|
+
* (`kind`, `source`), fetch-by-id, ETag revalidation,
|
|
11
|
+
* unknown id `404`, and the gate-off read (`404 not_found`
|
|
12
|
+
* when `endpointsSupported` is not advertised);
|
|
13
|
+
* mutable the create → read → duplicate → update → non-monotonic
|
|
14
|
+
* → delete round trip ({@link driveLifecycle} observes,
|
|
15
|
+
* {@link judgeLifecycle} is pure), built-ins read-only,
|
|
16
|
+
* writes authenticated, and the gate-off write;
|
|
17
|
+
* render deterministic hash, a changed binding changes the hash,
|
|
18
|
+
* hash shapes, the body present only under `full`, and an
|
|
19
|
+
* unbound required variable refused `validation_error`;
|
|
20
|
+
* packs pack-source listing coherent with `packsSupported`, the
|
|
21
|
+
* `meta.packName`/`packVersion` stamps, and the in-tree
|
|
22
|
+
* reference pack fetched by id (inapplicable when absent).
|
|
23
|
+
*
|
|
24
|
+
* Each leg OBSERVES and returns findings; it asserts nothing. The scenario maps
|
|
25
|
+
* findings to requirement ids, and `prompt-library-witness.test.ts` proves every
|
|
26
|
+
* leg against `prompt-double.ts` with one defect turned on.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import { randomBytes } from 'node:crypto';
|
|
30
|
+
import { driver, type OpenWOPResponse } from './driver.js';
|
|
31
|
+
import { readErrorCode } from './error-envelope.js';
|
|
32
|
+
import type { MajorProfile } from './major-profile.js';
|
|
33
|
+
|
|
34
|
+
export interface PromptFinding { readonly ok: boolean; readonly doc: string; readonly message: string }
|
|
35
|
+
export type Skip = { readonly kind: 'skip'; readonly disposition: 'inapplicable' | 'blocked'; readonly reason: string };
|
|
36
|
+
export type PromptOutcome = Skip | { readonly kind: 'observed'; readonly findings: readonly PromptFinding[] };
|
|
37
|
+
/** A schema validator, e.g. `v2Validator('prompt-template')`. */
|
|
38
|
+
export type Validate = (doc: unknown) => { ok: boolean; errors: string };
|
|
39
|
+
|
|
40
|
+
interface Variable { name?: unknown; required?: unknown; source?: unknown }
|
|
41
|
+
export interface Template { templateId?: unknown; version?: unknown; kind?: unknown; text?: unknown; variables?: unknown; meta?: { source?: unknown; packName?: unknown; packVersion?: unknown } }
|
|
42
|
+
|
|
43
|
+
const isRecord = (v: unknown): v is Record<string, unknown> => v !== null && typeof v === 'object' && !Array.isArray(v);
|
|
44
|
+
const f = (ok: boolean, doc: string, message: string): PromptFinding => ({ ok, doc, message });
|
|
45
|
+
const SHA = /^sha256:[0-9a-f]{64}$/;
|
|
46
|
+
const SEMVER = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/;
|
|
47
|
+
const DOC_PROMPTS = 'host-services.md §prompts';
|
|
48
|
+
const DOC_LIB = 'host-services.md §Library';
|
|
49
|
+
const DOC_GATE = 'errors.md §Unadvertised operations';
|
|
50
|
+
|
|
51
|
+
/** The in-tree reference prompt pack (`examples/packs/prompt-sample`). */
|
|
52
|
+
export const REFERENCE_PACK = 'vendor.openwop.prompt-sample';
|
|
53
|
+
export const REFERENCE_TEMPLATE = 'writer-system';
|
|
54
|
+
|
|
55
|
+
/** What the host advertises for `prompts` at this major. */
|
|
56
|
+
export interface PromptAdverts {
|
|
57
|
+
readonly family: Record<string, unknown> | null;
|
|
58
|
+
readonly endpoints: boolean;
|
|
59
|
+
readonly mutable: boolean;
|
|
60
|
+
readonly packs: boolean;
|
|
61
|
+
readonly observability: 'off' | 'hashed' | 'full';
|
|
62
|
+
readonly promptsPath: string;
|
|
63
|
+
readonly renderPath: string;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function promptAdverts(profile: MajorProfile, doc: unknown): PromptAdverts {
|
|
67
|
+
const family = profile.family(doc, 'prompts');
|
|
68
|
+
const promptsPath = profile.major >= 2 ? '/prompts' : '/v1/prompts';
|
|
69
|
+
const lib = isRecord(family?.['library']) ? (family!['library'] as Record<string, unknown>) : undefined;
|
|
70
|
+
const obs = family?.['observability'];
|
|
71
|
+
return {
|
|
72
|
+
family,
|
|
73
|
+
endpoints: family?.['endpointsSupported'] === true,
|
|
74
|
+
mutable: family?.['endpointsSupported'] === true && family['mutableLibrary'] === true,
|
|
75
|
+
packs: family?.['packsSupported'] === true,
|
|
76
|
+
observability: obs === 'off' || obs === 'full' ? obs : 'hashed',
|
|
77
|
+
promptsPath,
|
|
78
|
+
renderPath: typeof lib?.['renderEndpoint'] === 'string' ? (lib['renderEndpoint'] as string) : `${promptsPath}:render`,
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export function endpointsGate(a: PromptAdverts): Skip | null {
|
|
83
|
+
if (a.family === null) return { kind: 'skip', disposition: 'inapplicable', reason: 'the host does not advertise prompts' };
|
|
84
|
+
if (!a.endpoints) return { kind: 'skip', disposition: 'inapplicable', reason: 'prompts.endpointsSupported is not true — the /prompts* operations are not served' };
|
|
85
|
+
return null;
|
|
86
|
+
}
|
|
87
|
+
function mutableGate(a: PromptAdverts): Skip | null {
|
|
88
|
+
const g = endpointsGate(a); if (g) return g;
|
|
89
|
+
return a.mutable ? null : { kind: 'skip', disposition: 'inapplicable', reason: 'prompts.mutableLibrary is not true — the /prompts writes are not served' };
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const tplPath = (profile: MajorProfile, a: PromptAdverts, id: string, query = ''): string => `${a.promptsPath}/${profile.idSegment(id)}${query}`;
|
|
93
|
+
const items = (res: OpenWOPResponse): Template[] => {
|
|
94
|
+
const v = isRecord(res.json) ? res.json['items'] : undefined;
|
|
95
|
+
return Array.isArray(v) ? v.filter(isRecord) as Template[] : [];
|
|
96
|
+
};
|
|
97
|
+
const gateOff = (res: OpenWOPResponse, what: string, facet: string): PromptFinding[] => [
|
|
98
|
+
f(res.status === 404, DOC_GATE, `${what} with ${facet} unadvertised MUST answer 404 (got ${res.status})`),
|
|
99
|
+
f(readErrorCode(res.json) === 'not_found', DOC_GATE, `${what} with ${facet} unadvertised MUST carry not_found (got ${String(readErrorCode(res.json))})`),
|
|
100
|
+
];
|
|
101
|
+
|
|
102
|
+
// ── list-and-fetch ─────────────────────────────────────────────────────────
|
|
103
|
+
|
|
104
|
+
/** Leg: GET /prompts answers `{ items, nextCursor? }` and every item validates. */
|
|
105
|
+
export async function listLeg(a: PromptAdverts, validate: Validate): Promise<PromptOutcome> {
|
|
106
|
+
const g = endpointsGate(a); if (g) return g;
|
|
107
|
+
const res = await driver.get(a.promptsPath);
|
|
108
|
+
const body = isRecord(res.json) ? res.json : {};
|
|
109
|
+
const findings = [
|
|
110
|
+
f(res.status === 200, DOC_PROMPTS, `GET ${a.promptsPath} MUST answer 200 when endpointsSupported is advertised (got ${res.status})`),
|
|
111
|
+
f(Array.isArray(body['items']), DOC_PROMPTS, 'the list MUST carry an items array'),
|
|
112
|
+
f(body['nextCursor'] === undefined || typeof body['nextCursor'] === 'string', DOC_PROMPTS, `nextCursor MUST be an opaque string when present (got ${JSON.stringify(body['nextCursor'])})`),
|
|
113
|
+
];
|
|
114
|
+
for (const t of items(res)) {
|
|
115
|
+
const v = validate(t);
|
|
116
|
+
findings.push(f(v.ok, 'schemas PromptTemplate', `every listed template MUST validate (${String(t.templateId)}): ${v.errors}`));
|
|
117
|
+
}
|
|
118
|
+
return { kind: 'observed', findings };
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** Leg: `?kind=system` and `?source=host` narrow without breaking the envelope. */
|
|
122
|
+
export async function filterLeg(a: PromptAdverts): Promise<PromptOutcome> {
|
|
123
|
+
const g = endpointsGate(a); if (g) return g;
|
|
124
|
+
const byKind = await driver.get(`${a.promptsPath}?kind=system`);
|
|
125
|
+
const bySource = await driver.get(`${a.promptsPath}?source=host`);
|
|
126
|
+
return { kind: 'observed', findings: [
|
|
127
|
+
f(byKind.status === 200 && Array.isArray(isRecord(byKind.json) ? byKind.json['items'] : undefined), DOC_PROMPTS, `GET ?kind=system MUST answer 200 with items (got ${byKind.status})`),
|
|
128
|
+
...items(byKind).map((t) => f(t.kind === 'system', DOC_PROMPTS, `?kind=system MUST narrow to kind system (got ${String(t.kind)} on ${String(t.templateId)})`)),
|
|
129
|
+
f(bySource.status === 200 && Array.isArray(isRecord(bySource.json) ? bySource.json['items'] : undefined), DOC_PROMPTS, `GET ?source=host MUST answer 200 with items (got ${bySource.status})`),
|
|
130
|
+
...items(bySource).map((t) => f(t.meta?.source === 'host', DOC_PROMPTS, `?source=host MUST narrow to meta.source host (got ${String(t.meta?.source)} on ${String(t.templateId)})`)),
|
|
131
|
+
] };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** A listed template to fetch, host-sourced first. `null`: the library lists nothing. */
|
|
135
|
+
async function aTemplate(a: PromptAdverts): Promise<Template | null> {
|
|
136
|
+
const host = items(await driver.get(`${a.promptsPath}?source=host&limit=1`))[0];
|
|
137
|
+
return host ?? items(await driver.get(`${a.promptsPath}?limit=1`))[0] ?? null;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** Leg: a listed template is fetched by id. */
|
|
141
|
+
export async function fetchLeg(profile: MajorProfile, a: PromptAdverts, validate: Validate): Promise<PromptOutcome> {
|
|
142
|
+
const g = endpointsGate(a); if (g) return g;
|
|
143
|
+
const t = await aTemplate(a);
|
|
144
|
+
if (t === null || typeof t.templateId !== 'string') return { kind: 'skip', disposition: 'inapplicable', reason: 'the library lists no template, so there is none to fetch (an empty library is conformant)' };
|
|
145
|
+
const res = await driver.get(tplPath(profile, a, t.templateId));
|
|
146
|
+
const body = isRecord(res.json) ? res.json : {};
|
|
147
|
+
const v = validate(body);
|
|
148
|
+
return { kind: 'observed', findings: [
|
|
149
|
+
f(res.status === 200, DOC_PROMPTS, `GET ${a.promptsPath}/{templateId} MUST answer 200 for a listed template (got ${res.status})`),
|
|
150
|
+
f(body['templateId'] === t.templateId, DOC_PROMPTS, `the fetched template MUST be the one named (got ${String(body['templateId'])})`),
|
|
151
|
+
f(v.ok, 'schemas PromptTemplate', `the fetched template MUST validate: ${v.errors}`),
|
|
152
|
+
] };
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/** Leg: an ETag, when sent, revalidates to `304`. `getPromptTemplate` SHOULD send one; none is inapplicable. */
|
|
156
|
+
export async function etagLeg(profile: MajorProfile, a: PromptAdverts): Promise<PromptOutcome> {
|
|
157
|
+
const g = endpointsGate(a); if (g) return g;
|
|
158
|
+
const t = await aTemplate(a);
|
|
159
|
+
if (t === null || typeof t.templateId !== 'string') return { kind: 'skip', disposition: 'inapplicable', reason: 'the library lists no template, so there is none to revalidate' };
|
|
160
|
+
const first = await driver.get(tplPath(profile, a, t.templateId));
|
|
161
|
+
const etag = first.headers.get('etag');
|
|
162
|
+
if (first.status !== 200 || etag === null || etag.length === 0) return { kind: 'skip', disposition: 'inapplicable', reason: `getPromptTemplate sent no ETag (status ${first.status}) — a SHOULD, so there is nothing to revalidate` };
|
|
163
|
+
const again = await driver.get(tplPath(profile, a, t.templateId), { headers: { 'If-None-Match': etag } });
|
|
164
|
+
return { kind: 'observed', findings: [f(again.status === 304, `${DOC_LIB}; RFC 9110 §13.1.2`, `If-None-Match with the current ETag MUST answer 304 (got ${again.status})`)] };
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/** Leg: an unknown template id is `404` with the error envelope. */
|
|
168
|
+
export async function unknownLeg(profile: MajorProfile, a: PromptAdverts, validateEnvelope: Validate): Promise<PromptOutcome> {
|
|
169
|
+
const g = endpointsGate(a); if (g) return g;
|
|
170
|
+
const res = await driver.get(tplPath(profile, a, `conformance-unknown-${randomBytes(6).toString('hex')}`));
|
|
171
|
+
const v = validateEnvelope(res.json);
|
|
172
|
+
return { kind: 'observed', findings: [
|
|
173
|
+
f(res.status === 404, DOC_PROMPTS, `an unknown templateId MUST answer 404 (got ${res.status})`),
|
|
174
|
+
f(v.ok, 'errors.md; schemas ErrorEnvelope', `the 404 MUST carry the error envelope: ${v.errors}`),
|
|
175
|
+
] };
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** Leg: with `endpointsSupported` unadvertised, the read answers `404 not_found`. */
|
|
179
|
+
export async function gateOffReadLeg(a: PromptAdverts): Promise<PromptOutcome> {
|
|
180
|
+
if (a.endpoints) return { kind: 'skip', disposition: 'inapplicable', reason: 'prompts.endpointsSupported is advertised — the gate is on' };
|
|
181
|
+
return { kind: 'observed', findings: gateOff(await driver.get(a.promptsPath), `GET ${a.promptsPath}`, 'prompts.endpointsSupported') };
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// ── mutable library ────────────────────────────────────────────────────────
|
|
185
|
+
|
|
186
|
+
export interface LifecycleObservation {
|
|
187
|
+
readonly templateId: string;
|
|
188
|
+
readonly create: { status: number; location: string | null };
|
|
189
|
+
readonly read: { status: number; source: unknown; templateId: unknown };
|
|
190
|
+
readonly duplicate: number;
|
|
191
|
+
readonly update: { status: number; storedVersion: unknown };
|
|
192
|
+
readonly nonMonotonic: number;
|
|
193
|
+
readonly remove: { status: number; after: number };
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/** The round trip, observed. Asserts nothing. */
|
|
197
|
+
export async function driveLifecycle(profile: MajorProfile, a: PromptAdverts): Promise<Skip | LifecycleObservation> {
|
|
198
|
+
const g = mutableGate(a); if (g) return g;
|
|
199
|
+
const templateId = `conformance.user.lifecycle-${randomBytes(5).toString('hex')}`;
|
|
200
|
+
const at = tplPath(profile, a, templateId);
|
|
201
|
+
const body = (version: string, text: string): Record<string, unknown> => ({ templateId, version, kind: 'system', text });
|
|
202
|
+
const created = await driver.post(a.promptsPath, body('1.0.0', 'You are a conformance probe. {{tone}}'));
|
|
203
|
+
const read = await driver.get(at);
|
|
204
|
+
const dup = await driver.post(a.promptsPath, body('1.0.0', 'duplicate'));
|
|
205
|
+
const upd = await driver.put(at, body('1.1.0', 'You are a conformance probe v1.1. {{tone}}'));
|
|
206
|
+
const stored = await driver.get(at);
|
|
207
|
+
const back = await driver.put(at, body('0.9.0', 'cannot go back'));
|
|
208
|
+
const del = await driver.delete(at);
|
|
209
|
+
const after = await driver.get(at);
|
|
210
|
+
const r = isRecord(read.json) ? read.json : {};
|
|
211
|
+
const meta = isRecord(r['meta']) ? r['meta'] : {};
|
|
212
|
+
return {
|
|
213
|
+
templateId,
|
|
214
|
+
create: { status: created.status, location: created.headers.get('location') },
|
|
215
|
+
read: { status: read.status, source: meta['source'], templateId: r['templateId'] },
|
|
216
|
+
duplicate: dup.status,
|
|
217
|
+
update: { status: upd.status, storedVersion: isRecord(stored.json) ? stored.json['version'] : undefined },
|
|
218
|
+
nonMonotonic: back.status,
|
|
219
|
+
remove: { status: del.status, after: after.status },
|
|
220
|
+
};
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
export type LifecycleStep = 'create' | 'read' | 'duplicate' | 'update' | 'nonMonotonic' | 'remove';
|
|
224
|
+
|
|
225
|
+
/** The round trip, judged per step. Pure. */
|
|
226
|
+
export function judgeLifecycle(o: LifecycleObservation): Record<LifecycleStep, PromptFinding[]> {
|
|
227
|
+
return {
|
|
228
|
+
create: [
|
|
229
|
+
f(o.create.status === 201, DOC_LIB, `POST creates a user template: 201 (got ${o.create.status})`),
|
|
230
|
+
f(typeof o.create.location === 'string' && o.create.location.includes(o.templateId), 'openapi createPromptTemplate 201', `the 201 MUST carry a Location naming the new templateId (got ${String(o.create.location)})`),
|
|
231
|
+
],
|
|
232
|
+
read: [
|
|
233
|
+
f(o.read.status === 200 && o.read.templateId === o.templateId, DOC_LIB, `the created template MUST be readable (got ${o.read.status})`),
|
|
234
|
+
f(o.read.source === 'user', 'schemas PromptTemplate meta.source', `a template created through the API MUST carry meta.source user (got ${String(o.read.source)})`),
|
|
235
|
+
],
|
|
236
|
+
duplicate: [f(o.duplicate === 409, 'openapi createPromptTemplate 409', `a second POST of the same (templateId, version) MUST answer 409 (got ${o.duplicate})`)],
|
|
237
|
+
update: [
|
|
238
|
+
f(o.update.status === 200, DOC_LIB, `an update carrying a greater SemVer MUST be accepted: 200 (got ${o.update.status})`),
|
|
239
|
+
f(o.update.storedVersion === '1.1.0', DOC_LIB, `the stored version MUST reflect the update (got ${String(o.update.storedVersion)})`),
|
|
240
|
+
],
|
|
241
|
+
nonMonotonic: [f(o.nonMonotonic === 409, DOC_LIB, `an update MUST carry a greater SemVer; a lower one answers 409 (got ${o.nonMonotonic})`)],
|
|
242
|
+
remove: [
|
|
243
|
+
f(o.remove.status === 204, 'openapi deletePromptTemplate 204', `DELETE of a user template MUST answer 204 (got ${o.remove.status})`),
|
|
244
|
+
f(o.remove.after === 404, DOC_LIB, `a deleted template MUST read 404 (got ${o.remove.after})`),
|
|
245
|
+
],
|
|
246
|
+
};
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/** Leg: built-in templates are read-only — DELETE answers 403. */
|
|
250
|
+
export async function readOnlyLeg(profile: MajorProfile, a: PromptAdverts): Promise<PromptOutcome> {
|
|
251
|
+
const g = mutableGate(a); if (g) return g;
|
|
252
|
+
const t = items(await driver.get(`${a.promptsPath}?source=host&limit=1`))[0];
|
|
253
|
+
if (t === undefined || typeof t.templateId !== 'string') return { kind: 'skip', disposition: 'inapplicable', reason: 'the library lists no host built-in template to probe' };
|
|
254
|
+
// DELETE only, as at major 1: a PUT probe would overwrite a shared built-in on a defective host.
|
|
255
|
+
const del = await driver.delete(tplPath(profile, a, t.templateId));
|
|
256
|
+
return { kind: 'observed', findings: [f(del.status === 403, DOC_LIB, `built-in templates are read-only: DELETE MUST answer 403 (got ${del.status})`)] };
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/** Leg: writes MUST be authenticated. */
|
|
260
|
+
export async function unauthenticatedWriteLeg(a: PromptAdverts): Promise<PromptOutcome> {
|
|
261
|
+
const g = mutableGate(a); if (g) return g;
|
|
262
|
+
const templateId = `conformance.user.anon-${randomBytes(5).toString('hex')}`;
|
|
263
|
+
const res = await driver.post(a.promptsPath, { templateId, version: '1.0.0', kind: 'system', text: 'x' }, { authenticated: false });
|
|
264
|
+
return { kind: 'observed', findings: [f(res.status === 401, DOC_LIB, `an unauthenticated write MUST be refused 401 (got ${res.status})`)] };
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/** Leg: with `mutableLibrary` unadvertised, a write answers `404 not_found`. */
|
|
268
|
+
export async function gateOffWriteLeg(profile: MajorProfile, a: PromptAdverts): Promise<PromptOutcome> {
|
|
269
|
+
if (a.mutable) return { kind: 'skip', disposition: 'inapplicable', reason: 'prompts.mutableLibrary is advertised — the gate is on' };
|
|
270
|
+
const templateId = `conformance.user.gate-off-${randomBytes(5).toString('hex')}`;
|
|
271
|
+
const res = await driver.post(a.promptsPath, { templateId, version: '1.0.0', kind: 'system', text: 'x' });
|
|
272
|
+
if (res.status >= 200 && res.status < 300) await driver.delete(tplPath(profile, a, templateId)).catch(() => undefined);
|
|
273
|
+
return { kind: 'observed', findings: gateOff(res, `POST ${a.promptsPath}`, 'prompts.mutableLibrary') };
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
// ── render ─────────────────────────────────────────────────────────────────
|
|
277
|
+
|
|
278
|
+
const vars = (t: Template): Variable[] => (Array.isArray(t.variables) ? t.variables.filter(isRecord) as Variable[] : []);
|
|
279
|
+
const renderable = (t: Template): boolean => !vars(t).some((v) => v.source === 'secret' && v.required === true);
|
|
280
|
+
const bindings = (t: Template, value: (name: string) => string): Record<string, string> =>
|
|
281
|
+
Object.fromEntries(vars(t).filter((v) => v.source !== 'secret' && typeof v.name === 'string').map((v) => [v.name as string, value(v.name as string)]));
|
|
282
|
+
const refOf = (t: Template): string => `prompt:${String(t.templateId)}@${String(t.version)}`;
|
|
283
|
+
|
|
284
|
+
/** A host-listed template every required binding of which the suite can supply (no required secret). */
|
|
285
|
+
async function renderTarget(a: PromptAdverts, want?: (t: Template) => boolean): Promise<Template | null> {
|
|
286
|
+
const listed = items(await driver.get(`${a.promptsPath}?source=host&limit=200`)).filter(renderable);
|
|
287
|
+
const pool = listed.length > 0 ? listed : items(await driver.get(`${a.promptsPath}?limit=200`)).filter(renderable);
|
|
288
|
+
const ok = pool.filter((t) => typeof t.templateId === 'string' && typeof t.version === 'string' && (want ?? (() => true))(t));
|
|
289
|
+
return ok.find((t) => vars(t).some((v) => v.required === true && v.source !== 'secret')) ?? ok[0] ?? null;
|
|
290
|
+
}
|
|
291
|
+
const render = (a: PromptAdverts, ref: string, variables: Record<string, unknown>): Promise<OpenWOPResponse> => driver.post(a.renderPath, { ref, variables });
|
|
292
|
+
const rbody = (r: OpenWOPResponse): Record<string, unknown> => (isRecord(r.json) ? r.json : {});
|
|
293
|
+
const vh = (r: OpenWOPResponse): Record<string, unknown> => (isRecord(rbody(r)['variableHashes']) ? rbody(r)['variableHashes'] as Record<string, unknown> : {});
|
|
294
|
+
const noTarget: Skip = { kind: 'skip', disposition: 'inapplicable', reason: 'the library lists no template the suite can render (none, or every one needs a secret binding)' };
|
|
295
|
+
|
|
296
|
+
/** Leg: identical inputs render the identical hash and variableHashes. */
|
|
297
|
+
export async function deterministicLeg(a: PromptAdverts): Promise<PromptOutcome> {
|
|
298
|
+
const g = endpointsGate(a); if (g) return g;
|
|
299
|
+
const t = await renderTarget(a); if (t === null) return noTarget;
|
|
300
|
+
const v = bindings(t, (n) => `conformance-${n}-value`);
|
|
301
|
+
const one = await render(a, refOf(t), v);
|
|
302
|
+
const two = await render(a, refOf(t), v);
|
|
303
|
+
return { kind: 'observed', findings: [
|
|
304
|
+
f(one.status === 200 && two.status === 200, DOC_LIB, `renderPromptTemplate MUST answer 200 for ${refOf(t)} with every required binding (got ${one.status}, ${two.status})`),
|
|
305
|
+
f(typeof rbody(one)['hash'] === 'string' && rbody(one)['hash'] === rbody(two)['hash'], DOC_LIB, `identical (ref, variables) MUST render the identical hash (got ${String(rbody(one)['hash'])} vs ${String(rbody(two)['hash'])})`),
|
|
306
|
+
f(JSON.stringify(Object.entries(vh(one)).sort()) === JSON.stringify(Object.entries(vh(two)).sort()), DOC_LIB, 'identical (ref, variables) MUST render identical variableHashes'),
|
|
307
|
+
] };
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/** Leg: a changed binding changes the hash and its variableHash. */
|
|
311
|
+
export async function variesLeg(a: PromptAdverts): Promise<PromptOutcome> {
|
|
312
|
+
const g = endpointsGate(a); if (g) return g;
|
|
313
|
+
const t = await renderTarget(a); if (t === null) return noTarget;
|
|
314
|
+
const req = vars(t).find((v) => v.required === true && v.source !== 'secret' && typeof v.name === 'string');
|
|
315
|
+
if (req === undefined) return { kind: 'skip', disposition: 'inapplicable', reason: `no listed template has a required non-secret variable to toggle (${refOf(t)})` };
|
|
316
|
+
const name = req.name as string;
|
|
317
|
+
const v = bindings(t, (n) => `conformance-${n}-baseline`);
|
|
318
|
+
const base = await render(a, refOf(t), v);
|
|
319
|
+
const toggled = await render(a, refOf(t), { ...v, [name]: 'conformance-toggled-value' });
|
|
320
|
+
return { kind: 'observed', findings: [
|
|
321
|
+
f(base.status === 200 && toggled.status === 200, DOC_LIB, `renderPromptTemplate MUST answer 200 (got ${base.status}, ${toggled.status})`),
|
|
322
|
+
f(rbody(base)['hash'] !== rbody(toggled)['hash'], DOC_LIB, 'a changed binding MUST change the rendered hash'),
|
|
323
|
+
f(vh(base)[name] !== undefined && vh(base)[name] !== vh(toggled)[name], DOC_LIB, `a changed binding MUST change variableHashes.${name}`),
|
|
324
|
+
] };
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/** Leg: hash shapes, refs, and the body present only under `observability: full`. */
|
|
328
|
+
export async function renderShapeLeg(a: PromptAdverts): Promise<PromptOutcome> {
|
|
329
|
+
const g = endpointsGate(a); if (g) return g;
|
|
330
|
+
// A few-shot or schema-hint body is the one a kind-specific field would miss.
|
|
331
|
+
const t = (await renderTarget(a, (x) => x.kind === 'few-shot' || x.kind === 'schema-hint')) ?? (await renderTarget(a));
|
|
332
|
+
if (t === null) return noTarget;
|
|
333
|
+
const res = await render(a, refOf(t), bindings(t, () => 'conformance-value'));
|
|
334
|
+
const b = rbody(res);
|
|
335
|
+
const findings: PromptFinding[] = [
|
|
336
|
+
f(res.status === 200, DOC_LIB, `renderPromptTemplate MUST answer 200 for ${refOf(t)} (got ${res.status})`),
|
|
337
|
+
f(typeof b['hash'] === 'string' && SHA.test(b['hash']), 'openapi renderPromptTemplate 200', `hash MUST match sha256:<hex64> (got ${String(b['hash'])})`),
|
|
338
|
+
f(Array.isArray(b['refs']), 'openapi renderPromptTemplate 200', 'refs MUST be an array'),
|
|
339
|
+
...Object.entries(vh(res)).map(([k, h]) => f(typeof h === 'string' && SHA.test(h), 'openapi renderPromptTemplate 200', `variableHashes.${k} MUST match sha256:<hex64> (got ${String(h)})`)),
|
|
340
|
+
];
|
|
341
|
+
findings.push(a.observability === 'full'
|
|
342
|
+
? f(typeof b['composed'] === 'string' && b['composed'].length > 0, `${DOC_PROMPTS} §Composition; openapi renderPromptTemplate`, `under observability full, composed MUST carry the body for every kind (${String(t.kind)} ${refOf(t)})`)
|
|
343
|
+
: f(b['composed'] === undefined, `${DOC_PROMPTS} §Composition ("carrying bodies only under full"); openapi renderPromptTemplate`, `under observability ${a.observability}, composed MUST be absent (got ${typeof b['composed']})`));
|
|
344
|
+
return { kind: 'observed', findings };
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
/** Leg: an unbound required variable is refused `400 validation_error`. */
|
|
348
|
+
export async function unboundRequiredLeg(a: PromptAdverts): Promise<PromptOutcome> {
|
|
349
|
+
const g = endpointsGate(a); if (g) return g;
|
|
350
|
+
const t = await renderTarget(a); if (t === null) return noTarget;
|
|
351
|
+
const req = vars(t).find((v) => v.required === true && v.source !== 'secret' && typeof v.name === 'string');
|
|
352
|
+
if (req === undefined) return { kind: 'skip', disposition: 'inapplicable', reason: `no listed template has a required non-secret variable to leave unbound (${refOf(t)})` };
|
|
353
|
+
const v = bindings(t, (n) => `conformance-${n}`);
|
|
354
|
+
delete v[req.name as string];
|
|
355
|
+
const res = await render(a, refOf(t), v);
|
|
356
|
+
return { kind: 'observed', findings: [
|
|
357
|
+
f(res.status === 400, 'openapi renderPromptTemplate 400', `rendering with required variable ${String(req.name)} unbound MUST answer 400 (got ${res.status})`),
|
|
358
|
+
f(readErrorCode(res.json) === 'validation_error', 'openapi renderPromptTemplate 400', `the refusal MUST carry validation_error (got ${String(readErrorCode(res.json))})`),
|
|
359
|
+
] };
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
// ── packs ──────────────────────────────────────────────────────────────────
|
|
363
|
+
|
|
364
|
+
/** Leg: `?source=pack` lists only pack templates, none at all unless `packsSupported`, and at least one when the operator says a pack is installed. */
|
|
365
|
+
export async function packListLeg(a: PromptAdverts, opts: { requireInstalled?: boolean } = {}): Promise<PromptOutcome> {
|
|
366
|
+
const g = endpointsGate(a); if (g) return g;
|
|
367
|
+
const res = await driver.get(`${a.promptsPath}?source=pack`);
|
|
368
|
+
const listed = items(res);
|
|
369
|
+
const findings: PromptFinding[] = [
|
|
370
|
+
f(res.status === 200 && Array.isArray(isRecord(res.json) ? res.json['items'] : undefined), DOC_PROMPTS, `GET ?source=pack MUST answer 200 with items (got ${res.status})`),
|
|
371
|
+
...listed.map((t) => f(t.meta?.source === 'pack', DOC_PROMPTS, `?source=pack MUST narrow to meta.source pack (got ${String(t.meta?.source)} on ${String(t.templateId)})`)),
|
|
372
|
+
];
|
|
373
|
+
if (!a.packs) findings.push(f(listed.length === 0, 'capabilities.schema.json §prompts.packsSupported', `with packsSupported not advertised, packs are not loaded — ?source=pack MUST list none (got ${listed.length})`));
|
|
374
|
+
if (opts.requireInstalled === true) findings.push(f(listed.length > 0, 'RFC 0028 §B (operator: OPENWOP_TEST_PROMPT_PACK_INSTALLED=true)', 'the operator says a prompt pack is installed, so ?source=pack MUST list at least one template'));
|
|
375
|
+
return { kind: 'observed', findings };
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
/** Leg: every pack template carries `meta.packName` and `meta.packVersion`. */
|
|
379
|
+
export async function packStampLeg(a: PromptAdverts, validate: Validate): Promise<PromptOutcome> {
|
|
380
|
+
const g = endpointsGate(a); if (g) return g;
|
|
381
|
+
if (!a.packs) return { kind: 'skip', disposition: 'inapplicable', reason: 'prompts.packsSupported is not true — no pack template exists to stamp' };
|
|
382
|
+
const listed = items(await driver.get(`${a.promptsPath}?source=pack`)).filter((t) => t.meta?.source === 'pack');
|
|
383
|
+
if (listed.length === 0) return { kind: 'skip', disposition: 'inapplicable', reason: 'no prompt pack is installed — zero installed packs is conformant, and there is no template to judge' };
|
|
384
|
+
return { kind: 'observed', findings: listed.flatMap((t) => [
|
|
385
|
+
f(typeof t.meta?.packName === 'string' && t.meta.packName.length > 0, DOC_LIB, `a pack template MUST carry meta.packName (${String(t.templateId)})`),
|
|
386
|
+
f(typeof t.meta?.packVersion === 'string' && SEMVER.test(t.meta.packVersion), DOC_LIB, `a pack template MUST carry a SemVer meta.packVersion (${String(t.templateId)}: ${String(t.meta?.packVersion)})`),
|
|
387
|
+
f(validate(t).ok, 'schemas PromptTemplate', `a pack template MUST validate (${String(t.templateId)}): ${validate(t).errors}`),
|
|
388
|
+
]) };
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
/** Leg: the in-tree reference pack's template is fetched by id under its library. Inapplicable when that pack is not installed. */
|
|
392
|
+
export async function referencePackLeg(profile: MajorProfile, a: PromptAdverts): Promise<PromptOutcome> {
|
|
393
|
+
const g = endpointsGate(a); if (g) return g;
|
|
394
|
+
if (!a.packs) return { kind: 'skip', disposition: 'inapplicable', reason: 'prompts.packsSupported is not true' };
|
|
395
|
+
const listed = items(await driver.get(`${a.promptsPath}?source=pack`));
|
|
396
|
+
const ref = listed.find((t) => t.templateId === REFERENCE_TEMPLATE && t.meta?.packName === REFERENCE_PACK);
|
|
397
|
+
if (ref === undefined) return { kind: 'skip', disposition: 'inapplicable', reason: `the reference pack ${REFERENCE_PACK} is not installed — a host MAY install other packs or none` };
|
|
398
|
+
const res = await driver.get(tplPath(profile, a, REFERENCE_TEMPLATE, `?libraryId=${encodeURIComponent(REFERENCE_PACK)}`));
|
|
399
|
+
const b = isRecord(res.json) ? res.json : {};
|
|
400
|
+
const meta = isRecord(b['meta']) ? b['meta'] : {};
|
|
401
|
+
return { kind: 'observed', findings: [
|
|
402
|
+
f(res.status === 200, DOC_PROMPTS, `GET ${a.promptsPath}/${REFERENCE_TEMPLATE}?libraryId=${REFERENCE_PACK} MUST answer 200 for a listed pack template (got ${res.status})`),
|
|
403
|
+
f(b['templateId'] === REFERENCE_TEMPLATE, DOC_PROMPTS, `the fetched template MUST be ${REFERENCE_TEMPLATE} (got ${String(b['templateId'])})`),
|
|
404
|
+
f(meta['source'] === 'pack' && meta['packName'] === REFERENCE_PACK, DOC_LIB, `the fetched template MUST keep its pack provenance (got source ${String(meta['source'])}, packName ${String(meta['packName'])})`),
|
|
405
|
+
] };
|
|
406
|
+
}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The run execution-bounds witness (`runs.md` §`run` section; RFC 0058), shared
|
|
3
|
+
* by every major that gives a run a wire surface for `runTimeoutMs`.
|
|
4
|
+
*
|
|
5
|
+
* Three legs:
|
|
6
|
+
* shape advertised `limits.maxRunDurationMs` is an integer ≥ 1000 and
|
|
7
|
+
* `limits.maxLoopIterations` an integer ≥ 1, when present;
|
|
8
|
+
* refused an out-of-range `runTimeoutMs` (0, below the schema minimum of 1)
|
|
9
|
+
* is refused `400 validation_error` at create — unaided;
|
|
10
|
+
* breach a run of the `conformance-run-duration-breach` fixture with a
|
|
11
|
+
* 1 s `runTimeoutMs` ends `failed` with `run_timeout` and emits
|
|
12
|
+
* `cap.breached { kind: 'run-duration' }` with `observed > limit`.
|
|
13
|
+
*
|
|
14
|
+
* Two halves: the drivers observe and assert nothing; {@link judgeBreach} is pure,
|
|
15
|
+
* proven in `run-bounds-witness.test.ts`.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { driver } from './driver.js';
|
|
19
|
+
import { readErrorCode } from './error-envelope.js';
|
|
20
|
+
import { isFixtureAdvertised } from './fixtures.js';
|
|
21
|
+
import type { MajorProfile } from './major-profile.js';
|
|
22
|
+
|
|
23
|
+
export const TIMEOUT_FIXTURE = 'conformance-run-duration-breach';
|
|
24
|
+
const isRecord = (v: unknown): v is Record<string, unknown> => v !== null && typeof v === 'object' && !Array.isArray(v);
|
|
25
|
+
|
|
26
|
+
export interface BoundsFinding { readonly ok: boolean; readonly doc: string; readonly message: string }
|
|
27
|
+
export type BoundsOutcome =
|
|
28
|
+
| { readonly kind: 'skip'; readonly disposition: 'inapplicable' | 'blocked'; readonly reason: string }
|
|
29
|
+
| { readonly kind: 'observed'; readonly findings: readonly BoundsFinding[] };
|
|
30
|
+
|
|
31
|
+
const DOC = 'runs.md §run section';
|
|
32
|
+
const f = (ok: boolean, message: string): BoundsFinding => ({ ok, doc: DOC, message });
|
|
33
|
+
|
|
34
|
+
/** The `run` section fragment that carries a `runTimeoutMs` at this major. */
|
|
35
|
+
function runTimeout(profile: MajorProfile, ms: number): Record<string, unknown> {
|
|
36
|
+
return profile.major >= 2 ? { configurable: { version: 1, run: { runTimeoutMs: ms } } } : { configurable: { runTimeoutMs: ms } };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Leg 1, shape: judged from the discovery record alone. */
|
|
40
|
+
export function shapeLeg(profile: MajorProfile, discovery: unknown): BoundsOutcome {
|
|
41
|
+
const limits = profile.family(discovery, 'limits');
|
|
42
|
+
if (limits === null) return { kind: 'skip', disposition: 'inapplicable', reason: 'the host does not advertise limits' };
|
|
43
|
+
const findings: BoundsFinding[] = [];
|
|
44
|
+
const dur = limits['maxRunDurationMs'];
|
|
45
|
+
const loops = limits['maxLoopIterations'];
|
|
46
|
+
if (dur !== undefined) findings.push(f(Number.isInteger(dur) && (dur as number) >= 1000, `limits.maxRunDurationMs MUST be an integer >= 1000 when present (got ${JSON.stringify(dur)})`));
|
|
47
|
+
if (loops !== undefined) findings.push(f(Number.isInteger(loops) && (loops as number) >= 1, `limits.maxLoopIterations MUST be an integer >= 1 when present (got ${JSON.stringify(loops)})`));
|
|
48
|
+
if (findings.length === 0) return { kind: 'skip', disposition: 'inapplicable', reason: 'the host advertises neither limits.maxRunDurationMs nor limits.maxLoopIterations' };
|
|
49
|
+
return { kind: 'observed', findings };
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Leg 2, refused: an out-of-range runTimeoutMs is refused at create. Unaided: any advertised fixture names a valid workflow. */
|
|
53
|
+
export async function refusedLeg(profile: MajorProfile, workflowId: string | undefined): Promise<BoundsOutcome> {
|
|
54
|
+
if (workflowId === undefined) return { kind: 'skip', disposition: 'inapplicable', reason: 'the host advertises no fixture workflow to name in a create, so a refusal cannot be told from an unknown workflow' };
|
|
55
|
+
const res = await driver.post(profile.runsPath, { workflowId, ...runTimeout(profile, 0) });
|
|
56
|
+
if (res.status === 201 || res.status === 202) {
|
|
57
|
+
const runId = isRecord(res.json) ? res.json['runId'] : undefined;
|
|
58
|
+
if (typeof runId === 'string') await driver.post(`${profile.runsPath}/${profile.idSegment(runId)}/cancel`, {}).catch(() => undefined);
|
|
59
|
+
}
|
|
60
|
+
return { kind: 'observed', findings: [
|
|
61
|
+
f(res.status === 400, `runTimeoutMs: 0 (below the minimum of 1) MUST be refused 400 at create (got ${res.status})`),
|
|
62
|
+
f(readErrorCode(res.json) === 'validation_error', `the refusal MUST carry validation_error (got ${String(readErrorCode(res.json))})`),
|
|
63
|
+
] };
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export interface BreachObservation {
|
|
67
|
+
readonly createStatus: number;
|
|
68
|
+
readonly terminalStatus: string | undefined;
|
|
69
|
+
readonly errorCode: string | undefined;
|
|
70
|
+
readonly breach: Record<string, unknown> | undefined;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Leg 3, breach, observed: a fixture run with a 1 s runTimeoutMs. Asserts nothing. */
|
|
74
|
+
export async function driveBreach(profile: MajorProfile, opts: { timeoutMs?: number; pollMs?: number } = {}): Promise<Extract<BoundsOutcome, { kind: 'skip' }> | BreachObservation> {
|
|
75
|
+
if (!isFixtureAdvertised(TIMEOUT_FIXTURE)) return { kind: 'skip', disposition: 'inapplicable', reason: `fixture ${TIMEOUT_FIXTURE} is not advertised — the host does not claim a run long enough to breach a 1 s timeout` };
|
|
76
|
+
const created = await driver.post(profile.runsPath, { workflowId: TIMEOUT_FIXTURE, ...runTimeout(profile, 1000) });
|
|
77
|
+
const runId = isRecord(created.json) ? created.json['runId'] : undefined;
|
|
78
|
+
if ((created.status !== 201 && created.status !== 202) || typeof runId !== 'string') {
|
|
79
|
+
return { createStatus: created.status, terminalStatus: undefined, errorCode: readErrorCode(created.json), breach: undefined };
|
|
80
|
+
}
|
|
81
|
+
const deadline = Date.now() + (opts.timeoutMs ?? 20_000);
|
|
82
|
+
let snap: Record<string, unknown> = {};
|
|
83
|
+
for (;;) {
|
|
84
|
+
const r = await driver.get(`${profile.runsPath}/${profile.idSegment(runId)}`);
|
|
85
|
+
snap = isRecord(r.json) ? r.json : {};
|
|
86
|
+
if (['completed', 'failed', 'cancelled'].includes(String(snap['status'])) || Date.now() > deadline) break;
|
|
87
|
+
await new Promise((res) => setTimeout(res, opts.pollMs ?? 500));
|
|
88
|
+
}
|
|
89
|
+
const poll = await driver.get(`${profile.runsPath}/${profile.idSegment(runId)}/events/poll`);
|
|
90
|
+
const events = ((poll.json as { events?: unknown } | undefined)?.events as Array<Record<string, unknown>> | undefined) ?? [];
|
|
91
|
+
const capType = profile.eventType('cap.breached') ?? 'cap.breached';
|
|
92
|
+
const breach = events.find((e) => e['type'] === capType);
|
|
93
|
+
const err = isRecord(snap['error']) ? snap['error'] : undefined;
|
|
94
|
+
return { createStatus: created.status, terminalStatus: typeof snap['status'] === 'string' ? (snap['status'] as string) : undefined, errorCode: typeof err?.['code'] === 'string' ? (err['code'] as string) : undefined, breach: isRecord(breach?.['payload']) ? (breach!['payload'] as Record<string, unknown>) : undefined };
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** Leg 3, breach, judged. Pure. */
|
|
98
|
+
export function judgeBreach(o: BreachObservation): BoundsFinding[] {
|
|
99
|
+
const out = [f(o.createStatus === 201 || o.createStatus === 202, `a create with runTimeoutMs: 1000 MUST be accepted (got ${o.createStatus})`)];
|
|
100
|
+
if (!out[0]!.ok) return out;
|
|
101
|
+
const observed = o.breach?.['observed'];
|
|
102
|
+
const limit = o.breach?.['limit'];
|
|
103
|
+
out.push(
|
|
104
|
+
f(o.terminalStatus === 'failed', `a run exceeding runTimeoutMs MUST end failed (got ${String(o.terminalStatus)})`),
|
|
105
|
+
f(o.errorCode === 'run_timeout', `the run's error.code MUST be run_timeout (got ${String(o.errorCode)})`),
|
|
106
|
+
f(o.breach !== undefined, 'a cap.breached event MUST be emitted on the breach'),
|
|
107
|
+
f(o.breach?.['kind'] === 'run-duration', `cap.breached.kind MUST be run-duration (got ${String(o.breach?.['kind'])})`),
|
|
108
|
+
f(typeof observed === 'number' && typeof limit === 'number' && observed > limit, `cap.breached.observed MUST exceed limit (got observed=${String(observed)}, limit=${String(limit)})`),
|
|
109
|
+
);
|
|
110
|
+
return out;
|
|
111
|
+
}
|
package/src/lib/scratch-host.ts
CHANGED
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http';
|
|
17
17
|
import type { AddressInfo } from 'node:net';
|
|
18
18
|
import type { MajorProfile } from './major-profile.js';
|
|
19
|
+
import { unprojectBoundId } from './bound-id.js';
|
|
19
20
|
|
|
20
21
|
export interface ScriptedEvent { readonly type: string; readonly payload?: Record<string, unknown> | undefined }
|
|
21
22
|
export interface ScriptedRun {
|
|
@@ -112,7 +113,9 @@ export class ScratchHost {
|
|
|
112
113
|
if (req.method === 'POST' && path === `${runs}:bulk-cancel`) return this.send(res, 200, { results: [] });
|
|
113
114
|
|
|
114
115
|
const m = new RegExp(`^${runs}/([^/]+)(/events/poll|/events|/cancel)?$`).exec(path);
|
|
115
|
-
|
|
116
|
+
// A bound id arrives `~`-projected (identity.md §5) or percent-encoded; accept both.
|
|
117
|
+
const seg = m ? (m[1] as string) : '';
|
|
118
|
+
const stored = m ? this.runs.get(seg.includes('~') ? unprojectBoundId(seg) : decodeURIComponent(seg)) : undefined;
|
|
116
119
|
if (!m || stored === undefined) return this.send(res, 404, { error: 'not_found', message: 'no such route or run' });
|
|
117
120
|
const tail = m[2];
|
|
118
121
|
if (tail === '/cancel') return this.send(res, 200, { status: 'cancelled' });
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
* nothing; {@link judge} is pure.
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
|
+
import { projectBoundId } from './bound-id.js';
|
|
21
22
|
import { randomUUID } from 'node:crypto';
|
|
22
23
|
import { driver, type OpenWOPResponse } from './driver.js';
|
|
23
24
|
import { readErrorCode } from './error-envelope.js';
|
|
@@ -52,7 +53,8 @@ const isRecord = (v: unknown): v is Record<string, unknown> => v !== null && typ
|
|
|
52
53
|
async function http(fn: () => Promise<OpenWOPResponse>): Promise<OpenWOPResponse | null> {
|
|
53
54
|
try { return await fn(); } catch { return null; }
|
|
54
55
|
}
|
|
55
|
-
|
|
56
|
+
/** A v2 bound id as one path segment: the `~` projection (identity.md §5). */
|
|
57
|
+
const enc = (id: string): string => projectBoundId(id);
|
|
56
58
|
|
|
57
59
|
async function probe(action: ProbeAction, timeoutMs: number): Promise<ProbeOutcome | string> {
|
|
58
60
|
const table = `conformance_schema_${randomUUID().replace(/-/g, '').slice(0, 16)}`;
|