@openwop/openwop-conformance 2.45.13 → 2.45.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/README.md +2 -2
- package/dist/spec-artifacts.lock.json +2 -2
- package/package.json +2 -2
- package/requirements.json +1446 -4
- package/scenario-majors.json +65 -2
- package/schemas/CORPUS-STAMP.json +14 -12
- package/src/lib/backpressure-witness.ts +2 -2
- package/src/lib/budget-witness.ts +3 -3
- package/src/lib/envelope-schema-static.ts +166 -0
- package/src/lib/exhaustion-facet-witness.ts +1 -1
- package/src/lib/family-advert-witness.ts +212 -0
- package/src/lib/i18n-double.ts +99 -0
- package/src/lib/i18n-negotiation-witness.ts +173 -0
- package/src/lib/localized-content-witness.ts +49 -0
- package/src/lib/model-capability-insufficient-witness.ts +121 -0
- package/src/lib/prompt-double.ts +217 -0
- package/src/lib/prompt-events-witness.ts +211 -0
- package/src/lib/prompt-library-witness.ts +406 -0
- package/src/lib/run-bounds-witness.ts +111 -0
- package/src/lib/scratch-host.ts +4 -1
- package/src/lib/table-schema-witness.ts +3 -1
- package/src/scenarios/v2-ai-envelope-shape-static.test.ts +93 -0
- package/src/scenarios/v2-ai-providers-advertisement.test.ts +94 -0
- package/src/scenarios/v2-envelope-reasoning-shape.test.ts +81 -0
- package/src/scenarios/v2-envelope-tier-one-subset-static.test.ts +77 -0
- package/src/scenarios/v2-envelope-variant-discriminator-static.test.ts +57 -0
- package/src/scenarios/v2-envelopes-advertisement.test.ts +85 -0
- package/src/scenarios/v2-i18n-negotiation.test.ts +91 -0
- package/src/scenarios/v2-limits-envelope-caps-advertisement.test.ts +61 -0
- package/src/scenarios/v2-localized-content-delivery.test.ts +136 -0
- package/src/scenarios/v2-media-inline-cap-static.test.ts +59 -0
- package/src/scenarios/v2-model-capabilities-advertisement.test.ts +61 -0
- package/src/scenarios/v2-model-capability-insufficient.test.ts +63 -0
- package/src/scenarios/v2-prompt-all-four-kinds-events.test.ts +69 -0
- package/src/scenarios/v2-prompt-end-to-end-events.test.ts +71 -0
- package/src/scenarios/v2-prompt-list-and-fetch.test.ts +103 -0
- package/src/scenarios/v2-prompt-mutable-lifecycle.test.ts +130 -0
- package/src/scenarios/v2-prompt-pack-install.test.ts +78 -0
- package/src/scenarios/v2-prompt-render-deterministic.test.ts +87 -0
- package/src/scenarios/v2-prompt-resolution-chain-event.test.ts +46 -0
- package/src/scenarios/v2-prompt-template-shape.test.ts +112 -0
- package/src/scenarios/v2-run-execution-bounds.test.ts +67 -0
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A scratch double for locale negotiation at major 2 (`i18n.md`): a discovery
|
|
3
|
+
* document carrying an `i18n` record and a run read that answers an error
|
|
4
|
+
* envelope localized by `Accept-Language`. Each self-test in
|
|
5
|
+
* `i18n-negotiation-witness.test.ts` turns on ONE defect to show the witness
|
|
6
|
+
* convicts it; a standalone instance lets the scenario file run against it.
|
|
7
|
+
*
|
|
8
|
+
* A test double, not a host: it executes nothing, and nothing it does is
|
|
9
|
+
* evidence about any implementation.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http';
|
|
13
|
+
import type { AddressInfo } from 'node:net';
|
|
14
|
+
import { unprojectBoundId } from './bound-id.js';
|
|
15
|
+
|
|
16
|
+
export type I18nDefect =
|
|
17
|
+
| 'none' | 'malformed-400' | 'unsupported-406' | 'content-language-lies' | 'content-language-garbage'
|
|
18
|
+
| 'code-localized' | 'details-keys-localized' | 'details-locale-garbage';
|
|
19
|
+
|
|
20
|
+
/** The tenant the double treats as the caller's. */
|
|
21
|
+
export const OWN_TENANT = 'acme';
|
|
22
|
+
|
|
23
|
+
const MESSAGES: Record<string, string> = { en: 'No such run', es: 'No existe tal ejecución', 'pt-BR': 'Execução inexistente' };
|
|
24
|
+
|
|
25
|
+
export class I18nDouble {
|
|
26
|
+
defect: I18nDefect = 'none';
|
|
27
|
+
url = '';
|
|
28
|
+
private server: Server | undefined;
|
|
29
|
+
discovery: Record<string, unknown> = { i18n: { status: 'stable', since: '2.0', witness: 'witnessable-gated', defaultLocale: 'en', supportedLocales: ['en', 'es', 'pt-BR'] } };
|
|
30
|
+
|
|
31
|
+
async start(): Promise<string> {
|
|
32
|
+
this.server = createServer((req, res) => { this.handle(req, res); });
|
|
33
|
+
await new Promise<void>((r) => this.server!.listen(0, '127.0.0.1', r));
|
|
34
|
+
this.url = `http://127.0.0.1:${(this.server.address() as AddressInfo).port}`;
|
|
35
|
+
return this.url;
|
|
36
|
+
}
|
|
37
|
+
async stop(): Promise<void> { const s = this.server; if (s) { s.closeAllConnections(); await new Promise<void>((r) => s.close(() => r())); } }
|
|
38
|
+
reset(defect: I18nDefect): void { this.defect = defect; }
|
|
39
|
+
|
|
40
|
+
private supported(): string[] { return ((this.discovery['i18n'] as { supportedLocales?: string[] } | undefined)?.supportedLocales) ?? ['en']; }
|
|
41
|
+
|
|
42
|
+
/** RFC 9110 §12.5.4 best-effort: highest q wins, ties to order, then the language family, else the default. `null` = malformed. */
|
|
43
|
+
private negotiate(header: string | undefined): { used: string; malformed: boolean; firstRaw: string | undefined } {
|
|
44
|
+
if (header === undefined) return { used: 'en', malformed: false, firstRaw: undefined };
|
|
45
|
+
const entries: Array<{ tag: string; q: number; i: number }> = [];
|
|
46
|
+
let malformed = false;
|
|
47
|
+
header.split(',').forEach((part, i) => {
|
|
48
|
+
const [tag, ...params] = part.trim().split(';').map((s) => s.trim());
|
|
49
|
+
if (!tag || !/^[a-zA-Z]{1,8}(-[a-zA-Z0-9]{1,8})*$|^\*$/.test(tag)) { malformed = true; return; }
|
|
50
|
+
const qp = params.find((p) => p.startsWith('q='));
|
|
51
|
+
const q = qp === undefined ? 1 : Number(qp.slice(2));
|
|
52
|
+
if (Number.isNaN(q)) { malformed = true; return; }
|
|
53
|
+
entries.push({ tag, q, i });
|
|
54
|
+
});
|
|
55
|
+
entries.sort((x, y) => y.q - x.q || x.i - y.i);
|
|
56
|
+
const sup = this.supported();
|
|
57
|
+
for (const e of entries) {
|
|
58
|
+
// `*` (which Node's fetch sends by default) matches any locale: the default serves.
|
|
59
|
+
if (e.tag === '*') return { used: 'en', malformed, firstRaw: undefined };
|
|
60
|
+
const exact = sup.find((s) => s.toLowerCase() === e.tag.toLowerCase());
|
|
61
|
+
if (exact) return { used: exact, malformed, firstRaw: header.split(',')[0]?.split(';')[0]?.trim() };
|
|
62
|
+
const fam = sup.find((s) => s.toLowerCase() === (e.tag.split('-')[0] ?? '').toLowerCase());
|
|
63
|
+
if (fam) return { used: fam, malformed, firstRaw: header.split(',')[0]?.split(';')[0]?.trim() };
|
|
64
|
+
}
|
|
65
|
+
return { used: 'en', malformed, firstRaw: header.split(',')[0]?.split(';')[0]?.trim() };
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
private send(res: ServerResponse, status: number, body: unknown, headers: Record<string, string> = {}): void {
|
|
69
|
+
res.writeHead(status, { 'content-type': 'application/json', ...headers });
|
|
70
|
+
res.end(JSON.stringify(body));
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
private handle(req: IncomingMessage, res: ServerResponse): void {
|
|
74
|
+
const path = (req.url ?? '/').split('?')[0] ?? '/';
|
|
75
|
+
if (req.method === 'GET' && path === '/.well-known/openwop') return this.send(res, 200, this.discovery);
|
|
76
|
+
const m = /^\/runs\/([^/]+)$/.exec(path);
|
|
77
|
+
if (req.method !== 'GET' || !m) return this.send(res, 404, { error: 'not_found', message: `no route ${req.method} ${path}` });
|
|
78
|
+
|
|
79
|
+
const header = req.headers['accept-language'];
|
|
80
|
+
const n = this.negotiate(typeof header === 'string' ? header : undefined);
|
|
81
|
+
if (n.malformed && this.defect === 'malformed-400') return this.send(res, 400, { error: 'validation_error', message: 'bad Accept-Language' });
|
|
82
|
+
if (typeof header === 'string' && !n.malformed && n.firstRaw !== undefined && n.used === 'en' && !header.toLowerCase().startsWith('en') && this.defect === 'unsupported-406') {
|
|
83
|
+
return this.send(res, 406, { error: 'not_acceptable', message: 'locale not supported' });
|
|
84
|
+
}
|
|
85
|
+
const localized = n.used !== 'en';
|
|
86
|
+
let cl: string | undefined = typeof header === 'string' ? n.used : undefined;
|
|
87
|
+
if (this.defect === 'content-language-lies' && n.firstRaw !== undefined) cl = n.firstRaw;
|
|
88
|
+
if (this.defect === 'content-language-garbage' && cl !== undefined) cl = 'en_US!';
|
|
89
|
+
|
|
90
|
+
let id: string;
|
|
91
|
+
try { id = unprojectBoundId(m[1]!); } catch { return this.send(res, 400, { error: 'validation_error', message: 'bad id projection' }); }
|
|
92
|
+
const tenant = id.includes('/') ? id.split('/')[0] : OWN_TENANT;
|
|
93
|
+
const foreign = tenant !== OWN_TENANT;
|
|
94
|
+
const code = this.defect === 'code-localized' && localized ? (foreign ? 'inquilino_distinto' : 'no_encontrado') : (foreign ? 'id_tenant_mismatch' : 'not_found');
|
|
95
|
+
const details: Record<string, unknown> = this.defect === 'details-keys-localized' && localized ? { recurso: 'run' } : { resource: 'run' };
|
|
96
|
+
if (localized) details['locale'] = this.defect === 'details-locale-garbage' ? 'spanish!' : n.used;
|
|
97
|
+
return this.send(res, foreign ? 403 : 404, { error: code, message: MESSAGES[n.used] ?? MESSAGES['en'], details }, cl === undefined ? {} : { 'content-language': cl });
|
|
98
|
+
}
|
|
99
|
+
}
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The locale-negotiation witness (`i18n.md` §`Accept-Language`, §Fallback,
|
|
3
|
+
* §Error envelopes, §The `i18n` record), shared by every major that states
|
|
4
|
+
* those rules. What differs between majors (the run path, how an id becomes a
|
|
5
|
+
* path segment, how a family is advertised) is the profile row.
|
|
6
|
+
*
|
|
7
|
+
* Four legs:
|
|
8
|
+
* shape the advertised `i18n` record: BCP 47 tags, and
|
|
9
|
+
* `supportedLocales` contains `defaultLocale`;
|
|
10
|
+
* unsupported an `Accept-Language` the host does not support is answered
|
|
11
|
+
* exactly as the same request without it, and a
|
|
12
|
+
* `Content-Language`, when sent, names the default locale;
|
|
13
|
+
* malformed a malformed `Accept-Language` never yields `400` and is
|
|
14
|
+
* answered exactly as the same request without it;
|
|
15
|
+
* errorCode the `error` code and the `details` keys are identical under
|
|
16
|
+
* the default and a negotiated locale.
|
|
17
|
+
*
|
|
18
|
+
* The probe is a read of a run id no host minted. Its answer is not pinned
|
|
19
|
+
* (`404 not_found`, or at major 2 `403 id_tenant_mismatch` when the tenant
|
|
20
|
+
* segment is not the caller's): every live leg compares against the same
|
|
21
|
+
* request sent with no `Accept-Language`, so what is witnessed is that the
|
|
22
|
+
* header changes nothing a client routes on.
|
|
23
|
+
*
|
|
24
|
+
* Each leg OBSERVES and returns findings; it asserts nothing. The scenario maps
|
|
25
|
+
* findings to requirement ids, and `i18n-negotiation-witness.test.ts` proves
|
|
26
|
+
* every leg against `i18n-double.ts` with one defect turned on.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import { randomBytes } from 'node:crypto';
|
|
30
|
+
import { driver, type OpenWOPResponse } from './driver.js';
|
|
31
|
+
import { readErrorCode } from './error-envelope.js';
|
|
32
|
+
import type { MajorProfile } from './major-profile.js';
|
|
33
|
+
|
|
34
|
+
export interface I18nFinding { readonly ok: boolean; readonly doc: string; readonly message: string }
|
|
35
|
+
export type I18nOutcome =
|
|
36
|
+
| { readonly kind: 'skip'; readonly disposition: 'inapplicable' | 'blocked'; readonly reason: string }
|
|
37
|
+
| { readonly kind: 'observed'; readonly findings: readonly I18nFinding[] };
|
|
38
|
+
|
|
39
|
+
/** BCP 47 well-formedness as `capabilities.schema.json` §i18n pins it. */
|
|
40
|
+
export const BCP47 = /^[a-zA-Z]{2,3}(-[a-zA-Z0-9]{2,8}){0,3}$/;
|
|
41
|
+
/** A registered code (snake_case) or a vendor code (`errors.md`; the v2 envelope's vendor grammar). */
|
|
42
|
+
const ERROR_CODE = /^(?:[a-z][a-z0-9_]*|(?!openwop\.)[a-z][a-z0-9]*(?:-[a-z0-9]+)*\.[a-z][a-z0-9_]*)$/;
|
|
43
|
+
|
|
44
|
+
const isRecord = (v: unknown): v is Record<string, unknown> => v !== null && typeof v === 'object' && !Array.isArray(v);
|
|
45
|
+
const f = (ok: boolean, doc: string, message: string): I18nFinding => ({ ok, doc, message });
|
|
46
|
+
const lower = (s: string): string => s.toLowerCase();
|
|
47
|
+
const primary = (tag: string): string => lower(tag.split('-')[0] ?? tag);
|
|
48
|
+
|
|
49
|
+
export interface I18nAdverts {
|
|
50
|
+
readonly record: Record<string, unknown> | null;
|
|
51
|
+
/** `i18n.defaultLocale`, or `en` when omitted (`i18n.md` §The `i18n` record). */
|
|
52
|
+
readonly defaultLocale: string;
|
|
53
|
+
readonly supportedLocales: readonly string[];
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function i18nAdverts(profile: MajorProfile, doc: unknown): I18nAdverts {
|
|
57
|
+
const record = profile.family(doc, 'i18n');
|
|
58
|
+
const dl = record?.['defaultLocale'];
|
|
59
|
+
const sl = record?.['supportedLocales'];
|
|
60
|
+
return {
|
|
61
|
+
record,
|
|
62
|
+
defaultLocale: typeof dl === 'string' ? dl : 'en',
|
|
63
|
+
supportedLocales: Array.isArray(sl) ? sl.filter((t): t is string => typeof t === 'string') : [],
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const gate = (a: I18nAdverts): I18nOutcome | null =>
|
|
68
|
+
a.record === null ? { kind: 'skip', disposition: 'inapplicable', reason: 'the host does not advertise i18n — it serves one locale and ignores Accept-Language (i18n.md §The i18n record)' } : null;
|
|
69
|
+
|
|
70
|
+
/** Leg 1, shape: judged from the discovery record alone. */
|
|
71
|
+
export function shapeLeg(a: I18nAdverts): I18nOutcome {
|
|
72
|
+
const g = gate(a); if (g) return g;
|
|
73
|
+
const rec = a.record!;
|
|
74
|
+
const doc = 'i18n.md §Language tags + §The i18n record';
|
|
75
|
+
const findings: I18nFinding[] = [];
|
|
76
|
+
const dl = rec['defaultLocale'];
|
|
77
|
+
const sl = rec['supportedLocales'];
|
|
78
|
+
if (dl !== undefined) findings.push(f(typeof dl === 'string' && BCP47.test(dl), doc, `i18n.defaultLocale MUST be a BCP 47 tag (got ${JSON.stringify(dl)})`));
|
|
79
|
+
if (sl !== undefined) {
|
|
80
|
+
findings.push(f(Array.isArray(sl), doc, 'i18n.supportedLocales MUST be an array when present'));
|
|
81
|
+
for (const t of Array.isArray(sl) ? sl : []) findings.push(f(typeof t === 'string' && BCP47.test(t), doc, `every i18n.supportedLocales entry MUST be a BCP 47 tag (got ${JSON.stringify(t)})`));
|
|
82
|
+
// The schema states the containment "when both are present"; tags compare case-insensitively.
|
|
83
|
+
if (typeof dl === 'string' && Array.isArray(sl)) {
|
|
84
|
+
findings.push(f(sl.some((t) => typeof t === 'string' && lower(t) === lower(dl)), doc, `i18n.supportedLocales MUST contain defaultLocale ${dl} (got ${JSON.stringify(sl)})`));
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
if (findings.length === 0) return { kind: 'skip', disposition: 'inapplicable', reason: 'the i18n record carries neither defaultLocale nor supportedLocales — nothing to judge' };
|
|
88
|
+
return { kind: 'observed', findings };
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** A read of a run id no host minted. */
|
|
92
|
+
export function probePath(profile: MajorProfile): string {
|
|
93
|
+
const opaque = `never-minted-${randomBytes(8).toString('hex')}`;
|
|
94
|
+
return `${profile.runsPath}/${profile.idSegment(profile.major >= 2 ? `conformance-i18n/${opaque}` : `openwop-conformance-i18n-${opaque}`)}`;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** A syntactically valid tag the host does not list. */
|
|
98
|
+
export function unsupportedTag(supported: readonly string[]): string {
|
|
99
|
+
const have = new Set(supported.map(lower));
|
|
100
|
+
return ['tlh', 'kl-GL', 'mi-NZ', 'eu-ES'].find((c) => !have.has(lower(c)) && !have.has(primary(c))) ?? 'tlh';
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const get = (path: string, acceptLanguage?: string): Promise<OpenWOPResponse> =>
|
|
104
|
+
driver.get(path, acceptLanguage === undefined ? {} : { headers: { 'Accept-Language': acceptLanguage } });
|
|
105
|
+
|
|
106
|
+
/** The baseline answer must be an error a client can route on, or nothing can be compared. */
|
|
107
|
+
function baselineSkip(base: OpenWOPResponse): I18nOutcome | null {
|
|
108
|
+
if (base.status >= 400 && base.status < 500 && readErrorCode(base.json) !== undefined) return null;
|
|
109
|
+
return { kind: 'skip', disposition: 'blocked', reason: `the probe read answered ${base.status} without a 4xx error envelope, so there is no baseline to compare a negotiated answer against` };
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** Leg 2, unsupported: an unsupported locale changes nothing a client routes on, and Content-Language never lies. */
|
|
113
|
+
export async function unsupportedLeg(profile: MajorProfile, a: I18nAdverts): Promise<I18nOutcome> {
|
|
114
|
+
const g = gate(a); if (g) return g;
|
|
115
|
+
const path = probePath(profile);
|
|
116
|
+
const base = await get(path);
|
|
117
|
+
const b = baselineSkip(base); if (b) return b;
|
|
118
|
+
const tag = unsupportedTag(a.supportedLocales);
|
|
119
|
+
const res = await get(path, tag);
|
|
120
|
+
const findings: I18nFinding[] = [
|
|
121
|
+
f(res.status === base.status && res.status !== 400 && res.status !== 406, 'i18n.md §Accept-Language ("MUST NOT reject a request because it does not support the requested locale")', `Accept-Language: ${tag} (unsupported) MUST be answered as the same request without it (${base.status}); got ${res.status}`),
|
|
122
|
+
f(readErrorCode(res.json) === readErrorCode(base.json), 'i18n.md §Accept-Language + §Error envelopes', `the unsupported-locale answer MUST carry the same error code (${String(readErrorCode(base.json))}); got ${String(readErrorCode(res.json))}`),
|
|
123
|
+
];
|
|
124
|
+
const cl = res.headers.get('content-language');
|
|
125
|
+
if (cl !== null) {
|
|
126
|
+
findings.push(
|
|
127
|
+
f(BCP47.test(cl.trim()), 'i18n.md §Language tags', `Content-Language MUST be a BCP 47 tag (got ${JSON.stringify(cl)})`),
|
|
128
|
+
f(primary(cl.trim()) === primary(a.defaultLocale), 'i18n.md §Fallback rules 3–4', `for an unsupported Accept-Language the host uses its default; Content-Language MUST name the locale actually used (${a.defaultLocale}), never another (got ${cl})`),
|
|
129
|
+
);
|
|
130
|
+
}
|
|
131
|
+
return { kind: 'observed', findings };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** Leg 3, malformed: a malformed header never fails the request. */
|
|
135
|
+
export async function malformedLeg(profile: MajorProfile, a: I18nAdverts): Promise<I18nOutcome> {
|
|
136
|
+
const g = gate(a); if (g) return g;
|
|
137
|
+
const path = probePath(profile);
|
|
138
|
+
const base = await get(path);
|
|
139
|
+
const b = baselineSkip(base); if (b) return b;
|
|
140
|
+
const res = await get(path, ';;;not===a,,language q=;');
|
|
141
|
+
return { kind: 'observed', findings: [
|
|
142
|
+
f(res.status !== 400, 'i18n.md §Accept-Language ("A malformed value MUST NOT cause 400")', `a malformed Accept-Language MUST NOT cause 400 (got ${res.status})`),
|
|
143
|
+
f(res.status === base.status && readErrorCode(res.json) === readErrorCode(base.json), 'i18n.md §Accept-Language ("the host proceeds in its default locale")', `a malformed Accept-Language MUST be answered as the request without it (${base.status} ${String(readErrorCode(base.json))}); got ${res.status} ${String(readErrorCode(res.json))}`),
|
|
144
|
+
] };
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** `details` keys, less `locale`, which a host MAY add when it localizes the message. */
|
|
148
|
+
const detailsKeys = (json: unknown): string[] => {
|
|
149
|
+
const d = isRecord(json) ? json['details'] : undefined;
|
|
150
|
+
return isRecord(d) ? Object.keys(d).filter((k) => k !== 'locale').sort() : [];
|
|
151
|
+
};
|
|
152
|
+
|
|
153
|
+
/** Leg 4, errorCode: the machine-readable parts of an error envelope are locale-invariant. */
|
|
154
|
+
export async function errorCodeLeg(profile: MajorProfile, a: I18nAdverts): Promise<I18nOutcome> {
|
|
155
|
+
const g = gate(a); if (g) return g;
|
|
156
|
+
const path = probePath(profile);
|
|
157
|
+
const base = await get(path, a.defaultLocale);
|
|
158
|
+
const b = baselineSkip(base); if (b) return b;
|
|
159
|
+
const negotiated = a.supportedLocales.find((t) => lower(t) !== lower(a.defaultLocale)) ?? a.defaultLocale;
|
|
160
|
+
const res = await get(path, negotiated);
|
|
161
|
+
const code = readErrorCode(res.json);
|
|
162
|
+
const doc = 'i18n.md §Error envelopes';
|
|
163
|
+
const findings: I18nFinding[] = [
|
|
164
|
+
f(res.status === base.status, doc, `the status MUST NOT change with the negotiated locale (${a.defaultLocale} → ${base.status}; ${negotiated} → ${res.status})`),
|
|
165
|
+
f(typeof code === 'string' && ERROR_CODE.test(code), doc, `the error code is a registered identifier, never translated (got ${String(code)} under ${negotiated})`),
|
|
166
|
+
f(code === readErrorCode(base.json), doc, `the error code MUST be the same identifier in every locale (${a.defaultLocale} → ${String(readErrorCode(base.json))}; ${negotiated} → ${String(code)})`),
|
|
167
|
+
f(JSON.stringify(detailsKeys(res.json)) === JSON.stringify(detailsKeys(base.json)), doc, `details keys are schema field names and are not localized (${a.defaultLocale} → ${JSON.stringify(detailsKeys(base.json))}; ${negotiated} → ${JSON.stringify(detailsKeys(res.json))})`),
|
|
168
|
+
];
|
|
169
|
+
const d = isRecord(res.json) ? res.json['details'] : undefined;
|
|
170
|
+
const loc = isRecord(d) ? d['locale'] : undefined;
|
|
171
|
+
if (loc !== undefined) findings.push(f(typeof loc === 'string' && BCP47.test(loc), 'i18n.md §Language tags ("every locale field — is a BCP 47 tag")', `details.locale MUST be a BCP 47 tag (got ${JSON.stringify(loc)})`));
|
|
172
|
+
return { kind: 'observed', findings };
|
|
173
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The localized-content advertisement-coherence judge (`i18n.md` §Localized
|
|
3
|
+
* content, "Advertisement"; `capabilities.schema.json` §content), shared by
|
|
4
|
+
* every major that states it. Pure: it reads a discovery document and asserts
|
|
5
|
+
* nothing; the scenario maps the findings to requirement ids and
|
|
6
|
+
* `localized-content-witness.test.ts` proves each rule in both directions.
|
|
7
|
+
*
|
|
8
|
+
* A host advertising `content`:
|
|
9
|
+
* 1. advertises `i18n`;
|
|
10
|
+
* 2. has `content.baseLocale` equal to `i18n.defaultLocale` (`en` when omitted);
|
|
11
|
+
* 3. lists every locale of `baseLocale ∪ content.supportedLocales` in
|
|
12
|
+
* `i18n.supportedLocales`;
|
|
13
|
+
* 4. does not list `baseLocale` in `content.supportedLocales` (stated by the
|
|
14
|
+
* schema's `supportedLocales` description).
|
|
15
|
+
*
|
|
16
|
+
* Tags compare case-insensitively (`i18n.md` §Language tags).
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import type { MajorProfile } from './major-profile.js';
|
|
20
|
+
|
|
21
|
+
export interface ContentFinding { readonly ok: boolean; readonly doc: string; readonly message: string }
|
|
22
|
+
export type ContentOutcome =
|
|
23
|
+
| { readonly kind: 'skip'; readonly disposition: 'inapplicable' | 'blocked'; readonly reason: string }
|
|
24
|
+
| { readonly kind: 'observed'; readonly findings: readonly ContentFinding[] };
|
|
25
|
+
|
|
26
|
+
const strings = (v: unknown): string[] => (Array.isArray(v) ? v.filter((t): t is string => typeof t === 'string') : []);
|
|
27
|
+
const f = (ok: boolean, doc: string, message: string): ContentFinding => ({ ok, doc, message });
|
|
28
|
+
const DOC = 'i18n.md §Localized content ("Advertisement")';
|
|
29
|
+
|
|
30
|
+
export function coherenceLeg(profile: MajorProfile, discovery: unknown): ContentOutcome {
|
|
31
|
+
const content = profile.family(discovery, 'content');
|
|
32
|
+
if (content === null) return { kind: 'skip', disposition: 'inapplicable', reason: 'the host does not advertise content' };
|
|
33
|
+
const i18n = profile.family(discovery, 'i18n');
|
|
34
|
+
const findings: ContentFinding[] = [f(i18n !== null, DOC, 'a host advertising content MUST advertise i18n')];
|
|
35
|
+
if (i18n === null) return { kind: 'observed', findings };
|
|
36
|
+
|
|
37
|
+
const base = typeof content['baseLocale'] === 'string' ? (content['baseLocale'] as string) : undefined;
|
|
38
|
+
const supported = strings(content['supportedLocales']);
|
|
39
|
+
const def = typeof i18n['defaultLocale'] === 'string' ? (i18n['defaultLocale'] as string) : 'en';
|
|
40
|
+
// An i18n record without supportedLocales serves its default alone.
|
|
41
|
+
const i18nSet = new Set((i18n['supportedLocales'] === undefined ? [def] : strings(i18n['supportedLocales'])).map((t) => t.toLowerCase()));
|
|
42
|
+
|
|
43
|
+
findings.push(f(base !== undefined && base.toLowerCase() === def.toLowerCase(), DOC, `content.baseLocale MUST equal i18n.defaultLocale (got baseLocale ${String(base)}, defaultLocale ${def})`));
|
|
44
|
+
const resolvable = [...(base === undefined ? [] : [base]), ...supported];
|
|
45
|
+
const missing = resolvable.filter((t) => !i18nSet.has(t.toLowerCase()));
|
|
46
|
+
findings.push(f(missing.length === 0, DOC, `every locale in baseLocale ∪ content.supportedLocales MUST be in i18n.supportedLocales (missing ${JSON.stringify(missing)})`));
|
|
47
|
+
findings.push(f(base === undefined || !supported.some((t) => t.toLowerCase() === base.toLowerCase()), 'capabilities.schema.json §content.supportedLocales', `content.supportedLocales MUST NOT contain baseLocale ${String(base)} (got ${JSON.stringify(supported)})`));
|
|
48
|
+
return { kind: 'observed', findings };
|
|
49
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The model-capability refusal witness (`host-services.md` §modelCapabilities;
|
|
3
|
+
* RFC 0031 §B step 4 + §D), shared by every major that serves the
|
|
4
|
+
* `conformance-model-capability-insufficient` fixture.
|
|
5
|
+
*
|
|
6
|
+
* The fixture's one node declares `requiredModelCapabilities:
|
|
7
|
+
* ['nonexistent-capability-9b3f']` and no `fallbackModel`, so no model can meet
|
|
8
|
+
* it and no substitution is possible. A host advertising `modelCapabilities`
|
|
9
|
+
* MUST then "emit `model.capability-insufficient` and fail the run with
|
|
10
|
+
* `capability_not_provided`", and MUST NOT dispatch.
|
|
11
|
+
*
|
|
12
|
+
* Two legs over ONE run:
|
|
13
|
+
* refusal the run ends `failed` / `capability_not_provided`; the log
|
|
14
|
+
* carries the insufficient event, before `node.failed`, with a
|
|
15
|
+
* payload valid against `run-event-payloads#/$defs/modelCapabilityInsufficient`
|
|
16
|
+
* and `fallbackAttempted` not `true` (no fallback was declared);
|
|
17
|
+
* no-dispatch the log carries no `node.completed`, `provider.usage` or
|
|
18
|
+
* envelope-reliability event — the node never ran.
|
|
19
|
+
*
|
|
20
|
+
* Two halves: {@link drive} observes and asserts nothing; {@link judgeRefusal}
|
|
21
|
+
* and {@link judgeNoDispatch} are pure, proven in
|
|
22
|
+
* `model-capability-insufficient-witness.test.ts`.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { driver } from './driver.js';
|
|
26
|
+
import { readErrorCode } from './error-envelope.js';
|
|
27
|
+
import { isFixtureAdvertised } from './fixtures.js';
|
|
28
|
+
import type { MajorProfile } from './major-profile.js';
|
|
29
|
+
import { v2RefValidator } from './v2.js';
|
|
30
|
+
|
|
31
|
+
export const INSUFFICIENT_FIXTURE = 'conformance-model-capability-insufficient';
|
|
32
|
+
const DOC = 'host-services.md §modelCapabilities';
|
|
33
|
+
const isRecord = (v: unknown): v is Record<string, unknown> => v !== null && typeof v === 'object' && !Array.isArray(v);
|
|
34
|
+
|
|
35
|
+
export interface InsufficientFinding { readonly ok: boolean; readonly doc: string; readonly message: string }
|
|
36
|
+
export interface InsufficientSkip { readonly kind: 'skip'; readonly disposition: 'inapplicable' | 'blocked'; readonly reason: string }
|
|
37
|
+
export interface ObservedEvent { readonly type: string; readonly sequence: number; readonly payload: Record<string, unknown> | undefined }
|
|
38
|
+
export interface InsufficientObservation {
|
|
39
|
+
readonly createStatus: number;
|
|
40
|
+
readonly createCode: string | undefined;
|
|
41
|
+
readonly terminalStatus: string | undefined;
|
|
42
|
+
readonly errorCode: string | undefined;
|
|
43
|
+
/** The run's events, ordered by `sequence`. */
|
|
44
|
+
readonly events: readonly ObservedEvent[];
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** This major's name for an era-1 event; the era-1 name when the codemap has no row (the judge then looks for both). */
|
|
48
|
+
const nameAt = (profile: MajorProfile, era1: string): string => profile.eventType(era1) ?? era1;
|
|
49
|
+
|
|
50
|
+
/** One fixture run, observed. Asserts nothing. */
|
|
51
|
+
export async function drive(profile: MajorProfile, discovery: unknown, opts: { timeoutMs?: number; pollMs?: number } = {}): Promise<InsufficientSkip | InsufficientObservation> {
|
|
52
|
+
if (profile.family(discovery, 'modelCapabilities') === null) return { kind: 'skip', disposition: 'inapplicable', reason: 'the host does not advertise modelCapabilities — the dispatch gate binds only a host that does' };
|
|
53
|
+
if (!isFixtureAdvertised(INSUFFICIENT_FIXTURE)) return { kind: 'skip', disposition: 'inapplicable', reason: `fixture ${INSUFFICIENT_FIXTURE} is not advertised — the host claims no workflow whose node requires a capability no model offers` };
|
|
54
|
+
const created = await driver.post(profile.runsPath, { workflowId: INSUFFICIENT_FIXTURE });
|
|
55
|
+
const runId = isRecord(created.json) ? created.json['runId'] : undefined;
|
|
56
|
+
if ((created.status !== 201 && created.status !== 202) || typeof runId !== 'string') {
|
|
57
|
+
return { createStatus: created.status, createCode: readErrorCode(created.json), terminalStatus: undefined, errorCode: undefined, events: [] };
|
|
58
|
+
}
|
|
59
|
+
const base = `${profile.runsPath}/${profile.idSegment(runId)}`;
|
|
60
|
+
const deadline = Date.now() + (opts.timeoutMs ?? 20_000);
|
|
61
|
+
let snap: Record<string, unknown> = {};
|
|
62
|
+
for (;;) {
|
|
63
|
+
const r = await driver.get(base);
|
|
64
|
+
snap = isRecord(r.json) ? r.json : {};
|
|
65
|
+
if (['completed', 'failed', 'cancelled'].includes(String(snap['status'])) || Date.now() > deadline) break;
|
|
66
|
+
await new Promise((res) => setTimeout(res, opts.pollMs ?? 500));
|
|
67
|
+
}
|
|
68
|
+
const poll = await driver.get(`${base}/events/poll?timeout=1`);
|
|
69
|
+
const raw = isRecord(poll.json) && Array.isArray(poll.json['events']) ? (poll.json['events'] as unknown[]) : [];
|
|
70
|
+
const events = raw.filter(isRecord).map((e) => ({
|
|
71
|
+
type: String(e['type']),
|
|
72
|
+
sequence: typeof e['sequence'] === 'number' ? e['sequence'] : Number.MAX_SAFE_INTEGER,
|
|
73
|
+
payload: isRecord(e['payload']) ? e['payload'] : undefined,
|
|
74
|
+
})).sort((a, b) => a.sequence - b.sequence);
|
|
75
|
+
const err = isRecord(snap['error']) ? snap['error'] : undefined;
|
|
76
|
+
return {
|
|
77
|
+
createStatus: created.status,
|
|
78
|
+
createCode: undefined,
|
|
79
|
+
terminalStatus: typeof snap['status'] === 'string' ? snap['status'] : undefined,
|
|
80
|
+
errorCode: typeof err?.['code'] === 'string' ? err['code'] : undefined,
|
|
81
|
+
events,
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const f = (ok: boolean, message: string): InsufficientFinding => ({ ok, doc: DOC, message });
|
|
86
|
+
|
|
87
|
+
/** Leg 1, refusal. Pure. */
|
|
88
|
+
export function judgeRefusal(profile: MajorProfile, o: InsufficientObservation): InsufficientFinding[] {
|
|
89
|
+
const created = f(o.createStatus === 201 || o.createStatus === 202, `a create of the advertised fixture MUST be accepted; the refusal is at dispatch, not at create (got ${o.createStatus}${o.createCode ? ` ${o.createCode}` : ''})`);
|
|
90
|
+
if (!created.ok) return [created];
|
|
91
|
+
const insufficientType = nameAt(profile, 'model.capability.insufficient');
|
|
92
|
+
const nodeFailedType = nameAt(profile, 'node.failed');
|
|
93
|
+
const idx = o.events.findIndex((e) => e.type === insufficientType);
|
|
94
|
+
const failedIdx = o.events.findIndex((e) => e.type === nodeFailedType);
|
|
95
|
+
const ev = idx >= 0 ? o.events[idx] : undefined;
|
|
96
|
+
const out = [
|
|
97
|
+
created,
|
|
98
|
+
f(o.terminalStatus === 'failed', `a run whose node requires a capability no model offers MUST end failed (got ${String(o.terminalStatus)})`),
|
|
99
|
+
f(o.errorCode === 'capability_not_provided', `the run's error.code MUST be capability_not_provided (got ${String(o.errorCode)})`),
|
|
100
|
+
f(ev !== undefined, `the host MUST emit ${insufficientType} (types seen: ${JSON.stringify(o.events.map((e) => e.type))})`),
|
|
101
|
+
];
|
|
102
|
+
if (ev === undefined) return out;
|
|
103
|
+
out.push(f(failedIdx < 0 || idx < failedIdx, `${insufficientType} MUST precede ${nodeFailedType} (cause before effect; at ${idx} vs ${failedIdx})`));
|
|
104
|
+
if (profile.major >= 2) {
|
|
105
|
+
const r = v2RefValidator('run-event-payloads.schema.json#/$defs/modelCapabilityInsufficient')(ev.payload ?? {});
|
|
106
|
+
out.push(f(r.ok, `the ${insufficientType} payload MUST validate against run-event-payloads#/$defs/modelCapabilityInsufficient${r.ok ? '' : ` (${r.errors})`}`));
|
|
107
|
+
}
|
|
108
|
+
out.push(f(ev.payload?.['fallbackAttempted'] !== true, `fallbackAttempted MUST NOT be true: the fixture declares no fallbackModel, so no fallback was attempted (got ${JSON.stringify(ev.payload?.['fallbackAttempted'])})`));
|
|
109
|
+
return out;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** What a refused dispatch MUST NOT emit, by era-1 name. */
|
|
113
|
+
export const NOT_AFTER_REFUSAL = ['node.completed', 'provider.usage', 'envelope.retry.attempted', 'envelope.retry.exhausted', 'envelope.refusal', 'envelope.truncated', 'envelope.nlToFormat.engaged', 'envelope.recovery.applied'] as const;
|
|
114
|
+
|
|
115
|
+
/** Leg 2, no-dispatch. Pure. Looks for each event by this major's name AND its era-1 name. */
|
|
116
|
+
export function judgeNoDispatch(profile: MajorProfile, o: InsufficientObservation): InsufficientFinding[] {
|
|
117
|
+
if (o.createStatus !== 201 && o.createStatus !== 202) return [f(false, `a create of the advertised fixture MUST be accepted (got ${o.createStatus})`)];
|
|
118
|
+
const forbidden = new Set<string>(NOT_AFTER_REFUSAL.flatMap((n) => [n, nameAt(profile, n)]));
|
|
119
|
+
const leaked = o.events.filter((e) => forbidden.has(e.type)).map((e) => e.type);
|
|
120
|
+
return [f(leaked.length === 0, `a refused dispatch MUST NOT emit node completion, provider usage or envelope-reliability events — the node never ran (got ${JSON.stringify(leaked)})`)];
|
|
121
|
+
}
|