@openwop/openwop-conformance 1.124.0 → 1.127.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/scenario-disposition.js +14 -1
- package/package.json +1 -1
- package/schemas/CORPUS-STAMP.json +2 -2
- package/src/lib/scenario-disposition.ts +15 -1
- package/src/scenarios/a2a-1-0-task-roundtrip.test.ts +3 -1
- package/src/scenarios/capability-example-root-layout.test.ts +159 -0
- package/src/scenarios/conformance-execution-witness.test.ts +12 -0
- package/src/scenarios/core-manifest-and-extension-registry.test.ts +40 -14
- package/src/scenarios/cross-engine-append-behavior.test.ts +26 -5
- package/src/scenarios/cross-host-ancestry-endpoint.test.ts +8 -4
- package/src/scenarios/cross-host-causation-shape.test.ts +4 -2
- package/src/scenarios/multi-agent-memory-lifecycle.test.ts +8 -4
- package/src/scenarios/multi-region-idempotency-behavior.test.ts +29 -7
- package/src/scenarios/replay-divergence-at-refusal.test.ts +12 -6
- package/src/scenarios/replay-observable-sequence-determinism.test.ts +4 -2
- package/src/scenarios/sandbox-mvp-behavior.test.ts +20 -10
|
@@ -88,7 +88,9 @@ export function fileDisposition(states, gateReason, assertionCount) {
|
|
|
88
88
|
* - a zero-assertion "pass" ⇒ the file's noted reason (`softSkip` /
|
|
89
89
|
* `seamAbsent`: inapplicable | skipped | blocked) or a behaviorGate reason;
|
|
90
90
|
* - a zero-assertion "pass" with NO reason ⇒ `blocked` + the marker detail —
|
|
91
|
-
* an early return can never become a pass
|
|
91
|
+
* an early return can never become a pass;
|
|
92
|
+
* - every test `ctx.skip()`ped ⇒ the file's noted reason if it wrote one
|
|
93
|
+
* BEFORE skipping (`ctx.skip()` throws), else `blocked` + the marker.
|
|
92
94
|
*/
|
|
93
95
|
export function resolveFileRecord(states, gateReason, assertionCount, noted) {
|
|
94
96
|
let { disposition, detail } = fileDisposition(states, gateReason, assertionCount);
|
|
@@ -102,6 +104,17 @@ export function resolveFileRecord(states, gateReason, assertionCount, noted) {
|
|
|
102
104
|
detail = UNCLASSIFIED_RETURN_DETAIL;
|
|
103
105
|
}
|
|
104
106
|
}
|
|
107
|
+
else if (noted !== null &&
|
|
108
|
+
gateReason === undefined &&
|
|
109
|
+
states.length > 0 &&
|
|
110
|
+
states.every((s) => s === 'skip')) {
|
|
111
|
+
// Every test called `ctx.skip()` (vitest reports them as skipped, not as
|
|
112
|
+
// zero-assertion passes) and the file noted why first. Note-then-skip is
|
|
113
|
+
// the required order: `ctx.skip()` throws, so a note written after it is
|
|
114
|
+
// dead code — which is how seven files carried notes the ledger never saw.
|
|
115
|
+
disposition = noted.kind;
|
|
116
|
+
detail = noted.reason;
|
|
117
|
+
}
|
|
105
118
|
return detail === undefined ? { disposition } : { disposition, detail };
|
|
106
119
|
}
|
|
107
120
|
/**
|
package/package.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
2
|
"_comment": "Provenance of this vendored schemas/ copy. See conformance/README.md \u00a7\"Resolving the contract\". Compare against the stamp in your installed @openwop/openwop-conformance to detect a stale hand-copied contract.",
|
|
3
|
-
"suiteVersion": "1.
|
|
4
|
-
"corpusCommit": "
|
|
3
|
+
"suiteVersion": "1.127.0",
|
|
4
|
+
"corpusCommit": "5e2aefe837a81b82004c12c308cb1c3973c0233c"
|
|
5
5
|
}
|
|
@@ -94,7 +94,9 @@ export function fileDisposition(
|
|
|
94
94
|
* - a zero-assertion "pass" ⇒ the file's noted reason (`softSkip` /
|
|
95
95
|
* `seamAbsent`: inapplicable | skipped | blocked) or a behaviorGate reason;
|
|
96
96
|
* - a zero-assertion "pass" with NO reason ⇒ `blocked` + the marker detail —
|
|
97
|
-
* an early return can never become a pass
|
|
97
|
+
* an early return can never become a pass;
|
|
98
|
+
* - every test `ctx.skip()`ped ⇒ the file's noted reason if it wrote one
|
|
99
|
+
* BEFORE skipping (`ctx.skip()` throws), else `blocked` + the marker.
|
|
98
100
|
*/
|
|
99
101
|
export function resolveFileRecord(
|
|
100
102
|
states: readonly FileTestState[],
|
|
@@ -111,6 +113,18 @@ export function resolveFileRecord(
|
|
|
111
113
|
disposition = 'blocked';
|
|
112
114
|
detail = UNCLASSIFIED_RETURN_DETAIL;
|
|
113
115
|
}
|
|
116
|
+
} else if (
|
|
117
|
+
noted !== null &&
|
|
118
|
+
gateReason === undefined &&
|
|
119
|
+
states.length > 0 &&
|
|
120
|
+
states.every((s) => s === 'skip')
|
|
121
|
+
) {
|
|
122
|
+
// Every test called `ctx.skip()` (vitest reports them as skipped, not as
|
|
123
|
+
// zero-assertion passes) and the file noted why first. Note-then-skip is
|
|
124
|
+
// the required order: `ctx.skip()` throws, so a note written after it is
|
|
125
|
+
// dead code — which is how seven files carried notes the ledger never saw.
|
|
126
|
+
disposition = noted.kind;
|
|
127
|
+
detail = noted.reason;
|
|
114
128
|
}
|
|
115
129
|
return detail === undefined ? { disposition } : { disposition, detail };
|
|
116
130
|
}
|
|
@@ -53,7 +53,9 @@ async function claims10(): Promise<boolean> {
|
|
|
53
53
|
async function jsonrpc10Url(): Promise<string | null> {
|
|
54
54
|
const caps = await a2a();
|
|
55
55
|
if (typeof caps?.agentCardUrl !== 'string') return null;
|
|
56
|
-
|
|
56
|
+
// S18 (#1028): a header-less card GET returns the 0.3 shape while `a2a-0.3-legacy`
|
|
57
|
+
// is advertised; a 1.0 client asks for the 1.0 card explicitly (a2a-integration.md §C).
|
|
58
|
+
const res = await fetch(caps.agentCardUrl, { headers: { accept: 'application/json', 'A2A-Version': '1.0' } });
|
|
57
59
|
if (res.status !== 200) return null;
|
|
58
60
|
const card = (await res.json()) as { supportedInterfaces?: Array<{ url?: string; protocolBinding?: string; protocolVersion?: string }> };
|
|
59
61
|
const iface = (card.supportedInterfaces ?? []).find((i) => i.protocolBinding === 'JSONRPC' && i.protocolVersion === '1.0');
|
|
@@ -21,6 +21,27 @@
|
|
|
21
21
|
* otherwise legal unknown server-emitted property, and nothing here reads a
|
|
22
22
|
* host.
|
|
23
23
|
*
|
|
24
|
+
* **Canonical-typo leg (RFC 0149 §B, second bullet; UQ2 decided 2026-08-16).** A
|
|
25
|
+
* root key within edit distance ONE of a canonical family, in a discovery-shaped
|
|
26
|
+
* example, that is not itself canonical and not vendor-namespaced, is a typo
|
|
27
|
+
* (`compensaton`, `interupts`) — an implementer copying it advertises nothing.
|
|
28
|
+
* UQ2 asked what rule avoids false positives on legitimate extension names; the
|
|
29
|
+
* answer was measured, not guessed. Over every fenced root object in `spec/v1` +
|
|
30
|
+
* `RFCS/` (218 on 2026-08-16), plain distance-one produced six near-misses —
|
|
31
|
+
* `ts`/`fs`, `agent`/`agents`, `secret`/`secrets`, `context`/`content`,
|
|
32
|
+
* `prompt`/`prompts`, `schemaVersion`/`schemaVersions` — every one of them a key
|
|
33
|
+
* of an EVENT or RUN object, not a discovery document. So the predicate is
|
|
34
|
+
* *discovery-shaped*: every root key is canonical, vendor-namespaced
|
|
35
|
+
* (`host-extensions.md` §"Canonical prefixes"), the legacy `capabilities`
|
|
36
|
+
* wrapper, or within distance one of a canonical family — and at least one key
|
|
37
|
+
* is canonical-or-near. That excludes events (`ts`/`type`/`payload` are none of
|
|
38
|
+
* those) while still catching a typo-only snippet whose single key is misspelt.
|
|
39
|
+
* Scope: `spec/v1` and RFCs numbered >= 0149 (the rule's own RFC); older RFCs
|
|
40
|
+
* are the dated record, on the same boundary logic as the wrapper leg's 0073.
|
|
41
|
+
* Under that predicate and scope the corpus measured 53 discovery-shaped
|
|
42
|
+
* objects and 0 findings; the one out-of-scope near-miss is RFC 0109's
|
|
43
|
+
* `{ "agent": … }` payload fragment.
|
|
44
|
+
*
|
|
24
45
|
* `spec/v1/` and `RFCS/` ship in the repository, NOT in the published tarball,
|
|
25
46
|
* so this self-skips under the published layout. That asymmetry has produced
|
|
26
47
|
* three defects in this corpus — the `CORPUS-STAMP` gate, the link-checker's
|
|
@@ -66,6 +87,97 @@ function fencedExamples(dir: string): FencedExample[] {
|
|
|
66
87
|
return found;
|
|
67
88
|
}
|
|
68
89
|
|
|
90
|
+
/** RFC 0149 introduced the typo lint; examples in earlier RFCs are historical record. */
|
|
91
|
+
const TYPO_LINT_RFC = 149;
|
|
92
|
+
|
|
93
|
+
const SCHEMA_PATH =
|
|
94
|
+
V1_DIR === null ? null : pathResolve(V1_DIR, '..', '..', 'schemas', 'capabilities.schema.json');
|
|
95
|
+
|
|
96
|
+
/** The canonical families, read from the schema rather than hand-listed. */
|
|
97
|
+
function canonicalFamilies(): Set<string> {
|
|
98
|
+
if (SCHEMA_PATH === null) return new Set();
|
|
99
|
+
const schema = JSON.parse(readFileSync(SCHEMA_PATH, 'utf8')) as { properties?: Record<string, unknown> };
|
|
100
|
+
return new Set(Object.keys(schema.properties ?? {}));
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** `host-extensions.md` §"Canonical prefixes". */
|
|
104
|
+
function isVendorKey(key: string): boolean {
|
|
105
|
+
return /^x-host-/.test(key) || /^(vendor|private)\./.test(key);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** Levenshtein distance exactly one (one substitution, insertion, or deletion). */
|
|
109
|
+
export function withinOne(a: string, b: string): boolean {
|
|
110
|
+
if (a === b) return false;
|
|
111
|
+
if (Math.abs(a.length - b.length) > 1) return false;
|
|
112
|
+
if (a.length === b.length) {
|
|
113
|
+
let diff = 0;
|
|
114
|
+
for (let i = 0; i < a.length; i++) if (a[i] !== b[i] && ++diff > 1) return false;
|
|
115
|
+
return diff === 1;
|
|
116
|
+
}
|
|
117
|
+
const [short, long] = a.length < b.length ? [a, b] : [b, a];
|
|
118
|
+
let i = 0;
|
|
119
|
+
let j = 0;
|
|
120
|
+
let skipped = false;
|
|
121
|
+
while (i < short.length && j < long.length) {
|
|
122
|
+
if (short[i] === long[j]) {
|
|
123
|
+
i++;
|
|
124
|
+
j++;
|
|
125
|
+
} else if (skipped) {
|
|
126
|
+
return false;
|
|
127
|
+
} else {
|
|
128
|
+
skipped = true;
|
|
129
|
+
j++;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
return true;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export interface TypoFinding {
|
|
136
|
+
readonly key: string;
|
|
137
|
+
readonly near: readonly string[];
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* The UQ2 rule. Returns the near-miss root keys of a DISCOVERY-SHAPED object, or
|
|
142
|
+
* `null` when the object is not discovery-shaped (and so is out of scope: an
|
|
143
|
+
* event, a run body, a manifest). Exported so the predicate is pinned below.
|
|
144
|
+
*/
|
|
145
|
+
export function canonicalTypos(value: unknown, families: Set<string>): TypoFinding[] | null {
|
|
146
|
+
if (value === null || typeof value !== 'object' || Array.isArray(value)) return null;
|
|
147
|
+
const keys = Object.keys(value as Record<string, unknown>);
|
|
148
|
+
if (keys.length === 0) return null;
|
|
149
|
+
const near = new Map<string, string[]>();
|
|
150
|
+
for (const k of keys) {
|
|
151
|
+
if (families.has(k) || isVendorKey(k) || k === 'capabilities') continue;
|
|
152
|
+
const n = [...families].filter((f) => withinOne(k, f));
|
|
153
|
+
if (n.length === 0) return null; // a key that is none of the four kinds ⇒ not discovery-shaped
|
|
154
|
+
near.set(k, n);
|
|
155
|
+
}
|
|
156
|
+
if (!keys.some((k) => families.has(k) || near.has(k))) return null;
|
|
157
|
+
return [...near.entries()].map(([key, n]) => ({ key, near: n }));
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** Every parseable fenced json/jsonc root object under `dir`, with its source line. */
|
|
161
|
+
function fencedObjects(dir: string): { file: string; line: number; value: unknown }[] {
|
|
162
|
+
const out: { file: string; line: number; value: unknown }[] = [];
|
|
163
|
+
for (const name of readdirSync(dir).filter((f) => f.endsWith('.md')).sort()) {
|
|
164
|
+
const lines = readFileSync(join(dir, name), 'utf8').split('\n');
|
|
165
|
+
for (let i = 0; i < lines.length; i++) {
|
|
166
|
+
if (!/^```(json|jsonc)\s*$/.test(lines[i]!.trim())) continue;
|
|
167
|
+
const body: string[] = [];
|
|
168
|
+
let j = i + 1;
|
|
169
|
+
while (j < lines.length && lines[j]!.trim() !== '```') body.push(lines[j++]!);
|
|
170
|
+
try {
|
|
171
|
+
out.push({ file: name, line: i + 2, value: JSON.parse(body.join('\n')) });
|
|
172
|
+
} catch {
|
|
173
|
+
// Unparseable blocks are RFC 0150 §D's problem, not this gate's.
|
|
174
|
+
}
|
|
175
|
+
i = j;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return out;
|
|
179
|
+
}
|
|
180
|
+
|
|
69
181
|
describe.skipIf(V1_DIR === null)('RFC 0149 §B — discovery examples use the document-root layout', () => {
|
|
70
182
|
const v1Dir = V1_DIR as string;
|
|
71
183
|
|
|
@@ -110,4 +222,51 @@ describe.skipIf(V1_DIR === null)('RFC 0149 §B — discovery examples use the do
|
|
|
110
222
|
late.join('\n '),
|
|
111
223
|
).toEqual([]);
|
|
112
224
|
});
|
|
225
|
+
it('the UQ2 predicate is pinned: a typo-only snippet is flagged, an event object is out of scope, a vendor key is exempt', () => {
|
|
226
|
+
const fams = new Set(['compensation', 'interrupts', 'fs', 'agents', 'content']);
|
|
227
|
+
// A misspelt single-key discovery snippet: all keys near-canonical ⇒ in scope, flagged.
|
|
228
|
+
expect(canonicalTypos({ compensaton: { supported: true } }, fams)).toEqual([{ key: 'compensaton', near: ['compensation'] }]);
|
|
229
|
+
// Canonical + a typo ⇒ flagged.
|
|
230
|
+
expect(canonicalTypos({ compensation: {}, interupts: {} }, fams)).toEqual([{ key: 'interupts', near: ['interrupts'] }]);
|
|
231
|
+
// An event: `ts` is one from `fs` but `type`/`payload` are none of the four kinds ⇒ not discovery-shaped.
|
|
232
|
+
expect(canonicalTypos({ ts: 1, type: 'x', payload: {} }, fams)).toBeNull();
|
|
233
|
+
// Vendor-namespaced keys are exempt (RFC 0149 §B) and the legacy wrapper is the wrapper leg's business.
|
|
234
|
+
expect(canonicalTypos({ compensation: {}, 'x-host-acme-agent': {} }, fams)).toEqual([]);
|
|
235
|
+
expect(canonicalTypos({ capabilities: {} }, fams)).toBeNull();
|
|
236
|
+
// Distance exactly one, both directions.
|
|
237
|
+
expect(withinOne('agent', 'agents')).toBe(true);
|
|
238
|
+
expect(withinOne('agents', 'agent')).toBe(true);
|
|
239
|
+
expect(withinOne('agent', 'agentz')).toBe(true);
|
|
240
|
+
expect(withinOne('agent', 'agenzs')).toBe(false);
|
|
241
|
+
expect(withinOne('agents', 'agents')).toBe(false);
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
it('no discovery-shaped example in spec/v1 or a post-0149 RFC has a root key within one edit of a canonical family', () => {
|
|
245
|
+
const families = canonicalFamilies();
|
|
246
|
+
expect(families.size, 'capabilities.schema.json MUST declare families').toBeGreaterThan(50);
|
|
247
|
+
const findings: string[] = [];
|
|
248
|
+
let shaped = 0;
|
|
249
|
+
const scan = (dir: string, rel: string, minRfc: number | null): void => {
|
|
250
|
+
for (const { file, line, value } of fencedObjects(dir)) {
|
|
251
|
+
if (minRfc !== null) {
|
|
252
|
+
const n = Number.parseInt(file.slice(0, 4), 10);
|
|
253
|
+
if (!Number.isFinite(n) || n < minRfc) continue;
|
|
254
|
+
}
|
|
255
|
+
const typos = canonicalTypos(value, families);
|
|
256
|
+
if (typos === null) continue;
|
|
257
|
+
shaped++;
|
|
258
|
+
for (const t of typos) findings.push(`${rel}/${file}:${line} → \`${t.key}\` (did you mean ${t.near.map((x) => '`' + x + '`').join(' / ')}?)`);
|
|
259
|
+
}
|
|
260
|
+
};
|
|
261
|
+
scan(v1Dir, 'spec/v1', null);
|
|
262
|
+
if (RFCS_DIR !== null && existsSync(RFCS_DIR)) scan(RFCS_DIR, 'RFCS', TYPO_LINT_RFC);
|
|
263
|
+
expect(shaped, 'the scan MUST find discovery-shaped examples, or the leg is vacuous').toBeGreaterThan(20);
|
|
264
|
+
expect(
|
|
265
|
+
findings,
|
|
266
|
+
'RFC 0149 §B: a root key within one edit of a canonical family, in a discovery-shaped example, is a ' +
|
|
267
|
+
'typo — an implementer copying it advertises nothing. Vendor surface goes under `x-host-*` / ' +
|
|
268
|
+
'`vendor.*` / `private.*` (host-extensions.md).\n ' +
|
|
269
|
+
findings.join('\n '),
|
|
270
|
+
).toEqual([]);
|
|
271
|
+
});
|
|
113
272
|
});
|
|
@@ -51,6 +51,18 @@ describe('RFC 0148 §A — conformance-execution-witness: the runner record', ()
|
|
|
51
51
|
expect(resolveFileRecord(['pass'], undefined, 0, { kind: 'skipped', reason: 'operator opt-out' })).toEqual({ disposition: 'skipped', detail: 'operator opt-out' });
|
|
52
52
|
});
|
|
53
53
|
|
|
54
|
+
it('every test ctx.skip()ped takes the noted reason when one was written before the skip, else stays the blocked marker', () => {
|
|
55
|
+
// `ctx.skip()` throws — a `softSkip(...)` AFTER it is dead code. Seven files
|
|
56
|
+
// carried exactly that dead note and reported as unclassified for a suite minor.
|
|
57
|
+
expect(resolveFileRecord(['skip', 'skip'], undefined, 0, { kind: 'inapplicable', reason: 'sandbox not advertised' })).toEqual({ disposition: 'inapplicable', detail: 'sandbox not advertised' });
|
|
58
|
+
expect(resolveFileRecord(['skip'], undefined, 0, { kind: 'blocked', reason: 'simulator seam 404' })).toEqual({ disposition: 'blocked', detail: 'simulator seam 404' });
|
|
59
|
+
const bare = resolveFileRecord(['skip', 'skip'], undefined, 0, null);
|
|
60
|
+
expect(bare.disposition).toBe('blocked');
|
|
61
|
+
expect(bare.detail).toMatch(/every test skipped with no recorded reason/);
|
|
62
|
+
// A behaviorGate reason still wins over a note — the gate is the more specific record.
|
|
63
|
+
expect(resolveFileRecord(['skip'], 'inapplicable', 0, { kind: 'blocked', reason: 'x' }).disposition).toBe('inapplicable');
|
|
64
|
+
});
|
|
65
|
+
|
|
54
66
|
it('a behaviorGate reason resolves a zero-assertion pass to inapplicable/skipped; a note outranks nothing but never a witnessed pass', () => {
|
|
55
67
|
expect(resolveFileRecord(['pass'], 'inapplicable', 0, null).disposition).toBe('inapplicable');
|
|
56
68
|
expect(resolveFileRecord(['pass'], 'skipped', 0, null).disposition).toBe('skipped');
|
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
*/
|
|
25
25
|
|
|
26
26
|
import { describe, it, expect } from 'vitest';
|
|
27
|
-
import { readFileSync } from 'node:fs';
|
|
27
|
+
import { readFileSync, existsSync } from 'node:fs';
|
|
28
28
|
import { join } from 'node:path';
|
|
29
29
|
import { V1_DIR } from '../lib/paths.js';
|
|
30
30
|
import { PROFILE_FLOOR_SCENARIOS } from '../lib/profiles.js';
|
|
@@ -36,7 +36,7 @@ type Maturity = (typeof MATURITIES)[number];
|
|
|
36
36
|
interface Extension {
|
|
37
37
|
readonly id: string;
|
|
38
38
|
readonly maturity: Maturity;
|
|
39
|
-
readonly owningRfc: string;
|
|
39
|
+
readonly owningRfc: string | null;
|
|
40
40
|
readonly capabilityPath: string;
|
|
41
41
|
readonly dependsOn: readonly string[];
|
|
42
42
|
readonly securityTier: string;
|
|
@@ -190,35 +190,61 @@ describe.skipIf(V1_DIR === null)('RFC 0155 §C — extension registry', () => {
|
|
|
190
190
|
// either a core predicate field, covered by a record's capabilityPath, or
|
|
191
191
|
// listed as uncovered — and nothing is in two buckets.
|
|
192
192
|
const reg = registry as unknown as {
|
|
193
|
-
coverage?: {
|
|
193
|
+
coverage?: {
|
|
194
|
+
familiesTotal: number;
|
|
195
|
+
coreFields: string[];
|
|
196
|
+
metadataFields?: string[];
|
|
197
|
+
metadataRationale?: Record<string, string>;
|
|
198
|
+
covered: string[];
|
|
199
|
+
uncovered: string[];
|
|
200
|
+
};
|
|
194
201
|
extensions: Extension[];
|
|
195
202
|
};
|
|
196
203
|
expect(reg.coverage, 'RFC 0155 §C: the registry MUST carry a derived `coverage` block').toBeDefined();
|
|
197
204
|
const cov = reg.coverage as NonNullable<typeof reg.coverage>;
|
|
205
|
+
const metadata = cov.metadataFields ?? [];
|
|
198
206
|
const families = Object.keys((caps().properties as Record<string, unknown>) ?? {}).sort();
|
|
199
207
|
expect(cov.familiesTotal).toBe(families.length);
|
|
200
|
-
const all = [...cov.coreFields, ...cov.covered, ...cov.uncovered].sort();
|
|
201
|
-
expect(all, 'core + covered + uncovered MUST partition the family set exactly').toEqual(families);
|
|
208
|
+
const all = [...cov.coreFields, ...metadata, ...cov.covered, ...cov.uncovered].sort();
|
|
209
|
+
expect(all, 'core + metadata + covered + uncovered MUST partition the family set exactly').toEqual(families);
|
|
202
210
|
expect(new Set(all).size, 'no family may sit in two buckets').toBe(all.length);
|
|
203
211
|
const reached = new Set(reg.extensions.map((e) => e.capabilityPath.split('.')[0]));
|
|
204
212
|
for (const f of cov.covered) expect(reached.has(f), `${f} listed as covered MUST be reached by a record`).toBe(true);
|
|
205
213
|
for (const f of cov.uncovered) expect(reached.has(f), `${f} listed as uncovered MUST NOT be reached by a record`).toBe(false);
|
|
214
|
+
// Metadata is the one bucket a family can be moved INTO by hand, so it is
|
|
215
|
+
// the one that could hide an extension: every entry MUST carry a stated
|
|
216
|
+
// rationale, and no metadata key may carry a `supported` flag — a key that
|
|
217
|
+
// gates behaviour is a family, not a description of the document.
|
|
218
|
+
for (const f of metadata) {
|
|
219
|
+
expect(typeof cov.metadataRationale?.[f], `${f}: a metadata field MUST state why it is not an extension`).toBe('string');
|
|
220
|
+
const props = (caps().properties as Record<string, { properties?: Record<string, unknown> }>)[f]?.properties ?? {};
|
|
221
|
+
expect('supported' in props, `${f} is listed as metadata but carries a \`supported\` flag — that is an extension family`).toBe(false);
|
|
222
|
+
}
|
|
206
223
|
// The honest number, asserted so it cannot silently shrink by deletion of the
|
|
207
224
|
// uncovered list rather than by adding records.
|
|
208
|
-
expect(cov.uncovered.length + cov.covered.length + cov.coreFields.length).toBe(families.length);
|
|
225
|
+
expect(cov.uncovered.length + cov.covered.length + cov.coreFields.length + metadata.length).toBe(families.length);
|
|
209
226
|
});
|
|
210
227
|
|
|
211
|
-
it('every record names the RFC that owns it', () => {
|
|
228
|
+
it('every record names the RFC — or, for a v1 base advertisement, the spec document — that owns it', () => {
|
|
212
229
|
// Vendor extensions may not use an `openwop-*` id without an accepted RFC
|
|
213
|
-
// (§F). The owning RFC is what makes that checkable.
|
|
230
|
+
// (§F). The owning RFC is what makes that checkable. Six advertisements
|
|
231
|
+
// predate the RFC process (they shipped in the v1 base corpus: `secrets`,
|
|
232
|
+
// `webhooks`, `i18n`, `aiProviders`, `envelopeContracts`, `envelopeStrictness`);
|
|
233
|
+
// those carry `owningRfc: null` and an `owningDoc` under spec/v1/ that MUST
|
|
234
|
+
// exist — the steward's own corpus is the RFC-equivalent authority for them.
|
|
214
235
|
for (const e of (registry as NonNullable<typeof registry>).extensions) {
|
|
215
|
-
|
|
216
|
-
if (
|
|
217
|
-
expect(
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
236
|
+
const rec = e as Extension & { owningDoc?: string; securityTier?: string };
|
|
237
|
+
if (rec.owningRfc === null) {
|
|
238
|
+
expect(typeof rec.owningDoc, `${e.id}: \`owningRfc: null\` requires an \`owningDoc\``).toBe('string');
|
|
239
|
+
expect(rec.owningDoc, `${e.id}: owningDoc MUST be a spec/v1 document`).toMatch(/^spec\/v1\/[a-z0-9-]+\.md$/);
|
|
240
|
+
if (V1_DIR !== null) {
|
|
241
|
+
const file = join(V1_DIR, (rec.owningDoc as string).replace(/^spec\/v1\//, ''));
|
|
242
|
+
expect(existsSync(file), `${e.id}: owningDoc ${rec.owningDoc} MUST exist`).toBe(true);
|
|
243
|
+
}
|
|
244
|
+
} else {
|
|
245
|
+
expect(e.owningRfc, `${e.id} MUST name an owning RFC`).toMatch(/^\d{4}$/);
|
|
221
246
|
}
|
|
247
|
+
expect(['high', 'medium', 'low'], `${e.id}: securityTier is a closed enum`).toContain(rec.securityTier);
|
|
222
248
|
}
|
|
223
249
|
});
|
|
224
250
|
});
|
|
@@ -29,7 +29,8 @@
|
|
|
29
29
|
*/
|
|
30
30
|
|
|
31
31
|
import { describe, it, expect } from 'vitest';
|
|
32
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
32
|
+
import { softSkip, seamAbsent } from '../lib/soft-skip.js';
|
|
33
|
+
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
33
34
|
import { driver } from '../lib/driver.js';
|
|
34
35
|
|
|
35
36
|
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
@@ -69,12 +70,29 @@ async function resetLog(): Promise<number> {
|
|
|
69
70
|
return res.status;
|
|
70
71
|
}
|
|
71
72
|
|
|
73
|
+
/**
|
|
74
|
+
* The cross-engine harness seam answered 404. Whether that is `blocked` (the host
|
|
75
|
+
* advertises `eventLog.crossEngineOrdering.supported: true` and so made a claim
|
|
76
|
+
* this file cannot check) or `inapplicable` (no such advertisement) depends on
|
|
77
|
+
* discovery — record the honest one (RFC 0148 §A).
|
|
78
|
+
*/
|
|
79
|
+
async function noteHarnessAbsent(): Promise<void> {
|
|
80
|
+
const disco = await driver.get('/.well-known/openwop');
|
|
81
|
+
const el = capabilityFamily<{ crossEngineOrdering?: { supported?: unknown } }>(disco.json, 'eventLog');
|
|
82
|
+
if (el?.crossEngineOrdering?.supported === true) {
|
|
83
|
+
seamAbsent('host advertises `eventLog.crossEngineOrdering.supported: true` but `POST /v1/host/sample/test/cross-engine/reset` answered 404 — the ordering claim is unobservable (host-sample-test-seams.md)');
|
|
84
|
+
} else {
|
|
85
|
+
softSkip('inapplicable', 'optional advertisement — `eventLog.crossEngineOrdering` not advertised by this host, and the cross-engine harness seam is absent (RFC 0036 §B)');
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
72
89
|
describe.skipIf(HTTP_SKIP)('cross-engine-append-behavior: §B cross-engine ordering (RFC 0036)', () => {
|
|
73
90
|
it('interleaved appends from two engines converge to a single globally-ordered sequence', async (ctx) => {
|
|
74
91
|
const resetStatus = await resetLog();
|
|
75
92
|
if (resetStatus === 404) {
|
|
93
|
+
await noteHarnessAbsent();
|
|
76
94
|
ctx.skip(); // host doesn't expose the cross-engine harness seam
|
|
77
|
-
return
|
|
95
|
+
return;
|
|
78
96
|
}
|
|
79
97
|
expect(resetStatus).toBe(200);
|
|
80
98
|
|
|
@@ -125,8 +143,9 @@ describe.skipIf(HTTP_SKIP)('cross-engine-append-behavior: §B cross-engine order
|
|
|
125
143
|
it('lamport clocks monotonically advance across engines', async (ctx) => {
|
|
126
144
|
const resetStatus = await resetLog();
|
|
127
145
|
if (resetStatus === 404) {
|
|
146
|
+
await noteHarnessAbsent();
|
|
128
147
|
ctx.skip();
|
|
129
|
-
return
|
|
148
|
+
return;
|
|
130
149
|
}
|
|
131
150
|
expect(resetStatus).toBe(200);
|
|
132
151
|
|
|
@@ -153,8 +172,9 @@ describe.skipIf(HTTP_SKIP)('cross-engine-append-behavior: §B cross-engine order
|
|
|
153
172
|
it('lamport hint from engine A advances engine B past it', async (ctx) => {
|
|
154
173
|
const resetStatus = await resetLog();
|
|
155
174
|
if (resetStatus === 404) {
|
|
175
|
+
await noteHarnessAbsent();
|
|
156
176
|
ctx.skip();
|
|
157
|
-
return
|
|
177
|
+
return;
|
|
158
178
|
}
|
|
159
179
|
expect(resetStatus).toBe(200);
|
|
160
180
|
|
|
@@ -180,8 +200,9 @@ describe.skipIf(HTTP_SKIP)('cross-engine-append-behavior: §B cross-engine order
|
|
|
180
200
|
it('linearization is deterministic — same appends → same total order', async (ctx) => {
|
|
181
201
|
const resetStatus = await resetLog();
|
|
182
202
|
if (resetStatus === 404) {
|
|
203
|
+
await noteHarnessAbsent();
|
|
183
204
|
ctx.skip();
|
|
184
|
-
return
|
|
205
|
+
return;
|
|
185
206
|
}
|
|
186
207
|
expect(resetStatus).toBe(200);
|
|
187
208
|
|
|
@@ -69,8 +69,9 @@ describe.skipIf(HTTP_SKIP)('cross-host-ancestry-endpoint: behavioral (RFC 0040
|
|
|
69
69
|
const d = await readDiscovery();
|
|
70
70
|
const chc = capabilityFamily<{ executionModel?: { [k: string]: unknown; crossHostCausation?: Record<string, unknown>; replayDeterminism?: Record<string, unknown> } }>(d, 'multiAgent')?.executionModel?.crossHostCausation;
|
|
71
71
|
if (chc?.ancestryEndpointSupported !== true) {
|
|
72
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `chc?.ancestryEndpointSupported !== true` returned early');
|
|
72
73
|
ctx.skip();
|
|
73
|
-
return
|
|
74
|
+
return;
|
|
74
75
|
}
|
|
75
76
|
|
|
76
77
|
// Create a fresh top-level run via the host's conformance-dispatch-loop
|
|
@@ -78,8 +79,9 @@ describe.skipIf(HTTP_SKIP)('cross-host-ancestry-endpoint: behavioral (RFC 0040
|
|
|
78
79
|
// depend on the specific workflow).
|
|
79
80
|
const create = await driver.post('/v1/runs', { workflowId: 'conformance-dispatch-loop' });
|
|
80
81
|
if (create.status !== 201) {
|
|
82
|
+
softSkip('blocked', 'precondition not met — `create.status !== 201` returned early (seam, prior step, or fixture unavailable)');
|
|
81
83
|
ctx.skip();
|
|
82
|
-
return
|
|
84
|
+
return;
|
|
83
85
|
}
|
|
84
86
|
const runId = (create.json as { runId: string }).runId;
|
|
85
87
|
|
|
@@ -116,12 +118,14 @@ describe.skipIf(HTTP_SKIP)('cross-host-ancestry-endpoint: behavioral (RFC 0040
|
|
|
116
118
|
const d = await readDiscovery();
|
|
117
119
|
const chc = capabilityFamily<{ executionModel?: { [k: string]: unknown; crossHostCausation?: Record<string, unknown>; replayDeterminism?: Record<string, unknown> } }>(d, 'multiAgent')?.executionModel?.crossHostCausation;
|
|
118
120
|
if (chc?.supported !== true) {
|
|
121
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `chc?.supported !== true` returned early');
|
|
119
122
|
ctx.skip();
|
|
120
|
-
return
|
|
123
|
+
return;
|
|
121
124
|
}
|
|
122
125
|
if (chc.ancestryEndpointSupported === true) {
|
|
126
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `chc.ancestryEndpointSupported === true` returned early');
|
|
123
127
|
ctx.skip(); // covered by the test above
|
|
124
|
-
return
|
|
128
|
+
return;
|
|
125
129
|
}
|
|
126
130
|
|
|
127
131
|
// Use any runId — even a synthetic non-existent one. The endpoint should
|
|
@@ -62,13 +62,15 @@ describe.skipIf(HTTP_SKIP)('cross-host-causation-shape: advertisement shape (RFC
|
|
|
62
62
|
it('crossHostCausation (when present) conforms to RFC 0040 §D', async (ctx) => {
|
|
63
63
|
const d = await readDiscovery();
|
|
64
64
|
if (d === null) {
|
|
65
|
+
softSkip('blocked', 'precondition not met — `d === null` returned early (seam, prior step, or fixture unavailable)');
|
|
65
66
|
ctx.skip();
|
|
66
|
-
return
|
|
67
|
+
return;
|
|
67
68
|
}
|
|
68
69
|
const chc = capabilityFamily<{ executionModel?: { [k: string]: unknown; crossHostCausation?: Record<string, unknown>; replayDeterminism?: Record<string, unknown> } }>(d, 'multiAgent')?.executionModel?.crossHostCausation;
|
|
69
70
|
if (chc === undefined) {
|
|
71
|
+
softSkip('inapplicable', 'optional advertisement — `multiAgent.executionModel.crossHostCausation` not advertised by this host (RFC 0040 §D)');
|
|
70
72
|
ctx.skip(); // host doesn't advertise — soft-skip
|
|
71
|
-
return
|
|
73
|
+
return;
|
|
72
74
|
}
|
|
73
75
|
|
|
74
76
|
expect(
|
|
@@ -80,13 +80,15 @@ describe.skipIf(HTTP_SKIP)('multi-agent-memory-lifecycle: advertisement shape (R
|
|
|
80
80
|
it('crossChildMemoryConcurrency (when advertised) MUST be one of {strict, advisory}', async (ctx) => {
|
|
81
81
|
const d = await readDiscovery();
|
|
82
82
|
if (d === null) {
|
|
83
|
+
softSkip('blocked', 'precondition not met — `d === null` returned early (seam, prior step, or fixture unavailable)');
|
|
83
84
|
ctx.skip();
|
|
84
|
-
return
|
|
85
|
+
return;
|
|
85
86
|
}
|
|
86
87
|
const ccmc = capabilityFamily<{ executionModel?: { [k: string]: unknown; crossHostCausation?: Record<string, unknown>; replayDeterminism?: Record<string, unknown> } }>(d, 'multiAgent')?.executionModel?.crossChildMemoryConcurrency;
|
|
87
88
|
if (ccmc === undefined) {
|
|
89
|
+
softSkip('inapplicable', 'optional advertisement — `multiAgent.executionModel.crossChildMemoryConcurrency` not advertised by this host');
|
|
88
90
|
ctx.skip(); // optional advertisement — host hasn't opted in
|
|
89
|
-
return
|
|
91
|
+
return;
|
|
90
92
|
}
|
|
91
93
|
expect(
|
|
92
94
|
ccmc === 'strict' || ccmc === 'advisory',
|
|
@@ -134,16 +136,18 @@ describe.skipIf(HTTP_SKIP)('multi-agent-memory-lifecycle: behavioral (RFC 0039
|
|
|
134
136
|
it('MAE-3 replay snapshot refusal: fork mode:replay against a past-retention runId MUST return 422 replay_memory_snapshot_unavailable with documented envelope; silent substitution is non-conformant', async (ctx) => {
|
|
135
137
|
const d = await readDiscovery();
|
|
136
138
|
if (d === null) {
|
|
139
|
+
softSkip('blocked', 'precondition not met — `d === null` returned early (seam, prior step, or fixture unavailable)');
|
|
137
140
|
ctx.skip();
|
|
138
|
-
return
|
|
141
|
+
return;
|
|
139
142
|
}
|
|
140
143
|
const v = capabilityFamily<{ executionModel?: { [k: string]: unknown; crossHostCausation?: Record<string, unknown>; replayDeterminism?: Record<string, unknown> } }>(d, 'multiAgent')?.executionModel?.version;
|
|
141
144
|
const memorySupported = capabilityFamily<{ supported?: unknown }>(d, 'memory')?.supported;
|
|
142
145
|
const phase2OrLater = typeof v === 'number' && v >= 2;
|
|
143
146
|
const expiredRunId = process.env.OPENWOP_TEST_EXPIRED_REPLAY_RUN_ID;
|
|
144
147
|
if (!phase2OrLater || memorySupported !== true || !expiredRunId) {
|
|
148
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!phase2OrLater || memorySupported !== true || !expiredRunId` returned early');
|
|
145
149
|
ctx.skip();
|
|
146
|
-
return
|
|
150
|
+
return;
|
|
147
151
|
}
|
|
148
152
|
|
|
149
153
|
const fromSeq = 0;
|
|
@@ -31,7 +31,8 @@
|
|
|
31
31
|
*/
|
|
32
32
|
|
|
33
33
|
import { describe, it, expect } from 'vitest';
|
|
34
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
34
|
+
import { softSkip, seamAbsent } from '../lib/soft-skip.js';
|
|
35
|
+
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
35
36
|
import { driver } from '../lib/driver.js';
|
|
36
37
|
|
|
37
38
|
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
@@ -56,6 +57,21 @@ async function simulatePartition(claims: ConflictClaim[]): Promise<{ status: num
|
|
|
56
57
|
return { status: res.status, body: (res.json as ConvergenceResult) ?? {} };
|
|
57
58
|
}
|
|
58
59
|
|
|
60
|
+
/**
|
|
61
|
+
* The multi-region simulator seam answered 404. `blocked` when the host advertises
|
|
62
|
+
* a cross-region posture (`idempotency.crossRegion` present) it has made
|
|
63
|
+
* unobservable, `inapplicable` when it advertises none (RFC 0148 §A).
|
|
64
|
+
*/
|
|
65
|
+
async function noteSimulatorAbsent(): Promise<void> {
|
|
66
|
+
const disco = await driver.get('/.well-known/openwop');
|
|
67
|
+
const idem = capabilityFamily<{ crossRegion?: unknown }>(disco.json, 'idempotency');
|
|
68
|
+
if (idem?.crossRegion !== undefined) {
|
|
69
|
+
seamAbsent(`host advertises \`idempotency.crossRegion: ${String(idem.crossRegion)}\` but \`POST /v1/host/sample/test/multi-region/simulate-partition\` answered 404 — the convergence rule is unobservable (host-sample-test-seams.md §6)`);
|
|
70
|
+
} else {
|
|
71
|
+
softSkip('inapplicable', 'optional advertisement — `idempotency.crossRegion` not advertised by this host, and the multi-region simulator seam is absent (RFC 0036 §C)');
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
59
75
|
describe.skipIf(HTTP_SKIP)('multi-region-idempotency-behavior: convergence rule (RFC 0036 §C)', () => {
|
|
60
76
|
it('two-region conflict resolves to the lex-min runId per annex §"Convergence rule"', async (ctx) => {
|
|
61
77
|
const probe = await simulatePartition([
|
|
@@ -63,8 +79,9 @@ describe.skipIf(HTTP_SKIP)('multi-region-idempotency-behavior: convergence rule
|
|
|
63
79
|
{ runId: 'run-a-west', tenantId: 't1', endpoint: 'POST /v1/runs', key: 'idem-1', region: 'eu-west-1' },
|
|
64
80
|
]);
|
|
65
81
|
if (probe.status === 404) {
|
|
82
|
+
await noteSimulatorAbsent();
|
|
66
83
|
ctx.skip(); // host doesn't expose the simulator seam
|
|
67
|
-
return
|
|
84
|
+
return;
|
|
68
85
|
}
|
|
69
86
|
expect(
|
|
70
87
|
probe.status,
|
|
@@ -89,8 +106,9 @@ describe.skipIf(HTTP_SKIP)('multi-region-idempotency-behavior: convergence rule
|
|
|
89
106
|
{ runId: 'mmm-2', tenantId: 't1', endpoint: 'POST /v1/runs', key: 'idem-2', region: 'r3' },
|
|
90
107
|
]);
|
|
91
108
|
if (probe.status === 404) {
|
|
109
|
+
await noteSimulatorAbsent();
|
|
92
110
|
ctx.skip();
|
|
93
|
-
return
|
|
111
|
+
return;
|
|
94
112
|
}
|
|
95
113
|
expect(probe.status).toBe(200);
|
|
96
114
|
expect(
|
|
@@ -115,8 +133,9 @@ describe.skipIf(HTTP_SKIP)('multi-region-idempotency-behavior: convergence rule
|
|
|
115
133
|
{ runId: 'run-a', tenantId: 't1', endpoint: 'POST /v1/runs', key: 'idem-3', region: 'r2' },
|
|
116
134
|
]);
|
|
117
135
|
if (probe.status === 404) {
|
|
136
|
+
await noteSimulatorAbsent();
|
|
118
137
|
ctx.skip();
|
|
119
|
-
return
|
|
138
|
+
return;
|
|
120
139
|
}
|
|
121
140
|
expect(probe.status).toBe(200);
|
|
122
141
|
const redirects = probe.body.cacheRedirects ?? [];
|
|
@@ -144,8 +163,9 @@ describe.skipIf(HTTP_SKIP)('multi-region-idempotency-behavior: convergence rule
|
|
|
144
163
|
{ runId: 'run-a', tenantId: 't1', endpoint: 'POST /v1/runs', key: 'idem-4', region: 'r2' },
|
|
145
164
|
]);
|
|
146
165
|
if (probe.status === 404) {
|
|
166
|
+
await noteSimulatorAbsent();
|
|
147
167
|
ctx.skip();
|
|
148
|
-
return
|
|
168
|
+
return;
|
|
149
169
|
}
|
|
150
170
|
expect(probe.status).toBe(200);
|
|
151
171
|
expect(
|
|
@@ -165,8 +185,9 @@ describe.skipIf(HTTP_SKIP)('multi-region-idempotency-behavior: convergence rule
|
|
|
165
185
|
];
|
|
166
186
|
const p1 = await simulatePartition(claims);
|
|
167
187
|
if (p1.status === 404) {
|
|
188
|
+
await noteSimulatorAbsent();
|
|
168
189
|
ctx.skip();
|
|
169
|
-
return
|
|
190
|
+
return;
|
|
170
191
|
}
|
|
171
192
|
expect(p1.status).toBe(200);
|
|
172
193
|
const p2 = await simulatePartition([claims[2]!, claims[0]!, claims[1]!]);
|
|
@@ -190,8 +211,9 @@ describe.skipIf(HTTP_SKIP)('multi-region-idempotency-behavior: convergence rule
|
|
|
190
211
|
{ runId: 'r2', tenantId: 't2', endpoint: 'POST /v1/runs', key: 'idem-6', region: 'r2' },
|
|
191
212
|
]);
|
|
192
213
|
if (probe.status === 404) {
|
|
214
|
+
await noteSimulatorAbsent();
|
|
193
215
|
ctx.skip();
|
|
194
|
-
return
|
|
216
|
+
return;
|
|
195
217
|
}
|
|
196
218
|
expect(
|
|
197
219
|
probe.status,
|
|
@@ -78,13 +78,15 @@ describe.skipIf(HTTP_SKIP)('replay-divergence-at-refusal: advertisement shape (R
|
|
|
78
78
|
it('replayDeterminism (when present) conforms to RFC 0041 §D', async (ctx) => {
|
|
79
79
|
const d = await readDiscovery();
|
|
80
80
|
if (d === null) {
|
|
81
|
+
softSkip('blocked', 'precondition not met — `d === null` returned early (seam, prior step, or fixture unavailable)');
|
|
81
82
|
ctx.skip();
|
|
82
|
-
return
|
|
83
|
+
return;
|
|
83
84
|
}
|
|
84
85
|
const rd = capabilityFamily<{ executionModel?: { [k: string]: unknown; crossHostCausation?: Record<string, unknown>; replayDeterminism?: Record<string, unknown> } }>(d, 'multiAgent')?.executionModel?.replayDeterminism;
|
|
85
86
|
if (rd === undefined) {
|
|
87
|
+
softSkip('inapplicable', 'optional advertisement — `multiAgent.executionModel.replayDeterminism` not advertised by this host (RFC 0041 §D)');
|
|
86
88
|
ctx.skip(); // optional advertisement — host hasn't opted in
|
|
87
|
-
return
|
|
89
|
+
return;
|
|
88
90
|
}
|
|
89
91
|
|
|
90
92
|
expect(
|
|
@@ -192,8 +194,9 @@ describe.skipIf(HTTP_SKIP)('replay-divergence-at-refusal: behavioral (RFC 0041
|
|
|
192
194
|
{ content: validEnv, stopReason: 'end_turn' as const },
|
|
193
195
|
]);
|
|
194
196
|
if (programStatus === 404) {
|
|
197
|
+
softSkip('blocked', 'precondition not met — `programStatus === 404` returned early (seam, prior step, or fixture unavailable)');
|
|
195
198
|
ctx.skip(); // mock-AI program seam not exposed — soft-skip
|
|
196
|
-
return
|
|
199
|
+
return;
|
|
197
200
|
}
|
|
198
201
|
expect(programStatus).toBe(200);
|
|
199
202
|
|
|
@@ -201,8 +204,9 @@ describe.skipIf(HTTP_SKIP)('replay-divergence-at-refusal: behavioral (RFC 0041
|
|
|
201
204
|
workflowId: 'conformance-phase4-replay-divergence',
|
|
202
205
|
});
|
|
203
206
|
if (createRes.status === 404 || createRes.status === 422) {
|
|
207
|
+
softSkip('blocked', 'precondition not met — `createRes.status === 404 || createRes.status === 422` returned early (seam, prior step, or fixture unavailable)');
|
|
204
208
|
ctx.skip(); // fixture not advertised
|
|
205
|
-
return
|
|
209
|
+
return;
|
|
206
210
|
}
|
|
207
211
|
expect(createRes.status).toBe(201);
|
|
208
212
|
const sourceRunId = (createRes.json as { runId: string }).runId;
|
|
@@ -273,8 +277,9 @@ describe.skipIf(HTTP_SKIP)('replay-divergence-at-refusal: behavioral (RFC 0041
|
|
|
273
277
|
{ content: 'safety-refused-for-conformance', stopReason: 'safety' as const, refusalText: 'safety-refused-for-conformance' },
|
|
274
278
|
]);
|
|
275
279
|
if (programStatus === 404) {
|
|
280
|
+
softSkip('blocked', 'precondition not met — `programStatus === 404` returned early (seam, prior step, or fixture unavailable)');
|
|
276
281
|
ctx.skip();
|
|
277
|
-
return
|
|
282
|
+
return;
|
|
278
283
|
}
|
|
279
284
|
expect(programStatus).toBe(200);
|
|
280
285
|
|
|
@@ -282,8 +287,9 @@ describe.skipIf(HTTP_SKIP)('replay-divergence-at-refusal: behavioral (RFC 0041
|
|
|
282
287
|
workflowId: 'conformance-phase4-replay-divergence',
|
|
283
288
|
});
|
|
284
289
|
if (createRes.status === 404 || createRes.status === 422) {
|
|
290
|
+
softSkip('blocked', 'precondition not met — `createRes.status === 404 || createRes.status === 422` returned early (seam, prior step, or fixture unavailable)');
|
|
285
291
|
ctx.skip();
|
|
286
|
-
return
|
|
292
|
+
return;
|
|
287
293
|
}
|
|
288
294
|
expect(createRes.status).toBe(201);
|
|
289
295
|
const sourceRunId = (createRes.json as { runId: string }).runId;
|
|
@@ -41,7 +41,7 @@
|
|
|
41
41
|
*/
|
|
42
42
|
|
|
43
43
|
import { describe, it, expect } from 'vitest';
|
|
44
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
44
|
+
import { softSkip, seamAbsent } from '../lib/soft-skip.js';
|
|
45
45
|
import { driver } from '../lib/driver.js';
|
|
46
46
|
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
47
47
|
|
|
@@ -85,6 +85,7 @@ async function gateOnPhase4(ctx: { skip: () => void }): Promise<boolean> {
|
|
|
85
85
|
const em = capabilityFamily<{ executionModel?: ExecutionModelCaps }>(d, 'multiAgent')?.executionModel;
|
|
86
86
|
const version = typeof em?.version === 'number' ? em.version : 0;
|
|
87
87
|
if (em?.replayDeterminism?.supported !== true || version < 4) {
|
|
88
|
+
softSkip('inapplicable', 'optional advertisement — `multiAgent.executionModel.replayDeterminism.supported` with `version >= 4` not advertised by this host (RFC 0041 §C)');
|
|
88
89
|
ctx.skip();
|
|
89
90
|
return false;
|
|
90
91
|
}
|
|
@@ -152,7 +153,8 @@ function stripVolatile(ev: RunEventDoc): unknown {
|
|
|
152
153
|
async function startFixtureRun(ctx: { skip: () => void }): Promise<string | null> {
|
|
153
154
|
const create = await driver.post('/v1/runs', { workflowId: FIXTURE });
|
|
154
155
|
if (create.status === 404 || create.status === 422) {
|
|
155
|
-
|
|
156
|
+
seamAbsent(`fixture \`${FIXTURE}\` not registered on this host (POST /v1/runs → ${create.status}) while replayDeterminism is advertised`);
|
|
157
|
+
ctx.skip();
|
|
156
158
|
return null;
|
|
157
159
|
}
|
|
158
160
|
expect(create.status).toBe(201);
|
|
@@ -82,8 +82,9 @@ async function invoke(typeId: string, args: Record<string, unknown> = {}, allowe
|
|
|
82
82
|
describe.skipIf(HTTP_SKIP)('sandbox-mvp-behavior: RFC 0035 §B failure-mode invariants (node:vm MVP)', () => {
|
|
83
83
|
it('host-fs-escape — fs access from sandboxed code fails closed', async (ctx) => {
|
|
84
84
|
if (!(await isSandboxAdvertised())) {
|
|
85
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!(await isSandboxAdvertised())` returned early');
|
|
85
86
|
ctx.skip();
|
|
86
|
-
return
|
|
87
|
+
return;
|
|
87
88
|
}
|
|
88
89
|
const probe = await invoke('misbehave.fs-escape-read');
|
|
89
90
|
expect(probe.status).toBe(200);
|
|
@@ -105,8 +106,9 @@ describe.skipIf(HTTP_SKIP)('sandbox-mvp-behavior: RFC 0035 §B failure-mode inva
|
|
|
105
106
|
|
|
106
107
|
it('host-env-leak — process.env access from sandboxed code fails closed', async (ctx) => {
|
|
107
108
|
if (!(await isSandboxAdvertised())) {
|
|
109
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!(await isSandboxAdvertised())` returned early');
|
|
108
110
|
ctx.skip();
|
|
109
|
-
return
|
|
111
|
+
return;
|
|
110
112
|
}
|
|
111
113
|
const probe = await invoke('misbehave.env-leak');
|
|
112
114
|
expect(probe.status).toBe(200);
|
|
@@ -122,8 +124,9 @@ describe.skipIf(HTTP_SKIP)('sandbox-mvp-behavior: RFC 0035 §B failure-mode inva
|
|
|
122
124
|
|
|
123
125
|
it('network-escape — http/net access from sandboxed code fails closed', async (ctx) => {
|
|
124
126
|
if (!(await isSandboxAdvertised())) {
|
|
127
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!(await isSandboxAdvertised())` returned early');
|
|
125
128
|
ctx.skip();
|
|
126
|
-
return
|
|
129
|
+
return;
|
|
127
130
|
}
|
|
128
131
|
const probe = await invoke('misbehave.network-escape');
|
|
129
132
|
expect(probe.status).toBe(200);
|
|
@@ -139,8 +142,9 @@ describe.skipIf(HTTP_SKIP)('sandbox-mvp-behavior: RFC 0035 §B failure-mode inva
|
|
|
139
142
|
|
|
140
143
|
it('host-process-escape — child_process access from sandboxed code fails closed', async (ctx) => {
|
|
141
144
|
if (!(await isSandboxAdvertised())) {
|
|
145
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!(await isSandboxAdvertised())` returned early');
|
|
142
146
|
ctx.skip();
|
|
143
|
-
return
|
|
147
|
+
return;
|
|
144
148
|
}
|
|
145
149
|
const probe = await invoke('misbehave.process-escape');
|
|
146
150
|
expect(probe.status).toBe(200);
|
|
@@ -153,8 +157,9 @@ describe.skipIf(HTTP_SKIP)('sandbox-mvp-behavior: RFC 0035 §B failure-mode inva
|
|
|
153
157
|
|
|
154
158
|
it('sandbox-timeout — runaway loop terminated by wallClockLimitMs', async (ctx) => {
|
|
155
159
|
if (!(await isSandboxAdvertised())) {
|
|
160
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!(await isSandboxAdvertised())` returned early');
|
|
156
161
|
ctx.skip();
|
|
157
|
-
return
|
|
162
|
+
return;
|
|
158
163
|
}
|
|
159
164
|
const start = Date.now();
|
|
160
165
|
const probe = await invoke('misbehave.timeout');
|
|
@@ -179,8 +184,9 @@ describe.skipIf(HTTP_SKIP)('sandbox-mvp-behavior: RFC 0035 §B failure-mode inva
|
|
|
179
184
|
|
|
180
185
|
it('cross-pack-mutation — fresh vm context per invocation, no state leaks', async (ctx) => {
|
|
181
186
|
if (!(await isSandboxAdvertised())) {
|
|
187
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!(await isSandboxAdvertised())` returned early');
|
|
182
188
|
ctx.skip();
|
|
183
|
-
return
|
|
189
|
+
return;
|
|
184
190
|
}
|
|
185
191
|
const r1 = await invoke('misbehave.cross-pack-mutate');
|
|
186
192
|
const r2 = await invoke('misbehave.cross-pack-mutate');
|
|
@@ -205,8 +211,9 @@ describe.skipIf(HTTP_SKIP)('sandbox-mvp-behavior: RFC 0035 §B failure-mode inva
|
|
|
205
211
|
|
|
206
212
|
it('capability-gate-respected — host call NOT in allowedHostCalls fails with sandbox_capability_denied', async (ctx) => {
|
|
207
213
|
if (!(await isSandboxAdvertised())) {
|
|
214
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!(await isSandboxAdvertised())` returned early');
|
|
208
215
|
ctx.skip();
|
|
209
|
-
return
|
|
216
|
+
return;
|
|
210
217
|
}
|
|
211
218
|
const probe = await invoke('misbehave.capability-gate-violation', {}, []);
|
|
212
219
|
expect(probe.status).toBe(200);
|
|
@@ -228,8 +235,9 @@ describe.skipIf(HTTP_SKIP)('sandbox-mvp-behavior: RFC 0035 §B failure-mode inva
|
|
|
228
235
|
|
|
229
236
|
it('memory-exceeded — runaway allocation fails with sandbox_memory_exceeded', async (ctx) => {
|
|
230
237
|
if (!(await isSandboxAdvertised())) {
|
|
238
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!(await isSandboxAdvertised())` returned early');
|
|
231
239
|
ctx.skip();
|
|
232
|
-
return
|
|
240
|
+
return;
|
|
233
241
|
}
|
|
234
242
|
const probe = await invoke('misbehave.memory-bomb');
|
|
235
243
|
expect(probe.status).toBe(200);
|
|
@@ -255,8 +263,9 @@ describe.skipIf(HTTP_SKIP)('sandbox-mvp-behavior: RFC 0035 §B failure-mode inva
|
|
|
255
263
|
|
|
256
264
|
it('well-behaved.host-fetch — allowedHostCalls=[fetch] permits the host call', async (ctx) => {
|
|
257
265
|
if (!(await isSandboxAdvertised())) {
|
|
266
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!(await isSandboxAdvertised())` returned early');
|
|
258
267
|
ctx.skip();
|
|
259
|
-
return
|
|
268
|
+
return;
|
|
260
269
|
}
|
|
261
270
|
const probe = await invoke('well-behaved.host-fetch', {}, ['fetch']);
|
|
262
271
|
expect(probe.status).toBe(200);
|
|
@@ -271,8 +280,9 @@ describe.skipIf(HTTP_SKIP)('sandbox-mvp-behavior: RFC 0035 §B failure-mode inva
|
|
|
271
280
|
|
|
272
281
|
it('well-behaved.echo — sandboxed code returns args round-trip when no escape attempt', async (ctx) => {
|
|
273
282
|
if (!(await isSandboxAdvertised())) {
|
|
283
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!(await isSandboxAdvertised())` returned early');
|
|
274
284
|
ctx.skip();
|
|
275
|
-
return
|
|
285
|
+
return;
|
|
276
286
|
}
|
|
277
287
|
const probe = await invoke('well-behaved.echo', { input: 'hello-sandbox' });
|
|
278
288
|
expect(probe.status).toBe(200);
|