@openwop/openwop-conformance 1.124.0 → 1.125.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/scenario-disposition.js +14 -1
- package/package.json +1 -1
- package/schemas/CORPUS-STAMP.json +2 -2
- package/src/lib/scenario-disposition.ts +15 -1
- package/src/scenarios/conformance-execution-witness.test.ts +12 -0
- package/src/scenarios/cross-engine-append-behavior.test.ts +26 -5
- package/src/scenarios/cross-host-ancestry-endpoint.test.ts +8 -4
- package/src/scenarios/cross-host-causation-shape.test.ts +4 -2
- package/src/scenarios/multi-agent-memory-lifecycle.test.ts +8 -4
- package/src/scenarios/multi-region-idempotency-behavior.test.ts +29 -7
- package/src/scenarios/replay-divergence-at-refusal.test.ts +12 -6
- package/src/scenarios/replay-observable-sequence-determinism.test.ts +4 -2
- package/src/scenarios/sandbox-mvp-behavior.test.ts +20 -10
|
@@ -88,7 +88,9 @@ export function fileDisposition(states, gateReason, assertionCount) {
|
|
|
88
88
|
* - a zero-assertion "pass" ⇒ the file's noted reason (`softSkip` /
|
|
89
89
|
* `seamAbsent`: inapplicable | skipped | blocked) or a behaviorGate reason;
|
|
90
90
|
* - a zero-assertion "pass" with NO reason ⇒ `blocked` + the marker detail —
|
|
91
|
-
* an early return can never become a pass
|
|
91
|
+
* an early return can never become a pass;
|
|
92
|
+
* - every test `ctx.skip()`ped ⇒ the file's noted reason if it wrote one
|
|
93
|
+
* BEFORE skipping (`ctx.skip()` throws), else `blocked` + the marker.
|
|
92
94
|
*/
|
|
93
95
|
export function resolveFileRecord(states, gateReason, assertionCount, noted) {
|
|
94
96
|
let { disposition, detail } = fileDisposition(states, gateReason, assertionCount);
|
|
@@ -102,6 +104,17 @@ export function resolveFileRecord(states, gateReason, assertionCount, noted) {
|
|
|
102
104
|
detail = UNCLASSIFIED_RETURN_DETAIL;
|
|
103
105
|
}
|
|
104
106
|
}
|
|
107
|
+
else if (noted !== null &&
|
|
108
|
+
gateReason === undefined &&
|
|
109
|
+
states.length > 0 &&
|
|
110
|
+
states.every((s) => s === 'skip')) {
|
|
111
|
+
// Every test called `ctx.skip()` (vitest reports them as skipped, not as
|
|
112
|
+
// zero-assertion passes) and the file noted why first. Note-then-skip is
|
|
113
|
+
// the required order: `ctx.skip()` throws, so a note written after it is
|
|
114
|
+
// dead code — which is how seven files carried notes the ledger never saw.
|
|
115
|
+
disposition = noted.kind;
|
|
116
|
+
detail = noted.reason;
|
|
117
|
+
}
|
|
105
118
|
return detail === undefined ? { disposition } : { disposition, detail };
|
|
106
119
|
}
|
|
107
120
|
/**
|
package/package.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
2
|
"_comment": "Provenance of this vendored schemas/ copy. See conformance/README.md \u00a7\"Resolving the contract\". Compare against the stamp in your installed @openwop/openwop-conformance to detect a stale hand-copied contract.",
|
|
3
|
-
"suiteVersion": "1.
|
|
4
|
-
"corpusCommit": "
|
|
3
|
+
"suiteVersion": "1.125.0",
|
|
4
|
+
"corpusCommit": "13ab2a545d4e2a30c8da66354e39d4db7233acae"
|
|
5
5
|
}
|
|
@@ -94,7 +94,9 @@ export function fileDisposition(
|
|
|
94
94
|
* - a zero-assertion "pass" ⇒ the file's noted reason (`softSkip` /
|
|
95
95
|
* `seamAbsent`: inapplicable | skipped | blocked) or a behaviorGate reason;
|
|
96
96
|
* - a zero-assertion "pass" with NO reason ⇒ `blocked` + the marker detail —
|
|
97
|
-
* an early return can never become a pass
|
|
97
|
+
* an early return can never become a pass;
|
|
98
|
+
* - every test `ctx.skip()`ped ⇒ the file's noted reason if it wrote one
|
|
99
|
+
* BEFORE skipping (`ctx.skip()` throws), else `blocked` + the marker.
|
|
98
100
|
*/
|
|
99
101
|
export function resolveFileRecord(
|
|
100
102
|
states: readonly FileTestState[],
|
|
@@ -111,6 +113,18 @@ export function resolveFileRecord(
|
|
|
111
113
|
disposition = 'blocked';
|
|
112
114
|
detail = UNCLASSIFIED_RETURN_DETAIL;
|
|
113
115
|
}
|
|
116
|
+
} else if (
|
|
117
|
+
noted !== null &&
|
|
118
|
+
gateReason === undefined &&
|
|
119
|
+
states.length > 0 &&
|
|
120
|
+
states.every((s) => s === 'skip')
|
|
121
|
+
) {
|
|
122
|
+
// Every test called `ctx.skip()` (vitest reports them as skipped, not as
|
|
123
|
+
// zero-assertion passes) and the file noted why first. Note-then-skip is
|
|
124
|
+
// the required order: `ctx.skip()` throws, so a note written after it is
|
|
125
|
+
// dead code — which is how seven files carried notes the ledger never saw.
|
|
126
|
+
disposition = noted.kind;
|
|
127
|
+
detail = noted.reason;
|
|
114
128
|
}
|
|
115
129
|
return detail === undefined ? { disposition } : { disposition, detail };
|
|
116
130
|
}
|
|
@@ -51,6 +51,18 @@ describe('RFC 0148 §A — conformance-execution-witness: the runner record', ()
|
|
|
51
51
|
expect(resolveFileRecord(['pass'], undefined, 0, { kind: 'skipped', reason: 'operator opt-out' })).toEqual({ disposition: 'skipped', detail: 'operator opt-out' });
|
|
52
52
|
});
|
|
53
53
|
|
|
54
|
+
it('every test ctx.skip()ped takes the noted reason when one was written before the skip, else stays the blocked marker', () => {
|
|
55
|
+
// `ctx.skip()` throws — a `softSkip(...)` AFTER it is dead code. Seven files
|
|
56
|
+
// carried exactly that dead note and reported as unclassified for a suite minor.
|
|
57
|
+
expect(resolveFileRecord(['skip', 'skip'], undefined, 0, { kind: 'inapplicable', reason: 'sandbox not advertised' })).toEqual({ disposition: 'inapplicable', detail: 'sandbox not advertised' });
|
|
58
|
+
expect(resolveFileRecord(['skip'], undefined, 0, { kind: 'blocked', reason: 'simulator seam 404' })).toEqual({ disposition: 'blocked', detail: 'simulator seam 404' });
|
|
59
|
+
const bare = resolveFileRecord(['skip', 'skip'], undefined, 0, null);
|
|
60
|
+
expect(bare.disposition).toBe('blocked');
|
|
61
|
+
expect(bare.detail).toMatch(/every test skipped with no recorded reason/);
|
|
62
|
+
// A behaviorGate reason still wins over a note — the gate is the more specific record.
|
|
63
|
+
expect(resolveFileRecord(['skip'], 'inapplicable', 0, { kind: 'blocked', reason: 'x' }).disposition).toBe('inapplicable');
|
|
64
|
+
});
|
|
65
|
+
|
|
54
66
|
it('a behaviorGate reason resolves a zero-assertion pass to inapplicable/skipped; a note outranks nothing but never a witnessed pass', () => {
|
|
55
67
|
expect(resolveFileRecord(['pass'], 'inapplicable', 0, null).disposition).toBe('inapplicable');
|
|
56
68
|
expect(resolveFileRecord(['pass'], 'skipped', 0, null).disposition).toBe('skipped');
|
|
@@ -29,7 +29,8 @@
|
|
|
29
29
|
*/
|
|
30
30
|
|
|
31
31
|
import { describe, it, expect } from 'vitest';
|
|
32
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
32
|
+
import { softSkip, seamAbsent } from '../lib/soft-skip.js';
|
|
33
|
+
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
33
34
|
import { driver } from '../lib/driver.js';
|
|
34
35
|
|
|
35
36
|
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
@@ -69,12 +70,29 @@ async function resetLog(): Promise<number> {
|
|
|
69
70
|
return res.status;
|
|
70
71
|
}
|
|
71
72
|
|
|
73
|
+
/**
|
|
74
|
+
* The cross-engine harness seam answered 404. Whether that is `blocked` (the host
|
|
75
|
+
* advertises `eventLog.crossEngineOrdering.supported: true` and so made a claim
|
|
76
|
+
* this file cannot check) or `inapplicable` (no such advertisement) depends on
|
|
77
|
+
* discovery — record the honest one (RFC 0148 §A).
|
|
78
|
+
*/
|
|
79
|
+
async function noteHarnessAbsent(): Promise<void> {
|
|
80
|
+
const disco = await driver.get('/.well-known/openwop');
|
|
81
|
+
const el = capabilityFamily<{ crossEngineOrdering?: { supported?: unknown } }>(disco.json, 'eventLog');
|
|
82
|
+
if (el?.crossEngineOrdering?.supported === true) {
|
|
83
|
+
seamAbsent('host advertises `eventLog.crossEngineOrdering.supported: true` but `POST /v1/host/sample/test/cross-engine/reset` answered 404 — the ordering claim is unobservable (host-sample-test-seams.md)');
|
|
84
|
+
} else {
|
|
85
|
+
softSkip('inapplicable', 'optional advertisement — `eventLog.crossEngineOrdering` not advertised by this host, and the cross-engine harness seam is absent (RFC 0036 §B)');
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
72
89
|
describe.skipIf(HTTP_SKIP)('cross-engine-append-behavior: §B cross-engine ordering (RFC 0036)', () => {
|
|
73
90
|
it('interleaved appends from two engines converge to a single globally-ordered sequence', async (ctx) => {
|
|
74
91
|
const resetStatus = await resetLog();
|
|
75
92
|
if (resetStatus === 404) {
|
|
93
|
+
await noteHarnessAbsent();
|
|
76
94
|
ctx.skip(); // host doesn't expose the cross-engine harness seam
|
|
77
|
-
return
|
|
95
|
+
return;
|
|
78
96
|
}
|
|
79
97
|
expect(resetStatus).toBe(200);
|
|
80
98
|
|
|
@@ -125,8 +143,9 @@ describe.skipIf(HTTP_SKIP)('cross-engine-append-behavior: §B cross-engine order
|
|
|
125
143
|
it('lamport clocks monotonically advance across engines', async (ctx) => {
|
|
126
144
|
const resetStatus = await resetLog();
|
|
127
145
|
if (resetStatus === 404) {
|
|
146
|
+
await noteHarnessAbsent();
|
|
128
147
|
ctx.skip();
|
|
129
|
-
return
|
|
148
|
+
return;
|
|
130
149
|
}
|
|
131
150
|
expect(resetStatus).toBe(200);
|
|
132
151
|
|
|
@@ -153,8 +172,9 @@ describe.skipIf(HTTP_SKIP)('cross-engine-append-behavior: §B cross-engine order
|
|
|
153
172
|
it('lamport hint from engine A advances engine B past it', async (ctx) => {
|
|
154
173
|
const resetStatus = await resetLog();
|
|
155
174
|
if (resetStatus === 404) {
|
|
175
|
+
await noteHarnessAbsent();
|
|
156
176
|
ctx.skip();
|
|
157
|
-
return
|
|
177
|
+
return;
|
|
158
178
|
}
|
|
159
179
|
expect(resetStatus).toBe(200);
|
|
160
180
|
|
|
@@ -180,8 +200,9 @@ describe.skipIf(HTTP_SKIP)('cross-engine-append-behavior: §B cross-engine order
|
|
|
180
200
|
it('linearization is deterministic — same appends → same total order', async (ctx) => {
|
|
181
201
|
const resetStatus = await resetLog();
|
|
182
202
|
if (resetStatus === 404) {
|
|
203
|
+
await noteHarnessAbsent();
|
|
183
204
|
ctx.skip();
|
|
184
|
-
return
|
|
205
|
+
return;
|
|
185
206
|
}
|
|
186
207
|
expect(resetStatus).toBe(200);
|
|
187
208
|
|
|
@@ -69,8 +69,9 @@ describe.skipIf(HTTP_SKIP)('cross-host-ancestry-endpoint: behavioral (RFC 0040
|
|
|
69
69
|
const d = await readDiscovery();
|
|
70
70
|
const chc = capabilityFamily<{ executionModel?: { [k: string]: unknown; crossHostCausation?: Record<string, unknown>; replayDeterminism?: Record<string, unknown> } }>(d, 'multiAgent')?.executionModel?.crossHostCausation;
|
|
71
71
|
if (chc?.ancestryEndpointSupported !== true) {
|
|
72
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `chc?.ancestryEndpointSupported !== true` returned early');
|
|
72
73
|
ctx.skip();
|
|
73
|
-
return
|
|
74
|
+
return;
|
|
74
75
|
}
|
|
75
76
|
|
|
76
77
|
// Create a fresh top-level run via the host's conformance-dispatch-loop
|
|
@@ -78,8 +79,9 @@ describe.skipIf(HTTP_SKIP)('cross-host-ancestry-endpoint: behavioral (RFC 0040
|
|
|
78
79
|
// depend on the specific workflow).
|
|
79
80
|
const create = await driver.post('/v1/runs', { workflowId: 'conformance-dispatch-loop' });
|
|
80
81
|
if (create.status !== 201) {
|
|
82
|
+
softSkip('blocked', 'precondition not met — `create.status !== 201` returned early (seam, prior step, or fixture unavailable)');
|
|
81
83
|
ctx.skip();
|
|
82
|
-
return
|
|
84
|
+
return;
|
|
83
85
|
}
|
|
84
86
|
const runId = (create.json as { runId: string }).runId;
|
|
85
87
|
|
|
@@ -116,12 +118,14 @@ describe.skipIf(HTTP_SKIP)('cross-host-ancestry-endpoint: behavioral (RFC 0040
|
|
|
116
118
|
const d = await readDiscovery();
|
|
117
119
|
const chc = capabilityFamily<{ executionModel?: { [k: string]: unknown; crossHostCausation?: Record<string, unknown>; replayDeterminism?: Record<string, unknown> } }>(d, 'multiAgent')?.executionModel?.crossHostCausation;
|
|
118
120
|
if (chc?.supported !== true) {
|
|
121
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `chc?.supported !== true` returned early');
|
|
119
122
|
ctx.skip();
|
|
120
|
-
return
|
|
123
|
+
return;
|
|
121
124
|
}
|
|
122
125
|
if (chc.ancestryEndpointSupported === true) {
|
|
126
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `chc.ancestryEndpointSupported === true` returned early');
|
|
123
127
|
ctx.skip(); // covered by the test above
|
|
124
|
-
return
|
|
128
|
+
return;
|
|
125
129
|
}
|
|
126
130
|
|
|
127
131
|
// Use any runId — even a synthetic non-existent one. The endpoint should
|
|
@@ -62,13 +62,15 @@ describe.skipIf(HTTP_SKIP)('cross-host-causation-shape: advertisement shape (RFC
|
|
|
62
62
|
it('crossHostCausation (when present) conforms to RFC 0040 §D', async (ctx) => {
|
|
63
63
|
const d = await readDiscovery();
|
|
64
64
|
if (d === null) {
|
|
65
|
+
softSkip('blocked', 'precondition not met — `d === null` returned early (seam, prior step, or fixture unavailable)');
|
|
65
66
|
ctx.skip();
|
|
66
|
-
return
|
|
67
|
+
return;
|
|
67
68
|
}
|
|
68
69
|
const chc = capabilityFamily<{ executionModel?: { [k: string]: unknown; crossHostCausation?: Record<string, unknown>; replayDeterminism?: Record<string, unknown> } }>(d, 'multiAgent')?.executionModel?.crossHostCausation;
|
|
69
70
|
if (chc === undefined) {
|
|
71
|
+
softSkip('inapplicable', 'optional advertisement — `multiAgent.executionModel.crossHostCausation` not advertised by this host (RFC 0040 §D)');
|
|
70
72
|
ctx.skip(); // host doesn't advertise — soft-skip
|
|
71
|
-
return
|
|
73
|
+
return;
|
|
72
74
|
}
|
|
73
75
|
|
|
74
76
|
expect(
|
|
@@ -80,13 +80,15 @@ describe.skipIf(HTTP_SKIP)('multi-agent-memory-lifecycle: advertisement shape (R
|
|
|
80
80
|
it('crossChildMemoryConcurrency (when advertised) MUST be one of {strict, advisory}', async (ctx) => {
|
|
81
81
|
const d = await readDiscovery();
|
|
82
82
|
if (d === null) {
|
|
83
|
+
softSkip('blocked', 'precondition not met — `d === null` returned early (seam, prior step, or fixture unavailable)');
|
|
83
84
|
ctx.skip();
|
|
84
|
-
return
|
|
85
|
+
return;
|
|
85
86
|
}
|
|
86
87
|
const ccmc = capabilityFamily<{ executionModel?: { [k: string]: unknown; crossHostCausation?: Record<string, unknown>; replayDeterminism?: Record<string, unknown> } }>(d, 'multiAgent')?.executionModel?.crossChildMemoryConcurrency;
|
|
87
88
|
if (ccmc === undefined) {
|
|
89
|
+
softSkip('inapplicable', 'optional advertisement — `multiAgent.executionModel.crossChildMemoryConcurrency` not advertised by this host');
|
|
88
90
|
ctx.skip(); // optional advertisement — host hasn't opted in
|
|
89
|
-
return
|
|
91
|
+
return;
|
|
90
92
|
}
|
|
91
93
|
expect(
|
|
92
94
|
ccmc === 'strict' || ccmc === 'advisory',
|
|
@@ -134,16 +136,18 @@ describe.skipIf(HTTP_SKIP)('multi-agent-memory-lifecycle: behavioral (RFC 0039
|
|
|
134
136
|
it('MAE-3 replay snapshot refusal: fork mode:replay against a past-retention runId MUST return 422 replay_memory_snapshot_unavailable with documented envelope; silent substitution is non-conformant', async (ctx) => {
|
|
135
137
|
const d = await readDiscovery();
|
|
136
138
|
if (d === null) {
|
|
139
|
+
softSkip('blocked', 'precondition not met — `d === null` returned early (seam, prior step, or fixture unavailable)');
|
|
137
140
|
ctx.skip();
|
|
138
|
-
return
|
|
141
|
+
return;
|
|
139
142
|
}
|
|
140
143
|
const v = capabilityFamily<{ executionModel?: { [k: string]: unknown; crossHostCausation?: Record<string, unknown>; replayDeterminism?: Record<string, unknown> } }>(d, 'multiAgent')?.executionModel?.version;
|
|
141
144
|
const memorySupported = capabilityFamily<{ supported?: unknown }>(d, 'memory')?.supported;
|
|
142
145
|
const phase2OrLater = typeof v === 'number' && v >= 2;
|
|
143
146
|
const expiredRunId = process.env.OPENWOP_TEST_EXPIRED_REPLAY_RUN_ID;
|
|
144
147
|
if (!phase2OrLater || memorySupported !== true || !expiredRunId) {
|
|
148
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!phase2OrLater || memorySupported !== true || !expiredRunId` returned early');
|
|
145
149
|
ctx.skip();
|
|
146
|
-
return
|
|
150
|
+
return;
|
|
147
151
|
}
|
|
148
152
|
|
|
149
153
|
const fromSeq = 0;
|
|
@@ -31,7 +31,8 @@
|
|
|
31
31
|
*/
|
|
32
32
|
|
|
33
33
|
import { describe, it, expect } from 'vitest';
|
|
34
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
34
|
+
import { softSkip, seamAbsent } from '../lib/soft-skip.js';
|
|
35
|
+
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
35
36
|
import { driver } from '../lib/driver.js';
|
|
36
37
|
|
|
37
38
|
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
@@ -56,6 +57,21 @@ async function simulatePartition(claims: ConflictClaim[]): Promise<{ status: num
|
|
|
56
57
|
return { status: res.status, body: (res.json as ConvergenceResult) ?? {} };
|
|
57
58
|
}
|
|
58
59
|
|
|
60
|
+
/**
|
|
61
|
+
* The multi-region simulator seam answered 404. `blocked` when the host advertises
|
|
62
|
+
* a cross-region posture (`idempotency.crossRegion` present) it has made
|
|
63
|
+
* unobservable, `inapplicable` when it advertises none (RFC 0148 §A).
|
|
64
|
+
*/
|
|
65
|
+
async function noteSimulatorAbsent(): Promise<void> {
|
|
66
|
+
const disco = await driver.get('/.well-known/openwop');
|
|
67
|
+
const idem = capabilityFamily<{ crossRegion?: unknown }>(disco.json, 'idempotency');
|
|
68
|
+
if (idem?.crossRegion !== undefined) {
|
|
69
|
+
seamAbsent(`host advertises \`idempotency.crossRegion: ${String(idem.crossRegion)}\` but \`POST /v1/host/sample/test/multi-region/simulate-partition\` answered 404 — the convergence rule is unobservable (host-sample-test-seams.md §6)`);
|
|
70
|
+
} else {
|
|
71
|
+
softSkip('inapplicable', 'optional advertisement — `idempotency.crossRegion` not advertised by this host, and the multi-region simulator seam is absent (RFC 0036 §C)');
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
59
75
|
describe.skipIf(HTTP_SKIP)('multi-region-idempotency-behavior: convergence rule (RFC 0036 §C)', () => {
|
|
60
76
|
it('two-region conflict resolves to the lex-min runId per annex §"Convergence rule"', async (ctx) => {
|
|
61
77
|
const probe = await simulatePartition([
|
|
@@ -63,8 +79,9 @@ describe.skipIf(HTTP_SKIP)('multi-region-idempotency-behavior: convergence rule
|
|
|
63
79
|
{ runId: 'run-a-west', tenantId: 't1', endpoint: 'POST /v1/runs', key: 'idem-1', region: 'eu-west-1' },
|
|
64
80
|
]);
|
|
65
81
|
if (probe.status === 404) {
|
|
82
|
+
await noteSimulatorAbsent();
|
|
66
83
|
ctx.skip(); // host doesn't expose the simulator seam
|
|
67
|
-
return
|
|
84
|
+
return;
|
|
68
85
|
}
|
|
69
86
|
expect(
|
|
70
87
|
probe.status,
|
|
@@ -89,8 +106,9 @@ describe.skipIf(HTTP_SKIP)('multi-region-idempotency-behavior: convergence rule
|
|
|
89
106
|
{ runId: 'mmm-2', tenantId: 't1', endpoint: 'POST /v1/runs', key: 'idem-2', region: 'r3' },
|
|
90
107
|
]);
|
|
91
108
|
if (probe.status === 404) {
|
|
109
|
+
await noteSimulatorAbsent();
|
|
92
110
|
ctx.skip();
|
|
93
|
-
return
|
|
111
|
+
return;
|
|
94
112
|
}
|
|
95
113
|
expect(probe.status).toBe(200);
|
|
96
114
|
expect(
|
|
@@ -115,8 +133,9 @@ describe.skipIf(HTTP_SKIP)('multi-region-idempotency-behavior: convergence rule
|
|
|
115
133
|
{ runId: 'run-a', tenantId: 't1', endpoint: 'POST /v1/runs', key: 'idem-3', region: 'r2' },
|
|
116
134
|
]);
|
|
117
135
|
if (probe.status === 404) {
|
|
136
|
+
await noteSimulatorAbsent();
|
|
118
137
|
ctx.skip();
|
|
119
|
-
return
|
|
138
|
+
return;
|
|
120
139
|
}
|
|
121
140
|
expect(probe.status).toBe(200);
|
|
122
141
|
const redirects = probe.body.cacheRedirects ?? [];
|
|
@@ -144,8 +163,9 @@ describe.skipIf(HTTP_SKIP)('multi-region-idempotency-behavior: convergence rule
|
|
|
144
163
|
{ runId: 'run-a', tenantId: 't1', endpoint: 'POST /v1/runs', key: 'idem-4', region: 'r2' },
|
|
145
164
|
]);
|
|
146
165
|
if (probe.status === 404) {
|
|
166
|
+
await noteSimulatorAbsent();
|
|
147
167
|
ctx.skip();
|
|
148
|
-
return
|
|
168
|
+
return;
|
|
149
169
|
}
|
|
150
170
|
expect(probe.status).toBe(200);
|
|
151
171
|
expect(
|
|
@@ -165,8 +185,9 @@ describe.skipIf(HTTP_SKIP)('multi-region-idempotency-behavior: convergence rule
|
|
|
165
185
|
];
|
|
166
186
|
const p1 = await simulatePartition(claims);
|
|
167
187
|
if (p1.status === 404) {
|
|
188
|
+
await noteSimulatorAbsent();
|
|
168
189
|
ctx.skip();
|
|
169
|
-
return
|
|
190
|
+
return;
|
|
170
191
|
}
|
|
171
192
|
expect(p1.status).toBe(200);
|
|
172
193
|
const p2 = await simulatePartition([claims[2]!, claims[0]!, claims[1]!]);
|
|
@@ -190,8 +211,9 @@ describe.skipIf(HTTP_SKIP)('multi-region-idempotency-behavior: convergence rule
|
|
|
190
211
|
{ runId: 'r2', tenantId: 't2', endpoint: 'POST /v1/runs', key: 'idem-6', region: 'r2' },
|
|
191
212
|
]);
|
|
192
213
|
if (probe.status === 404) {
|
|
214
|
+
await noteSimulatorAbsent();
|
|
193
215
|
ctx.skip();
|
|
194
|
-
return
|
|
216
|
+
return;
|
|
195
217
|
}
|
|
196
218
|
expect(
|
|
197
219
|
probe.status,
|
|
@@ -78,13 +78,15 @@ describe.skipIf(HTTP_SKIP)('replay-divergence-at-refusal: advertisement shape (R
|
|
|
78
78
|
it('replayDeterminism (when present) conforms to RFC 0041 §D', async (ctx) => {
|
|
79
79
|
const d = await readDiscovery();
|
|
80
80
|
if (d === null) {
|
|
81
|
+
softSkip('blocked', 'precondition not met — `d === null` returned early (seam, prior step, or fixture unavailable)');
|
|
81
82
|
ctx.skip();
|
|
82
|
-
return
|
|
83
|
+
return;
|
|
83
84
|
}
|
|
84
85
|
const rd = capabilityFamily<{ executionModel?: { [k: string]: unknown; crossHostCausation?: Record<string, unknown>; replayDeterminism?: Record<string, unknown> } }>(d, 'multiAgent')?.executionModel?.replayDeterminism;
|
|
85
86
|
if (rd === undefined) {
|
|
87
|
+
softSkip('inapplicable', 'optional advertisement — `multiAgent.executionModel.replayDeterminism` not advertised by this host (RFC 0041 §D)');
|
|
86
88
|
ctx.skip(); // optional advertisement — host hasn't opted in
|
|
87
|
-
return
|
|
89
|
+
return;
|
|
88
90
|
}
|
|
89
91
|
|
|
90
92
|
expect(
|
|
@@ -192,8 +194,9 @@ describe.skipIf(HTTP_SKIP)('replay-divergence-at-refusal: behavioral (RFC 0041
|
|
|
192
194
|
{ content: validEnv, stopReason: 'end_turn' as const },
|
|
193
195
|
]);
|
|
194
196
|
if (programStatus === 404) {
|
|
197
|
+
softSkip('blocked', 'precondition not met — `programStatus === 404` returned early (seam, prior step, or fixture unavailable)');
|
|
195
198
|
ctx.skip(); // mock-AI program seam not exposed — soft-skip
|
|
196
|
-
return
|
|
199
|
+
return;
|
|
197
200
|
}
|
|
198
201
|
expect(programStatus).toBe(200);
|
|
199
202
|
|
|
@@ -201,8 +204,9 @@ describe.skipIf(HTTP_SKIP)('replay-divergence-at-refusal: behavioral (RFC 0041
|
|
|
201
204
|
workflowId: 'conformance-phase4-replay-divergence',
|
|
202
205
|
});
|
|
203
206
|
if (createRes.status === 404 || createRes.status === 422) {
|
|
207
|
+
softSkip('blocked', 'precondition not met — `createRes.status === 404 || createRes.status === 422` returned early (seam, prior step, or fixture unavailable)');
|
|
204
208
|
ctx.skip(); // fixture not advertised
|
|
205
|
-
return
|
|
209
|
+
return;
|
|
206
210
|
}
|
|
207
211
|
expect(createRes.status).toBe(201);
|
|
208
212
|
const sourceRunId = (createRes.json as { runId: string }).runId;
|
|
@@ -273,8 +277,9 @@ describe.skipIf(HTTP_SKIP)('replay-divergence-at-refusal: behavioral (RFC 0041
|
|
|
273
277
|
{ content: 'safety-refused-for-conformance', stopReason: 'safety' as const, refusalText: 'safety-refused-for-conformance' },
|
|
274
278
|
]);
|
|
275
279
|
if (programStatus === 404) {
|
|
280
|
+
softSkip('blocked', 'precondition not met — `programStatus === 404` returned early (seam, prior step, or fixture unavailable)');
|
|
276
281
|
ctx.skip();
|
|
277
|
-
return
|
|
282
|
+
return;
|
|
278
283
|
}
|
|
279
284
|
expect(programStatus).toBe(200);
|
|
280
285
|
|
|
@@ -282,8 +287,9 @@ describe.skipIf(HTTP_SKIP)('replay-divergence-at-refusal: behavioral (RFC 0041
|
|
|
282
287
|
workflowId: 'conformance-phase4-replay-divergence',
|
|
283
288
|
});
|
|
284
289
|
if (createRes.status === 404 || createRes.status === 422) {
|
|
290
|
+
softSkip('blocked', 'precondition not met — `createRes.status === 404 || createRes.status === 422` returned early (seam, prior step, or fixture unavailable)');
|
|
285
291
|
ctx.skip();
|
|
286
|
-
return
|
|
292
|
+
return;
|
|
287
293
|
}
|
|
288
294
|
expect(createRes.status).toBe(201);
|
|
289
295
|
const sourceRunId = (createRes.json as { runId: string }).runId;
|
|
@@ -41,7 +41,7 @@
|
|
|
41
41
|
*/
|
|
42
42
|
|
|
43
43
|
import { describe, it, expect } from 'vitest';
|
|
44
|
-
import { softSkip } from '../lib/soft-skip.js';
|
|
44
|
+
import { softSkip, seamAbsent } from '../lib/soft-skip.js';
|
|
45
45
|
import { driver } from '../lib/driver.js';
|
|
46
46
|
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
47
47
|
|
|
@@ -85,6 +85,7 @@ async function gateOnPhase4(ctx: { skip: () => void }): Promise<boolean> {
|
|
|
85
85
|
const em = capabilityFamily<{ executionModel?: ExecutionModelCaps }>(d, 'multiAgent')?.executionModel;
|
|
86
86
|
const version = typeof em?.version === 'number' ? em.version : 0;
|
|
87
87
|
if (em?.replayDeterminism?.supported !== true || version < 4) {
|
|
88
|
+
softSkip('inapplicable', 'optional advertisement — `multiAgent.executionModel.replayDeterminism.supported` with `version >= 4` not advertised by this host (RFC 0041 §C)');
|
|
88
89
|
ctx.skip();
|
|
89
90
|
return false;
|
|
90
91
|
}
|
|
@@ -152,7 +153,8 @@ function stripVolatile(ev: RunEventDoc): unknown {
|
|
|
152
153
|
async function startFixtureRun(ctx: { skip: () => void }): Promise<string | null> {
|
|
153
154
|
const create = await driver.post('/v1/runs', { workflowId: FIXTURE });
|
|
154
155
|
if (create.status === 404 || create.status === 422) {
|
|
155
|
-
|
|
156
|
+
seamAbsent(`fixture \`${FIXTURE}\` not registered on this host (POST /v1/runs → ${create.status}) while replayDeterminism is advertised`);
|
|
157
|
+
ctx.skip();
|
|
156
158
|
return null;
|
|
157
159
|
}
|
|
158
160
|
expect(create.status).toBe(201);
|
|
@@ -82,8 +82,9 @@ async function invoke(typeId: string, args: Record<string, unknown> = {}, allowe
|
|
|
82
82
|
describe.skipIf(HTTP_SKIP)('sandbox-mvp-behavior: RFC 0035 §B failure-mode invariants (node:vm MVP)', () => {
|
|
83
83
|
it('host-fs-escape — fs access from sandboxed code fails closed', async (ctx) => {
|
|
84
84
|
if (!(await isSandboxAdvertised())) {
|
|
85
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!(await isSandboxAdvertised())` returned early');
|
|
85
86
|
ctx.skip();
|
|
86
|
-
return
|
|
87
|
+
return;
|
|
87
88
|
}
|
|
88
89
|
const probe = await invoke('misbehave.fs-escape-read');
|
|
89
90
|
expect(probe.status).toBe(200);
|
|
@@ -105,8 +106,9 @@ describe.skipIf(HTTP_SKIP)('sandbox-mvp-behavior: RFC 0035 §B failure-mode inva
|
|
|
105
106
|
|
|
106
107
|
it('host-env-leak — process.env access from sandboxed code fails closed', async (ctx) => {
|
|
107
108
|
if (!(await isSandboxAdvertised())) {
|
|
109
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!(await isSandboxAdvertised())` returned early');
|
|
108
110
|
ctx.skip();
|
|
109
|
-
return
|
|
111
|
+
return;
|
|
110
112
|
}
|
|
111
113
|
const probe = await invoke('misbehave.env-leak');
|
|
112
114
|
expect(probe.status).toBe(200);
|
|
@@ -122,8 +124,9 @@ describe.skipIf(HTTP_SKIP)('sandbox-mvp-behavior: RFC 0035 §B failure-mode inva
|
|
|
122
124
|
|
|
123
125
|
it('network-escape — http/net access from sandboxed code fails closed', async (ctx) => {
|
|
124
126
|
if (!(await isSandboxAdvertised())) {
|
|
127
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!(await isSandboxAdvertised())` returned early');
|
|
125
128
|
ctx.skip();
|
|
126
|
-
return
|
|
129
|
+
return;
|
|
127
130
|
}
|
|
128
131
|
const probe = await invoke('misbehave.network-escape');
|
|
129
132
|
expect(probe.status).toBe(200);
|
|
@@ -139,8 +142,9 @@ describe.skipIf(HTTP_SKIP)('sandbox-mvp-behavior: RFC 0035 §B failure-mode inva
|
|
|
139
142
|
|
|
140
143
|
it('host-process-escape — child_process access from sandboxed code fails closed', async (ctx) => {
|
|
141
144
|
if (!(await isSandboxAdvertised())) {
|
|
145
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!(await isSandboxAdvertised())` returned early');
|
|
142
146
|
ctx.skip();
|
|
143
|
-
return
|
|
147
|
+
return;
|
|
144
148
|
}
|
|
145
149
|
const probe = await invoke('misbehave.process-escape');
|
|
146
150
|
expect(probe.status).toBe(200);
|
|
@@ -153,8 +157,9 @@ describe.skipIf(HTTP_SKIP)('sandbox-mvp-behavior: RFC 0035 §B failure-mode inva
|
|
|
153
157
|
|
|
154
158
|
it('sandbox-timeout — runaway loop terminated by wallClockLimitMs', async (ctx) => {
|
|
155
159
|
if (!(await isSandboxAdvertised())) {
|
|
160
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!(await isSandboxAdvertised())` returned early');
|
|
156
161
|
ctx.skip();
|
|
157
|
-
return
|
|
162
|
+
return;
|
|
158
163
|
}
|
|
159
164
|
const start = Date.now();
|
|
160
165
|
const probe = await invoke('misbehave.timeout');
|
|
@@ -179,8 +184,9 @@ describe.skipIf(HTTP_SKIP)('sandbox-mvp-behavior: RFC 0035 §B failure-mode inva
|
|
|
179
184
|
|
|
180
185
|
it('cross-pack-mutation — fresh vm context per invocation, no state leaks', async (ctx) => {
|
|
181
186
|
if (!(await isSandboxAdvertised())) {
|
|
187
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!(await isSandboxAdvertised())` returned early');
|
|
182
188
|
ctx.skip();
|
|
183
|
-
return
|
|
189
|
+
return;
|
|
184
190
|
}
|
|
185
191
|
const r1 = await invoke('misbehave.cross-pack-mutate');
|
|
186
192
|
const r2 = await invoke('misbehave.cross-pack-mutate');
|
|
@@ -205,8 +211,9 @@ describe.skipIf(HTTP_SKIP)('sandbox-mvp-behavior: RFC 0035 §B failure-mode inva
|
|
|
205
211
|
|
|
206
212
|
it('capability-gate-respected — host call NOT in allowedHostCalls fails with sandbox_capability_denied', async (ctx) => {
|
|
207
213
|
if (!(await isSandboxAdvertised())) {
|
|
214
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!(await isSandboxAdvertised())` returned early');
|
|
208
215
|
ctx.skip();
|
|
209
|
-
return
|
|
216
|
+
return;
|
|
210
217
|
}
|
|
211
218
|
const probe = await invoke('misbehave.capability-gate-violation', {}, []);
|
|
212
219
|
expect(probe.status).toBe(200);
|
|
@@ -228,8 +235,9 @@ describe.skipIf(HTTP_SKIP)('sandbox-mvp-behavior: RFC 0035 §B failure-mode inva
|
|
|
228
235
|
|
|
229
236
|
it('memory-exceeded — runaway allocation fails with sandbox_memory_exceeded', async (ctx) => {
|
|
230
237
|
if (!(await isSandboxAdvertised())) {
|
|
238
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!(await isSandboxAdvertised())` returned early');
|
|
231
239
|
ctx.skip();
|
|
232
|
-
return
|
|
240
|
+
return;
|
|
233
241
|
}
|
|
234
242
|
const probe = await invoke('misbehave.memory-bomb');
|
|
235
243
|
expect(probe.status).toBe(200);
|
|
@@ -255,8 +263,9 @@ describe.skipIf(HTTP_SKIP)('sandbox-mvp-behavior: RFC 0035 §B failure-mode inva
|
|
|
255
263
|
|
|
256
264
|
it('well-behaved.host-fetch — allowedHostCalls=[fetch] permits the host call', async (ctx) => {
|
|
257
265
|
if (!(await isSandboxAdvertised())) {
|
|
266
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!(await isSandboxAdvertised())` returned early');
|
|
258
267
|
ctx.skip();
|
|
259
|
-
return
|
|
268
|
+
return;
|
|
260
269
|
}
|
|
261
270
|
const probe = await invoke('well-behaved.host-fetch', {}, ['fetch']);
|
|
262
271
|
expect(probe.status).toBe(200);
|
|
@@ -271,8 +280,9 @@ describe.skipIf(HTTP_SKIP)('sandbox-mvp-behavior: RFC 0035 §B failure-mode inva
|
|
|
271
280
|
|
|
272
281
|
it('well-behaved.echo — sandboxed code returns args round-trip when no escape attempt', async (ctx) => {
|
|
273
282
|
if (!(await isSandboxAdvertised())) {
|
|
283
|
+
softSkip('inapplicable', 'capability or profile not advertised by this host — gate `!(await isSandboxAdvertised())` returned early');
|
|
274
284
|
ctx.skip();
|
|
275
|
-
return
|
|
285
|
+
return;
|
|
276
286
|
}
|
|
277
287
|
const probe = await invoke('well-behaved.echo', { input: 'hello-sandbox' });
|
|
278
288
|
expect(probe.status).toBe(200);
|