@openwop/openwop-conformance 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +2 -2
- package/api/asyncapi.yaml +8 -3
- package/api/openapi.yaml +305 -0
- package/coverage.md +29 -4
- package/fixtures/conformance-phase4-nondet-tool.json +53 -0
- package/fixtures/conformance-phase4-replay-divergence.json +40 -0
- package/fixtures.md +5 -3
- package/package.json +1 -1
- package/schemas/README.md +2 -0
- package/schemas/capabilities.schema.json +167 -3
- package/schemas/credential-reference.schema.json +21 -0
- package/schemas/node-pack-manifest.schema.json +112 -1
- package/schemas/run-diff-response.schema.json +64 -0
- package/schemas/run-event-payloads.schema.json +104 -2
- package/schemas/run-event.schema.json +8 -1
- package/schemas/run-snapshot.schema.json +11 -0
- package/src/lib/behavior-gate.ts +51 -0
- package/src/lib/driver.ts +13 -1
- package/src/lib/saml-idp.ts +179 -0
- package/src/scenarios/approval-gate-events.test.ts +61 -0
- package/src/scenarios/approval-gate-flow.test.ts +68 -0
- package/src/scenarios/auth-saml-profile.test.ts +119 -0
- package/src/scenarios/auth-scim-profile.test.ts +65 -0
- package/src/scenarios/authorization-fail-closed.test.ts +80 -0
- package/src/scenarios/authorization-roles-shape.test.ts +83 -0
- package/src/scenarios/connector-manifest-validity.test.ts +142 -0
- package/src/scenarios/credential-payload-redaction.test.ts +93 -0
- package/src/scenarios/credentials-capability-shape.test.ts +90 -0
- package/src/scenarios/cross-engine-append-behavior.test.ts +204 -0
- package/src/scenarios/cross-host-traceparent-propagation.test.ts +13 -6
- package/src/scenarios/cross-workspace-isolation.test.ts +72 -0
- package/src/scenarios/deadletter-capability-shape.test.ts +59 -0
- package/src/scenarios/deadletter-retry-exhaustion.test.ts +62 -0
- package/src/scenarios/experimental-tier-shape.test.ts +192 -0
- package/src/scenarios/identity-owner-shape.test.ts +64 -0
- package/src/scenarios/multi-agent-confidence-escalation.test.ts +13 -12
- package/src/scenarios/multi-agent-memory-lifecycle.test.ts +87 -12
- package/src/scenarios/multi-region-idempotency-behavior.test.ts +203 -0
- package/src/scenarios/oauth-capability-shape.test.ts +97 -0
- package/src/scenarios/oauth-connector-redaction.test.ts +91 -0
- package/src/scenarios/pack-registry-isolation.test.ts +108 -0
- package/src/scenarios/pack-registry-publish.test.ts +1 -1
- package/src/scenarios/prompt-mutation-workspace-membership-enforced.test.ts +126 -0
- package/src/scenarios/prompt-read-workspace-membership-enforced.test.ts +183 -0
- package/src/scenarios/replay-divergence-at-refusal.test.ts +187 -7
- package/src/scenarios/replay-observable-sequence-determinism.test.ts +20 -6
- package/src/scenarios/run-diff.test.ts +143 -0
- package/src/scenarios/sandbox-capability-gate-respected.test.ts +7 -1
- package/src/scenarios/sandbox-memory-cap.test.ts +7 -5
- package/src/scenarios/sandbox-mvp-behavior.test.ts +280 -0
- package/src/scenarios/sandbox-no-cross-pack-mutation.test.ts +7 -1
- package/src/scenarios/sandbox-no-host-env-leak.test.ts +5 -1
- package/src/scenarios/sandbox-no-host-fs-escape.test.ts +9 -1
- package/src/scenarios/sandbox-no-host-process-escape.test.ts +5 -1
- package/src/scenarios/sandbox-no-network-escape.test.ts +5 -1
- package/src/scenarios/sandbox-timeout-cap.test.ts +7 -5
- package/src/scenarios/scheduling-capability-shape.test.ts +81 -0
- package/src/scenarios/scheduling-cron-fires-once.test.ts +66 -0
- package/src/scenarios/secret-leakage-otel-attribute.test.ts +241 -0
- package/src/scenarios/spec-corpus-validity.test.ts +2 -2
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Pack-registry publish scenarios — `node-packs.md` §"PUT /v1/packs/{name}/-/{version}.tgz".
|
|
3
3
|
*
|
|
4
|
-
* Status: BEHAVIORAL (soft-skip). Per RFC 0025 (`
|
|
4
|
+
* Status: BEHAVIORAL (soft-skip). Per RFC 0025 (`Active` 2026-05-19),
|
|
5
5
|
* the conformance suite drives the documented 19-code error catalog
|
|
6
6
|
* via the test-mode mirror namespace `/v1/packs-test/*`, gated on
|
|
7
7
|
* `capabilities.packs.testMode.supported: true`. Each scenario soft-
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* prompt-mutation-workspace-membership-enforced — RFC 0028 Tier-2 §"Workspace
|
|
3
|
+
* membership on workspace-scoped writes" verification.
|
|
4
|
+
*
|
|
5
|
+
* Status: ACTIVE (capability-gated; behavioral when the host advertises
|
|
6
|
+
* `capabilities.prompts.mutableLibrary: true`). Hosts that don't advertise
|
|
7
|
+
* mutableLibrary soft-skip cleanly.
|
|
8
|
+
*
|
|
9
|
+
* The contract (spec/v1/prompts.md §"Discovery & distribution" §"REST
|
|
10
|
+
* endpoints" §"Workspace membership on workspace-scoped writes"):
|
|
11
|
+
*
|
|
12
|
+
* Hosts MUST verify that the authenticated principal is a member of the
|
|
13
|
+
* target workspace BEFORE honoring any POST / PUT / DELETE to a
|
|
14
|
+
* workspace-scoped /v1/prompts* resource. A workspaceId supplied by the
|
|
15
|
+
* caller (request body, URL, or query string) MUST NOT be trusted as
|
|
16
|
+
* authorization on its own. Non-members MUST be rejected fail-closed
|
|
17
|
+
* (typically 403) before any persistence occurs.
|
|
18
|
+
*
|
|
19
|
+
* The probe drives `POST /v1/prompts` with a `workspaceId` the conformance
|
|
20
|
+
* principal cannot be a member of (a cryptographically-unique random value
|
|
21
|
+
* by default; operator-overridable via `OPENWOP_TEST_NONMEMBER_WORKSPACE_ID`
|
|
22
|
+
* for hosts that need a specific synthetic workspace shape). The behavioral
|
|
23
|
+
* MUST is that the host refuses — NOT a 2xx. Any 4xx/5xx is acceptable
|
|
24
|
+
* (401 = auth not configured for this surface; 403 = membership check;
|
|
25
|
+
* 404 = endpoint absent; 422 = body validation; 501 = capability not
|
|
26
|
+
* provided). The failure mode this invariant guards against is a SILENT
|
|
27
|
+
* 2xx with a write to a workspace the caller doesn't belong to — that's the
|
|
28
|
+
* RFC 0028 Tier-2 vulnerability self-disclosed by an adopter on 2026-05-25.
|
|
29
|
+
*
|
|
30
|
+
* Why a random workspaceId is sufficient: a non-member workspace check is
|
|
31
|
+
* negative-space — the host MUST refuse for ANY workspace the principal
|
|
32
|
+
* isn't a member of, and a random UUID has astronomically-low collision
|
|
33
|
+
* probability with any real workspace membership grant.
|
|
34
|
+
*
|
|
35
|
+
* @see RFCS/0028-prompt-library-endpoints.md §"Post-promotion notes"
|
|
36
|
+
* @see spec/v1/prompts.md §"Security invariants" §prompt-mutation-workspace-membership-enforced
|
|
37
|
+
* @see spec/v1/auth.md §"Identity claims — tenant · workspace · principal"
|
|
38
|
+
* @see RFCS/0048-tenant-workspace-principal-identity-model.md §D
|
|
39
|
+
*/
|
|
40
|
+
|
|
41
|
+
import { describe, it, expect } from 'vitest';
|
|
42
|
+
import { randomUUID } from 'node:crypto';
|
|
43
|
+
import { driver } from '../lib/driver.js';
|
|
44
|
+
|
|
45
|
+
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
46
|
+
|
|
47
|
+
interface DiscoveryDoc {
|
|
48
|
+
capabilities?: {
|
|
49
|
+
prompts?: {
|
|
50
|
+
mutableLibrary?: unknown;
|
|
51
|
+
};
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async function readDiscovery(): Promise<DiscoveryDoc | null> {
|
|
56
|
+
try {
|
|
57
|
+
const res = await driver.get('/.well-known/openwop');
|
|
58
|
+
if (res.status !== 200) return null;
|
|
59
|
+
return res.json as DiscoveryDoc;
|
|
60
|
+
} catch {
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
describe.skipIf(HTTP_SKIP)(
|
|
66
|
+
'prompt-mutation-workspace-membership-enforced: writes to non-member workspaces MUST be refused (RFC 0028 Tier-2)',
|
|
67
|
+
() => {
|
|
68
|
+
it('POST /v1/prompts with a workspaceId the principal is not a member of MUST NOT succeed with 2xx', async (ctx) => {
|
|
69
|
+
const d = await readDiscovery();
|
|
70
|
+
if (d === null) {
|
|
71
|
+
ctx.skip();
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
const mutableLibrary = d.capabilities?.prompts?.mutableLibrary;
|
|
75
|
+
if (mutableLibrary !== true) {
|
|
76
|
+
ctx.skip();
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const nonMemberWorkspaceId =
|
|
81
|
+
process.env.OPENWOP_TEST_NONMEMBER_WORKSPACE_ID ??
|
|
82
|
+
`openwop-conformance-nonmember-${randomUUID()}`;
|
|
83
|
+
|
|
84
|
+
const res = await driver.post('/v1/prompts', {
|
|
85
|
+
workspaceId: nonMemberWorkspaceId,
|
|
86
|
+
templateId: `conformance-membership-probe-${randomUUID()}`,
|
|
87
|
+
version: '1.0.0',
|
|
88
|
+
kind: 'system',
|
|
89
|
+
text: 'conformance probe — SHOULD NOT persist',
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
// The conformance MUST: the host MUST NOT honor a write to a workspace
|
|
93
|
+
// the caller cannot prove membership of. Any refusal (4xx/5xx) is
|
|
94
|
+
// acceptable; a 2xx silent success is the failure mode that the RFC
|
|
95
|
+
// 0028 Tier-2 self-disclosed vulnerability demonstrated.
|
|
96
|
+
expect(
|
|
97
|
+
res.status,
|
|
98
|
+
driver.describe(
|
|
99
|
+
'spec/v1/prompts.md §Workspace membership on workspace-scoped reads and writes',
|
|
100
|
+
`mutating /v1/prompts MUST refuse a write to a non-member workspace; ` +
|
|
101
|
+
`got ${res.status} ${res.text.slice(0, 200)}`,
|
|
102
|
+
),
|
|
103
|
+
).toBeGreaterThanOrEqual(400);
|
|
104
|
+
|
|
105
|
+
// T1 canonicalization (2026-05-25): when the host CHOOSES 403 to
|
|
106
|
+
// signal the authz boundary, the response envelope MUST carry
|
|
107
|
+
// `error: "workspace_membership_required"` per rest-endpoints.md
|
|
108
|
+
// §"Common error codes". Hosts that refuse with other codes
|
|
109
|
+
// (401 if they treat the failure as authentication-level, 404 to
|
|
110
|
+
// avoid existence disclosure, 5xx on infra failure) have the
|
|
111
|
+
// refusal accepted above but the envelope shape is NOT constrained
|
|
112
|
+
// by this scenario — the canonical envelope is conditional on the
|
|
113
|
+
// 403 status code, not a forced upgrade.
|
|
114
|
+
if (res.status === 403) {
|
|
115
|
+
const body = res.json as { error?: unknown } | null;
|
|
116
|
+
expect(
|
|
117
|
+
body?.error,
|
|
118
|
+
driver.describe(
|
|
119
|
+
'spec/v1/rest-endpoints.md §Common error codes — workspace_membership_required',
|
|
120
|
+
`403 refusal of a workspace-scoped mutation MUST carry error: "workspace_membership_required"; got error: ${JSON.stringify(body?.error)}`,
|
|
121
|
+
),
|
|
122
|
+
).toBe('workspace_membership_required');
|
|
123
|
+
}
|
|
124
|
+
});
|
|
125
|
+
},
|
|
126
|
+
);
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* prompt-read-workspace-membership-enforced — RFC 0028 Tier-2 §"Workspace
|
|
3
|
+
* membership on workspace-scoped reads and writes" verification (READ path).
|
|
4
|
+
*
|
|
5
|
+
* Status: ACTIVE (capability-gated; behavioral when the host advertises
|
|
6
|
+
* `capabilities.prompts.supported: true` AND accepts `?workspaceId=` on
|
|
7
|
+
* `GET /v1/prompts`). Hosts that don't expose workspace-scoped reads
|
|
8
|
+
* (host-only template libraries with no workspace dimension) self-skip
|
|
9
|
+
* via response-shape detection.
|
|
10
|
+
*
|
|
11
|
+
* The contract (spec/v1/prompts.md §"Discovery & distribution" §"REST
|
|
12
|
+
* endpoints" §"Workspace membership on workspace-scoped reads and writes"):
|
|
13
|
+
*
|
|
14
|
+
* Read paths are NOT exempt from the workspace-membership invariant
|
|
15
|
+
* just because they don't write. A GET /v1/prompts?workspaceId=<not-mine>
|
|
16
|
+
* that returns another workspace's templates is a cross-tenant data leak
|
|
17
|
+
* with the same blast radius as a cross-tenant write. Hosts MUST verify
|
|
18
|
+
* the authenticated principal's workspace membership BEFORE returning
|
|
19
|
+
* workspace-scoped content.
|
|
20
|
+
*
|
|
21
|
+
* Gate per MyndHyve relay 2026-05-25 ("Option B"): probe ALL hosts that
|
|
22
|
+
* advertise `capabilities.prompts.supported: true` regardless of
|
|
23
|
+
* `mutableLibrary`; read-only hosts that expose `?workspaceId=` reads are
|
|
24
|
+
* NOT exempt from the symmetric authz invariant. Hosts that don't expose
|
|
25
|
+
* workspace-scoped reads at all self-skip via the response interpretation
|
|
26
|
+
* below (the suite avoids inventing a new capability field just for this
|
|
27
|
+
* gating concern).
|
|
28
|
+
*
|
|
29
|
+
* The probe drives `GET /v1/prompts?workspaceId=<random-uuid>` and
|
|
30
|
+
* interprets the response:
|
|
31
|
+
*
|
|
32
|
+
* - 4xx (any code) — PASS (refused). If 403 specifically, additionally
|
|
33
|
+
* pin `error === "workspace_membership_required"` per the canonical
|
|
34
|
+
* envelope in rest-endpoints.md §"Common error codes".
|
|
35
|
+
* - 200 with `templates: []` — PASS. The host correctly returned no
|
|
36
|
+
* content for a workspace the principal isn't a member of. A random
|
|
37
|
+
* UUID workspace also definitionally has no real content, so an empty
|
|
38
|
+
* result is the correct null answer.
|
|
39
|
+
* - 200 with `templates: [non-empty]` — FAIL. The host returned content
|
|
40
|
+
* for an unauthorized workspace. This is the cross-tenant data leak
|
|
41
|
+
* failure mode. (Note: this scenario uses a random workspaceId so any
|
|
42
|
+
* non-empty result is a leak — there can't legitimately be templates
|
|
43
|
+
* in a freshly-generated nonexistent workspace.)
|
|
44
|
+
* - 200 without a `templates[]` field, or a response shape that doesn't
|
|
45
|
+
* resemble the documented `/v1/prompts` list shape — SKIP with a
|
|
46
|
+
* diagnostic log. Indicates the host doesn't recognize `?workspaceId=`
|
|
47
|
+
* on this endpoint (e.g., host-only template library with no
|
|
48
|
+
* workspace dimension).
|
|
49
|
+
* - 5xx — PASS (refused; envelope shape unconstrained).
|
|
50
|
+
*
|
|
51
|
+
* Why a random workspaceId is sufficient: the assertion is negative-space.
|
|
52
|
+
* A host that correctly enforces membership MUST refuse for ANY workspace
|
|
53
|
+
* the principal isn't a member of, and a random UUID has astronomically-low
|
|
54
|
+
* collision probability with any real workspace membership grant. A host
|
|
55
|
+
* that returns templates from a random UUID workspace is leaking content
|
|
56
|
+
* from somewhere (host-built-in misclassified as workspace, or a silent
|
|
57
|
+
* fall-through to another workspace's content, or a query bug returning
|
|
58
|
+
* everything).
|
|
59
|
+
*
|
|
60
|
+
* @see RFCS/0028-prompt-library-endpoints.md §"Post-promotion notes"
|
|
61
|
+
* @see spec/v1/prompts.md §"Security invariants" §prompt-read-workspace-membership-enforced
|
|
62
|
+
* @see spec/v1/rest-endpoints.md §"Common error codes" §workspace_membership_required
|
|
63
|
+
* @see spec/v1/auth.md §"Identity claims — tenant · workspace · principal"
|
|
64
|
+
* @see RFCS/0048-tenant-workspace-principal-identity-model.md §D
|
|
65
|
+
*/
|
|
66
|
+
|
|
67
|
+
import { describe, it, expect } from 'vitest';
|
|
68
|
+
import { randomUUID } from 'node:crypto';
|
|
69
|
+
import { driver } from '../lib/driver.js';
|
|
70
|
+
|
|
71
|
+
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
72
|
+
|
|
73
|
+
interface DiscoveryDoc {
|
|
74
|
+
capabilities?: {
|
|
75
|
+
prompts?: {
|
|
76
|
+
supported?: unknown;
|
|
77
|
+
};
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
interface PromptListResponse {
|
|
82
|
+
templates?: unknown;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async function readDiscovery(): Promise<DiscoveryDoc | null> {
|
|
86
|
+
try {
|
|
87
|
+
const res = await driver.get('/.well-known/openwop');
|
|
88
|
+
if (res.status !== 200) return null;
|
|
89
|
+
return res.json as DiscoveryDoc;
|
|
90
|
+
} catch {
|
|
91
|
+
return null;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
describe.skipIf(HTTP_SKIP)(
|
|
96
|
+
'prompt-read-workspace-membership-enforced: workspace-scoped reads MUST NOT leak templates from another workspace (RFC 0028 Tier-2)',
|
|
97
|
+
() => {
|
|
98
|
+
it('GET /v1/prompts?workspaceId=<non-member> MUST refuse OR return empty templates[] — never another workspace\'s content', async (ctx) => {
|
|
99
|
+
const d = await readDiscovery();
|
|
100
|
+
if (d === null) {
|
|
101
|
+
ctx.skip();
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
const promptsSupported = d.capabilities?.prompts?.supported;
|
|
105
|
+
if (promptsSupported !== true) {
|
|
106
|
+
ctx.skip();
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const nonMemberWorkspaceId =
|
|
111
|
+
process.env.OPENWOP_TEST_NONMEMBER_WORKSPACE_ID ??
|
|
112
|
+
`openwop-conformance-nonmember-${randomUUID()}`;
|
|
113
|
+
|
|
114
|
+
const res = await driver.get(
|
|
115
|
+
`/v1/prompts?workspaceId=${encodeURIComponent(nonMemberWorkspaceId)}`,
|
|
116
|
+
);
|
|
117
|
+
|
|
118
|
+
// 4xx — refused. Acceptable shape for the membership-required failure
|
|
119
|
+
// (and any other refusal mode the host chooses: 401, 404 for
|
|
120
|
+
// existence-disclosure avoidance, etc).
|
|
121
|
+
if (res.status >= 400 && res.status < 500) {
|
|
122
|
+
// Canonical envelope on 403 per rest-endpoints.md §"Common error codes".
|
|
123
|
+
if (res.status === 403) {
|
|
124
|
+
const body = res.json as { error?: unknown } | null;
|
|
125
|
+
expect(
|
|
126
|
+
body?.error,
|
|
127
|
+
driver.describe(
|
|
128
|
+
'spec/v1/rest-endpoints.md §Common error codes — workspace_membership_required',
|
|
129
|
+
`403 refusal of a workspace-scoped read MUST carry error: "workspace_membership_required"; got error: ${JSON.stringify(body?.error)}`,
|
|
130
|
+
),
|
|
131
|
+
).toBe('workspace_membership_required');
|
|
132
|
+
}
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// 5xx — refused (infrastructure failure is acceptable; envelope shape
|
|
137
|
+
// unconstrained).
|
|
138
|
+
if (res.status >= 500) return;
|
|
139
|
+
|
|
140
|
+
// 2xx — must inspect the response body. The failure mode this
|
|
141
|
+
// invariant guards against is a 200 response that LEAKS templates
|
|
142
|
+
// from a workspace the principal isn't a member of.
|
|
143
|
+
if (res.status >= 200 && res.status < 300) {
|
|
144
|
+
const body = res.json as PromptListResponse | null;
|
|
145
|
+
if (
|
|
146
|
+
body === null ||
|
|
147
|
+
typeof body !== 'object' ||
|
|
148
|
+
!('templates' in body)
|
|
149
|
+
) {
|
|
150
|
+
// Host doesn't recognize `?workspaceId=` on this endpoint
|
|
151
|
+
// (response shape doesn't include the documented `templates[]`
|
|
152
|
+
// field). Soft-skip: this scenario probes hosts that expose
|
|
153
|
+
// workspace-scoped reads, and a host without that surface is
|
|
154
|
+
// simply out of scope.
|
|
155
|
+
ctx.skip();
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
const templates = body.templates;
|
|
159
|
+
if (!Array.isArray(templates)) {
|
|
160
|
+
// Same: unrecognized shape, skip.
|
|
161
|
+
ctx.skip();
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// A random non-member workspaceId can never legitimately contain
|
|
166
|
+
// templates the caller is authorized to see. Any non-empty result
|
|
167
|
+
// is a cross-tenant data leak.
|
|
168
|
+
expect(
|
|
169
|
+
templates.length,
|
|
170
|
+
driver.describe(
|
|
171
|
+
'spec/v1/prompts.md §Workspace membership on workspace-scoped reads and writes',
|
|
172
|
+
`GET /v1/prompts?workspaceId=<random-non-member> MUST NOT return any templates; got ${templates.length} templates which is a cross-tenant data leak (the random workspaceId is freshly generated per probe and cannot legitimately contain authorized content)`,
|
|
173
|
+
),
|
|
174
|
+
).toBe(0);
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// Other status codes (1xx, 3xx) — soft-skip with note. Not a clear
|
|
179
|
+
// signal either way.
|
|
180
|
+
ctx.skip();
|
|
181
|
+
});
|
|
182
|
+
},
|
|
183
|
+
);
|
|
@@ -27,9 +27,15 @@
|
|
|
27
27
|
* mock provider returning a valid envelope on the original run and a
|
|
28
28
|
* refusal on the replay (or vice-versa). Reference workflow-engine ships
|
|
29
29
|
* a mock-AI provider (`OPENWOP_MULTI_AGENT_EXECUTION_MODEL=true`); the
|
|
30
|
-
* Phase 4 wiring
|
|
31
|
-
*
|
|
32
|
-
*
|
|
30
|
+
* Phase 4 wiring (landed 2026-05-23 via commits `1fce55a` + `bba3b4a`)
|
|
31
|
+
* extends it with `checkReplayDivergence()` in the executor catch-path
|
|
32
|
+
* + symmetric success-path detection of envelope-kind divergence; emits
|
|
33
|
+
* `replay.divergedAtRefusal` event and fails the run with
|
|
34
|
+
* `error.code: 'replay_diverged_at_refusal'` when source vs replay
|
|
35
|
+
* differ at the same nodeId. Behavioral coverage is now real: 3
|
|
36
|
+
* assertions PASS against workflow-engine when Phase 4 advertisement
|
|
37
|
+
* is enabled (cover both divergence directions: original=valid +
|
|
38
|
+
* replay=refusal AND original=refusal + replay=valid).
|
|
33
39
|
*
|
|
34
40
|
* @see RFCS/0041-multi-agent-replay-under-nondeterminism.md §B
|
|
35
41
|
* @see spec/v1/replay.md §"Envelope-refusal recovery in replay (MAE-8 closure)"
|
|
@@ -113,6 +119,40 @@ describe.skipIf(HTTP_SKIP)('replay-divergence-at-refusal: advertisement shape (R
|
|
|
113
119
|
});
|
|
114
120
|
});
|
|
115
121
|
|
|
122
|
+
interface RunSnapshot {
|
|
123
|
+
status?: string;
|
|
124
|
+
error?: { code?: string; message?: string };
|
|
125
|
+
}
|
|
126
|
+
interface RunEventDoc {
|
|
127
|
+
type: string;
|
|
128
|
+
nodeId?: string;
|
|
129
|
+
sequence?: number;
|
|
130
|
+
payload?: Record<string, unknown>;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
async function pollUntilTerminal(runId: string): Promise<RunSnapshot> {
|
|
134
|
+
for (let i = 0; i < 50; i++) {
|
|
135
|
+
const r = await driver.get(`/v1/runs/${encodeURIComponent(runId)}`);
|
|
136
|
+
const snap = r.json as RunSnapshot;
|
|
137
|
+
if (snap.status === 'completed' || snap.status === 'failed' || snap.status === 'cancelled') {
|
|
138
|
+
return snap;
|
|
139
|
+
}
|
|
140
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
141
|
+
}
|
|
142
|
+
throw new Error(`run ${runId} did not reach terminal within 5s`);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
async function readEvents(runId: string): Promise<RunEventDoc[]> {
|
|
146
|
+
const r = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/events`);
|
|
147
|
+
const body = r.json as { events?: RunEventDoc[] };
|
|
148
|
+
return body.events ?? [];
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
async function programMock(nodeId: string, program: Array<Record<string, unknown>>): Promise<number> {
|
|
152
|
+
const r = await driver.post('/v1/host/sample/test/mock-ai/program', { nodeId, program });
|
|
153
|
+
return r.status;
|
|
154
|
+
}
|
|
155
|
+
|
|
116
156
|
describe.skipIf(HTTP_SKIP)('replay-divergence-at-refusal: behavioral (RFC 0041 §B MAE-8)', () => {
|
|
117
157
|
// Behavioral assertion drives a workflow whose mock-AI provider returns a
|
|
118
158
|
// valid envelope on the original run + a refusal on the replay (or
|
|
@@ -127,8 +167,148 @@ describe.skipIf(HTTP_SKIP)('replay-divergence-at-refusal: behavioral (RFC 0041
|
|
|
127
167
|
// originalEnvelopeKind === 'valid' AND replayEnvelopeKind === 'refusal'.
|
|
128
168
|
// 7. Assert NO silent substitution: the replay's continuation past the
|
|
129
169
|
// diverging node MUST NOT execute (run terminates at the divergence).
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
170
|
+
|
|
171
|
+
async function gateOnPhase4(ctx: { skip: () => void }): Promise<boolean> {
|
|
172
|
+
const d = await readDiscovery();
|
|
173
|
+
const rd = d?.capabilities?.multiAgent?.executionModel?.replayDeterminism;
|
|
174
|
+
if (rd?.supported !== true || rd?.refusalDivergenceEmission !== true) {
|
|
175
|
+
ctx.skip();
|
|
176
|
+
return false;
|
|
177
|
+
}
|
|
178
|
+
return true;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
it('Phase 4 host MUST emit replay.divergedAtRefusal + fail with replay_diverged_at_refusal when original=valid + replay=refusal', async (ctx) => {
|
|
182
|
+
if (!(await gateOnPhase4(ctx))) return;
|
|
183
|
+
|
|
184
|
+
const NODE_ID = 'structured-call';
|
|
185
|
+
// Original program: valid envelope. Replay program (set after the
|
|
186
|
+
// original completes): refusal. Programming twice is the spec-canonical
|
|
187
|
+
// pattern — see spec/v1/host-sample-test-seams.md §5.
|
|
188
|
+
const validEnv = '{"valid":true}';
|
|
189
|
+
const programStatus = await programMock(NODE_ID, [
|
|
190
|
+
{ content: validEnv, stopReason: 'end_turn' as const },
|
|
191
|
+
]);
|
|
192
|
+
if (programStatus === 404) {
|
|
193
|
+
ctx.skip(); // mock-AI program seam not exposed — soft-skip
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
expect(programStatus).toBe(200);
|
|
197
|
+
|
|
198
|
+
const createRes = await driver.post('/v1/runs', {
|
|
199
|
+
workflowId: 'conformance-phase4-replay-divergence',
|
|
200
|
+
});
|
|
201
|
+
if (createRes.status === 404 || createRes.status === 422) {
|
|
202
|
+
ctx.skip(); // fixture not advertised
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
205
|
+
expect(createRes.status).toBe(201);
|
|
206
|
+
const sourceRunId = (createRes.json as { runId: string }).runId;
|
|
207
|
+
const sourceTerminal = await pollUntilTerminal(sourceRunId);
|
|
208
|
+
expect(sourceTerminal.status).toBe('completed');
|
|
209
|
+
|
|
210
|
+
// Stage refusal for the replay's mock-AI dispatch.
|
|
211
|
+
await programMock(NODE_ID, [
|
|
212
|
+
{ content: 'safety-refused-for-conformance', stopReason: 'safety' as const, refusalText: 'safety-refused-for-conformance' },
|
|
213
|
+
]);
|
|
214
|
+
|
|
215
|
+
const forkRes = await driver.post(`/v1/runs/${encodeURIComponent(sourceRunId)}:fork`, {
|
|
216
|
+
fromSeq: 0,
|
|
217
|
+
mode: 'replay',
|
|
218
|
+
});
|
|
219
|
+
expect(forkRes.status).toBe(201);
|
|
220
|
+
const replayRunId = (forkRes.json as { runId: string }).runId;
|
|
221
|
+
const replayTerminal = await pollUntilTerminal(replayRunId);
|
|
222
|
+
|
|
223
|
+
expect(
|
|
224
|
+
replayTerminal.status,
|
|
225
|
+
driver.describe(
|
|
226
|
+
'RFCS/0041-multi-agent-replay-under-nondeterminism.md §B + spec/v1/rest-endpoints.md §"Common error codes"',
|
|
227
|
+
'replay MUST terminate `failed` when refusal-divergence is detected (silent substitution is non-conformant)',
|
|
228
|
+
),
|
|
229
|
+
).toBe('failed');
|
|
230
|
+
expect(
|
|
231
|
+
replayTerminal.error?.code,
|
|
232
|
+
driver.describe(
|
|
233
|
+
'spec/v1/rest-endpoints.md §"Common error codes" — replay_diverged_at_refusal',
|
|
234
|
+
'error.code MUST be `replay_diverged_at_refusal` per the canonical catalog',
|
|
235
|
+
),
|
|
236
|
+
).toBe('replay_diverged_at_refusal');
|
|
237
|
+
|
|
238
|
+
const replayEvents = await readEvents(replayRunId);
|
|
239
|
+
const divergenceEvent = replayEvents.find((e) => e.type === 'replay.divergedAtRefusal');
|
|
240
|
+
expect(
|
|
241
|
+
divergenceEvent,
|
|
242
|
+
driver.describe(
|
|
243
|
+
'schemas/run-event-payloads.schema.json §replayDivergedAtRefusal',
|
|
244
|
+
'replay event log MUST contain exactly one `replay.divergedAtRefusal` event identifying the divergence',
|
|
245
|
+
),
|
|
246
|
+
).toBeDefined();
|
|
247
|
+
expect(divergenceEvent?.payload?.sourceRunId).toBe(sourceRunId);
|
|
248
|
+
expect(divergenceEvent?.payload?.nodeId).toBe(NODE_ID);
|
|
249
|
+
expect(
|
|
250
|
+
divergenceEvent?.payload?.originalEnvelopeKind,
|
|
251
|
+
driver.describe(
|
|
252
|
+
'schemas/run-event-payloads.schema.json §replayDivergedAtRefusal.originalEnvelopeKind',
|
|
253
|
+
'originalEnvelopeKind MUST be `valid` (source run completed normally)',
|
|
254
|
+
),
|
|
255
|
+
).toBe('valid');
|
|
256
|
+
expect(
|
|
257
|
+
divergenceEvent?.payload?.replayEnvelopeKind,
|
|
258
|
+
driver.describe(
|
|
259
|
+
'schemas/run-event-payloads.schema.json §replayDivergedAtRefusal.replayEnvelopeKind',
|
|
260
|
+
'replayEnvelopeKind MUST be `refusal` (replay hit the refusal entry of the mock program)',
|
|
261
|
+
),
|
|
262
|
+
).toBe('refusal');
|
|
263
|
+
});
|
|
264
|
+
|
|
265
|
+
it('Phase 4 host MUST emit replay.divergedAtRefusal + fail with replay_diverged_at_refusal when original=refusal + replay=valid (symmetric case)', async (ctx) => {
|
|
266
|
+
if (!(await gateOnPhase4(ctx))) return;
|
|
267
|
+
|
|
268
|
+
const NODE_ID = 'structured-call';
|
|
269
|
+
// Symmetric: original=refusal, replay=valid.
|
|
270
|
+
const programStatus = await programMock(NODE_ID, [
|
|
271
|
+
{ content: 'safety-refused-for-conformance', stopReason: 'safety' as const, refusalText: 'safety-refused-for-conformance' },
|
|
272
|
+
]);
|
|
273
|
+
if (programStatus === 404) {
|
|
274
|
+
ctx.skip();
|
|
275
|
+
return;
|
|
276
|
+
}
|
|
277
|
+
expect(programStatus).toBe(200);
|
|
278
|
+
|
|
279
|
+
const createRes = await driver.post('/v1/runs', {
|
|
280
|
+
workflowId: 'conformance-phase4-replay-divergence',
|
|
281
|
+
});
|
|
282
|
+
if (createRes.status === 404 || createRes.status === 422) {
|
|
283
|
+
ctx.skip();
|
|
284
|
+
return;
|
|
285
|
+
}
|
|
286
|
+
expect(createRes.status).toBe(201);
|
|
287
|
+
const sourceRunId = (createRes.json as { runId: string }).runId;
|
|
288
|
+
const sourceTerminal = await pollUntilTerminal(sourceRunId);
|
|
289
|
+
// Source run fails because the LLM refused.
|
|
290
|
+
expect(sourceTerminal.status).toBe('failed');
|
|
291
|
+
|
|
292
|
+
// Stage valid envelope for the replay's mock-AI dispatch.
|
|
293
|
+
await programMock(NODE_ID, [
|
|
294
|
+
{ content: '{"valid":true}', stopReason: 'end_turn' as const },
|
|
295
|
+
]);
|
|
296
|
+
|
|
297
|
+
const forkRes = await driver.post(`/v1/runs/${encodeURIComponent(sourceRunId)}:fork`, {
|
|
298
|
+
fromSeq: 0,
|
|
299
|
+
mode: 'replay',
|
|
300
|
+
});
|
|
301
|
+
expect(forkRes.status).toBe(201);
|
|
302
|
+
const replayRunId = (forkRes.json as { runId: string }).runId;
|
|
303
|
+
const replayTerminal = await pollUntilTerminal(replayRunId);
|
|
304
|
+
|
|
305
|
+
expect(replayTerminal.status).toBe('failed');
|
|
306
|
+
expect(replayTerminal.error?.code).toBe('replay_diverged_at_refusal');
|
|
307
|
+
|
|
308
|
+
const replayEvents = await readEvents(replayRunId);
|
|
309
|
+
const divergenceEvent = replayEvents.find((e) => e.type === 'replay.divergedAtRefusal');
|
|
310
|
+
expect(divergenceEvent).toBeDefined();
|
|
311
|
+
expect(divergenceEvent?.payload?.originalEnvelopeKind).toBe('refusal');
|
|
312
|
+
expect(divergenceEvent?.payload?.replayEnvelopeKind).toBe('valid');
|
|
313
|
+
});
|
|
134
314
|
});
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* `capabilities.multiAgent.executionModel.version >= 4` AND
|
|
6
6
|
* `capabilities.multiAgent.executionModel.replayDeterminism.supported: true`.
|
|
7
7
|
*
|
|
8
|
-
* Asserts (behavioral, when a
|
|
8
|
+
* Asserts (behavioral, when a host advertises `version: 4` + the contract):
|
|
9
9
|
*
|
|
10
10
|
* 1. A `mode: replay` fork from event-log index `fromSeq` produces an
|
|
11
11
|
* event-log prefix `[0, fromSeq]` that is byte-equivalent to the
|
|
@@ -26,14 +26,14 @@
|
|
|
26
26
|
* Driving the assertion requires a workflow fixture whose tool call is
|
|
27
27
|
* pure-nondeterministic (different bytes on each call) but whose
|
|
28
28
|
* observable result is what gets cached. Reference workflow-engine ships
|
|
29
|
-
* `core.noop` + deterministic fixtures;
|
|
29
|
+
* `core.noop` + deterministic fixtures; the `version: 4` wiring needs a
|
|
30
30
|
* nondeterministic-tool fixture (e.g., `conformance-phase4-nondet-tool`).
|
|
31
31
|
* Until that lands, the cross-boundary assertion is surfaced as `it.todo`
|
|
32
32
|
* so test reporters track the gap.
|
|
33
33
|
*
|
|
34
34
|
* @see RFCS/0041-multi-agent-replay-under-nondeterminism.md §C
|
|
35
35
|
* @see spec/v1/replay.md §"Observable-output-sequence determinism vs bit-equivalent execution (MAE-9 closure)"
|
|
36
|
-
* @see spec/v1/multi-agent-execution.md §"
|
|
36
|
+
* @see spec/v1/multi-agent-execution.md §"Replay determinism under nondeterminism (RFC 0041)"
|
|
37
37
|
*/
|
|
38
38
|
|
|
39
39
|
import { describe, it } from 'vitest';
|
|
@@ -62,10 +62,19 @@ describe('replay-observable-sequence-determinism: prefix byte-equivalence (RFC 0
|
|
|
62
62
|
// 6. Read original + replay RunSnapshot at index N; assert
|
|
63
63
|
// variables + channels + status byte-equivalent.
|
|
64
64
|
// Surfaced as `todo` until the `conformance-phase4-nondet-tool`
|
|
65
|
-
// fixture ships in the suite — consistent with the sibling
|
|
65
|
+
// fixture ships in the suite — consistent with the sibling RFC 0041
|
|
66
66
|
// scenarios (`replay-divergence-at-refusal.test.ts`,
|
|
67
67
|
// `replay-llm-cache-key-portable.test.ts`).
|
|
68
|
-
|
|
68
|
+
// Marked out of stable profile via RFC 0042 §B (experimental tier):
|
|
69
|
+
// RFC 0041 §C remains Active, so its wire shape MAY shift compatibly
|
|
70
|
+
// within v1.x. Hosts that wire this assertion before RFC 0041 graduates
|
|
71
|
+
// to Accepted SHOULD advertise `multiAgent.executionModel.tier:
|
|
72
|
+
// 'experimental'` + `experimentalUntil` per RFC 0042 §A. Path-to-runnable
|
|
73
|
+
// requires: (a) host pure-replay observable-cache emission via the
|
|
74
|
+
// `:fork mode: replay` re-dispatch path and (b) the test seam endpoint
|
|
75
|
+
// contract for cache-hit-vs-fresh-call distinction (see
|
|
76
|
+
// `spec/v1/host-sample-test-seams.md` for the established seam pattern).
|
|
77
|
+
it.skip('original and replay event-log prefixes [0, fromSeq] MUST be byte-equivalent (modulo per-region clock + ULID-T entropy) — out of stable profile via RFC 0042');
|
|
69
78
|
});
|
|
70
79
|
|
|
71
80
|
describe('replay-observable-sequence-determinism: observable-result caching (RFC 0041 §C)', () => {
|
|
@@ -76,5 +85,10 @@ describe('replay-observable-sequence-determinism: observable-result caching (RFC
|
|
|
76
85
|
// this a valid determinism contract — bit-equivalent execution would
|
|
77
86
|
// require unbounded caching (rejected per RFC 0041 §"Alternatives
|
|
78
87
|
// considered" #2).
|
|
79
|
-
|
|
88
|
+
// Marked out of stable profile via RFC 0042 §B (experimental tier):
|
|
89
|
+
// see the prefix-byte-equivalence comment above for the same routing.
|
|
90
|
+
// This is RFC 0041 §C's load-bearing assertion; it lands as a runnable
|
|
91
|
+
// `it()` when RFC 0041 graduates to Accepted on first non-steward host
|
|
92
|
+
// adoption.
|
|
93
|
+
it.skip('replay of a workflow containing a nondeterministic tool call reproduces the original observable result, NOT a fresh call — out of stable profile via RFC 0042');
|
|
80
94
|
});
|