@openwop/openwop-conformance 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +31 -6
- package/api/grpc/openwop.proto +251 -0
- package/api/openapi.yaml +109 -3
- package/coverage.md +48 -9
- package/fixtures/conformance-configurable-schema.json +39 -0
- package/fixtures/conformance-subworkflow-parent.json +1 -1
- package/fixtures/conformance-wasm-pack-memory-cap-breach.json +23 -0
- package/fixtures/openwop-smoke-byok-roundtrip.json +25 -0
- package/fixtures.md +21 -0
- package/package.json +3 -1
- package/schemas/README.md +4 -0
- package/schemas/audit-verify-result.schema.json +90 -0
- package/schemas/capabilities.schema.json +293 -1
- package/schemas/node-pack-manifest.schema.json +4 -4
- package/schemas/pack-lockfile.schema.json +92 -0
- package/schemas/registry-version-manifest.schema.json +145 -0
- package/schemas/run-event-payloads.schema.json +2 -2
- package/schemas/security-advisory.schema.json +109 -0
- package/src/lib/a2a-fake-peer.ts +143 -56
- package/src/lib/behavior-gate.ts +68 -0
- package/src/lib/env.ts +10 -0
- package/src/lib/grpc-framing.test.ts +96 -0
- package/src/lib/grpc-framing.ts +76 -0
- package/src/lib/oidc-issuer.test.ts +328 -0
- package/src/lib/oidc-issuer.ts +241 -0
- package/src/lib/otel-collector-grpc.test.ts +191 -0
- package/src/lib/otel-collector.test.ts +303 -0
- package/src/lib/otel-collector.ts +318 -14
- package/src/lib/otlp-protobuf.test.ts +461 -0
- package/src/lib/otlp-protobuf.ts +529 -0
- package/src/scenarios/a2a-task-roundtrip.test.ts +147 -28
- package/src/scenarios/agentConfidenceEscalation.test.ts +1 -0
- package/src/scenarios/agentMemoryCrossTenantIsolation.test.ts +1 -0
- package/src/scenarios/agentMemoryRedactionContract.test.ts +1 -0
- package/src/scenarios/agentMemoryRoundTrip.test.ts +1 -0
- package/src/scenarios/agentMemoryTtlExpiry.test.ts +1 -0
- package/src/scenarios/agentMessageReducer.test.ts +1 -0
- package/src/scenarios/agentMetadata.test.ts +1 -0
- package/src/scenarios/agentPackExport.test.ts +1 -0
- package/src/scenarios/agentPackInstall.test.ts +1 -0
- package/src/scenarios/agentPackProvenance.test.ts +1 -0
- package/src/scenarios/audit-log-integrity.test.ts +3 -6
- package/src/scenarios/auth-api-key-rotation.test.ts +182 -0
- package/src/scenarios/auth-mtls.test.ts +274 -0
- package/src/scenarios/auth-oauth2-client-credentials.test.ts +259 -0
- package/src/scenarios/auth-oidc-user-bearer.test.ts +361 -0
- package/src/scenarios/bulk-cancel.test.ts +111 -0
- package/src/scenarios/configurable-schema.test.ts +48 -0
- package/src/scenarios/conversationCapabilityNegotiation.test.ts +1 -0
- package/src/scenarios/conversationLifecycle.test.ts +1 -0
- package/src/scenarios/conversationReplayDeterminism.test.ts +1 -0
- package/src/scenarios/conversationVsLegacySuspend.test.ts +1 -0
- package/src/scenarios/debug-bundle-truncation.test.ts +95 -0
- package/src/scenarios/discovery.test.ts +183 -0
- package/src/scenarios/http-client-ssrf.test.ts +71 -0
- package/src/scenarios/idempotency.test.ts +6 -0
- package/src/scenarios/idempotencyRetry.test.ts +3 -0
- package/src/scenarios/mcp-tool-roundtrip.test.ts +198 -34
- package/src/scenarios/mcp-toolcall-redaction.test.ts +66 -0
- package/src/scenarios/metric-emission.test.ts +113 -0
- package/src/scenarios/orchestratorConservativePath.test.ts +1 -0
- package/src/scenarios/orchestratorDispatch.test.ts +1 -0
- package/src/scenarios/orchestratorTermination.test.ts +1 -0
- package/src/scenarios/otel-emission-grpc.test.ts +98 -0
- package/src/scenarios/pause-resume.test.ts +119 -0
- package/src/scenarios/production-backpressure.test.ts +342 -0
- package/src/scenarios/production-retention-expiry.test.ts +164 -0
- package/src/scenarios/registry-public.test.ts +131 -0
- package/src/scenarios/replay-llm-cache-key.test.ts +35 -0
- package/src/scenarios/replay-retention-expiry.test.ts +178 -0
- package/src/scenarios/restart-during-run.test.ts +177 -0
- package/src/scenarios/spec-corpus-validity.test.ts +54 -26
- package/src/scenarios/staleClaim.test.ts +3 -0
- package/src/scenarios/wasm-pack-abi-version-rejection.test.ts +67 -10
- package/src/scenarios/wasm-pack-memory-cap.test.ts +64 -9
- package/src/scenarios/webhook-negative.test.ts +90 -0
- package/src/scenarios/webhook-signed-delivery.test.ts +178 -0
- package/src/setup.ts +25 -1
- package/vitest.config.ts +5 -1
|
@@ -0,0 +1,361 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RFC 0010 §D: openwop-auth-oidc-user-bearer profile.
|
|
3
|
+
*
|
|
4
|
+
* Verifies that hosts claiming the OIDC user-bearer profile satisfy
|
|
5
|
+
* `spec/v1/auth-profiles.md` §`openwop-auth-oidc-user-bearer`:
|
|
6
|
+
*
|
|
7
|
+
* 1. `capabilities.auth.profiles[]` includes
|
|
8
|
+
* `openwop-auth-oidc-user-bearer` and `oidc.supported`.
|
|
9
|
+
* 2. `oidc.issuers` is a non-empty array of URI strings; `audience`
|
|
10
|
+
* is a non-empty string when advertised; `supportedScopeMapping`
|
|
11
|
+
* (if present) is one of the canonical enum values;
|
|
12
|
+
* `introspectionIntervalSeconds` (if present) is a non-negative
|
|
13
|
+
* integer.
|
|
14
|
+
* 3. When `OPENWOP_TEST_OIDC_ISSUER_URL` is supplied, the scenario
|
|
15
|
+
* binds the synthetic OIDC issuer harness at that URL and exercises
|
|
16
|
+
* six host-side validation cases:
|
|
17
|
+
* a. Valid sub/iss/aud/exp → 201 on POST /v1/runs.
|
|
18
|
+
* b. Wrong `iss` → 401.
|
|
19
|
+
* c. Wrong `aud` → 401.
|
|
20
|
+
* d. Expired `exp` → 401.
|
|
21
|
+
* e. Unknown `kid` (header references a key not in JWKS) → 401.
|
|
22
|
+
* f. Insufficient scope (empty groups against a group-claim
|
|
23
|
+
* mapping host) → 403.
|
|
24
|
+
*
|
|
25
|
+
* The host MUST be pre-configured to trust `OPENWOP_TEST_OIDC_ISSUER_URL`
|
|
26
|
+
* as one of its `oidc.issuers`. The scenario binds the harness's JWKS
|
|
27
|
+
* + discovery endpoints on that URL's port so the host's introspection
|
|
28
|
+
* fetches succeed against this hermetic in-suite issuer.
|
|
29
|
+
*
|
|
30
|
+
* Cases (a) and (f) require the host's user-to-scope mapping policy to
|
|
31
|
+
* accept the harness's `sub`. The scenario soft-skips them with a
|
|
32
|
+
* warning when the host returns 403 to the "valid" token (no mapping)
|
|
33
|
+
* or when the host returns 401 to the "valid" token (host trust not
|
|
34
|
+
* actually wired up).
|
|
35
|
+
*
|
|
36
|
+
* @see RFCS/0010-auth-profile-conformance.md §D
|
|
37
|
+
* @see spec/v1/auth-profiles.md §`openwop-auth-oidc-user-bearer`
|
|
38
|
+
* @see conformance/src/lib/oidc-issuer.ts — synthetic harness
|
|
39
|
+
*/
|
|
40
|
+
|
|
41
|
+
import { afterAll, beforeAll, describe, it, expect } from 'vitest';
|
|
42
|
+
import { createServer, type Server } from 'node:http';
|
|
43
|
+
import { driver } from '../lib/driver.js';
|
|
44
|
+
import { behaviorGate } from '../lib/behavior-gate.js';
|
|
45
|
+
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
46
|
+
import {
|
|
47
|
+
createSyntheticOIDCIssuer,
|
|
48
|
+
type SyntheticOIDCIssuer,
|
|
49
|
+
} from '../lib/oidc-issuer.js';
|
|
50
|
+
|
|
51
|
+
interface OIDCCaps {
|
|
52
|
+
supported?: boolean;
|
|
53
|
+
issuers?: string[];
|
|
54
|
+
audience?: string;
|
|
55
|
+
supportedScopeMapping?: string;
|
|
56
|
+
introspectionIntervalSeconds?: number;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
interface AuthCaps {
|
|
60
|
+
profiles?: string[];
|
|
61
|
+
oidc?: OIDCCaps;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const PROFILE = 'openwop-auth-oidc-user-bearer';
|
|
65
|
+
const FIXTURE = 'conformance-noop';
|
|
66
|
+
|
|
67
|
+
async function readAuthCaps(): Promise<AuthCaps | undefined> {
|
|
68
|
+
const disco = await driver.get('/.well-known/openwop');
|
|
69
|
+
return (disco.json as { capabilities?: { auth?: AuthCaps } }).capabilities?.auth;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function isProfileAdvertised(auth: AuthCaps | undefined): boolean {
|
|
73
|
+
return (
|
|
74
|
+
Array.isArray(auth?.profiles) &&
|
|
75
|
+
auth.profiles.includes(PROFILE) &&
|
|
76
|
+
auth.oidc?.supported === true
|
|
77
|
+
);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
describe('auth-oidc-user-bearer: capability shape', () => {
|
|
81
|
+
it('host claiming OIDC profile advertises required fields', async () => {
|
|
82
|
+
const auth = await readAuthCaps();
|
|
83
|
+
|
|
84
|
+
if (!behaviorGate(PROFILE, isProfileAdvertised(auth))) {
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
expect(auth?.profiles?.includes(PROFILE), driver.describe(
|
|
89
|
+
'auth-profiles.md §`openwop-auth-oidc-user-bearer`',
|
|
90
|
+
'capabilities.auth.profiles MUST include openwop-auth-oidc-user-bearer when the profile is claimed',
|
|
91
|
+
)).toBe(true);
|
|
92
|
+
|
|
93
|
+
expect(auth?.oidc?.supported, driver.describe(
|
|
94
|
+
'auth-profiles.md §`openwop-auth-oidc-user-bearer`',
|
|
95
|
+
'capabilities.auth.oidc.supported MUST be true when the profile is claimed',
|
|
96
|
+
)).toBe(true);
|
|
97
|
+
|
|
98
|
+
expect(
|
|
99
|
+
Array.isArray(auth?.oidc?.issuers) && (auth?.oidc?.issuers?.length ?? 0) > 0,
|
|
100
|
+
driver.describe(
|
|
101
|
+
'capabilities.schema.json auth.oidc.issuers',
|
|
102
|
+
'issuers MUST be a non-empty array when the profile is claimed',
|
|
103
|
+
),
|
|
104
|
+
).toBe(true);
|
|
105
|
+
|
|
106
|
+
for (const issuer of auth?.oidc?.issuers ?? []) {
|
|
107
|
+
expect(
|
|
108
|
+
typeof issuer === 'string' && issuer.length > 0,
|
|
109
|
+
'each issuer entry MUST be a non-empty string',
|
|
110
|
+
).toBe(true);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
if (auth?.oidc?.audience !== undefined) {
|
|
114
|
+
expect(
|
|
115
|
+
typeof auth.oidc.audience === 'string' && auth.oidc.audience.length > 0,
|
|
116
|
+
'audience MUST be a non-empty string when advertised',
|
|
117
|
+
).toBe(true);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
if (auth?.oidc?.supportedScopeMapping !== undefined) {
|
|
121
|
+
expect(
|
|
122
|
+
['group-claim', 'scope-claim', 'host-acl'].includes(
|
|
123
|
+
auth.oidc.supportedScopeMapping,
|
|
124
|
+
),
|
|
125
|
+
driver.describe(
|
|
126
|
+
'capabilities.schema.json auth.oidc.supportedScopeMapping',
|
|
127
|
+
'supportedScopeMapping MUST be one of group-claim/scope-claim/host-acl',
|
|
128
|
+
),
|
|
129
|
+
).toBe(true);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
if (auth?.oidc?.introspectionIntervalSeconds !== undefined) {
|
|
133
|
+
expect(
|
|
134
|
+
Number.isInteger(auth.oidc.introspectionIntervalSeconds) &&
|
|
135
|
+
auth.oidc.introspectionIntervalSeconds >= 0,
|
|
136
|
+
'introspectionIntervalSeconds MUST be a non-negative integer when advertised',
|
|
137
|
+
).toBe(true);
|
|
138
|
+
}
|
|
139
|
+
});
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
describe('auth-oidc-user-bearer: harness-driven token validation', () => {
|
|
143
|
+
let server: Server | undefined;
|
|
144
|
+
let issuer: SyntheticOIDCIssuer | undefined;
|
|
145
|
+
let harnessUrl: string | undefined;
|
|
146
|
+
let harnessAudience: string | undefined;
|
|
147
|
+
let trustWired = false;
|
|
148
|
+
|
|
149
|
+
beforeAll(async () => {
|
|
150
|
+
const auth = await readAuthCaps();
|
|
151
|
+
if (!isProfileAdvertised(auth)) return;
|
|
152
|
+
|
|
153
|
+
harnessUrl = process.env.OPENWOP_TEST_OIDC_ISSUER_URL;
|
|
154
|
+
if (!harnessUrl) return;
|
|
155
|
+
|
|
156
|
+
harnessAudience = auth?.oidc?.audience ?? 'openwop-conformance';
|
|
157
|
+
|
|
158
|
+
issuer = createSyntheticOIDCIssuer({
|
|
159
|
+
issuer: harnessUrl,
|
|
160
|
+
audience: harnessAudience,
|
|
161
|
+
algorithm: 'RS256',
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
// Bind the harness's JWKS + discovery endpoints so the host can
|
|
165
|
+
// fetch them when validating tokens.
|
|
166
|
+
const parsed = new URL(harnessUrl);
|
|
167
|
+
const port = parsed.port ? Number.parseInt(parsed.port, 10) : 80;
|
|
168
|
+
|
|
169
|
+
server = createServer((req, res) => {
|
|
170
|
+
if (!issuer) {
|
|
171
|
+
res.writeHead(503);
|
|
172
|
+
res.end();
|
|
173
|
+
return;
|
|
174
|
+
}
|
|
175
|
+
if (req.url === '/.well-known/jwks.json') {
|
|
176
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
177
|
+
res.end(issuer.jwksJson);
|
|
178
|
+
} else if (req.url === '/.well-known/openid-configuration') {
|
|
179
|
+
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
180
|
+
res.end(issuer.discoveryJson);
|
|
181
|
+
} else {
|
|
182
|
+
res.writeHead(404);
|
|
183
|
+
res.end();
|
|
184
|
+
}
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
await new Promise<void>((resolve, reject) => {
|
|
188
|
+
server!.once('error', reject);
|
|
189
|
+
server!.listen(port, '127.0.0.1', () => resolve());
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
// Probe: does the host actually trust this harness? Mint a known-
|
|
193
|
+
// good token and see what the host returns.
|
|
194
|
+
if (isFixtureAdvertised(FIXTURE)) {
|
|
195
|
+
const probe = issuer.mint({ sub: 'conformance-suite', groups: ['openwop:operators'] });
|
|
196
|
+
const probeRes = await driver.post(
|
|
197
|
+
'/v1/runs',
|
|
198
|
+
{ workflowId: FIXTURE },
|
|
199
|
+
{
|
|
200
|
+
authenticated: false,
|
|
201
|
+
headers: { Authorization: `Bearer ${probe.token}` },
|
|
202
|
+
},
|
|
203
|
+
);
|
|
204
|
+
// Trust-wired status: host returns 201 (full success) or 403
|
|
205
|
+
// (token-valid-but-no-scope mapping). Both mean signature
|
|
206
|
+
// verification succeeded; the host trusts the issuer.
|
|
207
|
+
trustWired = probeRes.status === 201 || probeRes.status === 403;
|
|
208
|
+
}
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
afterAll(async () => {
|
|
212
|
+
if (server) {
|
|
213
|
+
await new Promise<void>((resolve) => server!.close(() => resolve()));
|
|
214
|
+
server = undefined;
|
|
215
|
+
}
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
it('wrong iss → 401', async () => {
|
|
219
|
+
if (!issuer || !trustWired) {
|
|
220
|
+
// eslint-disable-next-line no-console
|
|
221
|
+
console.warn(
|
|
222
|
+
'[auth-oidc-user-bearer] harness not wired or host trust not configured; skipping wrong-iss case',
|
|
223
|
+
);
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
const wrongIssIssuer = createSyntheticOIDCIssuer({
|
|
228
|
+
issuer: 'https://untrusted.example.invalid',
|
|
229
|
+
audience: harnessAudience ?? '',
|
|
230
|
+
});
|
|
231
|
+
const wrongIss = wrongIssIssuer.mint({ sub: 'attacker' });
|
|
232
|
+
|
|
233
|
+
const res = await driver.post(
|
|
234
|
+
'/v1/runs',
|
|
235
|
+
{ workflowId: FIXTURE },
|
|
236
|
+
{
|
|
237
|
+
authenticated: false,
|
|
238
|
+
headers: { Authorization: `Bearer ${wrongIss.token}` },
|
|
239
|
+
},
|
|
240
|
+
);
|
|
241
|
+
|
|
242
|
+
expect(res.status, driver.describe(
|
|
243
|
+
'auth-profiles.md §`openwop-auth-oidc-user-bearer`',
|
|
244
|
+
'token with non-trusted iss MUST return 401',
|
|
245
|
+
)).toBe(401);
|
|
246
|
+
});
|
|
247
|
+
|
|
248
|
+
it('wrong aud → 401', async () => {
|
|
249
|
+
if (!issuer || !trustWired) return;
|
|
250
|
+
const wrongAud = issuer.mint({ aud: 'wrong-audience', sub: 'attacker' });
|
|
251
|
+
const res = await driver.post(
|
|
252
|
+
'/v1/runs',
|
|
253
|
+
{ workflowId: FIXTURE },
|
|
254
|
+
{
|
|
255
|
+
authenticated: false,
|
|
256
|
+
headers: { Authorization: `Bearer ${wrongAud.token}` },
|
|
257
|
+
},
|
|
258
|
+
);
|
|
259
|
+
expect(res.status, driver.describe(
|
|
260
|
+
'auth-profiles.md §`openwop-auth-oidc-user-bearer`',
|
|
261
|
+
'token with wrong aud MUST return 401',
|
|
262
|
+
)).toBe(401);
|
|
263
|
+
});
|
|
264
|
+
|
|
265
|
+
it('expired exp → 401', async () => {
|
|
266
|
+
if (!issuer || !trustWired) return;
|
|
267
|
+
const expired = issuer.mint(
|
|
268
|
+
{ sub: 'conformance-suite' },
|
|
269
|
+
{ expiresInSeconds: -3600 },
|
|
270
|
+
);
|
|
271
|
+
const res = await driver.post(
|
|
272
|
+
'/v1/runs',
|
|
273
|
+
{ workflowId: FIXTURE },
|
|
274
|
+
{
|
|
275
|
+
authenticated: false,
|
|
276
|
+
headers: { Authorization: `Bearer ${expired.token}` },
|
|
277
|
+
},
|
|
278
|
+
);
|
|
279
|
+
expect(res.status, driver.describe(
|
|
280
|
+
'auth-profiles.md §`openwop-auth-oidc-user-bearer`',
|
|
281
|
+
'expired token (exp < now) MUST return 401',
|
|
282
|
+
)).toBe(401);
|
|
283
|
+
});
|
|
284
|
+
|
|
285
|
+
it('unknown kid → 401', async () => {
|
|
286
|
+
if (!issuer || !trustWired) return;
|
|
287
|
+
const unknownKid = issuer.mint(
|
|
288
|
+
{ sub: 'conformance-suite' },
|
|
289
|
+
{ keyId: 'openwop-conformance-key-NEVER-PUBLISHED' },
|
|
290
|
+
);
|
|
291
|
+
const res = await driver.post(
|
|
292
|
+
'/v1/runs',
|
|
293
|
+
{ workflowId: FIXTURE },
|
|
294
|
+
{
|
|
295
|
+
authenticated: false,
|
|
296
|
+
headers: { Authorization: `Bearer ${unknownKid.token}` },
|
|
297
|
+
},
|
|
298
|
+
);
|
|
299
|
+
expect(res.status, driver.describe(
|
|
300
|
+
'auth-profiles.md §`openwop-auth-oidc-user-bearer` + threat-model-auth-profiles.md A3',
|
|
301
|
+
'token referencing a kid not in JWKS MUST return 401',
|
|
302
|
+
)).toBe(401);
|
|
303
|
+
});
|
|
304
|
+
|
|
305
|
+
it('valid token → 201 or 403 (depending on host scope mapping)', async () => {
|
|
306
|
+
if (!issuer || !trustWired) return;
|
|
307
|
+
if (!isFixtureAdvertised(FIXTURE)) return;
|
|
308
|
+
|
|
309
|
+
const valid = issuer.mint({
|
|
310
|
+
sub: 'conformance-suite',
|
|
311
|
+
groups: ['openwop:operators'],
|
|
312
|
+
});
|
|
313
|
+
const res = await driver.post(
|
|
314
|
+
'/v1/runs',
|
|
315
|
+
{ workflowId: FIXTURE },
|
|
316
|
+
{
|
|
317
|
+
authenticated: false,
|
|
318
|
+
headers: { Authorization: `Bearer ${valid.token}` },
|
|
319
|
+
},
|
|
320
|
+
);
|
|
321
|
+
|
|
322
|
+
// 201: host trusts the token AND maps the sub to runs:create scope.
|
|
323
|
+
// 403: host trusts the token but the sub lacks the required scope.
|
|
324
|
+
// Both indicate the OIDC validation path succeeded; the scope
|
|
325
|
+
// decision is a separate host-side policy not normated by RFC 0010.
|
|
326
|
+
expect(
|
|
327
|
+
[201, 403].includes(res.status),
|
|
328
|
+
driver.describe(
|
|
329
|
+
'auth-profiles.md §`openwop-auth-oidc-user-bearer`',
|
|
330
|
+
'host-trusted token MUST yield 201 (mapped scope) or 403 (unmapped sub), NOT 401',
|
|
331
|
+
),
|
|
332
|
+
).toBe(true);
|
|
333
|
+
});
|
|
334
|
+
|
|
335
|
+
it('scope-insufficient → 403 (when host uses group-claim mapping)', async () => {
|
|
336
|
+
if (!issuer || !trustWired) return;
|
|
337
|
+
const auth = await readAuthCaps();
|
|
338
|
+
if (auth?.oidc?.supportedScopeMapping !== 'group-claim') {
|
|
339
|
+
// eslint-disable-next-line no-console
|
|
340
|
+
console.warn(
|
|
341
|
+
'[auth-oidc-user-bearer] host scope mapping is not group-claim; skipping scope-insufficient case',
|
|
342
|
+
);
|
|
343
|
+
return;
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
const noGroups = issuer.mint({ sub: 'conformance-suite', groups: [] });
|
|
347
|
+
const res = await driver.post(
|
|
348
|
+
'/v1/runs',
|
|
349
|
+
{ workflowId: FIXTURE },
|
|
350
|
+
{
|
|
351
|
+
authenticated: false,
|
|
352
|
+
headers: { Authorization: `Bearer ${noGroups.token}` },
|
|
353
|
+
},
|
|
354
|
+
);
|
|
355
|
+
|
|
356
|
+
expect(res.status, driver.describe(
|
|
357
|
+
'auth-profiles.md §`openwop-auth-oidc-user-bearer`',
|
|
358
|
+
'token-valid-but-empty-groups against group-claim host MUST return 403 (forbidden), NOT 401',
|
|
359
|
+
)).toBe(403);
|
|
360
|
+
});
|
|
361
|
+
});
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bulk-cancel scenario (closes R1 from rest-endpoints.md §Open spec gaps).
|
|
3
|
+
*
|
|
4
|
+
* Verifies `POST /v1/runs:bulk-cancel` per
|
|
5
|
+
* `spec/v1/rest-endpoints.md` §"POST /v1/runs:bulk-cancel":
|
|
6
|
+
*
|
|
7
|
+
* 1. Per-id results array shape (`{runId, ok, status?, error?}`).
|
|
8
|
+
* 2. Mixed-outcome request: known + unknown + already-terminal runIds
|
|
9
|
+
* MUST each surface their own outcome — partial failures do NOT
|
|
10
|
+
* block sibling cancellations.
|
|
11
|
+
* 3. Empty `runIds` array → 400 validation_error.
|
|
12
|
+
* 4. Oversized array (>100 by spec) → 400 validation_error with
|
|
13
|
+
* `details.maxRunIds`.
|
|
14
|
+
* 5. Idempotency: re-bulk-cancelling already-cancelled runs returns
|
|
15
|
+
* `ok: true, status: 'cancelled'` (idempotent), NOT an error.
|
|
16
|
+
*
|
|
17
|
+
* Normative reference: spec/v1/rest-endpoints.md §"POST /v1/runs:bulk-cancel"
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { describe, it, expect } from 'vitest';
|
|
21
|
+
import { driver } from '../lib/driver.js';
|
|
22
|
+
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
23
|
+
|
|
24
|
+
const CANCELLABLE = 'conformance-cancellable';
|
|
25
|
+
const NOOP = 'conformance-noop';
|
|
26
|
+
const SKIP =
|
|
27
|
+
!isFixtureAdvertised(CANCELLABLE) || !isFixtureAdvertised(NOOP);
|
|
28
|
+
|
|
29
|
+
interface BulkResult {
|
|
30
|
+
runId: string;
|
|
31
|
+
ok: boolean;
|
|
32
|
+
status?: 'cancelled' | 'cancelling';
|
|
33
|
+
error?: { code?: string; message?: string };
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
describe.skipIf(SKIP)('bulk-cancel: POST /v1/runs:bulk-cancel', () => {
|
|
37
|
+
it('mixed-outcome request returns per-id results in order', async () => {
|
|
38
|
+
// Spin up a long-running cancellable run + observe a known-bad id
|
|
39
|
+
// alongside it. The host MUST handle each independently.
|
|
40
|
+
const create = await driver.post('/v1/runs', {
|
|
41
|
+
workflowId: CANCELLABLE,
|
|
42
|
+
inputs: { delaySeconds: 30 },
|
|
43
|
+
});
|
|
44
|
+
expect(create.status).toBe(201);
|
|
45
|
+
const inflightRunId = (create.json as { runId: string }).runId;
|
|
46
|
+
|
|
47
|
+
const res = await driver.post('/v1/runs:bulk-cancel', {
|
|
48
|
+
runIds: [inflightRunId, 'run-does-not-exist-xxxxxxxx'],
|
|
49
|
+
reason: 'conformance bulk-cancel test',
|
|
50
|
+
});
|
|
51
|
+
expect(res.status, driver.describe(
|
|
52
|
+
'rest-endpoints.md §"POST /v1/runs:bulk-cancel"',
|
|
53
|
+
'top-level operation MUST return 200 when the request reached the host (per-id outcomes carry partial failure)',
|
|
54
|
+
)).toBe(200);
|
|
55
|
+
|
|
56
|
+
const body = res.json as { results: BulkResult[] };
|
|
57
|
+
expect(Array.isArray(body.results)).toBe(true);
|
|
58
|
+
expect(body.results.length, 'results MUST have one entry per request runId').toBe(2);
|
|
59
|
+
|
|
60
|
+
expect(body.results[0]!.runId, 'results order MUST mirror the request order').toBe(inflightRunId);
|
|
61
|
+
expect(body.results[0]!.ok).toBe(true);
|
|
62
|
+
expect(['cancelling', 'cancelled']).toContain(body.results[0]!.status);
|
|
63
|
+
|
|
64
|
+
expect(body.results[1]!.runId).toBe('run-does-not-exist-xxxxxxxx');
|
|
65
|
+
expect(body.results[1]!.ok, 'unknown runId entry MUST have ok=false').toBe(false);
|
|
66
|
+
expect(body.results[1]!.error?.code, driver.describe(
|
|
67
|
+
'rest-endpoints.md §"POST /v1/runs:bulk-cancel"',
|
|
68
|
+
'unknown runId outcomes carry `error.code === "not_found"`',
|
|
69
|
+
)).toBe('not_found');
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
it('empty runIds array returns 400 validation_error', async () => {
|
|
73
|
+
const res = await driver.post('/v1/runs:bulk-cancel', { runIds: [] });
|
|
74
|
+
expect(res.status).toBe(400);
|
|
75
|
+
const body = res.json as { error?: string };
|
|
76
|
+
expect(body.error).toBe('validation_error');
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
it('oversized runIds array returns 400 with details.maxRunIds', async () => {
|
|
80
|
+
// 101 entries — exceeds the recommended 100-entry cap.
|
|
81
|
+
const ids = Array.from({ length: 101 }, (_, i) => `run-overflow-${i}`);
|
|
82
|
+
const res = await driver.post('/v1/runs:bulk-cancel', { runIds: ids });
|
|
83
|
+
expect(res.status).toBe(400);
|
|
84
|
+
const body = res.json as { error?: string; details?: { maxRunIds?: number } };
|
|
85
|
+
expect(body.error).toBe('validation_error');
|
|
86
|
+
expect(typeof body.details?.maxRunIds, driver.describe(
|
|
87
|
+
'rest-endpoints.md §"POST /v1/runs:bulk-cancel"',
|
|
88
|
+
'over-cap request MUST carry details.maxRunIds disclosing the configured ceiling',
|
|
89
|
+
)).toBe('number');
|
|
90
|
+
expect(body.details!.maxRunIds!).toBeGreaterThanOrEqual(1);
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
it('re-bulk-cancel after first cancel is idempotent', async () => {
|
|
94
|
+
const create = await driver.post('/v1/runs', {
|
|
95
|
+
workflowId: CANCELLABLE,
|
|
96
|
+
inputs: { delaySeconds: 30 },
|
|
97
|
+
});
|
|
98
|
+
expect(create.status).toBe(201);
|
|
99
|
+
const runId = (create.json as { runId: string }).runId;
|
|
100
|
+
|
|
101
|
+
const first = await driver.post('/v1/runs:bulk-cancel', { runIds: [runId] });
|
|
102
|
+
expect(first.status).toBe(200);
|
|
103
|
+
const second = await driver.post('/v1/runs:bulk-cancel', { runIds: [runId] });
|
|
104
|
+
expect(second.status).toBe(200);
|
|
105
|
+
const body = second.json as { results: BulkResult[] };
|
|
106
|
+
expect(body.results[0]!.ok, driver.describe(
|
|
107
|
+
'rest-endpoints.md §"POST /v1/runs:bulk-cancel" §Idempotency',
|
|
108
|
+
're-cancelling an already-cancelling/cancelled run MUST be ok: true (idempotent)',
|
|
109
|
+
)).toBe(true);
|
|
110
|
+
});
|
|
111
|
+
});
|
|
@@ -73,4 +73,52 @@ describe('configurable-schema: per-workflow schema enforced', () => {
|
|
|
73
73
|
const body = create.json as { error?: string };
|
|
74
74
|
expect(body.error).toBe('validation_error');
|
|
75
75
|
});
|
|
76
|
+
|
|
77
|
+
it('configurable overlay matching configurableSchema is accepted', async () => {
|
|
78
|
+
const fixture = await pickFixture();
|
|
79
|
+
if (!fixture) return; // covered by skip warning above
|
|
80
|
+
|
|
81
|
+
const manifest = await driver.get(`/v1/workflows/${encodeURIComponent(fixture)}`);
|
|
82
|
+
const schema = (manifest.json as { configurableSchema?: Record<string, unknown> })
|
|
83
|
+
.configurableSchema;
|
|
84
|
+
if (!schema) return;
|
|
85
|
+
|
|
86
|
+
// Build a minimal valid overlay derived from the schema's first
|
|
87
|
+
// `properties.*` entry. This stays generic across fixtures: we pick
|
|
88
|
+
// the first integer property with a `minimum` (if present) and emit
|
|
89
|
+
// a value at that minimum. Falls back to {} when no usable property
|
|
90
|
+
// is declared.
|
|
91
|
+
const props = (schema.properties ?? {}) as Record<string, Record<string, unknown>>;
|
|
92
|
+
const overlay: Record<string, unknown> = {};
|
|
93
|
+
for (const [key, p] of Object.entries(props)) {
|
|
94
|
+
if (p.type === 'integer') {
|
|
95
|
+
const min = typeof p.minimum === 'number' ? p.minimum : 1;
|
|
96
|
+
overlay[key] = min;
|
|
97
|
+
break;
|
|
98
|
+
}
|
|
99
|
+
if (p.type === 'string') {
|
|
100
|
+
overlay[key] = 'conformance-test';
|
|
101
|
+
break;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const create = await driver.post('/v1/runs', {
|
|
106
|
+
workflowId: fixture,
|
|
107
|
+
configurable: overlay,
|
|
108
|
+
});
|
|
109
|
+
expect(create.status, driver.describe(
|
|
110
|
+
'run-options.md §"Per-workflow configurableSchema"',
|
|
111
|
+
'configurable matching the declared schema MUST be accepted (201)',
|
|
112
|
+
)).toBe(201);
|
|
113
|
+
|
|
114
|
+
const body = create.json as { runId?: string };
|
|
115
|
+
expect(typeof body.runId).toBe('string');
|
|
116
|
+
|
|
117
|
+
// Clean up so subsequent scenario runs don't accumulate state.
|
|
118
|
+
if (body.runId) {
|
|
119
|
+
await driver.post(`/v1/runs/${encodeURIComponent(body.runId)}/cancel`, {
|
|
120
|
+
reason: 'conformance-cleanup',
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
});
|
|
76
124
|
});
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Multi-Agent Shift Phase 4 — replay-fork of a conversation produces identical log.
|
|
3
|
+
* Normative reference: RFCS/0005-conversation.md
|
|
3
4
|
*
|
|
4
5
|
* Verifies that running `:fork` on a conversation-bearing run yields
|
|
5
6
|
* a child run whose conversation log (folded via the `message` reducer)
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Multi-Agent Shift Phase 4 — `conversation.exchange` differs from `clarification.requested`.
|
|
3
|
+
* Normative reference: RFCS/0005-conversation.md
|
|
3
4
|
*
|
|
4
5
|
* Verifies that `core.conversationGate.exchange` produces
|
|
5
6
|
* `conversation.exchanged` events in the run log — distinct from the
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Debug-bundle truncation contract (debug-bundle.md §"Bundle size limits").
|
|
3
|
+
*
|
|
4
|
+
* Verifies that when a bundle would exceed the host's size cap, the
|
|
5
|
+
* response surfaces `truncated: true` + a non-empty `truncatedReason`
|
|
6
|
+
* and the `events` array is a strict prefix of what the run produced.
|
|
7
|
+
*
|
|
8
|
+
* Driving truncation deterministically requires either a fixture that
|
|
9
|
+
* generates ≥ 8MB of events (impractical) or a host-implementation
|
|
10
|
+
* override. The SQLite reference host accepts a `?maxEvents=N` query
|
|
11
|
+
* parameter (host-implementation choice per the spec — "Hosts MAY
|
|
12
|
+
* raise the cap via implementation-defined configuration"). When
|
|
13
|
+
* neither the cap can be lowered nor a high-event fixture is available,
|
|
14
|
+
* this scenario soft-skips the assertion.
|
|
15
|
+
*
|
|
16
|
+
* @see spec/v1/debug-bundle.md §"Bundle size limits"
|
|
17
|
+
* @see spec/v1/production-profile.md §"Debug bundle behavior" (RFC 0009
|
|
18
|
+
* — this scenario satisfies the truncation-metadata predicate when
|
|
19
|
+
* the host advertises capabilities.production.debugBundle.truncationMetadata: true)
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { describe, it, expect } from 'vitest';
|
|
23
|
+
import { driver } from '../lib/driver.js';
|
|
24
|
+
import { pollUntilTerminal } from '../lib/polling.js';
|
|
25
|
+
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
26
|
+
|
|
27
|
+
// `conformance-multi-node` produces enough events (run.started, three
|
|
28
|
+
// node.started/completed pairs, run.completed = ~8 events) that
|
|
29
|
+
// `?maxEvents=2` reliably forces truncation.
|
|
30
|
+
const FIXTURE = 'conformance-multi-node';
|
|
31
|
+
|
|
32
|
+
describe('debug-bundle-truncation: truncated: true contract', () => {
|
|
33
|
+
it('host that supports ?maxEvents=N (or otherwise caps) surfaces truncated + truncatedReason', async () => {
|
|
34
|
+
if (!isFixtureAdvertised(FIXTURE)) {
|
|
35
|
+
// eslint-disable-next-line no-console
|
|
36
|
+
console.warn(
|
|
37
|
+
`[debug-bundle-truncation] ${FIXTURE} not advertised; skipping (host doesn't seed a multi-event fixture)`,
|
|
38
|
+
);
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const create = await driver.post('/v1/runs', { workflowId: FIXTURE });
|
|
43
|
+
expect(create.status).toBe(201);
|
|
44
|
+
const runId = (create.json as { runId: string }).runId;
|
|
45
|
+
await pollUntilTerminal(runId, { timeoutMs: 15_000 });
|
|
46
|
+
|
|
47
|
+
// First call: full bundle, so we know how many events the run produced.
|
|
48
|
+
const full = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/debug-bundle`);
|
|
49
|
+
expect(full.status).toBe(200);
|
|
50
|
+
const fullBody = full.json as { events?: unknown[]; truncated?: boolean };
|
|
51
|
+
const fullEventCount = (fullBody.events ?? []).length;
|
|
52
|
+
expect(fullEventCount, 'multi-node fixture MUST emit ≥ 3 events').toBeGreaterThanOrEqual(3);
|
|
53
|
+
expect(fullBody.truncated ?? false, 'baseline bundle MUST NOT be truncated').toBe(false);
|
|
54
|
+
|
|
55
|
+
// Force truncation via the host's optional maxEvents override.
|
|
56
|
+
const truncated = await driver.get(
|
|
57
|
+
`/v1/runs/${encodeURIComponent(runId)}/debug-bundle?maxEvents=2`,
|
|
58
|
+
);
|
|
59
|
+
|
|
60
|
+
// Hosts that don't honor `?maxEvents=` will return the full bundle
|
|
61
|
+
// (truncated: false). Soft-skip the assertion in that case so the
|
|
62
|
+
// suite remains forward-compatible with hosts using a different
|
|
63
|
+
// truncation-forcing mechanism.
|
|
64
|
+
const body = truncated.json as {
|
|
65
|
+
truncated?: boolean;
|
|
66
|
+
truncatedReason?: string;
|
|
67
|
+
events?: unknown[];
|
|
68
|
+
metrics?: { eventCount?: number };
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
if (body.truncated !== true) {
|
|
72
|
+
// eslint-disable-next-line no-console
|
|
73
|
+
console.warn(
|
|
74
|
+
'[debug-bundle-truncation] host does not honor ?maxEvents=; skipping truncated-shape assertions',
|
|
75
|
+
);
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
expect(typeof body.truncatedReason, driver.describe(
|
|
80
|
+
'debug-bundle.md §"Bundle size limits"',
|
|
81
|
+
'truncated: true MUST be accompanied by a non-empty truncatedReason string',
|
|
82
|
+
)).toBe('string');
|
|
83
|
+
expect((body.truncatedReason ?? '').length).toBeGreaterThan(0);
|
|
84
|
+
|
|
85
|
+
expect((body.events ?? []).length, driver.describe(
|
|
86
|
+
'debug-bundle.md §"Bundle size limits"',
|
|
87
|
+
'truncated events array MUST be a prefix (≤ maxEvents)',
|
|
88
|
+
)).toBeLessThanOrEqual(2);
|
|
89
|
+
|
|
90
|
+
expect(body.metrics?.eventCount, driver.describe(
|
|
91
|
+
'debug-bundle.md §"Bundle size limits"',
|
|
92
|
+
'metrics.eventCount MUST reflect the TOTAL event count, not the truncated length',
|
|
93
|
+
)).toBe(fullEventCount);
|
|
94
|
+
});
|
|
95
|
+
});
|