@openwop/openwop-conformance 1.18.0 → 1.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +2 -2
- package/api/.redocly.lint-ignore.yaml +22 -0
- package/api/openapi.yaml +13 -4
- package/coverage.md +2 -1
- package/dist/cli.js +235 -4
- package/dist/lib/paths.js +160 -0
- package/dist/lib/profiles.js +461 -0
- package/fixtures/conformance-agent-channel-dispatch.json +27 -0
- package/fixtures.md +15 -0
- package/package.json +1 -1
- package/schemas/README.md +1 -0
- package/schemas/capabilities.schema.json +5 -0
- package/schemas/conformance-certification-bundle.schema.json +86 -0
- package/src/cli.ts +268 -4
- package/src/lib/profiles.ts +85 -0
- package/src/scenarios/agent-channel-dispatch.test.ts +229 -0
- package/src/scenarios/auth-saml-profile.test.ts +51 -14
- package/src/scenarios/spec-corpus-validity.test.ts +183 -0
package/src/cli.ts
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
* openwop-conformance --offline # server-free subset only
|
|
12
12
|
* openwop-conformance --filter discovery # category filter
|
|
13
13
|
* openwop-conformance --base-url ... --api-key ... --filter "interrupt|cancellation"
|
|
14
|
+
* openwop-conformance --base-url ... --api-key ... --certify out.json # RFC 0089 bundle
|
|
14
15
|
*
|
|
15
16
|
* Environment variables override flags (per the conformance harness's
|
|
16
17
|
* existing convention):
|
|
@@ -25,7 +26,19 @@
|
|
|
25
26
|
|
|
26
27
|
import { spawnSync } from 'node:child_process';
|
|
27
28
|
import { fileURLToPath } from 'node:url';
|
|
28
|
-
import { dirname, resolve as resolvePath } from 'node:path';
|
|
29
|
+
import { dirname, resolve as resolvePath, join } from 'node:path';
|
|
30
|
+
import { createHash } from 'node:crypto';
|
|
31
|
+
import { readFileSync, writeFileSync, mkdtempSync, rmSync } from 'node:fs';
|
|
32
|
+
import { tmpdir } from 'node:os';
|
|
33
|
+
import Ajv2020 from 'ajv/dist/2020.js';
|
|
34
|
+
import addFormats from 'ajv-formats';
|
|
35
|
+
import { SCHEMAS_DIR } from './lib/paths.js';
|
|
36
|
+
import {
|
|
37
|
+
deriveProfiles,
|
|
38
|
+
isCoreStandard,
|
|
39
|
+
agentPlatformStatus,
|
|
40
|
+
type DiscoveryPayload,
|
|
41
|
+
} from './lib/profiles.js';
|
|
29
42
|
|
|
30
43
|
interface ParsedArgs {
|
|
31
44
|
readonly baseUrl: string | undefined;
|
|
@@ -35,6 +48,8 @@ interface ParsedArgs {
|
|
|
35
48
|
readonly help: boolean;
|
|
36
49
|
readonly impl: string | undefined;
|
|
37
50
|
readonly implVersion: string | undefined;
|
|
51
|
+
/** RFC 0089 — emit a conformance certification bundle to this path. */
|
|
52
|
+
readonly certify: string | undefined;
|
|
38
53
|
}
|
|
39
54
|
|
|
40
55
|
function parseArgs(argv: readonly string[]): ParsedArgs {
|
|
@@ -45,6 +60,7 @@ function parseArgs(argv: readonly string[]): ParsedArgs {
|
|
|
45
60
|
let help = false;
|
|
46
61
|
let impl: string | undefined;
|
|
47
62
|
let implVersion: string | undefined;
|
|
63
|
+
let certify: string | undefined;
|
|
48
64
|
|
|
49
65
|
for (let i = 0; i < argv.length; i++) {
|
|
50
66
|
const arg = argv[i] ?? '';
|
|
@@ -87,6 +103,9 @@ function parseArgs(argv: readonly string[]): ParsedArgs {
|
|
|
87
103
|
case '--implementation-version':
|
|
88
104
|
implVersion = nextValue();
|
|
89
105
|
break;
|
|
106
|
+
case '--certify':
|
|
107
|
+
certify = nextValue();
|
|
108
|
+
break;
|
|
90
109
|
default:
|
|
91
110
|
if (arg.startsWith('-')) {
|
|
92
111
|
// Unknown flag — pass through to vitest by ignoring here.
|
|
@@ -94,7 +113,7 @@ function parseArgs(argv: readonly string[]): ParsedArgs {
|
|
|
94
113
|
}
|
|
95
114
|
}
|
|
96
115
|
|
|
97
|
-
return { baseUrl, apiKey, offline, filter, help, impl, implVersion };
|
|
116
|
+
return { baseUrl, apiKey, offline, filter, help, impl, implVersion, certify };
|
|
98
117
|
}
|
|
99
118
|
|
|
100
119
|
const HELP_TEXT = `openwop-conformance — run the openwop conformance suite against a server
|
|
@@ -114,6 +133,14 @@ Implementation labels (cosmetic — surface in failure messages):
|
|
|
114
133
|
--impl <name> Implementation name (env: OPENWOP_IMPLEMENTATION_NAME)
|
|
115
134
|
--impl-version <version> Implementation version (env: OPENWOP_IMPLEMENTATION_VERSION)
|
|
116
135
|
|
|
136
|
+
Certification (RFC 0089):
|
|
137
|
+
--certify <out.json> Generate a machine-readable conformance certification
|
|
138
|
+
bundle: fetch /.well-known/openwop (captured verbatim +
|
|
139
|
+
SHA-256), derive claimedProfiles from it, run the suite
|
|
140
|
+
recording each scenario's terminal state, validate the
|
|
141
|
+
assembled bundle against the bundle schema, and write it
|
|
142
|
+
to <out.json>. Requires --base-url (and --api-key as usual).
|
|
143
|
+
|
|
117
144
|
Other:
|
|
118
145
|
--help, -h Show this message
|
|
119
146
|
|
|
@@ -121,9 +148,234 @@ Examples:
|
|
|
121
148
|
openwop-conformance --offline
|
|
122
149
|
openwop-conformance --base-url https://api.example.com --api-key hk_test_abc
|
|
123
150
|
openwop-conformance --filter "discovery|errors"
|
|
151
|
+
openwop-conformance --base-url https://api.example.com --api-key hk_test_abc \\
|
|
152
|
+
--certify certification-bundle.json
|
|
124
153
|
`;
|
|
125
154
|
|
|
126
|
-
|
|
155
|
+
/** This CLI package's own version — surfaced as `generator.version` + `suite.version`. */
|
|
156
|
+
function suiteVersion(): string {
|
|
157
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
158
|
+
const pkgPath = resolvePath(here, '..', 'package.json');
|
|
159
|
+
const pkg = JSON.parse(readFileSync(pkgPath, 'utf8')) as { version?: unknown };
|
|
160
|
+
return typeof pkg.version === 'string' ? pkg.version : '0.0.0';
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Deterministic canonical-JSON serialization (RFC 8785 spirit): object keys
|
|
165
|
+
* sorted lexicographically at every level, arrays preserved in order. Used to
|
|
166
|
+
* compute `discovery.sha256` so a verifier can re-derive the same digest from
|
|
167
|
+
* a live `/.well-known/openwop` fetch regardless of incidental key order.
|
|
168
|
+
*/
|
|
169
|
+
function canonicalJSON(value: unknown): string {
|
|
170
|
+
if (value === null || typeof value !== 'object') return JSON.stringify(value);
|
|
171
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJSON).join(',')}]`;
|
|
172
|
+
const obj = value as Record<string, unknown>;
|
|
173
|
+
const keys = Object.keys(obj).sort();
|
|
174
|
+
return `{${keys.map((k) => `${JSON.stringify(k)}:${canonicalJSON(obj[k])}`).join(',')}}`;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* The full set of profiles a discovery document derives — the closed
|
|
179
|
+
* `deriveProfiles` catalog plus the two operational annexes
|
|
180
|
+
* (`openwop-core-standard`, `openwop-agent-platform`) when their discovery
|
|
181
|
+
* predicate holds. This is `claimedProfiles`: a generated bundle MUST NOT
|
|
182
|
+
* claim a profile its own discovery document does not derive (RFC 0089 §B(1)).
|
|
183
|
+
*/
|
|
184
|
+
function claimedProfilesFor(doc: DiscoveryPayload): string[] {
|
|
185
|
+
const profiles: string[] = [...deriveProfiles(doc)];
|
|
186
|
+
if (isCoreStandard(doc)) profiles.push('openwop-core-standard');
|
|
187
|
+
if (agentPlatformStatus(doc) !== 'none') profiles.push('openwop-agent-platform');
|
|
188
|
+
return profiles;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** A single scenario test file's terminal state, derived from the vitest JSON report. */
|
|
192
|
+
type ScenarioState = 'passed' | 'failed' | 'skipped';
|
|
193
|
+
|
|
194
|
+
/** The subset of vitest's JSON reporter output we read. */
|
|
195
|
+
interface VitestJsonReport {
|
|
196
|
+
readonly testResults?: ReadonlyArray<{
|
|
197
|
+
readonly name?: string;
|
|
198
|
+
readonly assertionResults?: ReadonlyArray<{ readonly status?: string }>;
|
|
199
|
+
}>;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* Reduce a vitest JSON report into a per-scenario-file terminal state, keyed by
|
|
204
|
+
* the test-file basename (e.g. `discovery.test.ts`) to align with the basenames
|
|
205
|
+
* in `PROFILE_FLOOR_SCENARIOS`. A file is `passed` only if it ran AND had ≥1
|
|
206
|
+
* passing assertion AND zero failures (non-vacuous, per §C); a fully-skipped
|
|
207
|
+
* file is `skipped`; any failed assertion makes the file `failed`.
|
|
208
|
+
*/
|
|
209
|
+
function scenarioStatesFromReport(report: VitestJsonReport): Map<string, ScenarioState> {
|
|
210
|
+
const states = new Map<string, ScenarioState>();
|
|
211
|
+
for (const file of report.testResults ?? []) {
|
|
212
|
+
const name = file.name;
|
|
213
|
+
if (typeof name !== 'string') continue;
|
|
214
|
+
const basename = name.split('/').pop() ?? name;
|
|
215
|
+
const assertions = file.assertionResults ?? [];
|
|
216
|
+
let passes = 0;
|
|
217
|
+
let failures = 0;
|
|
218
|
+
let nonSkipped = 0;
|
|
219
|
+
for (const a of assertions) {
|
|
220
|
+
if (a.status === 'passed') {
|
|
221
|
+
passes++;
|
|
222
|
+
nonSkipped++;
|
|
223
|
+
} else if (a.status === 'failed') {
|
|
224
|
+
failures++;
|
|
225
|
+
nonSkipped++;
|
|
226
|
+
}
|
|
227
|
+
// `skipped` / `todo` / `pending` count toward neither pass nor fail.
|
|
228
|
+
}
|
|
229
|
+
let state: ScenarioState;
|
|
230
|
+
if (failures > 0) state = 'failed';
|
|
231
|
+
else if (passes > 0 && nonSkipped > 0) state = 'passed';
|
|
232
|
+
else state = 'skipped';
|
|
233
|
+
states.set(basename, state);
|
|
234
|
+
}
|
|
235
|
+
return states;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/** Generate + validate + write an RFC 0089 conformance certification bundle. */
|
|
239
|
+
async function runCertify(args: ParsedArgs, baseUrl: string, apiKey: string): Promise<never> {
|
|
240
|
+
const outPath = args.certify;
|
|
241
|
+
if (outPath === undefined) process.exit(2);
|
|
242
|
+
|
|
243
|
+
// (a) Fetch /.well-known/openwop verbatim + its canonical-JSON SHA-256.
|
|
244
|
+
const discoveryUrl = `${baseUrl.replace(/\/$/, '')}/.well-known/openwop`;
|
|
245
|
+
let document: DiscoveryPayload;
|
|
246
|
+
try {
|
|
247
|
+
const resp = await fetch(discoveryUrl, { headers: { Accept: 'application/json' } });
|
|
248
|
+
if (!resp.ok) {
|
|
249
|
+
process.stderr.write(
|
|
250
|
+
`openwop-conformance --certify: GET ${discoveryUrl} returned HTTP ${resp.status}.\n`,
|
|
251
|
+
);
|
|
252
|
+
process.exit(2);
|
|
253
|
+
}
|
|
254
|
+
document = (await resp.json()) as DiscoveryPayload;
|
|
255
|
+
} catch (err) {
|
|
256
|
+
process.stderr.write(
|
|
257
|
+
`openwop-conformance --certify: failed to fetch ${discoveryUrl}: ${String(err)}\n`,
|
|
258
|
+
);
|
|
259
|
+
process.exit(2);
|
|
260
|
+
}
|
|
261
|
+
const sha256 = createHash('sha256').update(canonicalJSON(document)).digest('hex');
|
|
262
|
+
|
|
263
|
+
// (b) Derive claimedProfiles from the captured document.
|
|
264
|
+
const claimedProfiles = claimedProfilesFor(document);
|
|
265
|
+
|
|
266
|
+
// (c) Run the suite, capturing per-scenario terminal state via the vitest
|
|
267
|
+
// JSON reporter. server-targeted scenarios live under src/scenarios/.
|
|
268
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
269
|
+
const conformanceRoot = resolvePath(here, '..');
|
|
270
|
+
const reportDir = mkdtempSync(join(tmpdir(), 'owp-certify-'));
|
|
271
|
+
const reportFile = join(reportDir, 'vitest-report.json');
|
|
272
|
+
const env: NodeJS.ProcessEnv = { ...process.env };
|
|
273
|
+
env.OPENWOP_BASE_URL = baseUrl;
|
|
274
|
+
env.OPENWOP_API_KEY = apiKey;
|
|
275
|
+
if (args.impl) env.OPENWOP_IMPLEMENTATION_NAME = args.impl;
|
|
276
|
+
if (args.implVersion) env.OPENWOP_IMPLEMENTATION_VERSION = args.implVersion;
|
|
277
|
+
|
|
278
|
+
const vitestArgs: string[] = [
|
|
279
|
+
'vitest',
|
|
280
|
+
'run',
|
|
281
|
+
'--config',
|
|
282
|
+
resolvePath(conformanceRoot, 'vitest.config.ts'),
|
|
283
|
+
'--reporter=json',
|
|
284
|
+
`--outputFile=${reportFile}`,
|
|
285
|
+
];
|
|
286
|
+
const runResult = spawnSync('npx', vitestArgs, { cwd: conformanceRoot, env, stdio: 'inherit' });
|
|
287
|
+
if (runResult.error) {
|
|
288
|
+
process.stderr.write(
|
|
289
|
+
`openwop-conformance --certify: failed to spawn vitest: ${String(runResult.error)}\n`,
|
|
290
|
+
);
|
|
291
|
+
process.exit(2);
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
let report: VitestJsonReport;
|
|
295
|
+
try {
|
|
296
|
+
report = JSON.parse(readFileSync(reportFile, 'utf8')) as VitestJsonReport;
|
|
297
|
+
} catch (err) {
|
|
298
|
+
process.stderr.write(
|
|
299
|
+
`openwop-conformance --certify: could not read vitest JSON report at ${reportFile}: ${String(err)}\n`,
|
|
300
|
+
);
|
|
301
|
+
process.exit(2);
|
|
302
|
+
} finally {
|
|
303
|
+
rmSync(reportDir, { recursive: true, force: true });
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
const states = scenarioStatesFromReport(report);
|
|
307
|
+
const passed: string[] = [];
|
|
308
|
+
const failed: string[] = [];
|
|
309
|
+
const skipped: string[] = [];
|
|
310
|
+
for (const [basename, state] of [...states.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
|
|
311
|
+
if (state === 'passed') passed.push(basename);
|
|
312
|
+
else if (state === 'failed') failed.push(basename);
|
|
313
|
+
else skipped.push(basename);
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
// (d) Assemble the bundle.
|
|
317
|
+
const version = suiteVersion();
|
|
318
|
+
const impl = (document as { implementation?: { name?: unknown; version?: unknown; vendor?: unknown } })
|
|
319
|
+
.implementation;
|
|
320
|
+
const hostName =
|
|
321
|
+
args.impl ?? (typeof impl?.name === 'string' ? impl.name : 'unknown-host');
|
|
322
|
+
const hostVersion =
|
|
323
|
+
args.implVersion ?? (typeof impl?.version === 'string' ? impl.version : '0.0.0');
|
|
324
|
+
const host: { name: string; version: string; vendor?: string } = {
|
|
325
|
+
name: hostName,
|
|
326
|
+
version: hostVersion,
|
|
327
|
+
};
|
|
328
|
+
if (typeof impl?.vendor === 'string') host.vendor = impl.vendor;
|
|
329
|
+
|
|
330
|
+
const bundle = {
|
|
331
|
+
bundleVersion: '1' as const,
|
|
332
|
+
generatedAt: new Date().toISOString(),
|
|
333
|
+
generator: { name: '@openwop/openwop-conformance --certify', version },
|
|
334
|
+
suite: { package: '@openwop/openwop-conformance' as const, version },
|
|
335
|
+
host,
|
|
336
|
+
discovery: { url: discoveryUrl, sha256, document },
|
|
337
|
+
claimedProfiles,
|
|
338
|
+
results: {
|
|
339
|
+
totals: {
|
|
340
|
+
passed: passed.length,
|
|
341
|
+
failed: failed.length,
|
|
342
|
+
skipped: skipped.length,
|
|
343
|
+
total: passed.length + failed.length + skipped.length,
|
|
344
|
+
},
|
|
345
|
+
passed,
|
|
346
|
+
failed,
|
|
347
|
+
skipped,
|
|
348
|
+
},
|
|
349
|
+
};
|
|
350
|
+
|
|
351
|
+
// (e) Validate against the bundle schema BEFORE writing.
|
|
352
|
+
const schema = JSON.parse(
|
|
353
|
+
readFileSync(join(SCHEMAS_DIR, 'conformance-certification-bundle.schema.json'), 'utf8'),
|
|
354
|
+
) as Record<string, unknown>;
|
|
355
|
+
const ajv = new Ajv2020({ allErrors: true, strict: false });
|
|
356
|
+
addFormats(ajv);
|
|
357
|
+
const validate = ajv.compile(schema);
|
|
358
|
+
if (!validate(bundle)) {
|
|
359
|
+
process.stderr.write(
|
|
360
|
+
'openwop-conformance --certify: assembled bundle FAILED schema validation:\n' +
|
|
361
|
+
`${JSON.stringify(validate.errors, null, 2)}\n`,
|
|
362
|
+
);
|
|
363
|
+
process.exit(2);
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
writeFileSync(outPath, `${JSON.stringify(bundle, null, 2)}\n`);
|
|
367
|
+
process.stdout.write(
|
|
368
|
+
`openwop-conformance --certify: wrote certification bundle to ${outPath}\n` +
|
|
369
|
+
` host: ${host.name}@${host.version}\n` +
|
|
370
|
+
` claimedProfiles: ${claimedProfiles.length > 0 ? claimedProfiles.join(', ') : '(none)'}\n` +
|
|
371
|
+
` results: ${passed.length} passed / ${failed.length} failed / ${skipped.length} skipped\n`,
|
|
372
|
+
);
|
|
373
|
+
// Exit code mirrors the suite outcome: a failing run still produces a bundle
|
|
374
|
+
// (the failures are recorded), but the process exit reflects pass/fail.
|
|
375
|
+
process.exit(failed.length > 0 ? 1 : 0);
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
async function main(): Promise<never> {
|
|
127
379
|
const args = parseArgs(process.argv.slice(2));
|
|
128
380
|
|
|
129
381
|
if (args.help) {
|
|
@@ -139,6 +391,18 @@ function main(): never {
|
|
|
139
391
|
if (args.impl) env.OPENWOP_IMPLEMENTATION_NAME = args.impl;
|
|
140
392
|
if (args.implVersion) env.OPENWOP_IMPLEMENTATION_VERSION = args.implVersion;
|
|
141
393
|
|
|
394
|
+
// RFC 0089 — certification-bundle generation requires a live host.
|
|
395
|
+
if (args.certify !== undefined) {
|
|
396
|
+
if (!env.OPENWOP_BASE_URL || !env.OPENWOP_API_KEY) {
|
|
397
|
+
process.stderr.write(
|
|
398
|
+
'openwop-conformance --certify: --base-url and --api-key are required.\n' +
|
|
399
|
+
'Run `openwop-conformance --help` for usage.\n',
|
|
400
|
+
);
|
|
401
|
+
process.exit(2);
|
|
402
|
+
}
|
|
403
|
+
return runCertify(args, env.OPENWOP_BASE_URL, env.OPENWOP_API_KEY);
|
|
404
|
+
}
|
|
405
|
+
|
|
142
406
|
if (!args.offline && (!env.OPENWOP_BASE_URL || !env.OPENWOP_API_KEY)) {
|
|
143
407
|
process.stderr.write(
|
|
144
408
|
'openwop-conformance: --base-url and --api-key are required (or use --offline).\n' +
|
|
@@ -184,4 +448,4 @@ function main(): never {
|
|
|
184
448
|
process.exit(result.status ?? 1);
|
|
185
449
|
}
|
|
186
450
|
|
|
187
|
-
main();
|
|
451
|
+
void main();
|
package/src/lib/profiles.ts
CHANGED
|
@@ -428,3 +428,88 @@ export function hasProfile(c: DiscoveryPayload, profile: ProfileName): boolean {
|
|
|
428
428
|
return isTriggerBridge(c);
|
|
429
429
|
}
|
|
430
430
|
}
|
|
431
|
+
|
|
432
|
+
// ── RFC 0089: conformance certification bundle — floor-scenario sets + verifier ──
|
|
433
|
+
//
|
|
434
|
+
// G1 (RFC 0089): the §B binding rule needs each profile's REQUIRED floor
|
|
435
|
+
// scenarios in a machine-readable form. Source of truth for
|
|
436
|
+
// `openwop-core-standard` is `core-standard-profile.md` §C — the nine named
|
|
437
|
+
// black-box floor scenarios plus the `interrupt-*` family. Keyed by profile
|
|
438
|
+
// name (string) because annex profiles like `openwop-core-standard` sit outside
|
|
439
|
+
// the closed `PROFILE_NAMES` catalog.
|
|
440
|
+
|
|
441
|
+
export interface ProfileFloor {
|
|
442
|
+
/** Scenario files (basenames) that MUST each appear in `results.passed`. */
|
|
443
|
+
readonly required: readonly string[];
|
|
444
|
+
/** Prefix groups where ≥1 matching passed scenario satisfies the group. */
|
|
445
|
+
readonly requiredAnyPrefix?: readonly string[];
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
export const PROFILE_FLOOR_SCENARIOS: Readonly<Record<string, ProfileFloor>> = {
|
|
449
|
+
'openwop-core-standard': {
|
|
450
|
+
required: [
|
|
451
|
+
'runs-lifecycle.test.ts',
|
|
452
|
+
'discovery.test.ts',
|
|
453
|
+
'auth.test.ts',
|
|
454
|
+
'eventOrdering.test.ts',
|
|
455
|
+
'failure-path.test.ts',
|
|
456
|
+
'idempotency.test.ts',
|
|
457
|
+
'idempotency-key-determinism.test.ts',
|
|
458
|
+
'webhook-negative.test.ts',
|
|
459
|
+
'audit-log-verification.test.ts',
|
|
460
|
+
],
|
|
461
|
+
requiredAnyPrefix: ['interrupt-'],
|
|
462
|
+
},
|
|
463
|
+
};
|
|
464
|
+
|
|
465
|
+
/** Is `profile` derivable from a discovery document? Maps a profile name to its predicate (RFC 0089 §B(1)). */
|
|
466
|
+
export function profileDerivable(c: DiscoveryPayload, profile: string): boolean {
|
|
467
|
+
if (profile === 'openwop-core-standard') return isCoreStandard(c);
|
|
468
|
+
if (profile === 'openwop-agent-platform') return agentPlatformStatus(c) !== 'none';
|
|
469
|
+
if ((PROFILE_NAMES as readonly string[]).includes(profile)) {
|
|
470
|
+
return deriveProfiles(c).includes(profile as ProfileName);
|
|
471
|
+
}
|
|
472
|
+
return false;
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
/** Minimal shape of an RFC 0089 certification bundle the verifier reads. */
|
|
476
|
+
export interface CertificationBundleLike {
|
|
477
|
+
readonly discovery: { readonly document: DiscoveryPayload };
|
|
478
|
+
readonly claimedProfiles: readonly string[];
|
|
479
|
+
readonly results: { readonly passed: readonly string[] };
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
export interface BundleProfileVerdict {
|
|
483
|
+
readonly profile: string;
|
|
484
|
+
/** §B(1): derivable from the captured discovery document. */
|
|
485
|
+
readonly derivable: boolean;
|
|
486
|
+
/** §B(2): every floor scenario for the profile is in `results.passed`. */
|
|
487
|
+
readonly floorProven: boolean;
|
|
488
|
+
readonly valid: boolean;
|
|
489
|
+
readonly missingFloor: readonly string[];
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
const scenarioBasename = (id: string): string => id.split('/').pop() ?? id;
|
|
493
|
+
|
|
494
|
+
/**
|
|
495
|
+
* Verify a bundle's claim for one profile per RFC 0089 §B. A consumer MUST
|
|
496
|
+
* re-derive (this function) rather than trust `claimedProfiles` verbatim.
|
|
497
|
+
*/
|
|
498
|
+
export function verifyBundleProfile(bundle: CertificationBundleLike, profile: string): BundleProfileVerdict {
|
|
499
|
+
const derivable = profileDerivable(bundle.discovery.document, profile);
|
|
500
|
+
const floor = PROFILE_FLOOR_SCENARIOS[profile];
|
|
501
|
+
const passed = new Set(bundle.results.passed.map(scenarioBasename));
|
|
502
|
+
const missingFloor = floor ? floor.required.filter((r) => !passed.has(scenarioBasename(r))) : [];
|
|
503
|
+
const prefixOk = (floor?.requiredAnyPrefix ?? []).every((p) => [...passed].some((s) => s.startsWith(p)));
|
|
504
|
+
const floorProven = missingFloor.length === 0 && prefixOk;
|
|
505
|
+
return { profile, derivable, floorProven, valid: derivable && floorProven, missingFloor };
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
/** Verify every profile in `bundle.claimedProfiles`; the bundle is valid iff all claims are valid. */
|
|
509
|
+
export function verifyBundle(bundle: CertificationBundleLike): {
|
|
510
|
+
readonly valid: boolean;
|
|
511
|
+
readonly verdicts: readonly BundleProfileVerdict[];
|
|
512
|
+
} {
|
|
513
|
+
const verdicts = bundle.claimedProfiles.map((p) => verifyBundleProfile(bundle, p));
|
|
514
|
+
return { valid: verdicts.every((v) => v.valid), verdicts };
|
|
515
|
+
}
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* agent-channel-dispatch (RFC 0082 §B) — PRODUCTION run-graph channel pin +
|
|
3
|
+
* replay reuse, exercised black-box.
|
|
4
|
+
*
|
|
5
|
+
* Complements `agent-deployment-lifecycle.test.ts` Leg 4, which drives the §B
|
|
6
|
+
* pin through the host-sample `deployment-transition` SEAM. This scenario
|
|
7
|
+
* proves the SAME contract from a real run graph (no seam): a canonical
|
|
8
|
+
* `POST /v1/runs` of a workflow whose node binds `agent.channel` MUST
|
|
9
|
+
* (1) resolve the channel to a concrete version at first resolution and
|
|
10
|
+
* record it as `resolvedChannel` + `resolvedAgentVersion` on
|
|
11
|
+
* `agent.invocation.started` (RFC 0077 recorded fact), and
|
|
12
|
+
* (2) on `POST /v1/runs/{runId}:fork {mode:"replay"}` RE-READ that recorded
|
|
13
|
+
* version — and MUST NOT re-resolve a since-moved channel
|
|
14
|
+
* per `agent-deployment.md §B` and `version-negotiation.md`
|
|
15
|
+
* §"Channel resolution + replay determinism". This graduates §B from
|
|
16
|
+
* seam-proven to production-path-proven.
|
|
17
|
+
*
|
|
18
|
+
* Leg 3 (the load-bearing non-re-resolution proof) MOVES the `stable` channel
|
|
19
|
+
* between the original run and a replay fork via the optional deployment
|
|
20
|
+
* seam, then asserts the fork STILL carries the ORIGINAL pin (not the moved
|
|
21
|
+
* version). It self-guards: it runs only when the seam exists AND the move is
|
|
22
|
+
* observable (a fresh run resolves to a different version); otherwise it logs
|
|
23
|
+
* and skips its strict assertion without failing.
|
|
24
|
+
*
|
|
25
|
+
* Gating (root-first per RFC 0073): soft-skips unless the host advertises
|
|
26
|
+
* `agents.deployment.supported:true` AND seeds+advertises the
|
|
27
|
+
* `conformance-agent-channel-dispatch` fixture AND advertises replay mode
|
|
28
|
+
* `replay`. Visible skip by default; hard-fails under
|
|
29
|
+
* `OPENWOP_REQUIRE_BEHAVIOR=true` (per `lib/behavior-gate.ts`). Hosts that omit
|
|
30
|
+
* `agents.deployment` MUST reject a channel-bearing ref with `validation_error`
|
|
31
|
+
* (`agent-ref.schema.json`) and so cannot seed the fixture — they gate out.
|
|
32
|
+
*
|
|
33
|
+
* Spec references:
|
|
34
|
+
* - https://github.com/openwop/openwop/blob/main/spec/v1/agent-deployment.md (§B)
|
|
35
|
+
* - https://github.com/openwop/openwop/blob/main/spec/v1/version-negotiation.md
|
|
36
|
+
* - https://github.com/openwop/openwop/blob/main/RFCS/0082-agent-deployment-lifecycle.md
|
|
37
|
+
*/
|
|
38
|
+
|
|
39
|
+
import { describe, it, expect } from 'vitest';
|
|
40
|
+
import { driver } from '../lib/driver.js';
|
|
41
|
+
import { pollUntilTerminal } from '../lib/polling.js';
|
|
42
|
+
import { behaviorGate } from '../lib/behavior-gate.js';
|
|
43
|
+
import { isFixtureAdvertised } from '../lib/fixtures.js';
|
|
44
|
+
import { readDeploymentCap, driveDeploymentTransition } from '../lib/agentDeployment.js';
|
|
45
|
+
|
|
46
|
+
const FIXTURE_ID = 'conformance-agent-channel-dispatch';
|
|
47
|
+
const BOUND_CHANNEL = 'stable';
|
|
48
|
+
const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
|
|
49
|
+
|
|
50
|
+
interface RunEventDoc {
|
|
51
|
+
eventId: string;
|
|
52
|
+
runId: string;
|
|
53
|
+
type: string;
|
|
54
|
+
payload: Record<string, unknown>;
|
|
55
|
+
sequence: number;
|
|
56
|
+
}
|
|
57
|
+
interface PollEventsResponse {
|
|
58
|
+
events: RunEventDoc[];
|
|
59
|
+
isComplete?: boolean;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
async function readAllEvents(runId: string): Promise<RunEventDoc[]> {
|
|
63
|
+
const res = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/events/poll?lastSequence=0`);
|
|
64
|
+
if (res.status !== 200) return [];
|
|
65
|
+
const body = res.json as PollEventsResponse;
|
|
66
|
+
return body.events ?? [];
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Advertised replay modes (root-level `replay.modes`, RFC 0073 / profiles.ts). */
|
|
70
|
+
async function fetchReplayModes(): Promise<readonly string[]> {
|
|
71
|
+
const res = await driver.get('/.well-known/openwop', { authenticated: false });
|
|
72
|
+
if (res.status !== 200) return [];
|
|
73
|
+
const replay = (res.json as { replay?: { supported?: unknown; modes?: unknown } })?.replay;
|
|
74
|
+
if (replay?.supported !== true || !Array.isArray(replay.modes)) return [];
|
|
75
|
+
return replay.modes.filter((m): m is string => typeof m === 'string');
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** First `agent.invocation.started` (by sequence) of a run, or null. */
|
|
79
|
+
async function firstInvocationStarted(runId: string): Promise<RunEventDoc | null> {
|
|
80
|
+
const events = (await readAllEvents(runId))
|
|
81
|
+
.filter((e) => e.type === 'agent.invocation.started')
|
|
82
|
+
.sort((a, b) => a.sequence - b.sequence);
|
|
83
|
+
return events[0] ?? null;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** Start the channel-bound fixture, wait for terminal, return its runId. */
|
|
87
|
+
async function startChannelRun(): Promise<string> {
|
|
88
|
+
const create = await driver.post('/v1/runs', { workflowId: FIXTURE_ID });
|
|
89
|
+
expect(
|
|
90
|
+
create.status,
|
|
91
|
+
driver.describe(
|
|
92
|
+
'agent-deployment.md §B',
|
|
93
|
+
`a host advertising agents.deployment + the ${FIXTURE_ID} fixture MUST accept a channel-bound run (201)`,
|
|
94
|
+
),
|
|
95
|
+
).toBe(201);
|
|
96
|
+
const runId = (create.json as { runId: string }).runId;
|
|
97
|
+
await pollUntilTerminal(runId, { timeoutMs: 15_000 });
|
|
98
|
+
return runId;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
describe.skipIf(HTTP_SKIP)('agent-channel-dispatch (RFC 0082 §B): production run-graph channel pin + replay reuse', () => {
|
|
102
|
+
it('resolves + records the channel pin on a real run and re-reads it on replay (never re-resolving a moved channel)', async (ctx) => {
|
|
103
|
+
const cap = await readDeploymentCap();
|
|
104
|
+
if (!behaviorGate('openwop-deployment-channel-dispatch', cap?.supported === true)) return;
|
|
105
|
+
if (!isFixtureAdvertised(FIXTURE_ID)) {
|
|
106
|
+
// Host advertises agents.deployment but hasn't seeded the channel-bound
|
|
107
|
+
// fixture — a host-config precondition, not a conformance failure.
|
|
108
|
+
ctx.skip();
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
const modes = await fetchReplayModes();
|
|
112
|
+
if (!modes.includes('replay')) {
|
|
113
|
+
ctx.skip();
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// ---- Leg 1: production-path channel resolution + recorded pin (§B) ----
|
|
118
|
+
const sourceRunId = await startChannelRun();
|
|
119
|
+
const started = await firstInvocationStarted(sourceRunId);
|
|
120
|
+
expect(
|
|
121
|
+
started !== null,
|
|
122
|
+
driver.describe(
|
|
123
|
+
'agent-deployment.md §B',
|
|
124
|
+
'a @channel-bound run MUST emit agent.invocation.started',
|
|
125
|
+
),
|
|
126
|
+
).toBe(true);
|
|
127
|
+
expect(
|
|
128
|
+
started!.payload.resolvedChannel === BOUND_CHANNEL,
|
|
129
|
+
driver.describe(
|
|
130
|
+
'agent-deployment.md §B',
|
|
131
|
+
`agent.invocation.started MUST carry the bound channel as resolvedChannel ("${BOUND_CHANNEL}")`,
|
|
132
|
+
),
|
|
133
|
+
).toBe(true);
|
|
134
|
+
const pinnedVersion = started!.payload.resolvedAgentVersion;
|
|
135
|
+
expect(
|
|
136
|
+
typeof pinnedVersion === 'string' && (pinnedVersion as string).length > 0,
|
|
137
|
+
driver.describe(
|
|
138
|
+
'agent-deployment.md §B',
|
|
139
|
+
'a @channel-bound run MUST record a concrete resolvedAgentVersion (the recorded fact a replay re-reads, RFC 0077)',
|
|
140
|
+
),
|
|
141
|
+
).toBe(true);
|
|
142
|
+
|
|
143
|
+
// ---- Leg 2: replay re-reads the recorded version --------------------
|
|
144
|
+
const fork1 = await driver.post(
|
|
145
|
+
`/v1/runs/${encodeURIComponent(sourceRunId)}:fork`,
|
|
146
|
+
{ fromSeq: 0, mode: 'replay' },
|
|
147
|
+
);
|
|
148
|
+
if (fork1.status === 501) {
|
|
149
|
+
// replay advertised but not implemented for this run — skip-equivalent.
|
|
150
|
+
ctx.skip();
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
expect(
|
|
154
|
+
fork1.status,
|
|
155
|
+
driver.describe('rest-endpoints.md POST /v1/runs/{runId}:fork', 'replay fork MUST return 201'),
|
|
156
|
+
).toBe(201);
|
|
157
|
+
const fork1RunId = (fork1.json as { runId: string }).runId;
|
|
158
|
+
await pollUntilTerminal(fork1RunId, { timeoutMs: 15_000 });
|
|
159
|
+
const fork1Started = await firstInvocationStarted(fork1RunId);
|
|
160
|
+
expect(
|
|
161
|
+
fork1Started !== null,
|
|
162
|
+
driver.describe('agent-deployment.md §B', 'a replay fork MUST re-emit agent.invocation.started'),
|
|
163
|
+
).toBe(true);
|
|
164
|
+
expect(
|
|
165
|
+
fork1Started!.payload.resolvedAgentVersion === pinnedVersion,
|
|
166
|
+
driver.describe(
|
|
167
|
+
'agent-deployment.md §B',
|
|
168
|
+
'a replay MUST re-read the recorded resolvedAgentVersion (NOT re-resolve the channel)',
|
|
169
|
+
),
|
|
170
|
+
).toBe(true);
|
|
171
|
+
|
|
172
|
+
// ---- Leg 3 (seam-guarded): move the channel, prove non-re-resolution -
|
|
173
|
+
// The strongest form of §B: after the original pin, MOVE `stable` to a new
|
|
174
|
+
// active version via the optional deployment seam. A replay fork of the
|
|
175
|
+
// ORIGINAL run MUST still carry the ORIGINAL pin — proving the host re-reads
|
|
176
|
+
// the recorded fact rather than re-resolving the (now-moved) channel.
|
|
177
|
+
const moved = await driveDeploymentTransition({
|
|
178
|
+
scenario: 'promote',
|
|
179
|
+
channel: BOUND_CHANNEL,
|
|
180
|
+
});
|
|
181
|
+
if (moved === null) {
|
|
182
|
+
// No deployment-transition seam — Leg 1+2 already give production-path
|
|
183
|
+
// evidence; the cross-move proof needs the seam. Honest skip of Leg 3.
|
|
184
|
+
// eslint-disable-next-line no-console
|
|
185
|
+
console.warn('[agent-channel-dispatch] deployment seam absent — skipping the channel-move non-re-resolution leg (Leg 3)');
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
// Confirm the move is OBSERVABLE: a fresh channel-bound run must now resolve
|
|
189
|
+
// to a DIFFERENT version. If it doesn't (canary split, no-op promote), we
|
|
190
|
+
// can't prove movement — skip the strict assertion rather than assert falsely.
|
|
191
|
+
const controlRunId = await startChannelRun();
|
|
192
|
+
const controlStarted = await firstInvocationStarted(controlRunId);
|
|
193
|
+
const movedVersion = controlStarted?.payload.resolvedAgentVersion;
|
|
194
|
+
if (typeof movedVersion !== 'string' || movedVersion === pinnedVersion) {
|
|
195
|
+
// eslint-disable-next-line no-console
|
|
196
|
+
console.warn('[agent-channel-dispatch] channel did not observably move — skipping Leg 3 strict assertion');
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
const fork2 = await driver.post(
|
|
200
|
+
`/v1/runs/${encodeURIComponent(sourceRunId)}:fork`,
|
|
201
|
+
{ fromSeq: 0, mode: 'replay' },
|
|
202
|
+
);
|
|
203
|
+
if (fork2.status === 501) {
|
|
204
|
+
ctx.skip();
|
|
205
|
+
return;
|
|
206
|
+
}
|
|
207
|
+
expect(
|
|
208
|
+
fork2.status,
|
|
209
|
+
driver.describe('rest-endpoints.md POST /v1/runs/{runId}:fork', 'replay fork MUST return 201'),
|
|
210
|
+
).toBe(201);
|
|
211
|
+
const fork2RunId = (fork2.json as { runId: string }).runId;
|
|
212
|
+
await pollUntilTerminal(fork2RunId, { timeoutMs: 15_000 });
|
|
213
|
+
const fork2Started = await firstInvocationStarted(fork2RunId);
|
|
214
|
+
expect(
|
|
215
|
+
fork2Started?.payload.resolvedAgentVersion === pinnedVersion,
|
|
216
|
+
driver.describe(
|
|
217
|
+
'agent-deployment.md §B',
|
|
218
|
+
'after the channel moves, a replay of the original run MUST still carry the ORIGINAL pin — never re-resolving the moved channel',
|
|
219
|
+
),
|
|
220
|
+
).toBe(true);
|
|
221
|
+
expect(
|
|
222
|
+
fork2Started?.payload.resolvedAgentVersion !== movedVersion,
|
|
223
|
+
driver.describe(
|
|
224
|
+
'agent-deployment.md §B',
|
|
225
|
+
'a replay MUST NOT resolve to the post-move version (proves the recorded fact is re-read, not re-resolved)',
|
|
226
|
+
),
|
|
227
|
+
).toBe(true);
|
|
228
|
+
});
|
|
229
|
+
});
|