@openwop/openwop-conformance 1.18.1 → 1.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/README.md +2 -2
- package/api/.redocly.lint-ignore.yaml +22 -0
- package/api/openapi.yaml +13 -4
- package/coverage.md +2 -1
- package/dist/cli.js +235 -4
- package/dist/lib/paths.js +160 -0
- package/dist/lib/profiles.js +461 -0
- package/fixtures/conformance-agent-channel-dispatch.json +27 -0
- package/fixtures.md +15 -0
- package/package.json +1 -1
- package/schemas/README.md +1 -0
- package/schemas/capabilities.schema.json +5 -0
- package/schemas/conformance-certification-bundle.schema.json +86 -0
- package/src/cli.ts +268 -4
- package/src/lib/profiles.ts +85 -0
- package/src/scenarios/agent-channel-dispatch.test.ts +229 -0
- package/src/scenarios/spec-corpus-validity.test.ts +183 -0
package/schemas/README.md
CHANGED
|
@@ -32,6 +32,7 @@
|
|
|
32
32
|
| `capabilities.schema.json` | `capabilities.md` | `/.well-known/openwop` response — protocolVersion + supportedEnvelopes + schemaVersions + limits + optional v1 discovery surface |
|
|
33
33
|
| `channel-written-payload.schema.json` | `channels-and-reducers.md` §Channel write event | Payload of the `channel.written` RunEvent — write input + reducer name |
|
|
34
34
|
| `chat-card-pack-manifest.schema.json` | `chat-card-packs.md` + RFC 0071 | DRAFT — manifest for `kind: "card"` registry packs (RFC 0071 Phase 2). Peer to the node/workflow-chain/prompt/artifact-type pack manifests; disjoint via the `kind` discriminator. Distributes AI chat cards: a prompt template + typed input subset bound to a typed `outputArtifactType`. |
|
|
35
|
+
| `conformance-certification-bundle.schema.json` | `conformance-certification.md` + RFC 0089 | DRAFT — machine-readable attestation binding a host's claimed profiles to the reproducible run that substantiates them (suite version + per-scenario pass list + host identity/commit + captured discovery document). Out-of-band; a consumer re-derives each claim via the §B binding rule. |
|
|
35
36
|
| `conversation-event.schema.json` | `channels-and-reducers.md` + conversation RFC | Multi-turn conversation event shape for orchestrator-driven HITL flows |
|
|
36
37
|
| `conversation-turn.schema.json` | `channels-and-reducers.md` + conversation RFC | Conversation turn shape for user/agent/system messages |
|
|
37
38
|
| `core-conformance-mock-agent-config.schema.json` | `node-packs.md` + RFC 0023 | Config shape for the conformance-only `core.conformance.mock-agent` typeId — drives `agent.*` event emission on cue (`mockReasoning` / `mockToolCalls` / `mockHandoff` / `mockDecision` / `mockConfidence`). Hosts MUST refuse this typeId for production tenants unless `capabilities.conformance.mockAgent` is advertised. |
|
|
@@ -769,6 +769,11 @@
|
|
|
769
769
|
"mockAgent": {
|
|
770
770
|
"type": "boolean",
|
|
771
771
|
"description": "RFC 0023 §B.2. When `true`, the host has registered the `core.conformance.mock-agent` typeId. The scenarios `agentReasoningEvents.test.ts` and `agentConfidenceEscalation.test.ts` rely on the typeId being reachable. Hosts that register the typeId only for workflow ids matching the conformance fixture prefix (`conformance-*`) and refuse it for other tenants MAY still advertise `true` — the advertisement says only that the typeId is reachable from the conformance suite, not that it is reachable from arbitrary workflows."
|
|
772
|
+
},
|
|
773
|
+
"certificationBundleUrl": {
|
|
774
|
+
"type": "string",
|
|
775
|
+
"format": "uri",
|
|
776
|
+
"description": "OPTIONAL (RFC 0089). URL of the host's most recent conformance certification bundle (`conformance-certification-bundle.schema.json`) — a machine-readable attestation binding this host's claimed profiles to the reproducible run that substantiates them. Omitting it is fully conformant; clients MUST tolerate its absence."
|
|
772
777
|
}
|
|
773
778
|
},
|
|
774
779
|
"additionalProperties": false
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://openwop.dev/spec/v1/conformance-certification-bundle.schema.json",
|
|
4
|
+
"title": "OpenWOP Conformance Certification Bundle",
|
|
5
|
+
"description": "RFC 0089. A machine-readable attestation binding a host's claimed profiles to the reproducible run that substantiates them: suite version, per-scenario pass list, host identity + commit, and the captured discovery document. An out-of-band artifact emitted by the conformance harness (not a runtime wire message). A consumer MUST re-derive each claimed profile from the embedded `discovery.document` rather than trusting `claimedProfiles` verbatim (RFC 0089 §B).",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"additionalProperties": false,
|
|
8
|
+
"required": ["bundleVersion", "generatedAt", "generator", "suite", "host", "discovery", "claimedProfiles", "results"],
|
|
9
|
+
"properties": {
|
|
10
|
+
"bundleVersion": {
|
|
11
|
+
"const": "1",
|
|
12
|
+
"description": "Certification-bundle format version. `1` for RFC 0089."
|
|
13
|
+
},
|
|
14
|
+
"generatedAt": {
|
|
15
|
+
"type": "string",
|
|
16
|
+
"format": "date-time",
|
|
17
|
+
"description": "RFC 3339 timestamp when the bundle was generated. Marks the bundle as a point-in-time attestation."
|
|
18
|
+
},
|
|
19
|
+
"generator": {
|
|
20
|
+
"type": "object",
|
|
21
|
+
"additionalProperties": false,
|
|
22
|
+
"required": ["name", "version"],
|
|
23
|
+
"properties": {
|
|
24
|
+
"name": { "type": "string", "description": "Tool that produced the bundle, e.g. `@openwop/openwop-conformance --certify`." },
|
|
25
|
+
"version": { "type": "string" }
|
|
26
|
+
}
|
|
27
|
+
},
|
|
28
|
+
"suite": {
|
|
29
|
+
"type": "object",
|
|
30
|
+
"additionalProperties": false,
|
|
31
|
+
"required": ["package", "version"],
|
|
32
|
+
"properties": {
|
|
33
|
+
"package": { "const": "@openwop/openwop-conformance" },
|
|
34
|
+
"version": { "type": "string", "description": "Exact conformance suite version the results were produced against." }
|
|
35
|
+
}
|
|
36
|
+
},
|
|
37
|
+
"host": {
|
|
38
|
+
"type": "object",
|
|
39
|
+
"additionalProperties": false,
|
|
40
|
+
"required": ["name", "version"],
|
|
41
|
+
"properties": {
|
|
42
|
+
"name": { "type": "string" },
|
|
43
|
+
"version": { "type": "string" },
|
|
44
|
+
"vendor": { "type": "string" },
|
|
45
|
+
"commit": { "type": "string", "description": "VCS commit / build id of the host under test, when known. Self-reported; authoritative only for independent-verifier-generated bundles (RFC 0089 §Security)." }
|
|
46
|
+
}
|
|
47
|
+
},
|
|
48
|
+
"discovery": {
|
|
49
|
+
"type": "object",
|
|
50
|
+
"additionalProperties": false,
|
|
51
|
+
"required": ["url", "sha256", "document"],
|
|
52
|
+
"properties": {
|
|
53
|
+
"url": { "type": "string", "format": "uri", "description": "The `/.well-known/openwop` URL fetched for this run." },
|
|
54
|
+
"sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$", "description": "SHA-256 of the canonical-JSON serialization of `document`, so a verifier can confirm it matches a live fetch." },
|
|
55
|
+
"document": { "type": "object", "description": "The verbatim `/.well-known/openwop` discovery document captured for this run. Profile derivation re-runs against THIS document." }
|
|
56
|
+
}
|
|
57
|
+
},
|
|
58
|
+
"claimedProfiles": {
|
|
59
|
+
"type": "array",
|
|
60
|
+
"items": { "type": "string" },
|
|
61
|
+
"minItems": 0,
|
|
62
|
+
"description": "Profiles the host claims. A bundle MUST NOT list a profile its own `discovery.document` does not derive (RFC 0089 §B(1))."
|
|
63
|
+
},
|
|
64
|
+
"results": {
|
|
65
|
+
"type": "object",
|
|
66
|
+
"additionalProperties": false,
|
|
67
|
+
"required": ["totals", "passed", "failed", "skipped"],
|
|
68
|
+
"properties": {
|
|
69
|
+
"totals": {
|
|
70
|
+
"type": "object",
|
|
71
|
+
"additionalProperties": false,
|
|
72
|
+
"required": ["passed", "failed", "skipped", "total"],
|
|
73
|
+
"properties": {
|
|
74
|
+
"passed": { "type": "integer", "minimum": 0 },
|
|
75
|
+
"failed": { "type": "integer", "minimum": 0 },
|
|
76
|
+
"skipped": { "type": "integer", "minimum": 0 },
|
|
77
|
+
"total": { "type": "integer", "minimum": 0 }
|
|
78
|
+
}
|
|
79
|
+
},
|
|
80
|
+
"passed": { "type": "array", "items": { "type": "string" }, "description": "Stable scenario IDs that passed non-vacuously. The generator MUST NOT list a scenario here that did not run non-vacuously." },
|
|
81
|
+
"failed": { "type": "array", "items": { "type": "string" } },
|
|
82
|
+
"skipped": { "type": "array", "items": { "type": "string" } }
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
package/src/cli.ts
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
* openwop-conformance --offline # server-free subset only
|
|
12
12
|
* openwop-conformance --filter discovery # category filter
|
|
13
13
|
* openwop-conformance --base-url ... --api-key ... --filter "interrupt|cancellation"
|
|
14
|
+
* openwop-conformance --base-url ... --api-key ... --certify out.json # RFC 0089 bundle
|
|
14
15
|
*
|
|
15
16
|
* Environment variables override flags (per the conformance harness's
|
|
16
17
|
* existing convention):
|
|
@@ -25,7 +26,19 @@
|
|
|
25
26
|
|
|
26
27
|
import { spawnSync } from 'node:child_process';
|
|
27
28
|
import { fileURLToPath } from 'node:url';
|
|
28
|
-
import { dirname, resolve as resolvePath } from 'node:path';
|
|
29
|
+
import { dirname, resolve as resolvePath, join } from 'node:path';
|
|
30
|
+
import { createHash } from 'node:crypto';
|
|
31
|
+
import { readFileSync, writeFileSync, mkdtempSync, rmSync } from 'node:fs';
|
|
32
|
+
import { tmpdir } from 'node:os';
|
|
33
|
+
import Ajv2020 from 'ajv/dist/2020.js';
|
|
34
|
+
import addFormats from 'ajv-formats';
|
|
35
|
+
import { SCHEMAS_DIR } from './lib/paths.js';
|
|
36
|
+
import {
|
|
37
|
+
deriveProfiles,
|
|
38
|
+
isCoreStandard,
|
|
39
|
+
agentPlatformStatus,
|
|
40
|
+
type DiscoveryPayload,
|
|
41
|
+
} from './lib/profiles.js';
|
|
29
42
|
|
|
30
43
|
interface ParsedArgs {
|
|
31
44
|
readonly baseUrl: string | undefined;
|
|
@@ -35,6 +48,8 @@ interface ParsedArgs {
|
|
|
35
48
|
readonly help: boolean;
|
|
36
49
|
readonly impl: string | undefined;
|
|
37
50
|
readonly implVersion: string | undefined;
|
|
51
|
+
/** RFC 0089 — emit a conformance certification bundle to this path. */
|
|
52
|
+
readonly certify: string | undefined;
|
|
38
53
|
}
|
|
39
54
|
|
|
40
55
|
function parseArgs(argv: readonly string[]): ParsedArgs {
|
|
@@ -45,6 +60,7 @@ function parseArgs(argv: readonly string[]): ParsedArgs {
|
|
|
45
60
|
let help = false;
|
|
46
61
|
let impl: string | undefined;
|
|
47
62
|
let implVersion: string | undefined;
|
|
63
|
+
let certify: string | undefined;
|
|
48
64
|
|
|
49
65
|
for (let i = 0; i < argv.length; i++) {
|
|
50
66
|
const arg = argv[i] ?? '';
|
|
@@ -87,6 +103,9 @@ function parseArgs(argv: readonly string[]): ParsedArgs {
|
|
|
87
103
|
case '--implementation-version':
|
|
88
104
|
implVersion = nextValue();
|
|
89
105
|
break;
|
|
106
|
+
case '--certify':
|
|
107
|
+
certify = nextValue();
|
|
108
|
+
break;
|
|
90
109
|
default:
|
|
91
110
|
if (arg.startsWith('-')) {
|
|
92
111
|
// Unknown flag — pass through to vitest by ignoring here.
|
|
@@ -94,7 +113,7 @@ function parseArgs(argv: readonly string[]): ParsedArgs {
|
|
|
94
113
|
}
|
|
95
114
|
}
|
|
96
115
|
|
|
97
|
-
return { baseUrl, apiKey, offline, filter, help, impl, implVersion };
|
|
116
|
+
return { baseUrl, apiKey, offline, filter, help, impl, implVersion, certify };
|
|
98
117
|
}
|
|
99
118
|
|
|
100
119
|
const HELP_TEXT = `openwop-conformance — run the openwop conformance suite against a server
|
|
@@ -114,6 +133,14 @@ Implementation labels (cosmetic — surface in failure messages):
|
|
|
114
133
|
--impl <name> Implementation name (env: OPENWOP_IMPLEMENTATION_NAME)
|
|
115
134
|
--impl-version <version> Implementation version (env: OPENWOP_IMPLEMENTATION_VERSION)
|
|
116
135
|
|
|
136
|
+
Certification (RFC 0089):
|
|
137
|
+
--certify <out.json> Generate a machine-readable conformance certification
|
|
138
|
+
bundle: fetch /.well-known/openwop (captured verbatim +
|
|
139
|
+
SHA-256), derive claimedProfiles from it, run the suite
|
|
140
|
+
recording each scenario's terminal state, validate the
|
|
141
|
+
assembled bundle against the bundle schema, and write it
|
|
142
|
+
to <out.json>. Requires --base-url (and --api-key as usual).
|
|
143
|
+
|
|
117
144
|
Other:
|
|
118
145
|
--help, -h Show this message
|
|
119
146
|
|
|
@@ -121,9 +148,234 @@ Examples:
|
|
|
121
148
|
openwop-conformance --offline
|
|
122
149
|
openwop-conformance --base-url https://api.example.com --api-key hk_test_abc
|
|
123
150
|
openwop-conformance --filter "discovery|errors"
|
|
151
|
+
openwop-conformance --base-url https://api.example.com --api-key hk_test_abc \\
|
|
152
|
+
--certify certification-bundle.json
|
|
124
153
|
`;
|
|
125
154
|
|
|
126
|
-
|
|
155
|
+
/** This CLI package's own version — surfaced as `generator.version` + `suite.version`. */
|
|
156
|
+
function suiteVersion(): string {
|
|
157
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
158
|
+
const pkgPath = resolvePath(here, '..', 'package.json');
|
|
159
|
+
const pkg = JSON.parse(readFileSync(pkgPath, 'utf8')) as { version?: unknown };
|
|
160
|
+
return typeof pkg.version === 'string' ? pkg.version : '0.0.0';
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Deterministic canonical-JSON serialization (RFC 8785 spirit): object keys
|
|
165
|
+
* sorted lexicographically at every level, arrays preserved in order. Used to
|
|
166
|
+
* compute `discovery.sha256` so a verifier can re-derive the same digest from
|
|
167
|
+
* a live `/.well-known/openwop` fetch regardless of incidental key order.
|
|
168
|
+
*/
|
|
169
|
+
function canonicalJSON(value: unknown): string {
|
|
170
|
+
if (value === null || typeof value !== 'object') return JSON.stringify(value);
|
|
171
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJSON).join(',')}]`;
|
|
172
|
+
const obj = value as Record<string, unknown>;
|
|
173
|
+
const keys = Object.keys(obj).sort();
|
|
174
|
+
return `{${keys.map((k) => `${JSON.stringify(k)}:${canonicalJSON(obj[k])}`).join(',')}}`;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* The full set of profiles a discovery document derives — the closed
|
|
179
|
+
* `deriveProfiles` catalog plus the two operational annexes
|
|
180
|
+
* (`openwop-core-standard`, `openwop-agent-platform`) when their discovery
|
|
181
|
+
* predicate holds. This is `claimedProfiles`: a generated bundle MUST NOT
|
|
182
|
+
* claim a profile its own discovery document does not derive (RFC 0089 §B(1)).
|
|
183
|
+
*/
|
|
184
|
+
function claimedProfilesFor(doc: DiscoveryPayload): string[] {
|
|
185
|
+
const profiles: string[] = [...deriveProfiles(doc)];
|
|
186
|
+
if (isCoreStandard(doc)) profiles.push('openwop-core-standard');
|
|
187
|
+
if (agentPlatformStatus(doc) !== 'none') profiles.push('openwop-agent-platform');
|
|
188
|
+
return profiles;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** A single scenario test file's terminal state, derived from the vitest JSON report. */
|
|
192
|
+
type ScenarioState = 'passed' | 'failed' | 'skipped';
|
|
193
|
+
|
|
194
|
+
/** The subset of vitest's JSON reporter output we read. */
|
|
195
|
+
interface VitestJsonReport {
|
|
196
|
+
readonly testResults?: ReadonlyArray<{
|
|
197
|
+
readonly name?: string;
|
|
198
|
+
readonly assertionResults?: ReadonlyArray<{ readonly status?: string }>;
|
|
199
|
+
}>;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* Reduce a vitest JSON report into a per-scenario-file terminal state, keyed by
|
|
204
|
+
* the test-file basename (e.g. `discovery.test.ts`) to align with the basenames
|
|
205
|
+
* in `PROFILE_FLOOR_SCENARIOS`. A file is `passed` only if it ran AND had ≥1
|
|
206
|
+
* passing assertion AND zero failures (non-vacuous, per §C); a fully-skipped
|
|
207
|
+
* file is `skipped`; any failed assertion makes the file `failed`.
|
|
208
|
+
*/
|
|
209
|
+
function scenarioStatesFromReport(report: VitestJsonReport): Map<string, ScenarioState> {
|
|
210
|
+
const states = new Map<string, ScenarioState>();
|
|
211
|
+
for (const file of report.testResults ?? []) {
|
|
212
|
+
const name = file.name;
|
|
213
|
+
if (typeof name !== 'string') continue;
|
|
214
|
+
const basename = name.split('/').pop() ?? name;
|
|
215
|
+
const assertions = file.assertionResults ?? [];
|
|
216
|
+
let passes = 0;
|
|
217
|
+
let failures = 0;
|
|
218
|
+
let nonSkipped = 0;
|
|
219
|
+
for (const a of assertions) {
|
|
220
|
+
if (a.status === 'passed') {
|
|
221
|
+
passes++;
|
|
222
|
+
nonSkipped++;
|
|
223
|
+
} else if (a.status === 'failed') {
|
|
224
|
+
failures++;
|
|
225
|
+
nonSkipped++;
|
|
226
|
+
}
|
|
227
|
+
// `skipped` / `todo` / `pending` count toward neither pass nor fail.
|
|
228
|
+
}
|
|
229
|
+
let state: ScenarioState;
|
|
230
|
+
if (failures > 0) state = 'failed';
|
|
231
|
+
else if (passes > 0 && nonSkipped > 0) state = 'passed';
|
|
232
|
+
else state = 'skipped';
|
|
233
|
+
states.set(basename, state);
|
|
234
|
+
}
|
|
235
|
+
return states;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/** Generate + validate + write an RFC 0089 conformance certification bundle. */
|
|
239
|
+
async function runCertify(args: ParsedArgs, baseUrl: string, apiKey: string): Promise<never> {
|
|
240
|
+
const outPath = args.certify;
|
|
241
|
+
if (outPath === undefined) process.exit(2);
|
|
242
|
+
|
|
243
|
+
// (a) Fetch /.well-known/openwop verbatim + its canonical-JSON SHA-256.
|
|
244
|
+
const discoveryUrl = `${baseUrl.replace(/\/$/, '')}/.well-known/openwop`;
|
|
245
|
+
let document: DiscoveryPayload;
|
|
246
|
+
try {
|
|
247
|
+
const resp = await fetch(discoveryUrl, { headers: { Accept: 'application/json' } });
|
|
248
|
+
if (!resp.ok) {
|
|
249
|
+
process.stderr.write(
|
|
250
|
+
`openwop-conformance --certify: GET ${discoveryUrl} returned HTTP ${resp.status}.\n`,
|
|
251
|
+
);
|
|
252
|
+
process.exit(2);
|
|
253
|
+
}
|
|
254
|
+
document = (await resp.json()) as DiscoveryPayload;
|
|
255
|
+
} catch (err) {
|
|
256
|
+
process.stderr.write(
|
|
257
|
+
`openwop-conformance --certify: failed to fetch ${discoveryUrl}: ${String(err)}\n`,
|
|
258
|
+
);
|
|
259
|
+
process.exit(2);
|
|
260
|
+
}
|
|
261
|
+
const sha256 = createHash('sha256').update(canonicalJSON(document)).digest('hex');
|
|
262
|
+
|
|
263
|
+
// (b) Derive claimedProfiles from the captured document.
|
|
264
|
+
const claimedProfiles = claimedProfilesFor(document);
|
|
265
|
+
|
|
266
|
+
// (c) Run the suite, capturing per-scenario terminal state via the vitest
|
|
267
|
+
// JSON reporter. server-targeted scenarios live under src/scenarios/.
|
|
268
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
269
|
+
const conformanceRoot = resolvePath(here, '..');
|
|
270
|
+
const reportDir = mkdtempSync(join(tmpdir(), 'owp-certify-'));
|
|
271
|
+
const reportFile = join(reportDir, 'vitest-report.json');
|
|
272
|
+
const env: NodeJS.ProcessEnv = { ...process.env };
|
|
273
|
+
env.OPENWOP_BASE_URL = baseUrl;
|
|
274
|
+
env.OPENWOP_API_KEY = apiKey;
|
|
275
|
+
if (args.impl) env.OPENWOP_IMPLEMENTATION_NAME = args.impl;
|
|
276
|
+
if (args.implVersion) env.OPENWOP_IMPLEMENTATION_VERSION = args.implVersion;
|
|
277
|
+
|
|
278
|
+
const vitestArgs: string[] = [
|
|
279
|
+
'vitest',
|
|
280
|
+
'run',
|
|
281
|
+
'--config',
|
|
282
|
+
resolvePath(conformanceRoot, 'vitest.config.ts'),
|
|
283
|
+
'--reporter=json',
|
|
284
|
+
`--outputFile=${reportFile}`,
|
|
285
|
+
];
|
|
286
|
+
const runResult = spawnSync('npx', vitestArgs, { cwd: conformanceRoot, env, stdio: 'inherit' });
|
|
287
|
+
if (runResult.error) {
|
|
288
|
+
process.stderr.write(
|
|
289
|
+
`openwop-conformance --certify: failed to spawn vitest: ${String(runResult.error)}\n`,
|
|
290
|
+
);
|
|
291
|
+
process.exit(2);
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
let report: VitestJsonReport;
|
|
295
|
+
try {
|
|
296
|
+
report = JSON.parse(readFileSync(reportFile, 'utf8')) as VitestJsonReport;
|
|
297
|
+
} catch (err) {
|
|
298
|
+
process.stderr.write(
|
|
299
|
+
`openwop-conformance --certify: could not read vitest JSON report at ${reportFile}: ${String(err)}\n`,
|
|
300
|
+
);
|
|
301
|
+
process.exit(2);
|
|
302
|
+
} finally {
|
|
303
|
+
rmSync(reportDir, { recursive: true, force: true });
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
const states = scenarioStatesFromReport(report);
|
|
307
|
+
const passed: string[] = [];
|
|
308
|
+
const failed: string[] = [];
|
|
309
|
+
const skipped: string[] = [];
|
|
310
|
+
for (const [basename, state] of [...states.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
|
|
311
|
+
if (state === 'passed') passed.push(basename);
|
|
312
|
+
else if (state === 'failed') failed.push(basename);
|
|
313
|
+
else skipped.push(basename);
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
// (d) Assemble the bundle.
|
|
317
|
+
const version = suiteVersion();
|
|
318
|
+
const impl = (document as { implementation?: { name?: unknown; version?: unknown; vendor?: unknown } })
|
|
319
|
+
.implementation;
|
|
320
|
+
const hostName =
|
|
321
|
+
args.impl ?? (typeof impl?.name === 'string' ? impl.name : 'unknown-host');
|
|
322
|
+
const hostVersion =
|
|
323
|
+
args.implVersion ?? (typeof impl?.version === 'string' ? impl.version : '0.0.0');
|
|
324
|
+
const host: { name: string; version: string; vendor?: string } = {
|
|
325
|
+
name: hostName,
|
|
326
|
+
version: hostVersion,
|
|
327
|
+
};
|
|
328
|
+
if (typeof impl?.vendor === 'string') host.vendor = impl.vendor;
|
|
329
|
+
|
|
330
|
+
const bundle = {
|
|
331
|
+
bundleVersion: '1' as const,
|
|
332
|
+
generatedAt: new Date().toISOString(),
|
|
333
|
+
generator: { name: '@openwop/openwop-conformance --certify', version },
|
|
334
|
+
suite: { package: '@openwop/openwop-conformance' as const, version },
|
|
335
|
+
host,
|
|
336
|
+
discovery: { url: discoveryUrl, sha256, document },
|
|
337
|
+
claimedProfiles,
|
|
338
|
+
results: {
|
|
339
|
+
totals: {
|
|
340
|
+
passed: passed.length,
|
|
341
|
+
failed: failed.length,
|
|
342
|
+
skipped: skipped.length,
|
|
343
|
+
total: passed.length + failed.length + skipped.length,
|
|
344
|
+
},
|
|
345
|
+
passed,
|
|
346
|
+
failed,
|
|
347
|
+
skipped,
|
|
348
|
+
},
|
|
349
|
+
};
|
|
350
|
+
|
|
351
|
+
// (e) Validate against the bundle schema BEFORE writing.
|
|
352
|
+
const schema = JSON.parse(
|
|
353
|
+
readFileSync(join(SCHEMAS_DIR, 'conformance-certification-bundle.schema.json'), 'utf8'),
|
|
354
|
+
) as Record<string, unknown>;
|
|
355
|
+
const ajv = new Ajv2020({ allErrors: true, strict: false });
|
|
356
|
+
addFormats(ajv);
|
|
357
|
+
const validate = ajv.compile(schema);
|
|
358
|
+
if (!validate(bundle)) {
|
|
359
|
+
process.stderr.write(
|
|
360
|
+
'openwop-conformance --certify: assembled bundle FAILED schema validation:\n' +
|
|
361
|
+
`${JSON.stringify(validate.errors, null, 2)}\n`,
|
|
362
|
+
);
|
|
363
|
+
process.exit(2);
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
writeFileSync(outPath, `${JSON.stringify(bundle, null, 2)}\n`);
|
|
367
|
+
process.stdout.write(
|
|
368
|
+
`openwop-conformance --certify: wrote certification bundle to ${outPath}\n` +
|
|
369
|
+
` host: ${host.name}@${host.version}\n` +
|
|
370
|
+
` claimedProfiles: ${claimedProfiles.length > 0 ? claimedProfiles.join(', ') : '(none)'}\n` +
|
|
371
|
+
` results: ${passed.length} passed / ${failed.length} failed / ${skipped.length} skipped\n`,
|
|
372
|
+
);
|
|
373
|
+
// Exit code mirrors the suite outcome: a failing run still produces a bundle
|
|
374
|
+
// (the failures are recorded), but the process exit reflects pass/fail.
|
|
375
|
+
process.exit(failed.length > 0 ? 1 : 0);
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
async function main(): Promise<never> {
|
|
127
379
|
const args = parseArgs(process.argv.slice(2));
|
|
128
380
|
|
|
129
381
|
if (args.help) {
|
|
@@ -139,6 +391,18 @@ function main(): never {
|
|
|
139
391
|
if (args.impl) env.OPENWOP_IMPLEMENTATION_NAME = args.impl;
|
|
140
392
|
if (args.implVersion) env.OPENWOP_IMPLEMENTATION_VERSION = args.implVersion;
|
|
141
393
|
|
|
394
|
+
// RFC 0089 — certification-bundle generation requires a live host.
|
|
395
|
+
if (args.certify !== undefined) {
|
|
396
|
+
if (!env.OPENWOP_BASE_URL || !env.OPENWOP_API_KEY) {
|
|
397
|
+
process.stderr.write(
|
|
398
|
+
'openwop-conformance --certify: --base-url and --api-key are required.\n' +
|
|
399
|
+
'Run `openwop-conformance --help` for usage.\n',
|
|
400
|
+
);
|
|
401
|
+
process.exit(2);
|
|
402
|
+
}
|
|
403
|
+
return runCertify(args, env.OPENWOP_BASE_URL, env.OPENWOP_API_KEY);
|
|
404
|
+
}
|
|
405
|
+
|
|
142
406
|
if (!args.offline && (!env.OPENWOP_BASE_URL || !env.OPENWOP_API_KEY)) {
|
|
143
407
|
process.stderr.write(
|
|
144
408
|
'openwop-conformance: --base-url and --api-key are required (or use --offline).\n' +
|
|
@@ -184,4 +448,4 @@ function main(): never {
|
|
|
184
448
|
process.exit(result.status ?? 1);
|
|
185
449
|
}
|
|
186
450
|
|
|
187
|
-
main();
|
|
451
|
+
void main();
|
package/src/lib/profiles.ts
CHANGED
|
@@ -428,3 +428,88 @@ export function hasProfile(c: DiscoveryPayload, profile: ProfileName): boolean {
|
|
|
428
428
|
return isTriggerBridge(c);
|
|
429
429
|
}
|
|
430
430
|
}
|
|
431
|
+
|
|
432
|
+
// ── RFC 0089: conformance certification bundle — floor-scenario sets + verifier ──
|
|
433
|
+
//
|
|
434
|
+
// G1 (RFC 0089): the §B binding rule needs each profile's REQUIRED floor
|
|
435
|
+
// scenarios in a machine-readable form. Source of truth for
|
|
436
|
+
// `openwop-core-standard` is `core-standard-profile.md` §C — the nine named
|
|
437
|
+
// black-box floor scenarios plus the `interrupt-*` family. Keyed by profile
|
|
438
|
+
// name (string) because annex profiles like `openwop-core-standard` sit outside
|
|
439
|
+
// the closed `PROFILE_NAMES` catalog.
|
|
440
|
+
|
|
441
|
+
export interface ProfileFloor {
|
|
442
|
+
/** Scenario files (basenames) that MUST each appear in `results.passed`. */
|
|
443
|
+
readonly required: readonly string[];
|
|
444
|
+
/** Prefix groups where ≥1 matching passed scenario satisfies the group. */
|
|
445
|
+
readonly requiredAnyPrefix?: readonly string[];
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
export const PROFILE_FLOOR_SCENARIOS: Readonly<Record<string, ProfileFloor>> = {
|
|
449
|
+
'openwop-core-standard': {
|
|
450
|
+
required: [
|
|
451
|
+
'runs-lifecycle.test.ts',
|
|
452
|
+
'discovery.test.ts',
|
|
453
|
+
'auth.test.ts',
|
|
454
|
+
'eventOrdering.test.ts',
|
|
455
|
+
'failure-path.test.ts',
|
|
456
|
+
'idempotency.test.ts',
|
|
457
|
+
'idempotency-key-determinism.test.ts',
|
|
458
|
+
'webhook-negative.test.ts',
|
|
459
|
+
'audit-log-verification.test.ts',
|
|
460
|
+
],
|
|
461
|
+
requiredAnyPrefix: ['interrupt-'],
|
|
462
|
+
},
|
|
463
|
+
};
|
|
464
|
+
|
|
465
|
+
/** Is `profile` derivable from a discovery document? Maps a profile name to its predicate (RFC 0089 §B(1)). */
|
|
466
|
+
export function profileDerivable(c: DiscoveryPayload, profile: string): boolean {
|
|
467
|
+
if (profile === 'openwop-core-standard') return isCoreStandard(c);
|
|
468
|
+
if (profile === 'openwop-agent-platform') return agentPlatformStatus(c) !== 'none';
|
|
469
|
+
if ((PROFILE_NAMES as readonly string[]).includes(profile)) {
|
|
470
|
+
return deriveProfiles(c).includes(profile as ProfileName);
|
|
471
|
+
}
|
|
472
|
+
return false;
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
/** Minimal shape of an RFC 0089 certification bundle the verifier reads. */
|
|
476
|
+
export interface CertificationBundleLike {
|
|
477
|
+
readonly discovery: { readonly document: DiscoveryPayload };
|
|
478
|
+
readonly claimedProfiles: readonly string[];
|
|
479
|
+
readonly results: { readonly passed: readonly string[] };
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
export interface BundleProfileVerdict {
|
|
483
|
+
readonly profile: string;
|
|
484
|
+
/** §B(1): derivable from the captured discovery document. */
|
|
485
|
+
readonly derivable: boolean;
|
|
486
|
+
/** §B(2): every floor scenario for the profile is in `results.passed`. */
|
|
487
|
+
readonly floorProven: boolean;
|
|
488
|
+
readonly valid: boolean;
|
|
489
|
+
readonly missingFloor: readonly string[];
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
const scenarioBasename = (id: string): string => id.split('/').pop() ?? id;
|
|
493
|
+
|
|
494
|
+
/**
|
|
495
|
+
* Verify a bundle's claim for one profile per RFC 0089 §B. A consumer MUST
|
|
496
|
+
* re-derive (this function) rather than trust `claimedProfiles` verbatim.
|
|
497
|
+
*/
|
|
498
|
+
export function verifyBundleProfile(bundle: CertificationBundleLike, profile: string): BundleProfileVerdict {
|
|
499
|
+
const derivable = profileDerivable(bundle.discovery.document, profile);
|
|
500
|
+
const floor = PROFILE_FLOOR_SCENARIOS[profile];
|
|
501
|
+
const passed = new Set(bundle.results.passed.map(scenarioBasename));
|
|
502
|
+
const missingFloor = floor ? floor.required.filter((r) => !passed.has(scenarioBasename(r))) : [];
|
|
503
|
+
const prefixOk = (floor?.requiredAnyPrefix ?? []).every((p) => [...passed].some((s) => s.startsWith(p)));
|
|
504
|
+
const floorProven = missingFloor.length === 0 && prefixOk;
|
|
505
|
+
return { profile, derivable, floorProven, valid: derivable && floorProven, missingFloor };
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
/** Verify every profile in `bundle.claimedProfiles`; the bundle is valid iff all claims are valid. */
|
|
509
|
+
export function verifyBundle(bundle: CertificationBundleLike): {
|
|
510
|
+
readonly valid: boolean;
|
|
511
|
+
readonly verdicts: readonly BundleProfileVerdict[];
|
|
512
|
+
} {
|
|
513
|
+
const verdicts = bundle.claimedProfiles.map((p) => verifyBundleProfile(bundle, p));
|
|
514
|
+
return { valid: verdicts.every((v) => v.valid), verdicts };
|
|
515
|
+
}
|