@openwop/openwop-conformance 1.151.0 → 1.153.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/coverage.md +1 -1
- package/dist/cli.js +7 -1
- package/dist/lib/requirement-ids.js +100 -0
- package/dist/lib/scenario-disposition.js +22 -0
- package/package.json +3 -1
- package/requirement-aliases.json +4 -0
- package/requirements.json +23554 -0
- package/schemas/CORPUS-STAMP.json +2 -2
- package/schemas/README.md +1 -1
- package/schemas/conformance-certification-bundle.schema.json +1 -1
- package/src/cli.ts +9 -1
- package/src/lib/requirement-ids.test.ts +83 -0
- package/src/lib/requirement-ids.ts +111 -0
- package/src/lib/saml-idp.test.ts +45 -1
- package/src/lib/scenario-disposition.ts +22 -0
- package/src/scenarios/auth-saml-profile.test.ts +7 -39
- package/src/setup.ts +84 -5
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
2
|
"_comment": "Provenance of this vendored schemas/ copy. See conformance/README.md \u00a7\"Resolving the contract\". Compare against the stamp in your installed @openwop/openwop-conformance to detect a stale hand-copied contract.",
|
|
3
|
-
"suiteVersion": "1.
|
|
4
|
-
"corpusCommit": "
|
|
3
|
+
"suiteVersion": "1.153.0",
|
|
4
|
+
"corpusCommit": "6c84ef16e2cae24b7e7b8d8bf09b9f46634bf9a5"
|
|
5
5
|
}
|
package/schemas/README.md
CHANGED
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
| `connection-pack-manifest.schema.json` | `connection-packs.md` + RFC 0095 | DRAFT — manifest for `kind: "connection"` registry packs (RFC 0095). Peer to the node/workflow-chain/prompt/artifact-type/chat-card pack manifests; disjoint via the `kind` discriminator. Distributes a portable provider definition — auth endpoints, read/write scope groups, exactly-one reach (`mcp`/`openapi`/`integration`) — that the RFC 0045/0047 `provider` string resolves against. Carries NO credential material (`connection-pack-no-credential-material`). |
|
|
47
47
|
| `frontend-plugin-manifest.schema.json` | `frontend-plugin-packs.md` + RFC 0117 | DRAFT — manifest for `kind: "frontend-plugin"` registry packs (RFC 0117). Peer to the other pack manifests; disjoint via the `kind` discriminator. Distributes a signed, SANDBOXED UI extension — one or more `uiPlugins[]` (a `pluginId`, a `surface`, an opaque `entry` bundle, a closed `hostApi` allowlist). A backend `runtime` member is FORBIDDEN (`not: {}`). Loaded only in a cross-origin sandbox (`frontend-plugin-isolation`). |
|
|
48
48
|
| `ui-plugin-message.schema.json` | `frontend-plugin-packs.md` + RFC 0117 | DRAFT — the `ui-plugin/1` `postMessage` host-RPC envelope (RFC 0117) between a sandboxed front-end plugin and its host: a plugin→host `request` (closed `method` allowlist), a host→plugin `response` (`ok`+`result` or `ok:false`+`error`), or a host→plugin `event`. Carries the optimistic-concurrency `version` token and the `artifact_conflict` + `currentVersion` shape (`frontend-plugin-rpc-allowlist` / `frontend-plugin-no-byok`). |
|
|
49
|
-
| `conformance-certification-bundle.schema.json` | `conformance-certification.md` + RFC 0089 |
|
|
49
|
+
| `conformance-certification-bundle.schema.json` | `conformance-certification.md` + RFC 0089 | **DEPRECATED** (RFC 0148; `--certify` defaults to bundle v2 since suite 1.152.0; v1 ceases to substantiate a new certification after 2026-11-10; removal scheduled for v2.0 per `spec/v1/deprecations.json`) — machine-readable attestation binding a host's claimed profiles to the reproducible run that substantiates them (suite version + per-scenario pass list + host identity/commit + captured discovery document). Out-of-band; a consumer re-derives each claim via the §B binding rule. |
|
|
50
50
|
| `conversation-event.schema.json` | `channels-and-reducers.md` + conversation RFC | Multi-turn conversation event shape for orchestrator-driven HITL flows |
|
|
51
51
|
| `conversation-turn.schema.json` | `channels-and-reducers.md` + conversation RFC | Conversation turn shape for user/agent/system messages |
|
|
52
52
|
| `core-conformance-mock-agent-config.schema.json` | `node-packs.md` + RFC 0023 | Config shape for the conformance-only `core.conformance.mock-agent` typeId — drives `agent.*` event emission on cue (`mockReasoning` / `mockToolCalls` / `mockHandoff` / `mockDecision` / `mockConfidence`). Hosts MUST refuse this typeId for production tenants unless `capabilities.conformance.mockAgent` is advertised. |
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
3
|
"$id": "https://openwop.dev/spec/v1/conformance-certification-bundle.schema.json",
|
|
4
4
|
"title": "OpenWOP Conformance Certification Bundle",
|
|
5
|
-
"description": "RFC 0089. A machine-readable attestation binding a host's claimed profiles to the reproducible run that substantiates them: suite version, per-scenario pass list, host identity + commit, and the captured discovery document. An out-of-band artifact emitted by the conformance harness (not a runtime wire message). A consumer MUST re-derive each claimed profile from the embedded `discovery.document` rather than trusting `claimedProfiles` verbatim (RFC 0089 §B).",
|
|
5
|
+
"description": "DEPRECATED (RFC 0148, suite 1.114.0): bundle v1 remains parseable but ceases to substantiate a new certification after 2026-11-10; emit `certification-bundle-v2.schema.json` (the `--certify` default since suite 1.152.0). Scheduled for removal at v2.0 — see spec/v1/deprecations.json. RFC 0089. A machine-readable attestation binding a host's claimed profiles to the reproducible run that substantiates them: suite version, per-scenario pass list, host identity + commit, and the captured discovery document. An out-of-band artifact emitted by the conformance harness (not a runtime wire message). A consumer MUST re-derive each claimed profile from the embedded `discovery.document` rather than trusting `claimedProfiles` verbatim (RFC 0089 §B).",
|
|
6
6
|
"type": "object",
|
|
7
7
|
"additionalProperties": false,
|
|
8
8
|
"required": ["bundleVersion", "generatedAt", "generator", "suite", "host", "discovery", "claimedProfiles", "results"],
|
package/src/cli.ts
CHANGED
|
@@ -81,7 +81,10 @@ function parseArgs(argv: readonly string[]): ParsedArgs {
|
|
|
81
81
|
let impl: string | undefined;
|
|
82
82
|
let implVersion: string | undefined;
|
|
83
83
|
let certify: string | undefined;
|
|
84
|
-
|
|
84
|
+
// Suite 1.152.0: bundle v2 (RFC 0148) is the default. v1 stays reachable via
|
|
85
|
+
// `--bundle-version 1` through the RFC 0148 migration window (ends
|
|
86
|
+
// 2026-11-10) and is removed at v2.0 (spec/v1/deprecations.json).
|
|
87
|
+
let bundleVersion: '1' | '2' = '2';
|
|
85
88
|
let maxWorkers: number | undefined = parseMaxWorkers(process.env.OPENWOP_MAX_WORKERS, 'OPENWOP_MAX_WORKERS');
|
|
86
89
|
|
|
87
90
|
for (let i = 0; i < argv.length; i++) {
|
|
@@ -132,6 +135,11 @@ function parseArgs(argv: readonly string[]): ParsedArgs {
|
|
|
132
135
|
process.exit(2);
|
|
133
136
|
}
|
|
134
137
|
bundleVersion = v;
|
|
138
|
+
if (v === '1') {
|
|
139
|
+
process.stderr.write(
|
|
140
|
+
'openwop-conformance: --bundle-version 1 is DEPRECATED (RFC 0148). A v1 bundle ceases to substantiate a new certification after 2026-11-10 and the format is removed at v2.0; omit the flag to emit bundle v2.\n',
|
|
141
|
+
);
|
|
142
|
+
}
|
|
135
143
|
break;
|
|
136
144
|
}
|
|
137
145
|
case '--certify':
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Suite self-tests for per-`it` requirement ids (RFC 0148 §A gap G3; v2
|
|
3
|
+
* charter Phase 1). These prove the ID GRAMMAR, not a host, so they live in
|
|
4
|
+
* `src/lib/` and produce no scenario ledger row.
|
|
5
|
+
*
|
|
6
|
+
* The generator (`scripts/generate-requirement-registry.mjs`) re-implements
|
|
7
|
+
* `slugTitle` in plain JS; the fixtures below are the contract both must meet,
|
|
8
|
+
* so a divergence shows up here before it shows up as a registry-check failure.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { describe, it, expect } from 'vitest';
|
|
12
|
+
import {
|
|
13
|
+
IT_ID_GRAMMAR,
|
|
14
|
+
ItIdAllocator,
|
|
15
|
+
itRequirementId,
|
|
16
|
+
req,
|
|
17
|
+
scenarioFileOfItId,
|
|
18
|
+
slugTitle,
|
|
19
|
+
takeExplicitRequirementId,
|
|
20
|
+
} from './requirement-ids.js';
|
|
21
|
+
|
|
22
|
+
describe('requirement-ids: slug grammar', () => {
|
|
23
|
+
const cases: ReadonlyArray<[string, string]> = [
|
|
24
|
+
['auth.profiles, when present, is an array of non-empty strings', 'auth-profiles-when-present-is-an-array-of-non-empty-strings'],
|
|
25
|
+
['REJECTS the alg-none assertion over the seam (synthetic IdP required)', 'rejects-the-alg-none-assertion-over-the-seam-synthetic-idp-required'],
|
|
26
|
+
[' --leading and trailing-- ', 'leading-and-trailing'],
|
|
27
|
+
['§B.2 fail-closed: `unbound` ⇒ 401', 'b-2-fail-closed-unbound-401'],
|
|
28
|
+
['', 'untitled'],
|
|
29
|
+
['!!!', 'untitled'],
|
|
30
|
+
];
|
|
31
|
+
for (const [title, slug] of cases) {
|
|
32
|
+
it(`slugs ${JSON.stringify(title)} → ${slug}`, () => {
|
|
33
|
+
expect(slugTitle(title)).toBe(slug);
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
it('caps the slug at 80 characters without a trailing hyphen', () => {
|
|
38
|
+
const long = 'word '.repeat(40);
|
|
39
|
+
const s = slugTitle(long);
|
|
40
|
+
expect(s.length).toBeLessThanOrEqual(80);
|
|
41
|
+
expect(s.endsWith('-')).toBe(false);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
it('every derived id matches IT_ID_GRAMMAR', () => {
|
|
45
|
+
for (const [title] of cases) {
|
|
46
|
+
expect(itRequirementId('auth-subject-link.test.ts', title)).toMatch(IT_ID_GRAMMAR);
|
|
47
|
+
}
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
it('maps an id back to its scenario file', () => {
|
|
51
|
+
expect(scenarioFileOfItId('openwop.it.auth-subject-link.leaver-deny')).toBe('auth-subject-link.test.ts');
|
|
52
|
+
expect(scenarioFileOfItId('openwop.it.auth-subject-link.leaver-deny~2')).toBe('auth-subject-link.test.ts');
|
|
53
|
+
expect(scenarioFileOfItId('openwop.floor.auth')).toBeNull();
|
|
54
|
+
expect(scenarioFileOfItId('openwop.scenario.auth')).toBeNull();
|
|
55
|
+
});
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
describe('requirement-ids: collision suffixing', () => {
|
|
59
|
+
it('first occurrence is bare; duplicates get ~2, ~3', () => {
|
|
60
|
+
const a = new ItIdAllocator();
|
|
61
|
+
expect(a.allocate('x.test.ts', 'same title')).toBe('openwop.it.x.same-title');
|
|
62
|
+
expect(a.allocate('x.test.ts', 'same title')).toBe('openwop.it.x.same-title~2');
|
|
63
|
+
expect(a.allocate('x.test.ts', 'Same Title!')).toBe('openwop.it.x.same-title~3');
|
|
64
|
+
expect('openwop.it.x.same-title~3').toMatch(IT_ID_GRAMMAR);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
it('reset() forgets prior titles', () => {
|
|
68
|
+
const a = new ItIdAllocator();
|
|
69
|
+
a.allocate('x.test.ts', 't');
|
|
70
|
+
a.reset();
|
|
71
|
+
expect(a.allocate('x.test.ts', 't')).toBe('openwop.it.x.t');
|
|
72
|
+
});
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
describe('requirement-ids: explicit override', () => {
|
|
76
|
+
it('req() sets the override for the current test and returns the citation message', () => {
|
|
77
|
+
takeExplicitRequirementId();
|
|
78
|
+
const msg = req('openwop.auth.subject-link.leaver-deny', 'auth-profiles.md §Subject linking', 'deactivation MUST deny');
|
|
79
|
+
expect(msg).toBe('auth-profiles.md §Subject linking: deactivation MUST deny');
|
|
80
|
+
expect(takeExplicitRequirementId()).toBe('openwop.auth.subject-link.leaver-deny');
|
|
81
|
+
expect(takeExplicitRequirementId()).toBeNull();
|
|
82
|
+
});
|
|
83
|
+
});
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-`it` requirement ids (RFC 0148 §A, gap G3; v2 charter Phase 1).
|
|
3
|
+
*
|
|
4
|
+
* The ledger has recorded ONE row per scenario FILE (`openwop.floor.<file>` /
|
|
5
|
+
* `openwop.scenario.<file>`). That granularity is what produces certification
|
|
6
|
+
* gap G8: a file that asserted a positive control and then soft-skipped the
|
|
7
|
+
* requirement still resolves to `executed-pass` for the whole file. The durable
|
|
8
|
+
* fix named in `scenario-disposition.ts` is per-`it` recording — this module
|
|
9
|
+
* supplies the ids for it.
|
|
10
|
+
*
|
|
11
|
+
* Why the id derives from the TEST TITLE and not from the `driver.describe`
|
|
12
|
+
* text: 190 of 1,756 citations interpolate the requirement text at run time
|
|
13
|
+
* (`docs/REQUIREMENT-REGISTRY-FEASIBILITY.md`), so no text-derived id can be
|
|
14
|
+
* stable for them. Test titles are literals in every scenario file, stable
|
|
15
|
+
* across hosts, and already the unit vitest reports on. The registry
|
|
16
|
+
* (`conformance/requirements.json`, generated) maps each id back to the
|
|
17
|
+
* citations found inside that test's body, so a bundle reader can still ask
|
|
18
|
+
* "which spec section did this row witness".
|
|
19
|
+
*
|
|
20
|
+
* Grammar: `openwop.it.<file-stem>.<title-slug>` where the stem is the scenario
|
|
21
|
+
* file name without `.test.ts` and the slug is the title lower-cased, every run
|
|
22
|
+
* of non-alphanumerics folded to one `-`, trimmed, and capped at 80 characters.
|
|
23
|
+
* A second test in the same file whose title slugs identically gets `~2`,
|
|
24
|
+
* `~3`, … (vitest allows duplicate titles; the ledger does not allow duplicate
|
|
25
|
+
* ids). A scenario MAY override the derived id for the current test with
|
|
26
|
+
* `req()` when it wants a hand-authored, registry-listed id.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
export const IT_ID_PREFIX = 'openwop.it.';
|
|
30
|
+
export const IT_ID_GRAMMAR = /^openwop\.it\.[a-z0-9]+(?:-[a-z0-9]+)*\.[a-z0-9]+(?:-[a-z0-9]+)*(?:~[2-9][0-9]*)?$/;
|
|
31
|
+
|
|
32
|
+
const MAX_SLUG = 80;
|
|
33
|
+
|
|
34
|
+
/** Title → slug: lower-case, non-alphanumeric runs → `-`, trimmed, capped. */
|
|
35
|
+
export function slugTitle(title: string): string {
|
|
36
|
+
const s = title
|
|
37
|
+
.toLowerCase()
|
|
38
|
+
.replace(/[^a-z0-9]+/g, '-')
|
|
39
|
+
.replace(/^-+|-+$/g, '');
|
|
40
|
+
const capped = s.length > MAX_SLUG ? s.slice(0, MAX_SLUG).replace(/-+$/g, '') : s;
|
|
41
|
+
return capped.length > 0 ? capped : 'untitled';
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** `auth-subject-link.test.ts` → `auth-subject-link`. */
|
|
45
|
+
export function fileStem(scenarioFile: string): string {
|
|
46
|
+
return scenarioFile.replace(/\.test\.ts$/, '');
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Derived id for one test, before collision suffixing. */
|
|
50
|
+
export function itRequirementId(scenarioFile: string, title: string): string {
|
|
51
|
+
return `${IT_ID_PREFIX}${fileStem(scenarioFile)}.${slugTitle(title)}`;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** The scenario file a per-`it` id belongs to, or null when the id is not one. */
|
|
55
|
+
export function scenarioFileOfItId(requirementId: string): string | null {
|
|
56
|
+
if (!requirementId.startsWith(IT_ID_PREFIX)) return null;
|
|
57
|
+
const rest = requirementId.slice(IT_ID_PREFIX.length);
|
|
58
|
+
const dot = rest.indexOf('.');
|
|
59
|
+
if (dot <= 0) return null;
|
|
60
|
+
return `${rest.slice(0, dot)}.test.ts`;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Collision suffixing: the first occurrence keeps the bare id, later ones get
|
|
65
|
+
* `~2`, `~3`, … Keyed per file so a worker running many files never crosses
|
|
66
|
+
* streams. Reset per file by `setup.ts` when the file finishes.
|
|
67
|
+
*/
|
|
68
|
+
export class ItIdAllocator {
|
|
69
|
+
private readonly seen = new Map<string, number>();
|
|
70
|
+
|
|
71
|
+
allocate(scenarioFile: string, title: string): string {
|
|
72
|
+
const base = itRequirementId(scenarioFile, title);
|
|
73
|
+
const n = (this.seen.get(base) ?? 0) + 1;
|
|
74
|
+
this.seen.set(base, n);
|
|
75
|
+
return n === 1 ? base : `${base}~${n}`;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
reset(): void {
|
|
79
|
+
this.seen.clear();
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// ---------------------------------------------------------------------------
|
|
84
|
+
// Explicit override for the current test.
|
|
85
|
+
// ---------------------------------------------------------------------------
|
|
86
|
+
|
|
87
|
+
let explicitId: string | null = null;
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Attach a hand-authored requirement id to the CURRENT test. The per-`it`
|
|
91
|
+
* ledger row for this test is recorded under `id` instead of the derived
|
|
92
|
+
* title id. Returns the usual `driver.describe`-style message so the call
|
|
93
|
+
* doubles as the assertion message:
|
|
94
|
+
*
|
|
95
|
+
* expect(x, req('openwop.auth.subject-link.leaver-deny', 'auth-profiles.md §Subject linking', 'deactivation MUST deny')).toBe(true)
|
|
96
|
+
*
|
|
97
|
+
* The id MUST be listed in `conformance/requirements.json` (the generator
|
|
98
|
+
* collects `req(` first-argument literals); an unlisted id fails the registry
|
|
99
|
+
* check, so a hand id cannot drift from the registry silently.
|
|
100
|
+
*/
|
|
101
|
+
export function req(id: string, specSection: string, requirement: string): string {
|
|
102
|
+
explicitId = id;
|
|
103
|
+
return `${specSection}: ${requirement}`;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** setup.ts reads and clears the override after each test. */
|
|
107
|
+
export function takeExplicitRequirementId(): string | null {
|
|
108
|
+
const id = explicitId;
|
|
109
|
+
explicitId = null;
|
|
110
|
+
return id;
|
|
111
|
+
}
|
package/src/lib/saml-idp.test.ts
CHANGED
|
@@ -19,10 +19,14 @@
|
|
|
19
19
|
*
|
|
20
20
|
* @see conformance/src/lib/saml-idp.ts
|
|
21
21
|
* @see RFCS/0163-subject-linking-hardening.md §B
|
|
22
|
+
*
|
|
23
|
+
* The RFC 0050 §A reference suite (1 positive + 6 negative variants against the
|
|
24
|
+
* same fixture) moved here from `scenarios/auth-saml-profile.test.ts` on
|
|
25
|
+
* 2026-09-02 (RFC 0163 gap G5) for the same reason.
|
|
22
26
|
*/
|
|
23
27
|
|
|
24
28
|
import { describe, it, expect } from 'vitest';
|
|
25
|
-
import { createSyntheticSamlIdp, DEFAULT_SAML_ENTITY_ID } from './saml-idp.js';
|
|
29
|
+
import { createSyntheticSamlIdp, DEFAULT_SAML_ENTITY_ID, type SamlVariant } from './saml-idp.js';
|
|
26
30
|
|
|
27
31
|
const COLLIDING_SUBJECT = 'idp-op-8f3a';
|
|
28
32
|
|
|
@@ -81,3 +85,43 @@ describe('saml-idp: two-trust-root fixture (RFC 0163 §B)', () => {
|
|
|
81
85
|
expect(idpB.verify(forged).valid, 'the original root MUST NOT verify it either').toBe(false);
|
|
82
86
|
});
|
|
83
87
|
});
|
|
88
|
+
|
|
89
|
+
describe('saml-idp: synthetic-IdP reference suite (RFC 0050 §A)', () => {
|
|
90
|
+
// Server-free: the bundled synthetic IdP (conformance/src/lib/saml-idp.ts)
|
|
91
|
+
// mints a valid assertion + the 6 negative variants, and its verify()
|
|
92
|
+
// implements the RFC 0050 §A MUST list. This proves each negative is
|
|
93
|
+
// detectably malformed and gives the suite a reference SAML validator.
|
|
94
|
+
// A host's real ACS validates the SAME assertions over the
|
|
95
|
+
// `auth/saml/validate` seam (scenarios/auth-saml-profile.test.ts, gated on
|
|
96
|
+
// OPENWOP_TEST_SAML_IDP_URL).
|
|
97
|
+
const idp = createSyntheticSamlIdp();
|
|
98
|
+
|
|
99
|
+
it('publishes a PEM signing certificate', () => {
|
|
100
|
+
expect(idp.certificatePem).toContain('BEGIN PUBLIC KEY');
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
it('accepts a valid signed, in-window, non-wrapped assertion', () => {
|
|
104
|
+
const r = idp.verify(idp.mint('valid'));
|
|
105
|
+
expect(r.valid, `expected valid; got reason=${r.reason}`).toBe(true);
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
const negatives: ReadonlyArray<[Exclude<SamlVariant, 'valid'>, string]> = [
|
|
109
|
+
['alg-none', 'alg-none'],
|
|
110
|
+
['unsigned', 'unsigned'],
|
|
111
|
+
['bad-signature', 'bad-signature'],
|
|
112
|
+
['expired', 'expired'],
|
|
113
|
+
['not-yet-valid', 'not-yet-valid'],
|
|
114
|
+
['signature-wrapping', 'signature-wrapping'],
|
|
115
|
+
];
|
|
116
|
+
|
|
117
|
+
for (const [variant, expectedReason] of negatives) {
|
|
118
|
+
it(`rejects the ${variant} assertion (RFC 0050 §A MUST)`, () => {
|
|
119
|
+
const r = idp.verify(idp.mint(variant));
|
|
120
|
+
expect(r.valid, `${variant} MUST be rejected`).toBe(false);
|
|
121
|
+
expect(
|
|
122
|
+
r.reason,
|
|
123
|
+
`${variant} MUST be rejected for the ${expectedReason} reason`,
|
|
124
|
+
).toBe(expectedReason);
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
});
|
|
@@ -26,6 +26,7 @@
|
|
|
26
26
|
* host or a vitest subprocess.
|
|
27
27
|
*/
|
|
28
28
|
|
|
29
|
+
import { scenarioFileOfItId } from './requirement-ids.js';
|
|
29
30
|
import { PROFILE_FLOOR_SCENARIOS } from './profiles.js';
|
|
30
31
|
import { requirementIdForScenario, requirementIdForPrefix, requirementsFor } from './requirement-registry.js';
|
|
31
32
|
import { UNCLASSIFIED_RETURN_DETAIL } from './soft-skip.js';
|
|
@@ -285,6 +286,27 @@ export function deriveRequirementDispositions(
|
|
|
285
286
|
rows.push(row);
|
|
286
287
|
}
|
|
287
288
|
|
|
289
|
+
// Per-`it` rows (suite 1.153.0): every ledger entry keyed `openwop.it.<file>.<slug>`
|
|
290
|
+
// becomes its own bundle row, attributed to its scenario file. Additive — the
|
|
291
|
+
// file-level and prefix rows above are unchanged, and the floors still key on
|
|
292
|
+
// them. This is the granularity RFC 0148 §A describes and the G8 fix.
|
|
293
|
+
const emitted = new Set(rows.map((r) => r.requirementId));
|
|
294
|
+
for (const e of [...ledger].sort((a, b) => a.requirementId.localeCompare(b.requirementId))) {
|
|
295
|
+
const file = scenarioFileOfItId(e.requirementId);
|
|
296
|
+
// Attribute only to files this run reported on: a worker's ledger can carry
|
|
297
|
+
// rows from files outside the certified set (the suite's own lib tests, or
|
|
298
|
+
// a filtered run), and those are not evidence about the host.
|
|
299
|
+
if (file === null || emitted.has(e.requirementId) || !reportStates.has(file)) continue;
|
|
300
|
+
emitted.add(e.requirementId);
|
|
301
|
+
rows.push({
|
|
302
|
+
requirementId: e.requirementId,
|
|
303
|
+
scenarioId: file,
|
|
304
|
+
disposition: e.disposition,
|
|
305
|
+
...(e.detail === undefined ? {} : { detail: e.detail }),
|
|
306
|
+
...(e.assertionCount === undefined ? {} : { assertionCount: e.assertionCount }),
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
|
|
288
310
|
const totals = { executedPass: 0, executedFail: 0, skipped: 0, inapplicable: 0, blocked: 0 };
|
|
289
311
|
for (const r of rows) {
|
|
290
312
|
if (r.disposition === 'executed-pass') totals.executedPass++;
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
|
|
24
24
|
import { describe, it, expect } from 'vitest';
|
|
25
25
|
import { driver } from '../lib/driver.js';
|
|
26
|
-
import {
|
|
26
|
+
import { type SamlVariant } from '../lib/saml-idp.js';
|
|
27
27
|
import { capabilityFamily } from '../lib/discovery-capabilities.js';
|
|
28
28
|
|
|
29
29
|
const SAML_PROFILE = 'openwop-auth-saml';
|
|
@@ -117,41 +117,9 @@ describe('auth-saml-profile: assertion validation (RFC 0050 §A — opt-in)', ()
|
|
|
117
117
|
}
|
|
118
118
|
});
|
|
119
119
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
// `auth/saml/validate` seam (gated above on OPENWOP_TEST_SAML_IDP_URL).
|
|
127
|
-
const idp = createSyntheticSamlIdp();
|
|
128
|
-
|
|
129
|
-
it('publishes a PEM signing certificate', () => {
|
|
130
|
-
expect(idp.certificatePem).toContain('BEGIN PUBLIC KEY');
|
|
131
|
-
});
|
|
132
|
-
|
|
133
|
-
it('accepts a valid signed, in-window, non-wrapped assertion', () => {
|
|
134
|
-
const r = idp.verify(idp.mint('valid'));
|
|
135
|
-
expect(r.valid, `expected valid; got reason=${r.reason}`).toBe(true);
|
|
136
|
-
});
|
|
137
|
-
|
|
138
|
-
const negatives: ReadonlyArray<[Exclude<SamlVariant, 'valid'>, string]> = [
|
|
139
|
-
['alg-none', 'alg-none'],
|
|
140
|
-
['unsigned', 'unsigned'],
|
|
141
|
-
['bad-signature', 'bad-signature'],
|
|
142
|
-
['expired', 'expired'],
|
|
143
|
-
['not-yet-valid', 'not-yet-valid'],
|
|
144
|
-
['signature-wrapping', 'signature-wrapping'],
|
|
145
|
-
];
|
|
146
|
-
|
|
147
|
-
for (const [variant, expectedReason] of negatives) {
|
|
148
|
-
it(`rejects the ${variant} assertion (RFC 0050 §A MUST)`, () => {
|
|
149
|
-
const r = idp.verify(idp.mint(variant));
|
|
150
|
-
expect(r.valid, `${variant} MUST be rejected`).toBe(false);
|
|
151
|
-
expect(
|
|
152
|
-
r.reason,
|
|
153
|
-
`${variant} MUST be rejected for the ${expectedReason} reason`,
|
|
154
|
-
).toBe(expectedReason);
|
|
155
|
-
});
|
|
156
|
-
}
|
|
157
|
-
});
|
|
120
|
+
// The RFC 0050 §A synthetic-IdP reference suite (1 positive + 6 negatives against
|
|
121
|
+
// the bundled fixture) moved to `conformance/src/lib/saml-idp.test.ts` on
|
|
122
|
+
// 2026-09-02 (RFC 0163 gap G5): it proves the FIXTURE, not a host, and a
|
|
123
|
+
// server-free block inside a scenario file records `executed-pass` in every
|
|
124
|
+
// host bundle (conformance-certification.md gap G8). This file keeps only the
|
|
125
|
+
// host-facing legs above.
|
package/src/setup.ts
CHANGED
|
@@ -30,11 +30,13 @@ import { setMultiAgentCapabilities } from './lib/multi-agent-capabilities.js';
|
|
|
30
30
|
import { OtelCollector, setCollector } from './lib/otel-collector.js';
|
|
31
31
|
import { McpFakeServer, setMcpFakeServer } from './lib/mcp-fake-server.js';
|
|
32
32
|
import { A2AFakePeer, setA2AFakePeer } from './lib/a2a-fake-peer.js';
|
|
33
|
-
import { afterAll, afterEach, beforeAll, expect } from 'vitest';
|
|
33
|
+
import { afterAll, afterEach, beforeAll, beforeEach, expect } from 'vitest';
|
|
34
34
|
import { basename } from 'node:path';
|
|
35
35
|
import { recordRequirement, hasRequirement, journalLength, journalSince } from './lib/requirement-ledger.js';
|
|
36
36
|
import { requirementIdForFile, resolveFileRecord, type FileTestState } from './lib/scenario-disposition.js';
|
|
37
37
|
import { softSkipDisposition } from './lib/soft-skip.js';
|
|
38
|
+
import { ItIdAllocator, takeExplicitRequirementId } from './lib/requirement-ids.js';
|
|
39
|
+
import { SPEC_COHERENCE_SCENARIOS, SPEC_COHERENCE_DETAIL } from './lib/spec-coherence.js';
|
|
38
40
|
import type { DiscoveryPayload } from './lib/profiles.js';
|
|
39
41
|
|
|
40
42
|
const SUITE_INIT_TIMEOUT_MS = 5_000;
|
|
@@ -225,6 +227,22 @@ await maybeStartA2AFakePeer();
|
|
|
225
227
|
const _fileStates = new Map<string, FileTestState[]>();
|
|
226
228
|
const _fileAssertions = new Map<string, number>();
|
|
227
229
|
const _ledgerMarks = new Map<string, number>();
|
|
230
|
+
// Per-`it` recording (v2 charter Phase 1, suite 1.153.0 — the durable G8 fix
|
|
231
|
+
// named in scenario-disposition.ts). Each test gets its own ledger row under
|
|
232
|
+
// `openwop.it.<file>.<title-slug>` so a file that asserted a positive control
|
|
233
|
+
// and soft-skipped the requirement no longer certifies the requirement. The
|
|
234
|
+
// file-level row is RETAINED (floors key on it); the per-`it` rows are
|
|
235
|
+
// additive bundle rows (RFC 0148 §C `requirements[]` accepts any id).
|
|
236
|
+
const _itAllocators = new Map<string, ItIdAllocator>();
|
|
237
|
+
const _itMarks = new Map<string, number>();
|
|
238
|
+
const _itAssertionsBefore = new Map<string, number>();
|
|
239
|
+
function _assertionCalls(): number {
|
|
240
|
+
try {
|
|
241
|
+
return (expect.getState() as { assertionCalls?: number }).assertionCalls ?? 0;
|
|
242
|
+
} catch {
|
|
243
|
+
return 0;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
228
246
|
function _fileOf(task: { file?: { filepath?: string; name?: string } } | undefined): string | null {
|
|
229
247
|
const f = task?.file?.filepath ?? task?.file?.name;
|
|
230
248
|
return typeof f === 'string' && f.length > 0 ? basename(f) : null;
|
|
@@ -236,6 +254,14 @@ beforeAll(({}, suite) => {
|
|
|
236
254
|
const file = _fileOf({ file: s.file ?? s });
|
|
237
255
|
if (file !== null && !_ledgerMarks.has(file)) _ledgerMarks.set(file, journalLength());
|
|
238
256
|
});
|
|
257
|
+
beforeEach(({ task }) => {
|
|
258
|
+
const file = _fileOf(task as { file?: { filepath?: string; name?: string } });
|
|
259
|
+
if (file === null) return;
|
|
260
|
+
// Window for this test's own gate decisions and its own assertion count.
|
|
261
|
+
_itMarks.set(file, journalLength());
|
|
262
|
+
_itAssertionsBefore.set(file, _assertionCalls());
|
|
263
|
+
takeExplicitRequirementId(); // clear any override left by a test that threw before afterEach
|
|
264
|
+
});
|
|
239
265
|
afterEach(({ task }) => {
|
|
240
266
|
const file = _fileOf(task as { file?: { filepath?: string; name?: string } });
|
|
241
267
|
if (file === null) return;
|
|
@@ -248,13 +274,63 @@ afterEach(({ task }) => {
|
|
|
248
274
|
// A leg that early-returns from a gate makes zero, and a file of such legs is
|
|
249
275
|
// an `executed-pass` with assertionCount 0 — visible, and unclassified for a
|
|
250
276
|
// claimed floor.
|
|
251
|
-
|
|
277
|
+
//
|
|
278
|
+
// `expect.getState().assertionCalls` is per-test in vitest (reset at each
|
|
279
|
+
// test start), so it is this test's count; the file total is the sum.
|
|
280
|
+
const calls = _assertionCalls();
|
|
281
|
+
_fileAssertions.set(file, (_fileAssertions.get(file) ?? 0) + calls);
|
|
282
|
+
|
|
283
|
+
// Per-`it` row. Disposition follows RFC 0148 §A at test granularity:
|
|
284
|
+
// fail → executed-fail (detail = the first error message)
|
|
285
|
+
// skip (ctx.skip / it.skip) → the gate's recorded reason since this test began, else `skipped`
|
|
286
|
+
// pass with ≥1 assertion → executed-pass
|
|
287
|
+
// pass with 0 assertions → the gate's recorded reason (softSkip / seamAbsent / behaviorGate)
|
|
288
|
+
// since this test began, else `blocked` (unclassified return)
|
|
289
|
+
if (!file.endsWith('.test.ts')) return;
|
|
290
|
+
// Only SCENARIO files get per-`it` rows. The setup hooks run for every file in
|
|
291
|
+
// the worker, including `src/lib/*.test.ts` — the suite's own self-tests,
|
|
292
|
+
// which prove fixtures and helpers, not a host, and must never become bundle
|
|
293
|
+
// evidence (that is why RFC 0163 G5 / RFC 0050 G1 moved them out of scenarios).
|
|
294
|
+
const filepath = (task as { file?: { filepath?: string } }).file?.filepath ?? '';
|
|
295
|
+
if (!/[\\/]src[\\/]scenarios[\\/]/.test(filepath)) {
|
|
296
|
+
takeExplicitRequirementId();
|
|
297
|
+
return;
|
|
298
|
+
}
|
|
299
|
+
const explicit = takeExplicitRequirementId();
|
|
300
|
+
const alloc = _itAllocators.get(file) ?? new ItIdAllocator();
|
|
301
|
+
_itAllocators.set(file, alloc);
|
|
302
|
+
const itId = explicit ?? alloc.allocate(file, task.name);
|
|
303
|
+
const since = journalSince(_itMarks.get(file) ?? 0);
|
|
304
|
+
const gate = since.find((e) => e.disposition === 'inapplicable') ?? since.find((e) => e.disposition === 'skipped');
|
|
305
|
+
let disposition: 'executed-pass' | 'executed-fail' | 'skipped' | 'inapplicable' | 'blocked';
|
|
306
|
+
let detail: string | undefined;
|
|
307
|
+
if (SPEC_COHERENCE_SCENARIOS.has(file) || SPEC_COHERENCE_SCENARIOS.has(file.replace(/\.test\.ts$/, ''))) {
|
|
308
|
+
// Corpus-coherence scenario: it reads spec/v1 and asserts nothing about a
|
|
309
|
+
// host, in any layout. Its rows are `inapplicable` to every host — the same
|
|
310
|
+
// rule `resolveFileRecord` applies to the file in the published layout.
|
|
311
|
+
disposition = 'inapplicable';
|
|
312
|
+
detail = SPEC_COHERENCE_DETAIL;
|
|
313
|
+
} else if (state === 'fail') {
|
|
314
|
+
disposition = 'executed-fail';
|
|
315
|
+
const err = (task.result?.errors ?? [])[0] as { message?: string } | undefined;
|
|
316
|
+
detail = `the test executed and failed: ${(err?.message ?? 'no message').slice(0, 300)}`;
|
|
317
|
+
} else if (state === 'pass' && calls > 0) {
|
|
318
|
+
disposition = 'executed-pass';
|
|
319
|
+
} else if (gate !== undefined) {
|
|
320
|
+
disposition = gate.disposition as 'skipped' | 'inapplicable';
|
|
321
|
+
detail = gate.detail ?? `${gate.disposition} (gate recorded no reason)`;
|
|
322
|
+
} else if (state === 'pass') {
|
|
323
|
+
disposition = 'blocked';
|
|
324
|
+
detail = 'unclassified return: the test passed with zero assertions and recorded no reason — RFC 0148 §A resolves it to blocked, never to a pass';
|
|
325
|
+
} else {
|
|
326
|
+
disposition = 'skipped';
|
|
327
|
+
detail = 'vitest skipped the test (ctx.skip / it.skip) without a recorded gate reason';
|
|
328
|
+
}
|
|
252
329
|
try {
|
|
253
|
-
|
|
330
|
+
recordRequirement(itId, disposition, detail, { assertionCount: calls });
|
|
254
331
|
} catch {
|
|
255
|
-
/*
|
|
332
|
+
/* never fail a test for bookkeeping */
|
|
256
333
|
}
|
|
257
|
-
_fileAssertions.set(file, (_fileAssertions.get(file) ?? 0) + calls);
|
|
258
334
|
});
|
|
259
335
|
afterAll(({}, suite) => {
|
|
260
336
|
// vitest 4: the suite/file task is the SECOND argument. For a file-level
|
|
@@ -306,5 +382,8 @@ afterAll(({}, suite) => {
|
|
|
306
382
|
_fileStates.delete(file);
|
|
307
383
|
_fileAssertions.delete(file);
|
|
308
384
|
_ledgerMarks.delete(file);
|
|
385
|
+
_itAllocators.delete(file);
|
|
386
|
+
_itMarks.delete(file);
|
|
387
|
+
_itAssertionsBefore.delete(file);
|
|
309
388
|
});
|
|
310
389
|
|