@openwop/openwop-conformance 1.106.1 → 1.124.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/coverage.md +6 -0
- package/dist/cli.js +89 -45
- package/dist/lib/certification-bundle-verify.js +274 -0
- package/dist/lib/profiles.js +102 -19
- package/dist/lib/requirement-ledger.js +182 -0
- package/dist/lib/requirement-registry.js +110 -0
- package/dist/lib/scenario-disposition.js +249 -0
- package/dist/lib/soft-skip.js +82 -0
- package/fixtures/conformance-a2a-task-roundtrip.json +1 -1
- package/fixtures.md +1 -1
- package/package.json +1 -1
- package/schemas/CORPUS-STAMP.json +2 -2
- package/schemas/README.md +1 -0
- package/schemas/capabilities.schema.json +2 -2
- package/schemas/compensation-policy.schema.json +64 -0
- package/schemas/run-snapshot.schema.json +5 -0
- package/schemas/workflow-chain-pack-manifest.schema.json +125 -0
- package/schemas/workflow-definition.schema.json +4 -0
- package/src/cli.ts +99 -45
- package/src/lib/a2a-fake-peer.ts +290 -110
- package/src/lib/certification-bundle-verify.ts +356 -0
- package/src/lib/error-envelope.ts +77 -0
- package/src/lib/llm-cache-key-recipe.ts +6 -5
- package/src/lib/mcp-fake-server.ts +235 -72
- package/src/lib/profiles.ts +132 -19
- package/src/lib/requirement-ledger.ts +98 -1
- package/src/lib/requirement-registry.ts +47 -4
- package/src/lib/scenario-disposition.ts +291 -0
- package/src/lib/soft-skip.ts +89 -0
- package/src/lib/workflow-chain-expansion.ts +189 -0
- package/src/scenarios/a2a-1-0-agent-card.test.ts +179 -0
- package/src/scenarios/a2a-1-0-task-roundtrip.test.ts +109 -0
- package/src/scenarios/a2a-card-runtime-consistency.test.ts +167 -0
- package/src/scenarios/a2a-peer-authority.test.ts +99 -0
- package/src/scenarios/a2a-task-roundtrip.test.ts +18 -6
- package/src/scenarios/agent-loop-iteration-monotonic.test.ts +3 -2
- package/src/scenarios/agent-loop-stateful-resume.test.ts +3 -2
- package/src/scenarios/agent-loop-version5-shape.test.ts +2 -1
- package/src/scenarios/agent-loop-workspace-snapshot.test.ts +4 -3
- package/src/scenarios/approval-gate-flow.test.ts +5 -4
- package/src/scenarios/artifact-type-store-without-render.test.ts +2 -1
- package/src/scenarios/auth-scim-profile.test.ts +5 -4
- package/src/scenarios/authorization-fail-closed.test.ts +4 -3
- package/src/scenarios/certification-bundle-non-vacuous.test.ts +249 -0
- package/src/scenarios/certification-bundle-redaction.test.ts +182 -0
- package/src/scenarios/certification-floor-enforcement.test.ts +30 -6
- package/src/scenarios/chain-compensation-expansion.test.ts +243 -0
- package/src/scenarios/commitment-fired.test.ts +3 -2
- package/src/scenarios/compensation-behavior.test.ts +63 -0
- package/src/scenarios/compensation-profile.test.ts +144 -1
- package/src/scenarios/conformance-advertised-seam-required.test.ts +118 -0
- package/src/scenarios/conformance-execution-witness.test.ts +104 -0
- package/src/scenarios/conversationCapabilityNegotiation.test.ts +3 -2
- package/src/scenarios/core-manifest-and-extension-registry.test.ts +26 -0
- package/src/scenarios/credential-payload-redaction.test.ts +4 -3
- package/src/scenarios/credentials-capability-shape.test.ts +4 -3
- package/src/scenarios/cross-engine-append-behavior.test.ts +5 -4
- package/src/scenarios/cross-engine-append-ordering.test.ts +3 -2
- package/src/scenarios/cross-host-ancestry-endpoint.test.ts +5 -4
- package/src/scenarios/cross-host-causation-shape.test.ts +3 -2
- package/src/scenarios/cross-workspace-isolation.test.ts +4 -3
- package/src/scenarios/data-residency-admission.test.ts +6 -6
- package/src/scenarios/deadletter-retry-exhaustion.test.ts +5 -4
- package/src/scenarios/distillation-index-roundtrip.test.ts +3 -2
- package/src/scenarios/distillation-secret-carryforward.test.ts +3 -2
- package/src/scenarios/distillation-shape.test.ts +2 -1
- package/src/scenarios/distillation-stable-archive.test.ts +4 -3
- package/src/scenarios/distillation-token-budget.test.ts +4 -3
- package/src/scenarios/envelope-recovery-applied.test.ts +3 -2
- package/src/scenarios/envelope-refusal-shape.test.ts +4 -3
- package/src/scenarios/error-envelope-canonical-shape.test.ts +64 -0
- package/src/scenarios/feedback-correction-redaction.test.ts +4 -3
- package/src/scenarios/feedback-cross-tenant-isolation.test.ts +4 -3
- package/src/scenarios/feedback-fork-not-copied.test.ts +6 -5
- package/src/scenarios/feedback-on-terminal-run.test.ts +4 -3
- package/src/scenarios/feedback-record-and-list.test.ts +4 -3
- package/src/scenarios/feedback-unsupported-501.test.ts +3 -2
- package/src/scenarios/fixtures-valid.test.ts +16 -18
- package/src/scenarios/fs-path-traversal.test.ts +3 -2
- package/src/scenarios/heartbeat-capability-shape.test.ts +2 -1
- package/src/scenarios/heartbeat-fires-once-per-tick.test.ts +4 -3
- package/src/scenarios/heartbeat-idempotent-no-spam.test.ts +6 -5
- package/src/scenarios/heartbeat-runtime-bound.test.ts +4 -3
- package/src/scenarios/mcp-2026-07-28-discover.test.ts +207 -0
- package/src/scenarios/mcp-cache-tenant-scope.test.ts +66 -0
- package/src/scenarios/mcp-current-auth-boundary.test.ts +55 -0
- package/src/scenarios/mcp-extension-opacity.test.ts +79 -0
- package/src/scenarios/mcp-mrtr-roundtrip.test.ts +140 -0
- package/src/scenarios/mcp-stateless-request.test.ts +77 -0
- package/src/scenarios/mcp-tool-roundtrip.test.ts +4 -3
- package/src/scenarios/mcp-toolcall-redaction.test.ts +2 -1
- package/src/scenarios/memory-attribution-emits-on-write.test.ts +6 -5
- package/src/scenarios/memory-attribution-no-content.test.ts +4 -3
- package/src/scenarios/memory-attribution-replay-stable.test.ts +6 -5
- package/src/scenarios/memory-attribution-tenant-scoped.test.ts +4 -3
- package/src/scenarios/memory-compaction-event-emitted.test.ts +3 -2
- package/src/scenarios/memory-compaction-provenance-tag.test.ts +6 -5
- package/src/scenarios/memory-compaction-sr1-carry-forward.test.ts +4 -3
- package/src/scenarios/memory-consolidation-idempotent.test.ts +4 -3
- package/src/scenarios/memory-injection-budget.test.ts +4 -3
- package/src/scenarios/metric-emission.test.ts +4 -3
- package/src/scenarios/multi-agent-confidence-escalation.test.ts +5 -4
- package/src/scenarios/multi-agent-handoff-state-machine.test.ts +4 -3
- package/src/scenarios/multi-agent-memory-lifecycle.test.ts +5 -4
- package/src/scenarios/multi-party-conversation-behavioral.test.ts +3 -2
- package/src/scenarios/multi-region-idempotency-behavior.test.ts +7 -6
- package/src/scenarios/multi-region-idempotency.test.ts +4 -3
- package/src/scenarios/normative-example-extraction.test.ts +242 -0
- package/src/scenarios/oauth-authorization-code-roundtrip.test.ts +4 -3
- package/src/scenarios/oauth-capability-shape.test.ts +4 -3
- package/src/scenarios/oauth-connector-redaction.test.ts +4 -3
- package/src/scenarios/openapi-asyncapi-sdk-parity.test.ts +309 -0
- package/src/scenarios/org-position-no-authority-escalation.test.ts +2 -1
- package/src/scenarios/otel-emission-grpc.test.ts +4 -3
- package/src/scenarios/otel-emission.test.ts +4 -3
- package/src/scenarios/otel-trace-propagation-subworkflow.test.ts +5 -4
- package/src/scenarios/otel-trace-propagation.test.ts +4 -3
- package/src/scenarios/pack-registry-isolation.test.ts +4 -3
- package/src/scenarios/pack-registry-publish.test.ts +65 -3
- package/src/scenarios/pack-registry.test.ts +22 -0
- package/src/scenarios/profile-discovery-core-alias.test.ts +137 -0
- package/src/scenarios/profileDerivation.test.ts +2 -1
- package/src/scenarios/provider-usage.test.ts +2 -2
- package/src/scenarios/rate-limit-envelope.test.ts +2 -1
- package/src/scenarios/registry-public.test.ts +6 -5
- package/src/scenarios/replay-divergence-at-refusal.test.ts +9 -8
- package/src/scenarios/replay-llm-cache-key-portable.test.ts +19 -13
- package/src/scenarios/replay-llm-cache-key.test.ts +120 -47
- package/src/scenarios/replay-observable-sequence-determinism.test.ts +5 -4
- package/src/scenarios/replay-side-effect-suppression.test.ts +20 -1
- package/src/scenarios/requirement-ledger.test.ts +29 -6
- package/src/scenarios/runner-ledger.test.ts +279 -0
- package/src/scenarios/runtime-requires-install-gate.test.ts +6 -5
- package/src/scenarios/sandbox-memory-cap.test.ts +3 -2
- package/src/scenarios/sandbox-mvp-behavior.test.ts +11 -10
- package/src/scenarios/sandbox-no-host-fs-escape.test.ts +2 -1
- package/src/scenarios/sandbox-timeout-cap.test.ts +3 -2
- package/src/scenarios/stream-text-fixture.test.ts +5 -4
- package/src/scenarios/subrun-approval-fail-closed.test.ts +3 -2
- package/src/scenarios/subrun-approval-gate.test.ts +4 -3
- package/src/scenarios/subrun-attestation-shape.test.ts +2 -1
- package/src/scenarios/subrun-checksum-stable.test.ts +4 -3
- package/src/scenarios/table-schema-enforcement.test.ts +2 -2
- package/src/scenarios/tool-hooks-authorization-fail-closed.test.ts +3 -2
- package/src/scenarios/tool-hooks-content-free.test.ts +3 -2
- package/src/scenarios/tool-hooks-rate-limit.test.ts +3 -2
- package/src/scenarios/tool-hooks-secret-redaction.test.ts +3 -2
- package/src/scenarios/tool-hooks-shape.test.ts +2 -1
- package/src/scenarios/trigger-bridge-delivery.test.ts +3 -2
- package/src/scenarios/version-fold.test.ts +5 -4
- package/src/scenarios/voice-streamref-tenant-bound.test.ts +2 -1
- package/src/scenarios/voice-transcription-streaming.test.ts +2 -1
- package/src/scenarios/voice-transcription-unadvertised.test.ts +2 -1
- package/src/scenarios/wasm-pack-abi-version-rejection.test.ts +5 -4
- package/src/scenarios/wasm-pack-invoke-completed.test.ts +3 -2
- package/src/scenarios/wasm-pack-invoke-suspended.test.ts +3 -2
- package/src/scenarios/wasm-pack-load.test.ts +4 -3
- package/src/scenarios/wasm-pack-memory-cap.test.ts +4 -3
- package/src/scenarios/wasm-pack-replay-determinism.test.ts +3 -2
- package/src/scenarios/webhook-signed-delivery.test.ts +4 -3
- package/src/scenarios/workflow-primary-output-annotation.test.ts +13 -23
- package/src/scenarios/workload-identity-behavior.test.ts +7 -4
- package/src/scenarios/workspace-cross-tenant-isolation-blackbox.test.ts +3 -2
- package/src/scenarios/workspace-cross-tenant-isolation.test.ts +3 -2
- package/src/setup.ts +95 -1
package/dist/lib/profiles.js
CHANGED
|
@@ -19,7 +19,24 @@
|
|
|
19
19
|
* Closed v1.x catalog. Adding a profile requires an RFC per
|
|
20
20
|
* `RFCS/0001-rfc-process.md`.
|
|
21
21
|
*/
|
|
22
|
+
/**
|
|
23
|
+
* RFC 0155 §A/§E — deprecated profile aliases, keyed by alias → canonical id.
|
|
24
|
+
* A certification bundle v2 (`certification-bundle-v2.schema.json`) MUST name
|
|
25
|
+
* canonical ids in `claimedProfiles`; an alias that also derives goes in
|
|
26
|
+
* `aliases`, never in `claimedProfiles` — a badge cannot be substantiated by a
|
|
27
|
+
* name that no longer means what it did.
|
|
28
|
+
*/
|
|
29
|
+
export const DEPRECATED_PROFILE_ALIASES = {
|
|
30
|
+
'openwop-core': 'openwop-discovery-core',
|
|
31
|
+
};
|
|
22
32
|
export const PROFILE_NAMES = [
|
|
33
|
+
// RFC 0155 §A (2026-08-16): `openwop-discovery-core` is the CANONICAL name of
|
|
34
|
+
// the discovery predicate; `openwop-core` is its DEPRECATED ALIAS for all of
|
|
35
|
+
// v1 and derives exactly when the canonical name derives — both or neither.
|
|
36
|
+
// The predicate is a discovery-payload check and says nothing about whether a
|
|
37
|
+
// run can be started, suspended, or replayed; the executable floor is
|
|
38
|
+
// `openwop-core-standard` (`core-standard-profile.md`, RFC 0088).
|
|
39
|
+
'openwop-discovery-core',
|
|
23
40
|
'openwop-core',
|
|
24
41
|
'openwop-interrupts',
|
|
25
42
|
'openwop-stream-sse',
|
|
@@ -383,8 +400,10 @@ export function isCoreStandard(c) {
|
|
|
383
400
|
*/
|
|
384
401
|
export function deriveProfiles(c) {
|
|
385
402
|
const result = [];
|
|
386
|
-
if (isCore(c))
|
|
387
|
-
result.push('openwop-core');
|
|
403
|
+
if (isCore(c)) {
|
|
404
|
+
result.push('openwop-discovery-core');
|
|
405
|
+
result.push('openwop-core'); // deprecated alias — never alone (RFC 0155 §A)
|
|
406
|
+
}
|
|
388
407
|
if (isInterrupts(c))
|
|
389
408
|
result.push('openwop-interrupts');
|
|
390
409
|
if (isStreamSse(c))
|
|
@@ -412,7 +431,8 @@ export function deriveProfiles(c) {
|
|
|
412
431
|
*/
|
|
413
432
|
export function hasProfile(c, profile) {
|
|
414
433
|
switch (profile) {
|
|
415
|
-
case 'openwop-core':
|
|
434
|
+
case 'openwop-discovery-core':
|
|
435
|
+
case 'openwop-core': // deprecated alias of `openwop-discovery-core` (RFC 0155 §A)
|
|
416
436
|
return isCore(c);
|
|
417
437
|
case 'openwop-interrupts':
|
|
418
438
|
return isInterrupts(c);
|
|
@@ -447,7 +467,19 @@ export const PROFILE_FLOOR_SCENARIOS = {
|
|
|
447
467
|
'idempotency.test.ts',
|
|
448
468
|
'idempotency-key-determinism.test.ts',
|
|
449
469
|
'webhook-negative.test.ts',
|
|
450
|
-
|
|
470
|
+
// Removed 2026-08-16 (S6): since RFC 0088/0089 this list carried
|
|
471
|
+
// `audit-log-verification.test.ts` — a file that has NEVER existed. The
|
|
472
|
+
// RFC 0148 §A ledger surfaced it on its first end-to-end `--certify` run
|
|
473
|
+
// as an unclassified floor requirement. The scenario it meant is
|
|
474
|
+
// `audit-log-integrity.test.ts`, which soft-skips unless the host
|
|
475
|
+
// advertises the OPTIONAL annex profile `openwop-audit-log-integrity`
|
|
476
|
+
// (auth-profiles.md §"Audit-log integrity" — `GET /v1/audit/verify` is
|
|
477
|
+
// REQUIRED-when-profile-claimed, not a core MUST). §C's membership rule
|
|
478
|
+
// is "no soft-skip, no seam, no env-gate", and §B's predicate does not
|
|
479
|
+
// include that profile, so by RFC 0088's own rule the row was never a
|
|
480
|
+
// floor scenario. Audit-log integrity remains required by
|
|
481
|
+
// `openwop-production` (PROFILE-DECISION-GUIDE) and certifiable on its
|
|
482
|
+
// own profile; it is not part of `openwop-core-standard`.
|
|
451
483
|
],
|
|
452
484
|
requiredAnyPrefix: ['interrupt-'],
|
|
453
485
|
},
|
|
@@ -460,7 +492,8 @@ export const PROFILE_FLOOR_SCENARIOS = {
|
|
|
460
492
|
// against nothing (RFC 0148 §C, gap G6).
|
|
461
493
|
// `profiles.md` §`openwop-core` is a pure discovery-payload predicate — it
|
|
462
494
|
// names no runtime scenario, so the predicate IS the whole claim.
|
|
463
|
-
'openwop-core': { required: [], discoveryOnly: true },
|
|
495
|
+
'openwop-discovery-core': { required: [], discoveryOnly: true },
|
|
496
|
+
'openwop-core': { required: [], discoveryOnly: true }, // deprecated alias (RFC 0155 §A)
|
|
464
497
|
// `profiles.md` §`openwop-fixtures`: "The profile is discovery-payload-only —
|
|
465
498
|
// it confirms the host claims SOME fixture, not that any specific fixture is
|
|
466
499
|
// wired."
|
|
@@ -475,18 +508,55 @@ export const PROFILE_FLOOR_SCENARIOS = {
|
|
|
475
508
|
'openwop-stream-poll': { required: ['stream-modes.test.ts'] },
|
|
476
509
|
// `profiles.md` §`openwop-node-packs`: "a host passes `openwop-node-packs`
|
|
477
510
|
// when it passes those scenarios."
|
|
478
|
-
|
|
479
|
-
//
|
|
480
|
-
//
|
|
481
|
-
|
|
482
|
-
//
|
|
483
|
-
//
|
|
484
|
-
//
|
|
485
|
-
//
|
|
486
|
-
//
|
|
487
|
-
//
|
|
488
|
-
//
|
|
489
|
-
//
|
|
511
|
+
// RFC 0025 makes the `/v1/packs/*` read surface unconditional-when-shipped
|
|
512
|
+
// with NO discovery advert, and `profiles.md` §openwop-node-packs says the
|
|
513
|
+
// profile "is derivable from which scenarios pass" — so it is runtime-derived.
|
|
514
|
+
'openwop-node-packs': { required: ['pack-registry.test.ts', 'pack-registry-publish.test.ts'], runtimeDerived: true },
|
|
515
|
+
// ── Transcribed 2026-08-16 (RFC 0148 §C gap G7) ─────────────────────────────
|
|
516
|
+
// Each floor below is named in `profiles.md`'s section for the profile
|
|
517
|
+
// ("Floor (RFC 0148 §C)") and obeys `core-standard-profile.md` §C's membership
|
|
518
|
+
// rule: black-box, no seam, no soft-skip on the profile's OWN advert. Fixture-
|
|
519
|
+
// gated scenarios stay in (the conformance fixture IS the black-box seam and
|
|
520
|
+
// its absence records honestly as blocked/skipped), capability-gated ones are
|
|
521
|
+
// in only where the gate is the profile predicate itself.
|
|
522
|
+
// `profiles.md` §`openwop-interrupts`: the `clarification.request` envelope is
|
|
523
|
+
// the canonical interrupt envelope; the interrupt family proves resume.
|
|
524
|
+
'openwop-interrupts': { required: ['interrupt-clarification.test.ts'], requiredAnyPrefix: ['interrupt-'] },
|
|
525
|
+
// `profiles.md` §`openwop-secrets`: credential resolution per `run-options.md`
|
|
526
|
+
// §"Credential references" — the BYOK canary round-trip (`fixtures.md`
|
|
527
|
+
// §conformance-secrets-roundtrip, SR-1) is the profile's proof.
|
|
528
|
+
'openwop-secrets': { required: ['byok-roundtrip.test.ts'] },
|
|
529
|
+
// `profiles.md` §`openwop-provider-policy`: the four-mode taxonomy shape and
|
|
530
|
+
// its enforcement on the wire.
|
|
531
|
+
'openwop-provider-policy': { required: ['policies.test.ts', 'providerPolicyEnforcement.test.ts'] },
|
|
532
|
+
// `profiles.md` §`openwop-memory`: capability model shape, write attribution
|
|
533
|
+
// (RFC 0080 §A) and the degraded-projection contract (RFC 0080 §C).
|
|
534
|
+
'openwop-memory': {
|
|
535
|
+
required: [
|
|
536
|
+
'memory-capability-model-shape.test.ts',
|
|
537
|
+
'memory-attribution-shape.test.ts',
|
|
538
|
+
'memory-attribution-emits-on-write.test.ts',
|
|
539
|
+
'memory-degraded-projection.test.ts',
|
|
540
|
+
],
|
|
541
|
+
},
|
|
542
|
+
// `profiles.md` §`openwop-trigger-bridge`: "The runtime conformance scenarios
|
|
543
|
+
// (`trigger-bridge-delivery.test.ts`, profile-gated) verify …" plus the shape.
|
|
544
|
+
'openwop-trigger-bridge': { required: ['trigger-bridge-shape.test.ts', 'trigger-bridge-delivery.test.ts'] },
|
|
545
|
+
// `profiles.md` §`openwop-replay-fork`: "This profile gates
|
|
546
|
+
// `replayDeterminism.test.ts` and `replay-fork.test.ts` … the conformance
|
|
547
|
+
// scenarios pass on whichever mode the host advertises." Discovery-conditional
|
|
548
|
+
// on `replay.modes` (RFC 0073 root family).
|
|
549
|
+
'openwop-replay-fork': {
|
|
550
|
+
required: [],
|
|
551
|
+
conditional: [
|
|
552
|
+
// `replay` mode carries caveat 1 (a replay MUST NOT re-fire external
|
|
553
|
+
// effects — unconditional, `replay.md`); the suppression scenario records
|
|
554
|
+
// `blocked` when no probeable mechanism is advertised, so a
|
|
555
|
+
// `sideEffectSuppression: none` host does NOT certify on determinism alone.
|
|
556
|
+
{ path: 'replay.modes', includes: 'replay', required: ['replayDeterminism.test.ts', 'replay-side-effect-suppression.test.ts'] },
|
|
557
|
+
{ path: 'replay.modes', includes: 'branch', required: ['replay-fork.test.ts'] },
|
|
558
|
+
],
|
|
559
|
+
},
|
|
490
560
|
};
|
|
491
561
|
/** Is `profile` derivable from a discovery document? Maps a profile name to its predicate (RFC 0089 §B(1)). */
|
|
492
562
|
export function profileDerivable(c, profile) {
|
|
@@ -517,9 +587,22 @@ export function verifyBundleProfile(bundle, profile) {
|
|
|
517
587
|
return { profile, derivable, floorProven: false, valid: false, missingFloor: [], floorUnspecified: true };
|
|
518
588
|
}
|
|
519
589
|
const passed = new Set(bundle.results.passed.map(scenarioBasename));
|
|
520
|
-
|
|
590
|
+
// Discovery-conditional floors (RFC 0148 §C G7): evaluate the conditional
|
|
591
|
+
// branches against the bundle's own captured discovery document.
|
|
592
|
+
const requiredFiles = [...floor.required];
|
|
593
|
+
for (const c of floor.conditional ?? []) {
|
|
594
|
+
let cur = bundle.discovery.document;
|
|
595
|
+
for (const seg of c.path.split('.'))
|
|
596
|
+
cur = cur !== null && typeof cur === 'object' ? cur[seg] : undefined;
|
|
597
|
+
if (Array.isArray(cur) && cur.includes(c.includes))
|
|
598
|
+
requiredFiles.push(...c.required);
|
|
599
|
+
}
|
|
600
|
+
const missingFloor = requiredFiles.filter((r) => !passed.has(scenarioBasename(r)));
|
|
521
601
|
const prefixOk = (floor.requiredAnyPrefix ?? []).every((p) => [...passed].some((s) => s.startsWith(p)));
|
|
522
|
-
|
|
602
|
+
// A conditional floor none of whose branches matched requires nothing — that
|
|
603
|
+
// is unprovable for a non-discovery-only profile, not proven.
|
|
604
|
+
const evaluable = floor.discoveryOnly === true || requiredFiles.length > 0 || (floor.requiredAnyPrefix ?? []).length > 0;
|
|
605
|
+
const floorProven = evaluable && missingFloor.length === 0 && prefixOk;
|
|
523
606
|
return {
|
|
524
607
|
profile,
|
|
525
608
|
derivable,
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RFC 0148 §A — the requirement execution ledger.
|
|
3
|
+
*
|
|
4
|
+
* The program exists because a green bundle could overstate behavior that never
|
|
5
|
+
* ran. Every instance found so far shares one shape: **something absent was
|
|
6
|
+
* treated as something proven.** `verifyBundleProfile()` derived `floorProven`
|
|
7
|
+
* from `[].every(...)` over an undefined floor. A gated subtest 404'd and
|
|
8
|
+
* soft-skipped. A scenario early-returned and the file still counted as passed.
|
|
9
|
+
*
|
|
10
|
+
* So the ledger inverts the default. A requirement has **no** disposition until
|
|
11
|
+
* a scenario explicitly records one, and a requirement with no recorded
|
|
12
|
+
* disposition resolves to `blocked` — never to `executed-pass`. Silence is
|
|
13
|
+
* evidence of nothing, and the data structure now says so rather than relying on
|
|
14
|
+
* each author to remember it.
|
|
15
|
+
*
|
|
16
|
+
* That is the whole mechanism. Everything else here is bookkeeping.
|
|
17
|
+
*/
|
|
18
|
+
import { appendFileSync, readFileSync, existsSync } from 'node:fs';
|
|
19
|
+
export const DISPOSITIONS = [
|
|
20
|
+
'executed-pass',
|
|
21
|
+
'executed-fail',
|
|
22
|
+
'skipped',
|
|
23
|
+
'inapplicable',
|
|
24
|
+
'blocked',
|
|
25
|
+
];
|
|
26
|
+
/**
|
|
27
|
+
* Dispositions that permit a profile to certify. `blocked` is deliberately NOT
|
|
28
|
+
* here: RFC 0148 §A says a blocked requirement in a claimed profile invalidates
|
|
29
|
+
* that profile's certification, because "we could not check" and "we checked and
|
|
30
|
+
* it holds" are the two states this program exists to stop conflating.
|
|
31
|
+
*/
|
|
32
|
+
export const CERTIFIABLE = [
|
|
33
|
+
'executed-pass',
|
|
34
|
+
'skipped',
|
|
35
|
+
'inapplicable',
|
|
36
|
+
];
|
|
37
|
+
const ledger = new Map();
|
|
38
|
+
/**
|
|
39
|
+
* Append-only journal of every recording in call order (duplicates included).
|
|
40
|
+
* `setup.ts` marks the journal length before a file's tests and reads what was
|
|
41
|
+
* recorded since, so a file's own gate decisions can be found even when the
|
|
42
|
+
* same requirement id was recorded earlier in the worker.
|
|
43
|
+
*/
|
|
44
|
+
const journal = [];
|
|
45
|
+
/**
|
|
46
|
+
* Record a requirement's outcome. Recording the same id twice with different
|
|
47
|
+
* dispositions throws: RFC 0148 §A says **exactly one** disposition per
|
|
48
|
+
* requirement, and a silent last-write-wins would let a later soft-skip
|
|
49
|
+
* overwrite an earlier real failure — the failure mode in reverse.
|
|
50
|
+
*/
|
|
51
|
+
export function recordRequirement(requirementId, disposition, detail, extras) {
|
|
52
|
+
const prior = ledger.get(requirementId);
|
|
53
|
+
if (prior !== undefined && prior.disposition !== disposition) {
|
|
54
|
+
throw new Error(`RFC 0148 §A: ${requirementId} already recorded as '${prior.disposition}', now '${disposition}'. ` +
|
|
55
|
+
'Exactly one disposition per requirement per run.');
|
|
56
|
+
}
|
|
57
|
+
if (disposition !== 'executed-pass' && (detail === undefined || detail.trim() === '')) {
|
|
58
|
+
throw new Error(`RFC 0148 §A: ${requirementId} recorded as '${disposition}' without a reason. ` +
|
|
59
|
+
'Anything other than executed-pass MUST say why, or the ledger records an outcome nobody can act on.');
|
|
60
|
+
}
|
|
61
|
+
const entry = {
|
|
62
|
+
requirementId,
|
|
63
|
+
disposition,
|
|
64
|
+
...(detail === undefined ? {} : { detail }),
|
|
65
|
+
...(extras?.assertionCount === undefined ? {} : { assertionCount: extras.assertionCount }),
|
|
66
|
+
};
|
|
67
|
+
ledger.set(requirementId, entry);
|
|
68
|
+
journal.push(entry);
|
|
69
|
+
// File sink (RFC 0148 acceptance item 2, S6). The in-memory map lives in a
|
|
70
|
+
// vitest worker; the `--certify` runner is a separate process reading a
|
|
71
|
+
// per-file JSON report. When the runner sets OPENWOP_LEDGER_PATH the worker
|
|
72
|
+
// appends every recording as one JSONL line so the runner can build
|
|
73
|
+
// requirement-level dispositions from what scenarios actually recorded —
|
|
74
|
+
// rather than inferring them from per-file pass/fail/skip, which cannot
|
|
75
|
+
// tell `skipped` from `inapplicable` from `blocked`. Best-effort: a sink
|
|
76
|
+
// failure must never turn a real assertion into a crash.
|
|
77
|
+
const sink = process.env['OPENWOP_LEDGER_PATH'];
|
|
78
|
+
if (sink !== undefined && sink !== '' && sink !== suspendedSink) {
|
|
79
|
+
try {
|
|
80
|
+
appendFileSync(sink, JSON.stringify(entry) + '\n');
|
|
81
|
+
}
|
|
82
|
+
catch {
|
|
83
|
+
/* ignore — the in-memory ledger is still authoritative for this worker */
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Read a ledger JSONL file written through the sink above. Duplicate lines for
|
|
89
|
+
* the same id (a scenario re-recording the same disposition, or several files
|
|
90
|
+
* gating the same profile) collapse to one entry; a CONFLICT (two different
|
|
91
|
+
* dispositions for one id across workers) resolves to the least certifiable
|
|
92
|
+
* — `executed-fail` > `blocked` > `executed-pass` > `skipped` > `inapplicable`
|
|
93
|
+
* — because "one worker said it failed" outranks "another said it passed", and
|
|
94
|
+
* an unresolvable disagreement must never round toward certification.
|
|
95
|
+
* Returns an empty array when the file is absent.
|
|
96
|
+
*/
|
|
97
|
+
export function readLedgerFile(path) {
|
|
98
|
+
if (!existsSync(path))
|
|
99
|
+
return [];
|
|
100
|
+
const rank = { 'executed-fail': 0, blocked: 1, 'executed-pass': 2, skipped: 3, inapplicable: 4 };
|
|
101
|
+
const merged = new Map();
|
|
102
|
+
for (const line of readFileSync(path, 'utf8').split('\n')) {
|
|
103
|
+
if (line.trim() === '')
|
|
104
|
+
continue;
|
|
105
|
+
let e;
|
|
106
|
+
try {
|
|
107
|
+
e = JSON.parse(line);
|
|
108
|
+
}
|
|
109
|
+
catch {
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
if (typeof e.requirementId !== 'string' || !DISPOSITIONS.includes(e.disposition))
|
|
113
|
+
continue;
|
|
114
|
+
const prior = merged.get(e.requirementId);
|
|
115
|
+
if (prior === undefined || rank[e.disposition] < rank[prior.disposition])
|
|
116
|
+
merged.set(e.requirementId, e);
|
|
117
|
+
}
|
|
118
|
+
return [...merged.values()].sort((a, b) => a.requirementId.localeCompare(b.requirementId));
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* The disposition for a requirement. **Absent resolves to `blocked`, never to a
|
|
122
|
+
* pass.** This is the inversion the whole section turns on: a scenario that
|
|
123
|
+
* returned early, threw and swallowed, or was never written leaves no entry, and
|
|
124
|
+
* the honest reading of no entry is "this was not exercised".
|
|
125
|
+
*/
|
|
126
|
+
export function dispositionOf(requirementId) {
|
|
127
|
+
return ledger.get(requirementId)?.disposition ?? 'blocked';
|
|
128
|
+
}
|
|
129
|
+
export function entryOf(requirementId) {
|
|
130
|
+
return (ledger.get(requirementId) ?? {
|
|
131
|
+
requirementId,
|
|
132
|
+
disposition: 'blocked',
|
|
133
|
+
detail: 'no disposition recorded — the requirement was not exercised',
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
export function snapshot() {
|
|
137
|
+
return [...ledger.values()].sort((a, b) => a.requirementId.localeCompare(b.requirementId));
|
|
138
|
+
}
|
|
139
|
+
/** Test-support only. Production runs record once and read once. */
|
|
140
|
+
let suspendedSink;
|
|
141
|
+
/**
|
|
142
|
+
* Scenarios that exercise `recordRequirement` itself (the ledger's own unit
|
|
143
|
+
* tests) record REAL requirement ids as fixtures — e.g. the whole core-standard
|
|
144
|
+
* floor as `executed-pass`. With the file sink on, those fixture rows would
|
|
145
|
+
* reach the runner's ledger and, under the reader's conflict rank, out-vote a
|
|
146
|
+
* genuine `inapplicable` recorded by the scenario that owns the requirement.
|
|
147
|
+
* Such a test file calls this in `beforeAll` and the returned restore in
|
|
148
|
+
* `afterAll`: recordings made in between are kept in memory but NOT appended to
|
|
149
|
+
* the sink that was active at the time of the call. A different path set by the
|
|
150
|
+
* test itself (its own scratch sink) is unaffected.
|
|
151
|
+
*/
|
|
152
|
+
export function suspendSinkForFixtures() {
|
|
153
|
+
suspendedSink = process.env['OPENWOP_LEDGER_PATH'];
|
|
154
|
+
return () => {
|
|
155
|
+
suspendedSink = undefined;
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
/** Number of recordings so far in this worker (a mark for `journalSince`). */
|
|
159
|
+
export function journalLength() {
|
|
160
|
+
return journal.length;
|
|
161
|
+
}
|
|
162
|
+
/** Every recording made since `mark` (from `journalLength()`), in call order. */
|
|
163
|
+
export function journalSince(mark) {
|
|
164
|
+
return journal.slice(mark);
|
|
165
|
+
}
|
|
166
|
+
export function resetLedger() {
|
|
167
|
+
ledger.clear();
|
|
168
|
+
journal.length = 0;
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Whether a profile's requirements permit certification. A profile with **no**
|
|
172
|
+
* requirements is NOT certifiable by this function — an empty requirement set
|
|
173
|
+
* is the `[].every(...)` shape that started all of this, and callers must
|
|
174
|
+
* distinguish "no floor by design" (`discoveryOnly`) from "no floor written yet"
|
|
175
|
+
* before reaching here.
|
|
176
|
+
*/
|
|
177
|
+
export function verifyProfileRequirements(profile, requirementIds) {
|
|
178
|
+
const blocking = requirementIds
|
|
179
|
+
.map(entryOf)
|
|
180
|
+
.filter((e) => !CERTIFIABLE.includes(e.disposition));
|
|
181
|
+
return { profile, certifiable: requirementIds.length > 0 && blocking.length === 0, blocking };
|
|
182
|
+
}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RFC 0148 §A — stable requirement IDs for the certification floor.
|
|
3
|
+
*
|
|
4
|
+
* §A requires "every normative conformance assertion included in a certifiable
|
|
5
|
+
* profile" to carry a stable `requirementId`. This registry is the first tranche:
|
|
6
|
+
* the floor scenarios that `PROFILE_FLOOR_SCENARIOS` already makes certification
|
|
7
|
+
* depend on. It is deliberately NOT a sweep over all 421 scenario files.
|
|
8
|
+
*
|
|
9
|
+
* The reason is the measurement in `docs/RFC-LIFECYCLE-COHERENCE.md`, applied to
|
|
10
|
+
* a different surface: a gate that fires hundreds of times on its first run gets
|
|
11
|
+
* disabled rather than fixed. Tagging every assertion in one pass would produce a
|
|
12
|
+
* registry nobody could review, and an unreviewed requirement ID is worth less
|
|
13
|
+
* than no requirement ID — it looks like coverage.
|
|
14
|
+
*
|
|
15
|
+
* So the scope is exactly what certification consumes today. `requirementsFor()`
|
|
16
|
+
* returns the IDs a profile's claim rests on, and anything outside this registry
|
|
17
|
+
* is honestly outside §A's coverage rather than silently assumed covered.
|
|
18
|
+
*
|
|
19
|
+
* Adding a requirement is deliberately cheap; adding it *without* a scenario
|
|
20
|
+
* recording a disposition for it is deliberately loud, because the ledger
|
|
21
|
+
* resolves an unrecorded requirement to `blocked`.
|
|
22
|
+
*/
|
|
23
|
+
import { PROFILE_FLOOR_SCENARIOS } from './profiles.js';
|
|
24
|
+
/** `runs-lifecycle.test.ts` → `openwop.floor.runs-lifecycle`. */
|
|
25
|
+
export function requirementIdForScenario(scenarioFile) {
|
|
26
|
+
return `openwop.floor.${scenarioFile.replace(/\.test\.ts$/, '')}`;
|
|
27
|
+
}
|
|
28
|
+
/** Prefix groups become one requirement: `interrupt-` → `openwop.floor.any.interrupt-`. */
|
|
29
|
+
export function requirementIdForPrefix(prefix) {
|
|
30
|
+
return `openwop.floor.any.${prefix}`;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* The requirement IDs a profile's certification rests on.
|
|
34
|
+
*
|
|
35
|
+
* Returns `null` — not an empty array — when the corpus has no floor for the
|
|
36
|
+
* profile. An empty array would flow into `verifyProfileRequirements()` and read
|
|
37
|
+
* as "nothing blocking", which is the `[].every(...)` shape this program exists
|
|
38
|
+
* to close. `null` forces the caller to decide between `discoveryOnly` (an empty
|
|
39
|
+
* floor by design) and unspecified (no floor written yet).
|
|
40
|
+
*/
|
|
41
|
+
/** Read a dot-path (RFC 0073 root families) out of a discovery document. */
|
|
42
|
+
function readPath(doc, path) {
|
|
43
|
+
let cur = doc;
|
|
44
|
+
for (const seg of path.split('.')) {
|
|
45
|
+
if (cur === null || typeof cur !== 'object')
|
|
46
|
+
return undefined;
|
|
47
|
+
cur = cur[seg];
|
|
48
|
+
}
|
|
49
|
+
return cur;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* The scenario FILES a profile's floor requires against a given discovery
|
|
53
|
+
* document — the unconditional `required` list plus every `conditional` branch
|
|
54
|
+
* whose `path` array includes its `includes` value. `null` when the floor is
|
|
55
|
+
* conditional and no document was supplied (unevaluable ≠ empty), or when the
|
|
56
|
+
* profile has no floor at all.
|
|
57
|
+
*/
|
|
58
|
+
export function floorFilesFor(profile, document) {
|
|
59
|
+
const floor = PROFILE_FLOOR_SCENARIOS[profile];
|
|
60
|
+
if (floor === undefined)
|
|
61
|
+
return null;
|
|
62
|
+
const files = [...floor.required];
|
|
63
|
+
if (floor.conditional !== undefined && floor.conditional.length > 0) {
|
|
64
|
+
if (document === undefined)
|
|
65
|
+
return null;
|
|
66
|
+
for (const c of floor.conditional) {
|
|
67
|
+
const arr = readPath(document, c.path);
|
|
68
|
+
if (Array.isArray(arr) && arr.includes(c.includes))
|
|
69
|
+
files.push(...c.required);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return [...new Set(files)];
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Requirement ids for a profile's floor. `document` is needed for a
|
|
76
|
+
* discovery-conditional floor (RFC 0148 §C G7 — `openwop-replay-fork`): without
|
|
77
|
+
* it such a floor is UNEVALUABLE and this returns `null`, never `[]`.
|
|
78
|
+
*/
|
|
79
|
+
export function requirementsFor(profile, document) {
|
|
80
|
+
const floor = PROFILE_FLOOR_SCENARIOS[profile];
|
|
81
|
+
if (floor === undefined)
|
|
82
|
+
return null;
|
|
83
|
+
if (floor.discoveryOnly === true)
|
|
84
|
+
return [];
|
|
85
|
+
const files = floorFilesFor(profile, document);
|
|
86
|
+
if (files === null)
|
|
87
|
+
return null;
|
|
88
|
+
return [
|
|
89
|
+
...files.map(requirementIdForScenario),
|
|
90
|
+
...(floor.requiredAnyPrefix ?? []).map(requirementIdForPrefix),
|
|
91
|
+
];
|
|
92
|
+
}
|
|
93
|
+
/** Every registered requirement ID across every profile with a runtime floor. */
|
|
94
|
+
export function allRequirements() {
|
|
95
|
+
const ids = new Set();
|
|
96
|
+
for (const [profile, floor] of Object.entries(PROFILE_FLOOR_SCENARIOS)) {
|
|
97
|
+
if (floor.discoveryOnly === true)
|
|
98
|
+
continue;
|
|
99
|
+
// every branch of a conditional floor is a registered requirement
|
|
100
|
+
for (const f of floor.required)
|
|
101
|
+
ids.add(requirementIdForScenario(f));
|
|
102
|
+
for (const c of floor.conditional ?? [])
|
|
103
|
+
for (const f of c.required)
|
|
104
|
+
ids.add(requirementIdForScenario(f));
|
|
105
|
+
for (const p of floor.requiredAnyPrefix ?? [])
|
|
106
|
+
ids.add(requirementIdForPrefix(p));
|
|
107
|
+
void profile;
|
|
108
|
+
}
|
|
109
|
+
return [...ids].sort();
|
|
110
|
+
}
|