@openwop/openwop-conformance 1.106.1 → 1.124.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. package/README.md +3 -3
  2. package/coverage.md +6 -0
  3. package/dist/cli.js +89 -45
  4. package/dist/lib/certification-bundle-verify.js +274 -0
  5. package/dist/lib/profiles.js +102 -19
  6. package/dist/lib/requirement-ledger.js +182 -0
  7. package/dist/lib/requirement-registry.js +110 -0
  8. package/dist/lib/scenario-disposition.js +249 -0
  9. package/dist/lib/soft-skip.js +82 -0
  10. package/fixtures/conformance-a2a-task-roundtrip.json +1 -1
  11. package/fixtures.md +1 -1
  12. package/package.json +1 -1
  13. package/schemas/CORPUS-STAMP.json +2 -2
  14. package/schemas/README.md +1 -0
  15. package/schemas/capabilities.schema.json +2 -2
  16. package/schemas/compensation-policy.schema.json +64 -0
  17. package/schemas/run-snapshot.schema.json +5 -0
  18. package/schemas/workflow-chain-pack-manifest.schema.json +125 -0
  19. package/schemas/workflow-definition.schema.json +4 -0
  20. package/src/cli.ts +99 -45
  21. package/src/lib/a2a-fake-peer.ts +290 -110
  22. package/src/lib/certification-bundle-verify.ts +356 -0
  23. package/src/lib/error-envelope.ts +77 -0
  24. package/src/lib/llm-cache-key-recipe.ts +6 -5
  25. package/src/lib/mcp-fake-server.ts +235 -72
  26. package/src/lib/profiles.ts +132 -19
  27. package/src/lib/requirement-ledger.ts +98 -1
  28. package/src/lib/requirement-registry.ts +47 -4
  29. package/src/lib/scenario-disposition.ts +291 -0
  30. package/src/lib/soft-skip.ts +89 -0
  31. package/src/lib/workflow-chain-expansion.ts +189 -0
  32. package/src/scenarios/a2a-1-0-agent-card.test.ts +179 -0
  33. package/src/scenarios/a2a-1-0-task-roundtrip.test.ts +109 -0
  34. package/src/scenarios/a2a-card-runtime-consistency.test.ts +167 -0
  35. package/src/scenarios/a2a-peer-authority.test.ts +99 -0
  36. package/src/scenarios/a2a-task-roundtrip.test.ts +18 -6
  37. package/src/scenarios/agent-loop-iteration-monotonic.test.ts +3 -2
  38. package/src/scenarios/agent-loop-stateful-resume.test.ts +3 -2
  39. package/src/scenarios/agent-loop-version5-shape.test.ts +2 -1
  40. package/src/scenarios/agent-loop-workspace-snapshot.test.ts +4 -3
  41. package/src/scenarios/approval-gate-flow.test.ts +5 -4
  42. package/src/scenarios/artifact-type-store-without-render.test.ts +2 -1
  43. package/src/scenarios/auth-scim-profile.test.ts +5 -4
  44. package/src/scenarios/authorization-fail-closed.test.ts +4 -3
  45. package/src/scenarios/certification-bundle-non-vacuous.test.ts +249 -0
  46. package/src/scenarios/certification-bundle-redaction.test.ts +182 -0
  47. package/src/scenarios/certification-floor-enforcement.test.ts +30 -6
  48. package/src/scenarios/chain-compensation-expansion.test.ts +243 -0
  49. package/src/scenarios/commitment-fired.test.ts +3 -2
  50. package/src/scenarios/compensation-behavior.test.ts +63 -0
  51. package/src/scenarios/compensation-profile.test.ts +144 -1
  52. package/src/scenarios/conformance-advertised-seam-required.test.ts +118 -0
  53. package/src/scenarios/conformance-execution-witness.test.ts +104 -0
  54. package/src/scenarios/conversationCapabilityNegotiation.test.ts +3 -2
  55. package/src/scenarios/core-manifest-and-extension-registry.test.ts +26 -0
  56. package/src/scenarios/credential-payload-redaction.test.ts +4 -3
  57. package/src/scenarios/credentials-capability-shape.test.ts +4 -3
  58. package/src/scenarios/cross-engine-append-behavior.test.ts +5 -4
  59. package/src/scenarios/cross-engine-append-ordering.test.ts +3 -2
  60. package/src/scenarios/cross-host-ancestry-endpoint.test.ts +5 -4
  61. package/src/scenarios/cross-host-causation-shape.test.ts +3 -2
  62. package/src/scenarios/cross-workspace-isolation.test.ts +4 -3
  63. package/src/scenarios/data-residency-admission.test.ts +6 -6
  64. package/src/scenarios/deadletter-retry-exhaustion.test.ts +5 -4
  65. package/src/scenarios/distillation-index-roundtrip.test.ts +3 -2
  66. package/src/scenarios/distillation-secret-carryforward.test.ts +3 -2
  67. package/src/scenarios/distillation-shape.test.ts +2 -1
  68. package/src/scenarios/distillation-stable-archive.test.ts +4 -3
  69. package/src/scenarios/distillation-token-budget.test.ts +4 -3
  70. package/src/scenarios/envelope-recovery-applied.test.ts +3 -2
  71. package/src/scenarios/envelope-refusal-shape.test.ts +4 -3
  72. package/src/scenarios/error-envelope-canonical-shape.test.ts +64 -0
  73. package/src/scenarios/feedback-correction-redaction.test.ts +4 -3
  74. package/src/scenarios/feedback-cross-tenant-isolation.test.ts +4 -3
  75. package/src/scenarios/feedback-fork-not-copied.test.ts +6 -5
  76. package/src/scenarios/feedback-on-terminal-run.test.ts +4 -3
  77. package/src/scenarios/feedback-record-and-list.test.ts +4 -3
  78. package/src/scenarios/feedback-unsupported-501.test.ts +3 -2
  79. package/src/scenarios/fixtures-valid.test.ts +16 -18
  80. package/src/scenarios/fs-path-traversal.test.ts +3 -2
  81. package/src/scenarios/heartbeat-capability-shape.test.ts +2 -1
  82. package/src/scenarios/heartbeat-fires-once-per-tick.test.ts +4 -3
  83. package/src/scenarios/heartbeat-idempotent-no-spam.test.ts +6 -5
  84. package/src/scenarios/heartbeat-runtime-bound.test.ts +4 -3
  85. package/src/scenarios/mcp-2026-07-28-discover.test.ts +207 -0
  86. package/src/scenarios/mcp-cache-tenant-scope.test.ts +66 -0
  87. package/src/scenarios/mcp-current-auth-boundary.test.ts +55 -0
  88. package/src/scenarios/mcp-extension-opacity.test.ts +79 -0
  89. package/src/scenarios/mcp-mrtr-roundtrip.test.ts +140 -0
  90. package/src/scenarios/mcp-stateless-request.test.ts +77 -0
  91. package/src/scenarios/mcp-tool-roundtrip.test.ts +4 -3
  92. package/src/scenarios/mcp-toolcall-redaction.test.ts +2 -1
  93. package/src/scenarios/memory-attribution-emits-on-write.test.ts +6 -5
  94. package/src/scenarios/memory-attribution-no-content.test.ts +4 -3
  95. package/src/scenarios/memory-attribution-replay-stable.test.ts +6 -5
  96. package/src/scenarios/memory-attribution-tenant-scoped.test.ts +4 -3
  97. package/src/scenarios/memory-compaction-event-emitted.test.ts +3 -2
  98. package/src/scenarios/memory-compaction-provenance-tag.test.ts +6 -5
  99. package/src/scenarios/memory-compaction-sr1-carry-forward.test.ts +4 -3
  100. package/src/scenarios/memory-consolidation-idempotent.test.ts +4 -3
  101. package/src/scenarios/memory-injection-budget.test.ts +4 -3
  102. package/src/scenarios/metric-emission.test.ts +4 -3
  103. package/src/scenarios/multi-agent-confidence-escalation.test.ts +5 -4
  104. package/src/scenarios/multi-agent-handoff-state-machine.test.ts +4 -3
  105. package/src/scenarios/multi-agent-memory-lifecycle.test.ts +5 -4
  106. package/src/scenarios/multi-party-conversation-behavioral.test.ts +3 -2
  107. package/src/scenarios/multi-region-idempotency-behavior.test.ts +7 -6
  108. package/src/scenarios/multi-region-idempotency.test.ts +4 -3
  109. package/src/scenarios/normative-example-extraction.test.ts +242 -0
  110. package/src/scenarios/oauth-authorization-code-roundtrip.test.ts +4 -3
  111. package/src/scenarios/oauth-capability-shape.test.ts +4 -3
  112. package/src/scenarios/oauth-connector-redaction.test.ts +4 -3
  113. package/src/scenarios/openapi-asyncapi-sdk-parity.test.ts +309 -0
  114. package/src/scenarios/org-position-no-authority-escalation.test.ts +2 -1
  115. package/src/scenarios/otel-emission-grpc.test.ts +4 -3
  116. package/src/scenarios/otel-emission.test.ts +4 -3
  117. package/src/scenarios/otel-trace-propagation-subworkflow.test.ts +5 -4
  118. package/src/scenarios/otel-trace-propagation.test.ts +4 -3
  119. package/src/scenarios/pack-registry-isolation.test.ts +4 -3
  120. package/src/scenarios/pack-registry-publish.test.ts +65 -3
  121. package/src/scenarios/pack-registry.test.ts +22 -0
  122. package/src/scenarios/profile-discovery-core-alias.test.ts +137 -0
  123. package/src/scenarios/profileDerivation.test.ts +2 -1
  124. package/src/scenarios/provider-usage.test.ts +2 -2
  125. package/src/scenarios/rate-limit-envelope.test.ts +2 -1
  126. package/src/scenarios/registry-public.test.ts +6 -5
  127. package/src/scenarios/replay-divergence-at-refusal.test.ts +9 -8
  128. package/src/scenarios/replay-llm-cache-key-portable.test.ts +19 -13
  129. package/src/scenarios/replay-llm-cache-key.test.ts +120 -47
  130. package/src/scenarios/replay-observable-sequence-determinism.test.ts +5 -4
  131. package/src/scenarios/replay-side-effect-suppression.test.ts +20 -1
  132. package/src/scenarios/requirement-ledger.test.ts +29 -6
  133. package/src/scenarios/runner-ledger.test.ts +279 -0
  134. package/src/scenarios/runtime-requires-install-gate.test.ts +6 -5
  135. package/src/scenarios/sandbox-memory-cap.test.ts +3 -2
  136. package/src/scenarios/sandbox-mvp-behavior.test.ts +11 -10
  137. package/src/scenarios/sandbox-no-host-fs-escape.test.ts +2 -1
  138. package/src/scenarios/sandbox-timeout-cap.test.ts +3 -2
  139. package/src/scenarios/stream-text-fixture.test.ts +5 -4
  140. package/src/scenarios/subrun-approval-fail-closed.test.ts +3 -2
  141. package/src/scenarios/subrun-approval-gate.test.ts +4 -3
  142. package/src/scenarios/subrun-attestation-shape.test.ts +2 -1
  143. package/src/scenarios/subrun-checksum-stable.test.ts +4 -3
  144. package/src/scenarios/table-schema-enforcement.test.ts +2 -2
  145. package/src/scenarios/tool-hooks-authorization-fail-closed.test.ts +3 -2
  146. package/src/scenarios/tool-hooks-content-free.test.ts +3 -2
  147. package/src/scenarios/tool-hooks-rate-limit.test.ts +3 -2
  148. package/src/scenarios/tool-hooks-secret-redaction.test.ts +3 -2
  149. package/src/scenarios/tool-hooks-shape.test.ts +2 -1
  150. package/src/scenarios/trigger-bridge-delivery.test.ts +3 -2
  151. package/src/scenarios/version-fold.test.ts +5 -4
  152. package/src/scenarios/voice-streamref-tenant-bound.test.ts +2 -1
  153. package/src/scenarios/voice-transcription-streaming.test.ts +2 -1
  154. package/src/scenarios/voice-transcription-unadvertised.test.ts +2 -1
  155. package/src/scenarios/wasm-pack-abi-version-rejection.test.ts +5 -4
  156. package/src/scenarios/wasm-pack-invoke-completed.test.ts +3 -2
  157. package/src/scenarios/wasm-pack-invoke-suspended.test.ts +3 -2
  158. package/src/scenarios/wasm-pack-load.test.ts +4 -3
  159. package/src/scenarios/wasm-pack-memory-cap.test.ts +4 -3
  160. package/src/scenarios/wasm-pack-replay-determinism.test.ts +3 -2
  161. package/src/scenarios/webhook-signed-delivery.test.ts +4 -3
  162. package/src/scenarios/workflow-primary-output-annotation.test.ts +13 -23
  163. package/src/scenarios/workload-identity-behavior.test.ts +7 -4
  164. package/src/scenarios/workspace-cross-tenant-isolation-blackbox.test.ts +3 -2
  165. package/src/scenarios/workspace-cross-tenant-isolation.test.ts +3 -2
  166. package/src/setup.ts +95 -1
package/coverage.md CHANGED
@@ -75,6 +75,12 @@
75
75
 
76
76
  ---
77
77
 
78
+ ## Requirement dispositions and certification (RFC 0148 §A/§C — the coverage guide)
79
+
80
+ Since suite `1.114.0` every scenario file records a **disposition** into the RFC 0148 §A ledger (`OPENWOP_LEDGER_PATH`, set by `--certify`): `executed-pass` (with `assertionCount` > 0), `executed-fail`, `skipped` (operator opt-out via `OPENWOP_OPTED_OUT_PROFILES`), `inapplicable` (capability / profile / fixture not advertised — `behaviorGate` or `softSkip('inapplicable', …)`), or `blocked` (seam absent, prior step failed, precondition unmet — `seamAbsent` / `softSkip('blocked', …)`). A test that returns before its first `expect` is a pass with **zero assertions**; the runner never records that as a pass — it takes the file's noted reason, or resolves it to `blocked` with a fixed marker (`UNCLASSIFIED_RETURN_DETAIL`), which certification still treats as an *unclassified return*. Rules for authors: `CONTRIBUTING.md` §Conformance suite ("never return early in silence").
81
+
82
+ **How a claim certifies (bundle v2, `--certify --bundle-version 2`):** `claimedProfiles` are derived from the captured discovery document (canonical ids; `openwop-core` goes in `aliases`); every catalog profile has a floor in `PROFILE_FLOOR_SCENARIOS` (`discoveryOnly`, plain, `runtimeDerived` for `openwop-node-packs` — claimed only when held — or discovery-`conditional` for `openwop-replay-fork`); a profile is *certifiable* when every floor row is `executed-pass` (witnessed) / `skipped` / `inapplicable`; a floor row that is `blocked` or `executed-fail` makes it *not certifiable* (valid evidence, honest state); a floor row with **no ledger row, a zero-assertion pass, or the runner marker** is an *unclassified return* and **rejects** the claim (`--certify` exits 3, bundle still written). The consumer verifier `verifyBundleV2` re-derives all of this from the bundle's own rows and document (`certification-bundle-non-vacuous.test.ts`), and both emitter and verifier scrub / reject the SR-1 secret canary (`certification-bundle-redaction.test.ts`). Under `OPENWOP_REQUIRE_BEHAVIOR=true` an advertised profile with a missing seam **fails** instead of recording `blocked` (`conformance-advertised-seam-required.test.ts`); an early return can never become a pass at any layer (`conformance-execution-witness.test.ts`). Strict runs need the host's opt-out list (its conformance boot sets it) or every unadvertised gated profile fails — those totals are not findings.
83
+
78
84
  ## Capability-gated scenarios: shape vs behavior
79
85
 
80
86
  The scenario groups in the table below (one row per group; count the rows — the number is no longer hard-coded here because it drifts) validate optional profiles where the host's discovery advertisement is well-formed (shape grade) but no reference host yet implements the profile end-to-end (behavior grade is `host-pending`). Default suite runs skip these with a warning; set `OPENWOP_REQUIRE_BEHAVIOR=true` to convert skips into hard failures.
package/dist/cli.js CHANGED
@@ -32,7 +32,10 @@ import { tmpdir } from 'node:os';
32
32
  import Ajv2020 from 'ajv/dist/2020.js';
33
33
  import addFormats from 'ajv-formats';
34
34
  import { SCHEMAS_DIR } from './lib/paths.js';
35
- import { deriveProfiles, isCoreStandard, agentPlatformStatus, } from './lib/profiles.js';
35
+ import { readLedgerFile } from './lib/requirement-ledger.js';
36
+ import { deriveRequirementDispositions } from './lib/scenario-disposition.js';
37
+ import { scrubEvidence, evidenceSecretsFromEnv, verifyBundleV2 } from './lib/certification-bundle-verify.js';
38
+ import { deriveProfiles, isCoreStandard, agentPlatformStatus, DEPRECATED_PROFILE_ALIASES, } from './lib/profiles.js';
36
39
  function parseArgs(argv) {
37
40
  let baseUrl;
38
41
  let apiKey;
@@ -249,9 +252,14 @@ async function runCertify(args, baseUrl, apiKey) {
249
252
  const conformanceRoot = resolvePath(here, '..');
250
253
  const reportDir = mkdtempSync(join(tmpdir(), 'owp-certify-'));
251
254
  const reportFile = join(reportDir, 'vitest-report.json');
255
+ // RFC 0148 §A ledger sink (S6): every scenario file records its disposition
256
+ // (and assertion count) here; the runner reads it after the run so bundle v2
257
+ // rows come from what scenarios RECORDED, not from per-file pass/fail/skip.
258
+ const ledgerFile = join(reportDir, 'requirement-ledger.jsonl');
252
259
  const env = { ...process.env };
253
260
  env.OPENWOP_BASE_URL = baseUrl;
254
261
  env.OPENWOP_API_KEY = apiKey;
262
+ env.OPENWOP_LEDGER_PATH = ledgerFile;
255
263
  if (args.impl)
256
264
  env.OPENWOP_IMPLEMENTATION_NAME = args.impl;
257
265
  if (args.implVersion)
@@ -270,6 +278,7 @@ async function runCertify(args, baseUrl, apiKey) {
270
278
  process.exit(2);
271
279
  }
272
280
  let report;
281
+ const ledgerEntries = readLedgerFile(ledgerFile);
273
282
  try {
274
283
  report = JSON.parse(readFileSync(reportFile, 'utf8'));
275
284
  }
@@ -343,6 +352,18 @@ async function runCertify(args, baseUrl, apiKey) {
343
352
  // a large `blocked` count and therefore does NOT certify. That is not a defect
344
353
  // in the emitter. It is the true state of the evidence, which v1 was unable to
345
354
  // represent and therefore reported as a clean skip.
355
+ // (d1) RFC 0148 acceptance item 2 — requirement-level dispositions from the
356
+ // ledger, and rejection of unclassified returns for a claimed profile.
357
+ const derived = deriveRequirementDispositions(states, ledgerEntries, claimedProfiles, document);
358
+ const notHeld = new Set(derived.verdicts.filter((v) => v.runtimeDerived && !v.held).map((v) => v.profile));
359
+ const rejectedProfiles = derived.verdicts.filter((v) => v.unclassified.length > 0);
360
+ if (rejectedProfiles.length > 0) {
361
+ process.stderr.write('openwop-conformance --certify: REJECTING certification — a claimed profile has UNCLASSIFIED floor requirements\n' +
362
+ ' (no disposition recorded, or an executed-pass with assertionCount 0 — a witness of nothing). RFC 0148 §A\n' +
363
+ ' resolves an unclassified requirement to `blocked`, never to a pass; the bundle is still written so the\n' +
364
+ ' evidence is inspectable, but it does not certify and this process exits 3.\n' +
365
+ rejectedProfiles.map((v) => ` - ${v.profile}: ${v.unclassified.join(', ')}\n`).join(''));
366
+ }
346
367
  if (args.bundleVersion === '2') {
347
368
  const scenarioIds = [...passed, ...failed, ...skipped].sort();
348
369
  const manifestSha = createHash('sha256').update(scenarioIds.join('\n'), 'utf8').digest('hex');
@@ -352,27 +373,15 @@ async function runCertify(args, baseUrl, apiKey) {
352
373
  const configSha = createHash('sha256')
353
374
  .update(`${args.baseUrl}\n${sha256}\n${process.env['OPENWOP_REQUIRE_BEHAVIOR'] ?? ''}`, 'utf8')
354
375
  .digest('hex');
355
- const requirements = [
356
- ...passed.map((f) => ({
357
- requirementId: `openwop.scenario.${f.replace(/\.test\.ts$/, '')}`,
358
- scenarioId: f,
359
- disposition: 'executed-pass',
360
- })),
361
- ...failed.map((f) => ({
362
- requirementId: `openwop.scenario.${f.replace(/\.test\.ts$/, '')}`,
363
- scenarioId: f,
364
- disposition: 'executed-fail',
365
- detail: 'the scenario executed and failed',
366
- })),
367
- ...skipped.map((f) => ({
368
- requirementId: `openwop.scenario.${f.replace(/\.test\.ts$/, '')}`,
369
- scenarioId: f,
370
- disposition: 'blocked',
371
- detail: 'runner cannot classify a skipped file: the RFC 0148 §A requirement ledger is not ' +
372
- 'populated during a run, so skipped/inapplicable/blocked are indistinguishable here. ' +
373
- '§A resolves an unclassifiable requirement to `blocked`, never to a pass.',
374
- })),
375
- ];
376
+ // Rows come from the ledger (S6). A file that recorded nothing is
377
+ // `blocked` — unclassified — and, if it sits on a claimed floor, rejected above.
378
+ const requirements = derived.requirements.map((r) => ({
379
+ requirementId: r.requirementId,
380
+ scenarioId: r.scenarioId,
381
+ disposition: r.disposition,
382
+ ...(r.detail === undefined ? {} : { detail: r.detail }),
383
+ ...(r.assertionCount === undefined ? {} : { assertionCount: r.assertionCount }),
384
+ }));
376
385
  const v2 = {
377
386
  bundleVersion: '2',
378
387
  generatedAt: new Date().toISOString(),
@@ -380,60 +389,95 @@ async function runCertify(args, baseUrl, apiKey) {
380
389
  suite: { package: '@openwop/openwop-conformance', version },
381
390
  host,
382
391
  discovery: { url: discoveryUrl, sha256, document },
383
- claimedProfiles,
392
+ // RFC 0155 §E: canonical ids only in `claimedProfiles`; a deprecated
393
+ // alias that also derives (`openwop-core`, always alongside
394
+ // `openwop-discovery-core`) is reported in `aliases`, never as a claim.
395
+ // A runtime-derived profile (`PROFILE_FLOOR_SCENARIOS[p].runtimeDerived`,
396
+ // today `openwop-node-packs`) is claimed only when the host HOLDS it — every
397
+ // floor row a witnessed pass. Its discovery predicate is `openwop-core`, so
398
+ // deriving the claim from discovery alone made every core host "claim" a
399
+ // registry it never advertised (RFC 0025: the read surface has no advert).
400
+ claimedProfiles: claimedProfiles.filter((p) => !(p in DEPRECATED_PROFILE_ALIASES) && !notHeld.has(p)),
401
+ ...(claimedProfiles.some((p) => p in DEPRECATED_PROFILE_ALIASES)
402
+ ? { aliases: claimedProfiles.filter((p) => p in DEPRECATED_PROFILE_ALIASES) }
403
+ : {}),
384
404
  results: {
385
- totals: {
386
- executedPass: passed.length,
387
- executedFail: failed.length,
388
- skipped: 0,
389
- inapplicable: 0,
390
- blocked: skipped.length,
391
- },
405
+ totals: derived.totals,
392
406
  requirements,
393
407
  },
394
408
  scenarioManifestSha256: manifestSha,
395
409
  targetConfigurationSha256: configSha,
396
410
  };
411
+ // RFC 0148 §C: secret canaries never enter evidence. Scrub the finished
412
+ // document with the credential this run was handed, every OPENWOP_*
413
+ // key/token/secret in the environment, and the conformance canary — a
414
+ // scenario's `detail` string, the captured discovery document, or a host
415
+ // field could carry any of them, and the emitter is the last place that
416
+ // can guarantee they do not ship.
417
+ const secrets = evidenceSecretsFromEnv(process.env, [apiKey]);
418
+ const scrubbed = scrubEvidence(v2, secrets);
419
+ const v2Out = scrubbed.value;
420
+ if (scrubbed.redactedAt.length > 0) {
421
+ process.stderr.write(`openwop-conformance --certify: REDACTED ${scrubbed.redactedAt.length} evidence field(s) that carried a configured secret or the conformance canary: ${scrubbed.redactedAt.slice(0, 8).join(', ')}${scrubbed.redactedAt.length > 8 ? ', …' : ''}\n`);
422
+ }
423
+ // Self-audit with the consumer verifier: the emitter MUST NOT write a
424
+ // document the verifier would reject on shape (duplicate rows, totals that
425
+ // disagree with rows, unknown dispositions, a canary). Rejections scoped to
426
+ // a claimed profile (unwitnessed / vacuous) are the exit-3 case below and
427
+ // are reported through `derived`; a bundle-wide rejection here is a bug in
428
+ // this emitter and exits 2.
429
+ const selfAudit = verifyBundleV2(v2Out);
430
+ if (selfAudit.rejections.length > 0) {
431
+ process.stderr.write('openwop-conformance --certify: assembled v2 bundle FAILED self-verification (emitter defect):\n' +
432
+ selfAudit.rejections.map((r) => ` - [${r.kind}] ${r.detail}`).join('\n') +
433
+ '\n');
434
+ process.exit(2);
435
+ }
397
436
  const v2Schema = JSON.parse(readFileSync(join(SCHEMAS_DIR, 'certification-bundle-v2.schema.json'), 'utf8'));
398
437
  const v2Ajv = new Ajv2020({ allErrors: true, strict: false });
399
438
  addFormats(v2Ajv);
400
439
  const v2Validate = v2Ajv.compile(v2Schema);
401
- if (!v2Validate(v2)) {
440
+ if (!v2Validate(v2Out)) {
402
441
  process.stderr.write('openwop-conformance --certify: assembled v2 bundle FAILED schema validation:\n' +
403
442
  `${JSON.stringify(v2Validate.errors, null, 2)}\n`);
404
443
  process.exit(2);
405
444
  }
406
- writeFileSync(outPath, `${JSON.stringify(v2, null, 2)}\n`);
445
+ writeFileSync(outPath, `${JSON.stringify(v2Out, null, 2)}\n`);
407
446
  process.stdout.write(`openwop-conformance --certify: wrote bundle v2 to ${outPath}\n` +
408
447
  ` host: ${host.name}@${host.version}\n` +
409
- ` executed-pass ${passed.length} / executed-fail ${failed.length} / blocked ${skipped.length}\n` +
410
- (skipped.length > 0
411
- ? ` NOTE: ${skipped.length} requirement(s) are 'blocked', not 'skipped'. The runner cannot\n` +
412
- ` classify a skipped file until scenarios record into the RFC 0148 §A ledger, and §A\n` +
413
- ` resolves an unclassifiable requirement to 'blocked' rather than to a pass. A bundle\n` +
414
- ` with blocked > 0 does NOT certifythat is the true state of the evidence, not a\n` +
415
- ` defect in this emitter.\n`
416
- : ''));
417
- process.exit(failed.length > 0 ? 1 : 0);
448
+ ` executed-pass ${derived.totals.executedPass} / executed-fail ${derived.totals.executedFail} / skipped ${derived.totals.skipped} / inapplicable ${derived.totals.inapplicable} / blocked ${derived.totals.blocked}\n` +
449
+ (derived.ledgerPresent
450
+ ? ` dispositions come from the RFC 0148 §A ledger (${ledgerEntries.length} entries recorded by the scenarios)\n`
451
+ : ` NOTE: no ledger was recorded every skipped file is 'blocked' (unclassifiable), which is the honest reading\n`) +
452
+ (derived.totals.blocked > 0 ? ` a bundle with blocked > 0 does NOT certify — that is the state of the evidence, not a defect in this emitter\n` : '') +
453
+ derived.verdicts.map((v) => ` ${v.profile}: ${v.runtimeDerived && !v.held ? `not held (runtime-derived dropped from claimedProfiles; floor rows not witnessed passes: ${v.blocking.join(', ')})` : v.certifiable ? 'certifiable' : 'NOT certifiable'}${v.unclassified.length > 0 ? ` (unclassified: ${v.unclassified.length})` : ''}\n`).join(''));
454
+ process.exit(rejectedProfiles.length > 0 ? 3 : failed.length > 0 ? 1 : 0);
418
455
  }
419
456
  // (e) Validate against the bundle schema BEFORE writing.
420
457
  const schema = JSON.parse(readFileSync(join(SCHEMAS_DIR, 'conformance-certification-bundle.schema.json'), 'utf8'));
421
458
  const ajv = new Ajv2020({ allErrors: true, strict: false });
422
459
  addFormats(ajv);
423
460
  const validate = ajv.compile(schema);
424
- if (!validate(bundle)) {
461
+ const bundleScrub = scrubEvidence(bundle, evidenceSecretsFromEnv(process.env, [apiKey]));
462
+ if (bundleScrub.redactedAt.length > 0) {
463
+ process.stderr.write(`openwop-conformance --certify: REDACTED ${bundleScrub.redactedAt.length} evidence field(s) that carried a configured secret or the conformance canary\n`);
464
+ }
465
+ const bundleOut = bundleScrub.value;
466
+ if (!validate(bundleOut)) {
425
467
  process.stderr.write('openwop-conformance --certify: assembled bundle FAILED schema validation:\n' +
426
468
  `${JSON.stringify(validate.errors, null, 2)}\n`);
427
469
  process.exit(2);
428
470
  }
429
- writeFileSync(outPath, `${JSON.stringify(bundle, null, 2)}\n`);
471
+ writeFileSync(outPath, `${JSON.stringify(bundleOut, null, 2)}\n`);
430
472
  process.stdout.write(`openwop-conformance --certify: wrote certification bundle to ${outPath}\n` +
431
473
  ` host: ${host.name}@${host.version}\n` +
432
474
  ` claimedProfiles: ${claimedProfiles.length > 0 ? claimedProfiles.join(', ') : '(none)'}\n` +
433
475
  ` results: ${passed.length} passed / ${failed.length} failed / ${skipped.length} skipped\n`);
434
476
  // Exit code mirrors the suite outcome: a failing run still produces a bundle
435
- // (the failures are recorded), but the process exit reflects pass/fail.
436
- process.exit(failed.length > 0 ? 1 : 0);
477
+ // (the failures are recorded), but the process exit reflects pass/fail — and
478
+ // 3 when a claimed profile has unclassified floor requirements (RFC 0148 §A;
479
+ // v1 bundles cannot express the distinction, so the exit code carries it).
480
+ process.exit(rejectedProfiles.length > 0 ? 3 : failed.length > 0 ? 1 : 0);
437
481
  }
438
482
  async function main() {
439
483
  const args = parseArgs(process.argv.slice(2));
@@ -0,0 +1,274 @@
1
+ /**
2
+ * RFC 0148 §A/§C — consumer-side verification of a certification bundle v2,
3
+ * and the evidence scrubber the emitter runs before a bundle is written.
4
+ *
5
+ * `verifyBundle()` in `profiles.ts` reads bundle v1 (`results.passed` lists).
6
+ * v2 replaced the lists with per-requirement dispositions, and a v2 consumer
7
+ * MUST re-derive from those rows rather than trust `claimedProfiles` — RFC
8
+ * 0089 §B still binds, and RFC 0148 adds what v1 could not say:
9
+ *
10
+ * - a required row that is MISSING is an unwitnessed requirement — the bundle
11
+ * is REJECTED as evidence, not merely "not certified";
12
+ * - an `executed-pass` with `assertionCount: 0` (or with no count at all) is
13
+ * a witness of nothing — REJECTED;
14
+ * - two rows for one requirement is exactly-one-disposition-per-requirement
15
+ * broken — REJECTED;
16
+ * - totals that disagree with the rows they summarize are a tampered or
17
+ * hand-edited witness — REJECTED;
18
+ * - the conformance secret canary anywhere in the document is evidence that
19
+ * leaked a secret — REJECTED (`threat-model-secret-leakage.md` §SR-1);
20
+ * - a required row that is honestly `blocked` or `executed-fail` is VALID
21
+ * evidence of a claim that does NOT certify. That is the state of the
22
+ * host, not a defect in the bundle, and the two verdicts are kept apart.
23
+ *
24
+ * Server-free; no I/O.
25
+ */
26
+ import { createHash } from 'node:crypto';
27
+ import { PROFILE_FLOOR_SCENARIOS, profileDerivable, } from './profiles.js';
28
+ import { CERTIFIABLE, DISPOSITIONS } from './requirement-ledger.js';
29
+ import { floorFilesFor, requirementIdForPrefix, requirementIdForScenario } from './requirement-registry.js';
30
+ import { UNCLASSIFIED_RETURN_DETAIL } from './soft-skip.js';
31
+ /**
32
+ * The BYOK conformance canary (`fixtures.md` §conformance-secrets-roundtrip;
33
+ * SR-1). Its raw value MUST NOT appear on any observable channel — variables,
34
+ * events, debug bundles, logs — and a certification bundle is one more
35
+ * observable channel. A verifier can check for THIS literal without knowing
36
+ * any operator secret, which is why it is the one canary the verifier binds.
37
+ */
38
+ export const CONFORMANCE_SECRET_CANARY = 'openwop-conformance-canary-secret';
39
+ const scenarioBasename = (id) => id.split('/').pop() ?? id;
40
+ /** Walk every string in a JSON value. */
41
+ function* strings(value, path = '$') {
42
+ if (typeof value === 'string') {
43
+ yield { path, value };
44
+ return;
45
+ }
46
+ if (Array.isArray(value)) {
47
+ for (let i = 0; i < value.length; i += 1)
48
+ yield* strings(value[i], `${path}[${i}]`);
49
+ return;
50
+ }
51
+ if (value !== null && typeof value === 'object') {
52
+ for (const [k, v] of Object.entries(value)) {
53
+ yield { path: `${path}.${k}`, value: k };
54
+ yield* strings(v, `${path}.${k}`);
55
+ }
56
+ }
57
+ }
58
+ /** Where (JSON paths) a literal appears anywhere in the value — keys included. */
59
+ export function findLiteral(value, literal) {
60
+ const hits = [];
61
+ if (literal === '')
62
+ return hits;
63
+ for (const s of strings(value))
64
+ if (s.value.includes(literal))
65
+ hits.push(s.path);
66
+ return hits;
67
+ }
68
+ /**
69
+ * Required requirement ids for a claimed profile, derived from the SAME floor
70
+ * map the emitter uses. For a `requiredAnyPrefix` entry the requirement is
71
+ * satisfied by any row whose scenario matches the prefix; the id recorded for
72
+ * it is the prefix requirement id, and the rows that satisfy it are the
73
+ * matching scenario rows.
74
+ */
75
+ function requiredFor(profile, document) {
76
+ const floor = PROFILE_FLOOR_SCENARIOS[profile];
77
+ if (floor === undefined)
78
+ return null;
79
+ // Discovery-conditional floors (G7) are evaluated against the bundle's own
80
+ // captured document — the same evidence the claim was derived from.
81
+ const files = floorFilesFor(profile, document);
82
+ if (files === null)
83
+ return null;
84
+ return {
85
+ files: [...files],
86
+ prefixes: [...(floor.requiredAnyPrefix ?? [])],
87
+ discoveryOnly: floor.discoveryOnly === true,
88
+ };
89
+ }
90
+ function isWitnessedPass(row) {
91
+ return row.disposition === 'executed-pass' && typeof row.assertionCount === 'number' && row.assertionCount > 0;
92
+ }
93
+ /**
94
+ * The emitter's own RFC 0148 §A resolution of a zero-assertion file that noted
95
+ * no reason: recorded `blocked` with a fixed marker detail. Honest as a row;
96
+ * for a REQUIRED requirement it is still an unclassified return, and a claim
97
+ * carrying it is rejected exactly as a vacuous pass would be.
98
+ */
99
+ function isUnclassifiedReturn(row) {
100
+ return row.disposition === 'blocked' && row.detail === UNCLASSIFIED_RETURN_DETAIL;
101
+ }
102
+ /**
103
+ * Verify a v2 bundle. Pure; the caller validates against
104
+ * `certification-bundle-v2.schema.json` separately (this function tolerates a
105
+ * schema-invalid document and reports what it can).
106
+ */
107
+ export function verifyBundleV2(bundle) {
108
+ const rejections = [];
109
+ if (bundle.bundleVersion !== '2') {
110
+ rejections.push({ kind: 'not-v2', detail: `bundleVersion is ${JSON.stringify(bundle.bundleVersion)}, expected "2"` });
111
+ }
112
+ const rows = bundle.results?.requirements ?? [];
113
+ const byId = new Map();
114
+ for (const r of rows) {
115
+ if (!DISPOSITIONS.includes(r.disposition)) {
116
+ rejections.push({ kind: 'unknown-disposition', requirementId: r.requirementId, detail: `${r.requirementId}: disposition ${JSON.stringify(r.disposition)} is not one of the five RFC 0148 §A names` });
117
+ }
118
+ if (r.disposition !== 'executed-pass' && (r.detail === undefined || r.detail.trim() === '')) {
119
+ rejections.push({ kind: 'reason-missing', requirementId: r.requirementId, detail: `${r.requirementId}: '${r.disposition}' recorded without a reason (RFC 0148 §A)` });
120
+ }
121
+ const arr = byId.get(r.requirementId) ?? [];
122
+ arr.push(r);
123
+ byId.set(r.requirementId, arr);
124
+ }
125
+ for (const [id, arr] of byId) {
126
+ if (arr.length > 1)
127
+ rejections.push({ kind: 'duplicate-requirement', requirementId: id, detail: `${id}: ${arr.length} rows — exactly one disposition per requirement per run (RFC 0148 §A)` });
128
+ }
129
+ // Totals MUST summarize the rows. A hand-edited total is a tampered witness.
130
+ const count = (d) => rows.filter((r) => r.disposition === d).length;
131
+ const t = bundle.results?.totals;
132
+ if (t !== undefined) {
133
+ const expected = { executedPass: count('executed-pass'), executedFail: count('executed-fail'), skipped: count('skipped'), inapplicable: count('inapplicable'), blocked: count('blocked') };
134
+ for (const k of Object.keys(expected)) {
135
+ if (t[k] !== expected[k])
136
+ rejections.push({ kind: 'totals-mismatch', detail: `totals.${k} is ${String(t[k])} but the rows count ${expected[k]}` });
137
+ }
138
+ }
139
+ // The canary MUST NOT be anywhere in evidence — keys included.
140
+ for (const p of findLiteral(bundle, CONFORMANCE_SECRET_CANARY)) {
141
+ rejections.push({ kind: 'secret-canary', detail: `conformance secret canary present at ${p} — evidence leaked a secret (SR-1)` });
142
+ }
143
+ const profiles = (bundle.claimedProfiles ?? []).map((profile) => {
144
+ const derivable = profileDerivable(bundle.discovery?.document ?? {}, profile);
145
+ const req = requiredFor(profile, (bundle.discovery?.document ?? {}));
146
+ if (req === null) {
147
+ return { profile, derivable, floorUnspecified: true, required: [], rejections: [], notCertifiable: [], evidenceValid: true, certified: false };
148
+ }
149
+ const required = [];
150
+ const profileRejections = [];
151
+ const notCertifiable = [];
152
+ for (const file of req.files) {
153
+ const id = requirementIdForScenario(scenarioBasename(file));
154
+ required.push(id);
155
+ const arr = byId.get(id) ?? [];
156
+ const row = arr[0];
157
+ if (row === undefined) {
158
+ profileRejections.push({ kind: 'unwitnessed-requirement', profile, requirementId: id, detail: `${id}: no row — an unwitnessed floor requirement (RFC 0148 §A resolves it to blocked; a claim carrying it is rejected)` });
159
+ continue;
160
+ }
161
+ if (row.disposition === 'executed-pass' && !isWitnessedPass(row)) {
162
+ profileRejections.push({ kind: 'vacuous-pass', profile, requirementId: id, detail: `${id}: executed-pass with assertionCount ${String(row.assertionCount)} — a witness of nothing` });
163
+ continue;
164
+ }
165
+ if (isUnclassifiedReturn(row)) {
166
+ profileRejections.push({ kind: 'unwitnessed-requirement', profile, requirementId: id, detail: `${id}: every test returned early with zero assertions and no recorded reason — an unclassified return the emitter resolved to blocked (RFC 0148 §A); a claim carrying it is rejected` });
167
+ continue;
168
+ }
169
+ if (!CERTIFIABLE.includes(row.disposition))
170
+ notCertifiable.push(id);
171
+ }
172
+ for (const prefix of req.prefixes) {
173
+ const id = requirementIdForPrefix(prefix);
174
+ required.push(id);
175
+ // Satisfied by the rows of matching scenarios; the emitter also writes a
176
+ // summary row under the prefix id. Either form is accepted; the rule is
177
+ // the same — every matching row witnessed, at least one present.
178
+ const matching = rows.filter((r) => scenarioBasename(r.scenarioId).startsWith(prefix) && r.requirementId !== id);
179
+ const summary = byId.get(id)?.[0];
180
+ if (matching.length === 0 && summary === undefined) {
181
+ profileRejections.push({ kind: 'unwitnessed-requirement', profile, requirementId: id, detail: `${id}: no scenario matching '${prefix}' recorded a row` });
182
+ continue;
183
+ }
184
+ const considered = matching.length > 0 ? matching : [summary];
185
+ const vacuous = considered.filter((r) => (r.disposition === 'executed-pass' && !isWitnessedPass(r)) || isUnclassifiedReturn(r));
186
+ if (vacuous.length > 0) {
187
+ profileRejections.push({ kind: 'vacuous-pass', profile, requirementId: id, detail: `${id}: ${vacuous.map((r) => r.scenarioId).join(', ')} executed-pass with no witnessed assertion` });
188
+ continue;
189
+ }
190
+ // `requiredAnyPrefix`: ANY witnessed pass among the matching scenarios
191
+ // satisfies the requirement (the emitter's summary row says the same).
192
+ if (!considered.some((r) => isWitnessedPass(r)))
193
+ notCertifiable.push(id);
194
+ }
195
+ const evidenceValid = profileRejections.length === 0;
196
+ const certified = derivable && evidenceValid && notCertifiable.length === 0 && (req.discoveryOnly || required.length > 0);
197
+ return { profile, derivable, floorUnspecified: false, required, rejections: profileRejections, notCertifiable, evidenceValid, certified };
198
+ });
199
+ const evidenceValid = rejections.length === 0 && profiles.every((p) => p.evidenceValid);
200
+ const certified = evidenceValid && profiles.length > 0 && profiles.every((p) => p.certified);
201
+ return { evidenceValid, certified, rejections, profiles };
202
+ }
203
+ /* ────────────────────────────────────────────────────────────────────────── */
204
+ /**
205
+ * Replacement for a scrubbed secret: a stable, non-reversible marker that
206
+ * still lets two occurrences of the same secret be recognized as the same.
207
+ */
208
+ export function redactionMarker(secret) {
209
+ return `«redacted:${createHash('sha256').update(secret).digest('hex').slice(0, 12)}»`;
210
+ }
211
+ /**
212
+ * Replace every occurrence of every secret in every string of `value` (keys
213
+ * included) with `redactionMarker(secret)`. Empty / whitespace-only secrets
214
+ * are ignored — scrubbing "" would blank every string. The emitter runs this
215
+ * over the finished bundle with the credentials it was given plus the
216
+ * conformance canary; RFC 0148 §C says secret canaries never enter evidence,
217
+ * and the only way to make that true regardless of what a scenario put in a
218
+ * `detail` string is to scrub the document, not to trust the scenarios.
219
+ */
220
+ export function scrubEvidence(value, secrets) {
221
+ const live = [...new Set(secrets.filter((s) => typeof s === 'string' && s.trim() !== ''))].sort((a, b) => b.length - a.length);
222
+ const redactedAt = [];
223
+ if (live.length === 0)
224
+ return { value, redactedAt };
225
+ const scrubString = (s, path) => {
226
+ let out = s;
227
+ let hit = false;
228
+ for (const secret of live) {
229
+ if (out.includes(secret)) {
230
+ out = out.split(secret).join(redactionMarker(secret));
231
+ hit = true;
232
+ }
233
+ }
234
+ if (hit)
235
+ redactedAt.push(path);
236
+ return out;
237
+ };
238
+ const walk = (v, path) => {
239
+ if (typeof v === 'string')
240
+ return scrubString(v, path);
241
+ if (Array.isArray(v))
242
+ return v.map((x, i) => walk(x, `${path}[${i}]`));
243
+ if (v !== null && typeof v === 'object') {
244
+ const out = {};
245
+ for (const [k, x] of Object.entries(v)) {
246
+ out[scrubString(k, `${path}.${k}`)] = walk(x, `${path}.${k}`);
247
+ }
248
+ return out;
249
+ }
250
+ return v;
251
+ };
252
+ return { value: walk(value, '$'), redactedAt };
253
+ }
254
+ /**
255
+ * The secrets a `--certify` run must keep out of its own evidence: the
256
+ * credential it was handed, plus every `OPENWOP_*` environment value that
257
+ * names a key/token/secret/password, plus the conformance canary.
258
+ */
259
+ export function evidenceSecretsFromEnv(env, extra = []) {
260
+ const out = new Set();
261
+ for (const [k, v] of Object.entries(env)) {
262
+ if (!k.startsWith('OPENWOP_'))
263
+ continue;
264
+ if (!/(KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL)/.test(k))
265
+ continue;
266
+ if (v !== undefined && v.trim() !== '')
267
+ out.add(v);
268
+ }
269
+ for (const s of extra)
270
+ if (s !== undefined && s.trim() !== '')
271
+ out.add(s);
272
+ out.add(CONFORMANCE_SECRET_CANARY);
273
+ return [...out];
274
+ }