@openwop/openwop-conformance 1.106.0 → 1.123.0

package/schemas/workflow-definition.schema.json

@@ -814,6 +814,10 @@
           "type": "integer",
           "minimum": 1,
           "default": 5
+        },
+        "compensation": {
+          "$ref": "compensation-policy.schema.json",
+          "description": "RFC 0151 §B — the workflow-level compensation policy (WHEN an unwind starts and HOW it runs). OPTIONAL and capability-gated: a host that advertises `capabilities.compensation` MUST validate it at registration; a host that does not MUST refuse the workflow with `capability_required` rather than accept a policy it will never honour. Node-level `nodes[].compensation` declarations remain valid without it — they describe the inverse action; this key requests the unwind."
         }
       }
     }

package/src/cli.ts

@@ -33,10 +33,14 @@ import { tmpdir } from 'node:os';
 import Ajv2020 from 'ajv/dist/2020.js';
 import addFormats from 'ajv-formats';
 import { SCHEMAS_DIR } from './lib/paths.js';
+import { readLedgerFile } from './lib/requirement-ledger.js';
+import { deriveRequirementDispositions } from './lib/scenario-disposition.js';
+import { scrubEvidence, evidenceSecretsFromEnv, verifyBundleV2 } from './lib/certification-bundle-verify.js';
 import {
   deriveProfiles,
   isCoreStandard,
   agentPlatformStatus,
+  DEPRECATED_PROFILE_ALIASES,
   type DiscoveryPayload,
 } from './lib/profiles.js';
 
@@ -284,9 +288,14 @@ async function runCertify(args: ParsedArgs, baseUrl: string, apiKey: string): Pr
   const conformanceRoot = resolvePath(here, '..');
   const reportDir = mkdtempSync(join(tmpdir(), 'owp-certify-'));
   const reportFile = join(reportDir, 'vitest-report.json');
+  // RFC 0148 §A ledger sink (S6): every scenario file records its disposition
+  // (and assertion count) here; the runner reads it after the run so bundle v2
+  // rows come from what scenarios RECORDED, not from per-file pass/fail/skip.
+  const ledgerFile = join(reportDir, 'requirement-ledger.jsonl');
   const env: NodeJS.ProcessEnv = { ...process.env };
   env.OPENWOP_BASE_URL = baseUrl;
   env.OPENWOP_API_KEY = apiKey;
+  env.OPENWOP_LEDGER_PATH = ledgerFile;
   if (args.impl) env.OPENWOP_IMPLEMENTATION_NAME = args.impl;
   if (args.implVersion) env.OPENWOP_IMPLEMENTATION_VERSION = args.implVersion;
 
@@ -307,6 +316,7 @@ async function runCertify(args: ParsedArgs, baseUrl: string, apiKey: string): Pr
   }
 
   let report: VitestJsonReport;
+  const ledgerEntries = readLedgerFile(ledgerFile);
   try {
     report = JSON.parse(readFileSync(reportFile, 'utf8')) as VitestJsonReport;
   } catch (err) {
@@ -382,6 +392,21 @@ async function runCertify(args: ParsedArgs, baseUrl: string, apiKey: string): Pr
   // a large `blocked` count and therefore does NOT certify. That is not a defect
   // in the emitter. It is the true state of the evidence, which v1 was unable to
   // represent and therefore reported as a clean skip.
+  // (d1) RFC 0148 acceptance item 2 — requirement-level dispositions from the
+  // ledger, and rejection of unclassified returns for a claimed profile.
+  const derived = deriveRequirementDispositions(states, ledgerEntries, claimedProfiles, document as Record<string, unknown>);
+  const notHeld = new Set(derived.verdicts.filter((v) => v.runtimeDerived && !v.held).map((v) => v.profile));
+  const rejectedProfiles = derived.verdicts.filter((v) => v.unclassified.length > 0);
+  if (rejectedProfiles.length > 0) {
+    process.stderr.write(
+      'openwop-conformance --certify: REJECTING certification — a claimed profile has UNCLASSIFIED floor requirements\n' +
+        '  (no disposition recorded, or an executed-pass with assertionCount 0 — a witness of nothing). RFC 0148 §A\n' +
+        '  resolves an unclassified requirement to `blocked`, never to a pass; the bundle is still written so the\n' +
+        '  evidence is inspectable, but it does not certify and this process exits 3.\n' +
+        rejectedProfiles.map((v) => `  - ${v.profile}: ${v.unclassified.join(', ')}\n`).join(''),
+    );
+  }
+
   if (args.bundleVersion === '2') {
     const scenarioIds = [...passed, ...failed, ...skipped].sort();
     const manifestSha = createHash('sha256').update(scenarioIds.join('\n'), 'utf8').digest('hex');
@@ -392,28 +417,15 @@ async function runCertify(args: ParsedArgs, baseUrl: string, apiKey: string): Pr
       .update(`${args.baseUrl}\n${sha256}\n${process.env['OPENWOP_REQUIRE_BEHAVIOR'] ?? ''}`, 'utf8')
       .digest('hex');
 
-    const requirements = [
-      ...passed.map((f) => ({
-        requirementId: `openwop.scenario.${f.replace(/\.test\.ts$/, '')}`,
-        scenarioId: f,
-        disposition: 'executed-pass' as const,
-      })),
-      ...failed.map((f) => ({
-        requirementId: `openwop.scenario.${f.replace(/\.test\.ts$/, '')}`,
-        scenarioId: f,
-        disposition: 'executed-fail' as const,
-        detail: 'the scenario executed and failed',
-      })),
-      ...skipped.map((f) => ({
-        requirementId: `openwop.scenario.${f.replace(/\.test\.ts$/, '')}`,
-        scenarioId: f,
-        disposition: 'blocked' as const,
-        detail:
-          'runner cannot classify a skipped file: the RFC 0148 §A requirement ledger is not ' +
-          'populated during a run, so skipped/inapplicable/blocked are indistinguishable here. ' +
-          '§A resolves an unclassifiable requirement to `blocked`, never to a pass.',
-      })),
-    ];
+    // Rows come from the ledger (S6). A file that recorded nothing is
+    // `blocked` — unclassified — and, if it sits on a claimed floor, rejected above.
+    const requirements = derived.requirements.map((r) => ({
+      requirementId: r.requirementId,
+      scenarioId: r.scenarioId,
+      disposition: r.disposition,
+      ...(r.detail === undefined ? {} : { detail: r.detail }),
+      ...(r.assertionCount === undefined ? {} : { assertionCount: r.assertionCount }),
+    }));
 
     const v2 = {
       bundleVersion: '2' as const,
@@ -422,48 +434,81 @@ async function runCertify(args: ParsedArgs, baseUrl: string, apiKey: string): Pr
       suite: { package: '@openwop/openwop-conformance' as const, version },
       host,
       discovery: { url: discoveryUrl, sha256, document },
-      claimedProfiles,
+      // RFC 0155 §E: canonical ids only in `claimedProfiles`; a deprecated
+      // alias that also derives (`openwop-core`, always alongside
+      // `openwop-discovery-core`) is reported in `aliases`, never as a claim.
+      // A runtime-derived profile (`PROFILE_FLOOR_SCENARIOS[p].runtimeDerived`,
+      // today `openwop-node-packs`) is claimed only when the host HOLDS it — every
+      // floor row a witnessed pass. Its discovery predicate is `openwop-core`, so
+      // deriving the claim from discovery alone made every core host "claim" a
+      // registry it never advertised (RFC 0025: the read surface has no advert).
+      claimedProfiles: claimedProfiles.filter((p) => !(p in DEPRECATED_PROFILE_ALIASES) && !notHeld.has(p)),
+      ...(claimedProfiles.some((p) => p in DEPRECATED_PROFILE_ALIASES)
+        ? { aliases: claimedProfiles.filter((p) => p in DEPRECATED_PROFILE_ALIASES) }
+        : {}),
       results: {
-        totals: {
-          executedPass: passed.length,
-          executedFail: failed.length,
-          skipped: 0,
-          inapplicable: 0,
-          blocked: skipped.length,
-        },
+        totals: derived.totals,
         requirements,
       },
       scenarioManifestSha256: manifestSha,
       targetConfigurationSha256: configSha,
     };
 
+    // RFC 0148 §C: secret canaries never enter evidence. Scrub the finished
+    // document with the credential this run was handed, every OPENWOP_*
+    // key/token/secret in the environment, and the conformance canary — a
+    // scenario's `detail` string, the captured discovery document, or a host
+    // field could carry any of them, and the emitter is the last place that
+    // can guarantee they do not ship.
+    const secrets = evidenceSecretsFromEnv(process.env, [apiKey]);
+    const scrubbed = scrubEvidence(v2, secrets);
+    const v2Out = scrubbed.value;
+    if (scrubbed.redactedAt.length > 0) {
+      process.stderr.write(
+        `openwop-conformance --certify: REDACTED ${scrubbed.redactedAt.length} evidence field(s) that carried a configured secret or the conformance canary: ${scrubbed.redactedAt.slice(0, 8).join(', ')}${scrubbed.redactedAt.length > 8 ? ', …' : ''}\n`,
+      );
+    }
+    // Self-audit with the consumer verifier: the emitter MUST NOT write a
+    // document the verifier would reject on shape (duplicate rows, totals that
+    // disagree with rows, unknown dispositions, a canary). Rejections scoped to
+    // a claimed profile (unwitnessed / vacuous) are the exit-3 case below and
+    // are reported through `derived`; a bundle-wide rejection here is a bug in
+    // this emitter and exits 2.
+    const selfAudit = verifyBundleV2(v2Out);
+    if (selfAudit.rejections.length > 0) {
+      process.stderr.write(
+        'openwop-conformance --certify: assembled v2 bundle FAILED self-verification (emitter defect):\n' +
+          selfAudit.rejections.map((r) => `  - [${r.kind}] ${r.detail}`).join('\n') +
+          '\n',
+      );
+      process.exit(2);
+    }
+
     const v2Schema = JSON.parse(
       readFileSync(join(SCHEMAS_DIR, 'certification-bundle-v2.schema.json'), 'utf8'),
     ) as Record<string, unknown>;
     const v2Ajv = new Ajv2020({ allErrors: true, strict: false });
     addFormats(v2Ajv);
     const v2Validate = v2Ajv.compile(v2Schema);
-    if (!v2Validate(v2)) {
+    if (!v2Validate(v2Out)) {
       process.stderr.write(
         'openwop-conformance --certify: assembled v2 bundle FAILED schema validation:\n' +
           `${JSON.stringify(v2Validate.errors, null, 2)}\n`,
       );
       process.exit(2);
     }
-    writeFileSync(outPath, `${JSON.stringify(v2, null, 2)}\n`);
+    writeFileSync(outPath, `${JSON.stringify(v2Out, null, 2)}\n`);
     process.stdout.write(
       `openwop-conformance --certify: wrote bundle v2 to ${outPath}\n` +
         `  host: ${host.name}@${host.version}\n` +
-        `  executed-pass ${passed.length} / executed-fail ${failed.length} / blocked ${skipped.length}\n` +
-        (skipped.length > 0
-          ? `  NOTE: ${skipped.length} requirement(s) are 'blocked', not 'skipped'. The runner cannot\n` +
-            `  classify a skipped file until scenarios record into the RFC 0148 §A ledger, and §A\n` +
-            `  resolves an unclassifiable requirement to 'blocked' rather than to a pass. A bundle\n` +
-            `  with blocked > 0 does NOT certify — that is the true state of the evidence, not a\n` +
-            `  defect in this emitter.\n`
-          : ''),
+        `  executed-pass ${derived.totals.executedPass} / executed-fail ${derived.totals.executedFail} / skipped ${derived.totals.skipped} / inapplicable ${derived.totals.inapplicable} / blocked ${derived.totals.blocked}\n` +
+        (derived.ledgerPresent
+          ? `  dispositions come from the RFC 0148 §A ledger (${ledgerEntries.length} entries recorded by the scenarios)\n`
+          : `  NOTE: no ledger was recorded — every skipped file is 'blocked' (unclassifiable), which is the honest reading\n`) +
+        (derived.totals.blocked > 0 ? `  a bundle with blocked > 0 does NOT certify — that is the state of the evidence, not a defect in this emitter\n` : '') +
+        derived.verdicts.map((v) => `  ${v.profile}: ${v.runtimeDerived && !v.held ? `not held (runtime-derived — dropped from claimedProfiles; floor rows not witnessed passes: ${v.blocking.join(', ')})` : v.certifiable ? 'certifiable' : 'NOT certifiable'}${v.unclassified.length > 0 ? ` (unclassified: ${v.unclassified.length})` : ''}\n`).join(''),
     );
-    process.exit(failed.length > 0 ? 1 : 0);
+    process.exit(rejectedProfiles.length > 0 ? 3 : failed.length > 0 ? 1 : 0);
   }
 
   // (e) Validate against the bundle schema BEFORE writing.
@@ -473,7 +518,14 @@ async function runCertify(args: ParsedArgs, baseUrl: string, apiKey: string): Pr
   const ajv = new Ajv2020({ allErrors: true, strict: false });
   addFormats(ajv);
   const validate = ajv.compile(schema);
-  if (!validate(bundle)) {
+  const bundleScrub = scrubEvidence(bundle, evidenceSecretsFromEnv(process.env, [apiKey]));
+  if (bundleScrub.redactedAt.length > 0) {
+    process.stderr.write(
+      `openwop-conformance --certify: REDACTED ${bundleScrub.redactedAt.length} evidence field(s) that carried a configured secret or the conformance canary\n`,
+    );
+  }
+  const bundleOut = bundleScrub.value;
+  if (!validate(bundleOut)) {
     process.stderr.write(
       'openwop-conformance --certify: assembled bundle FAILED schema validation:\n' +
         `${JSON.stringify(validate.errors, null, 2)}\n`,
@@ -481,7 +533,7 @@ async function runCertify(args: ParsedArgs, baseUrl: string, apiKey: string): Pr
     process.exit(2);
   }
 
-  writeFileSync(outPath, `${JSON.stringify(bundle, null, 2)}\n`);
+  writeFileSync(outPath, `${JSON.stringify(bundleOut, null, 2)}\n`);
   process.stdout.write(
     `openwop-conformance --certify: wrote certification bundle to ${outPath}\n` +
       `  host: ${host.name}@${host.version}\n` +
@@ -489,8 +541,10 @@ async function runCertify(args: ParsedArgs, baseUrl: string, apiKey: string): Pr
       `  results: ${passed.length} passed / ${failed.length} failed / ${skipped.length} skipped\n`,
   );
   // Exit code mirrors the suite outcome: a failing run still produces a bundle
-  // (the failures are recorded), but the process exit reflects pass/fail.
-  process.exit(failed.length > 0 ? 1 : 0);
+  // (the failures are recorded), but the process exit reflects pass/fail — and
+  // 3 when a claimed profile has unclassified floor requirements (RFC 0148 §A;
+  // v1 bundles cannot express the distinction, so the exit code carries it).
+  process.exit(rejectedProfiles.length > 0 ? 3 : failed.length > 0 ? 1 : 0);
 }
 
 async function main(): Promise<never> {