@openwop/openwop-conformance 1.106.1 → 1.124.0

package/src/scenarios/workflow-primary-output-annotation.test.ts

@@ -20,35 +20,25 @@
 import { describe, it, expect } from 'vitest';
 import Ajv2020 from 'ajv/dist/2020.js';
 import addFormats from 'ajv-formats';
-import { readFileSync } from 'node:fs';
+import { readFileSync, readdirSync } from 'node:fs';
 import { join } from 'node:path';
 import { SCHEMAS_DIR } from '../lib/paths.js';
 
 function compileWorkflowDefinition(): ReturnType<Ajv2020['compile']> {
   const ajv = new Ajv2020({ strict: false, allErrors: true });
   addFormats(ajv);
-  // Register cross-file `$ref` targets — same pattern as
-  // `fixtures-valid.test.ts`. Without these, Ajv throws
-  // `missingRef` when compiling `workflow-definition.schema.json`
-  // because it references agent-ref + prompt-ref by URL.
-  const agentRefSchema = JSON.parse(
-    readFileSync(join(SCHEMAS_DIR, 'agent-ref.schema.json'), 'utf8'),
-  ) as Record<string, unknown>;
-  const promptRefSchema = JSON.parse(
-    readFileSync(join(SCHEMAS_DIR, 'prompt-ref.schema.json'), 'utf8'),
-  ) as Record<string, unknown>;
-  const promptKindSchema = JSON.parse(
-    readFileSync(join(SCHEMAS_DIR, 'prompt-kind.schema.json'), 'utf8'),
-  ) as Record<string, unknown>;
-  ajv.addSchema(agentRefSchema, 'agent-ref.schema.json');
-  ajv.addSchema(promptRefSchema, 'prompt-ref.schema.json');
-  ajv.addSchema(promptRefSchema, './prompt-ref.schema.json');
-  ajv.addSchema(promptKindSchema, 'prompt-kind.schema.json');
-  ajv.addSchema(promptKindSchema, './prompt-kind.schema.json');
-  const schema = JSON.parse(
-    readFileSync(join(SCHEMAS_DIR, 'workflow-definition.schema.json'), 'utf8'),
-  ) as Record<string, unknown>;
-  return ajv.compile(schema);
+  // Register EVERY schema in the corpus (S14, 2026-08-16) — a fixed ref list is
+  // a claim about the schema graph that nothing keeps true; the fourth cross-file
+  // `$ref` (#1009) broke the previous list at describe time.
+  for (const file of readdirSync(SCHEMAS_DIR).filter((f) => f.endsWith('.schema.json'))) {
+    const s = JSON.parse(readFileSync(join(SCHEMAS_DIR, file), 'utf8')) as Record<string, unknown>;
+    ajv.addSchema(s, file);
+    ajv.addSchema(s, `./${file}`);
+  }
+  return (
+    ajv.getSchema('workflow-definition.schema.json') ??
+    ajv.compile(JSON.parse(readFileSync(join(SCHEMAS_DIR, 'workflow-definition.schema.json'), 'utf8')) as Record<string, unknown>)
+  );
 }
 
 /** Build the minimal-required shape of a WorkflowDefinition. Tests

package/src/scenarios/workload-identity-behavior.test.ts

@@ -33,6 +33,7 @@
  */
 
 import { describe, it, expect } from 'vitest';
+import { readErrorCode, readRetriable } from '../lib/error-envelope.js';
 import { driver } from '../lib/driver.js';
 import { behaviorGate } from '../lib/behavior-gate.js';
 import { capabilityFamily } from '../lib/discovery-capabilities.js';
@@ -76,7 +77,7 @@ async function resolve(body: Record<string, unknown>): Promise<{ status: number;
 }
 
 function reasonOf(json: unknown): string | undefined {
-  return (json as { error?: { code?: string } })?.error?.code;
+  return readErrorCode(json);
 }
 
 const IDENTITY = { scheme: 'spiffe', subject: 'spiffe://example/dispatcher', issuer: 'spiffe://example' };
@@ -154,9 +155,11 @@ describe('RFC 0154 §A — workload identity resolution (capability-gated behavi
     if (!behaviorGate(PROFILE, (await caps())?.supported === true)) return;
     const r = await resolve({ identity: { scheme: 'spiffe', subject: 'spiffe://example/unknown' } });
     if (r === null || r.status < 400) return;
-    const err = (r.json as { error?: { code?: string; retriable?: boolean } }).error;
+    // canonical flat envelope: code = `error`, retriable = `details.retriable`
+    // (the legacy nested `error.{code,retriable}` the §20 catalog prescribed until
+    // 2026-08-16 is tolerated by the helpers for the deprecation window)
     expect(
-      err?.retriable,
+      readRetriable(r.json),
       driver.describe(
         'spec/v1/host-sample-test-seams.md §20',
         'an identity that does not resolve will not resolve on retry. Marking it retriable invites ' +
@@ -166,7 +169,7 @@ describe('RFC 0154 §A — workload identity resolution (capability-gated behavi
     expect(
       ['identity_unverified', 'identity_unresolvable', 'audience_mismatch', 'delegation_expired', 'sender_constraint_missing'],
       driver.describe('RFCS/0154 §A', 'failures use a closed reason vocabulary'),
-    ).toContain(err?.code);
+    ).toContain(readErrorCode(r.json));
   });
 
   it('a resolution response carries no credential material', async () => {

package/src/scenarios/workspace-cross-tenant-isolation-blackbox.test.ts

@@ -24,6 +24,7 @@
  * @see SECURITY/invariants.yaml workspace-cross-tenant-isolation
  */
 import { describe, it, expect } from 'vitest';
+import { softSkip } from '../lib/soft-skip.js';
 import { randomUUID } from 'node:crypto';
 import { driver } from '../lib/driver.js';
 import { capabilityFamily } from '../lib/discovery-capabilities.js';
@@ -42,11 +43,11 @@ const asTenantB = { headers: { Authorization: `Bearer ${tenantBKey ?? ''}` } };
 
 describe('workspace-cross-tenant-isolation (black-box): a §C file MUST NOT leak across owners (RFC 0059 §E WCT-1)', () => {
   it('a file written by owner A is unreadable + un-enumerable by a second-tenant credential', async () => {
-    if (!(await workspaceSupported())) return; // capability not advertised — skip
+    if (!(await workspaceSupported())) return softSkip('inapplicable', 'capability not advertised — skip');
     if (!tenantBKey) {
       // eslint-disable-next-line no-console
       console.warn('[workspace-cross-tenant-isolation-blackbox] OPENWOP_TEST_TENANT_B_API_KEY not supplied; skipping the production-path cross-tenant assertion');
-      return;
+      return softSkip('blocked', '[workspace-cross-tenant-isolation-blackbox] OPENWOP_TEST_TENANT_B_API_KEY not supplied; skipping the production-path cross-tenant assertion');
     }
 
     const path = `wct-blackbox-${randomUUID()}.md`;

package/src/scenarios/workspace-cross-tenant-isolation.test.ts

@@ -19,6 +19,7 @@
  */
 
 import { describe, it, expect } from 'vitest';
+import { softSkip } from '../lib/soft-skip.js';
 import { driver } from '../lib/driver.js';
 import { capabilityFamily } from '../lib/discovery-capabilities.js';
 
@@ -40,11 +41,11 @@ const SEAM_PATH = 'WCT1-SECRET.md';
 
 describe('workspace-cross-tenant-isolation: a workspace file MUST NOT leak across owners (RFC 0059 §E WCT-1)', () => {
   it('a file written under {tenant A, workspace A} is not readable under a different owner', async () => {
-    if (!(await workspaceSupported())) return; // capability not advertised — skip
+    if (!(await workspaceSupported())) return softSkip('inapplicable', 'capability not advertised — skip');
 
     // Owner A writes a file.
     const put = await seam({ tenant: 'wct1-tenant-a', workspace: 'ws-a', op: 'put', path: SEAM_PATH, content: 'A-only secret body' });
-    if (put.status === 404) return; // seam unwired — soft-skip the behavioral probe
+    if (put.status === 404) return softSkip('blocked', 'seam unwired — soft-skip the behavioral probe');
     expect(put.status, driver.describe('agent-workspace.md §C PUT', 'seam put MUST succeed for the owning workspace')).toBe(200);
 
     // A DIFFERENT workspace (same tenant) MUST NOT read it.

package/src/setup.ts

@@ -30,6 +30,11 @@ import { setMultiAgentCapabilities } from './lib/multi-agent-capabilities.js';
 import { OtelCollector, setCollector } from './lib/otel-collector.js';
 import { McpFakeServer, setMcpFakeServer } from './lib/mcp-fake-server.js';
 import { A2AFakePeer, setA2AFakePeer } from './lib/a2a-fake-peer.js';
+import { afterAll, afterEach, beforeAll, expect } from 'vitest';
+import { basename } from 'node:path';
+import { recordRequirement, journalLength, journalSince } from './lib/requirement-ledger.js';
+import { requirementIdForFile, resolveFileRecord, type FileTestState } from './lib/scenario-disposition.js';
+import { softSkipDisposition } from './lib/soft-skip.js';
 import type { DiscoveryPayload } from './lib/profiles.js';
 
 const SUITE_INIT_TIMEOUT_MS = 5_000;
@@ -181,7 +186,15 @@ async function maybeStartA2AFakePeer(): Promise<void> {
   }
   const portEnv = process.env.OPENWOP_A2A_FAKE_PEER_PORT;
   const requestedPort = portEnv ? Number(portEnv) : 0;
-  const peer = new A2AFakePeer();
+  // Dual-era since 1.112.0. Order = card shape for a header-less GET (first
+  // wins); both eras are spoken on the RPC path. Default keeps the 0.3 card so
+  // today's 0.3 clients (which read `card.url`) keep working; set
+  // OPENWOP_A2A_FAKE_PEER_VERSIONS=1.0,0.3 to serve the 1.0 card by default.
+  const versionsEnv = process.env.OPENWOP_A2A_FAKE_PEER_VERSIONS;
+  const versions = versionsEnv
+    ? (versionsEnv.split(',').map((v) => v.trim()).filter((v) => v === '1.0' || v === '0.3') as Array<'1.0' | '0.3'>)
+    : undefined;
+  const peer = new A2AFakePeer(versions && versions.length > 0 ? { protocolVersions: versions } : undefined);
   await peer.start(requestedPort);
   setA2AFakePeer(peer);
   // eslint-disable-next-line no-console
@@ -195,3 +208,84 @@ await loadHostFixtures();
 await maybeStartOtelCollector();
 await maybeStartMcpFakeServer();
 await maybeStartA2AFakePeer();
+
+// ─── RFC 0148 §A file-level disposition recording (S6, acceptance item 2) ─────
+//
+// Every scenario FILE records exactly one disposition for its own requirement
+// id when it finishes. `afterEach` collects per-test states; `afterAll` folds
+// them (fileDisposition) and records — `executed-fail` / `executed-pass` /
+// `inapplicable` or `skipped` (when a behaviorGate in the file recorded that
+// reason for the profile it gates on) / else `blocked` (an all-skipped file
+// with no reason is an unclassified return, and §A resolves it to blocked).
+// When the runner set OPENWOP_LEDGER_PATH the recording also lands in the JSONL
+// sink, which is how `--certify` gets requirement-level dispositions instead of
+// inferring them from per-file pass/fail/skip.
+//
+// Setup-file hooks apply to every file in the worker; state is keyed by file.
+const _fileStates = new Map<string, FileTestState[]>();
+const _fileAssertions = new Map<string, number>();
+const _ledgerMarks = new Map<string, number>();
+function _fileOf(task: { file?: { filepath?: string; name?: string } } | undefined): string | null {
+  const f = task?.file?.filepath ?? task?.file?.name;
+  return typeof f === 'string' && f.length > 0 ? basename(f) : null;
+}
+beforeAll(({}, suite) => {
+  // Mark the ledger journal BEFORE the file's tests run, so a behaviorGate
+  // decision made by the very first test is inside the file's window.
+  const s = suite as unknown as { filepath?: string; name?: string; file?: { filepath?: string; name?: string } };
+  const file = _fileOf({ file: s.file ?? s });
+  if (file !== null && !_ledgerMarks.has(file)) _ledgerMarks.set(file, journalLength());
+});
+afterEach(({ task }) => {
+  const file = _fileOf(task as { file?: { filepath?: string; name?: string } });
+  if (file === null) return;
+  if (!_ledgerMarks.has(file)) _ledgerMarks.set(file, 0);
+  const state = task.result?.state;
+  const arr = _fileStates.get(file) ?? [];
+  arr.push(state === 'pass' ? 'pass' : state === 'fail' ? 'fail' : 'skip');
+  _fileStates.set(file, arr);
+  // RFC 0148 §C assertionCount: how many `expect` calls this test actually made.
+  // A leg that early-returns from a gate makes zero, and a file of such legs is
+  // an `executed-pass` with assertionCount 0 — visible, and unclassified for a
+  // claimed floor.
+  let calls = 0;
+  try {
+    calls = (expect.getState() as { assertionCalls?: number }).assertionCalls ?? 0;
+  } catch {
+    /* no state — count 0 */
+  }
+  _fileAssertions.set(file, (_fileAssertions.get(file) ?? 0) + calls);
+});
+afterAll(({}, suite) => {
+  // vitest 4: the suite/file task is the SECOND argument. For a file-level
+  // afterAll registered from a setup file, `suite` is the File task itself.
+  const s = suite as unknown as { filepath?: string; name?: string; file?: { filepath?: string; name?: string } };
+  const file = _fileOf({ file: s.file ?? s });
+  if (file === null || !file.endsWith('.test.ts')) return;
+  const states = _fileStates.get(file) ?? [];
+  // Did a behaviorGate in this file record inapplicable/skipped for its profile?
+  const mark = _ledgerMarks.get(file) ?? 0;
+  const since = journalSince(mark);
+  const gateReason = since.some((e) => e.disposition === 'inapplicable')
+    ? 'inapplicable'
+    : since.some((e) => e.disposition === 'skipped')
+      ? 'skipped'
+      : undefined;
+  const assertionCount = _fileAssertions.get(file) ?? 0;
+  // RFC 0148 §A: a pass with zero assertions is an unclassified return. If the
+  // file said why (softSkip / seamAbsent / behaviorGate), that is its
+  // disposition; if it said nothing, the runner resolves it to `blocked` with
+  // the marker detail — never to a pass. Floors still REJECT that row, so the
+  // honest bundle row and the pressure to say why both survive. The rule is
+  // `resolveFileRecord` (pinned by conformance-execution-witness.test.ts).
+  const { disposition, detail } = resolveFileRecord(states, gateReason, assertionCount, softSkipDisposition(file));
+  try {
+    recordRequirement(requirementIdForFile(file), disposition, detail, { assertionCount });
+  } catch {
+    /* a scenario that recorded its own file id first wins; never fail the file for bookkeeping */
+  }
+  _fileStates.delete(file);
+  _fileAssertions.delete(file);
+  _ledgerMarks.delete(file);
+});
+