@openwop/openwop-conformance 1.14.0 → 1.16.0

package/src/scenarios/replay-observable-sequence-determinism.test.ts

@@ -8,87 +8,229 @@
  * Asserts (behavioral, when a host advertises `version: 4` + the contract):
  *
  *   1. A `mode: replay` fork from event-log index `fromSeq` produces an
- *      event-log prefix `[0, fromSeq]` that is byte-equivalent to the
- *      original run's prefix (modulo per-region clock fields per RFC 0036
- *      §E and ULID component-T entropy when ULIDs are minted fresh).
+ *      observable event-log prefix `[0, fromSeq]` that is byte-equivalent
+ *      to the original run's prefix (modulo volatile per-event fields:
+ *      eventId/ULID entropy, per-region `observedAt` clocks per RFC 0036
+ *      §E, and the run id itself).
  *
- *   2. The replay's `RunSnapshot.variables`, `RunSnapshot.channels`, and
- *      `RunSnapshot.status` at the boundary index are byte-equivalent to
- *      the original.
+ *   2. (Crucially per §C.) The replay reproduces the OBSERVABLE RESULT of
+ *      a nondeterministic tool node EVEN WHEN a fresh call would produce
+ *      different bytes. The `conformance-phase4-nondet-tool` fixture's
+ *      first node declares `config.nondeterministic: true`; a `version: 4`
+ *      host MUST replay the original event-log entries for that node
+ *      (cache the observable result) rather than re-executing it, so the
+ *      node's terminal payload is identical across original + replay.
  *
- *   3. (Crucially per §C.) The replay reproduces observable output EVEN
- *      WHEN the underlying tool call would have produced different bytes.
- *      The reference test uses a mock tool that returns a fresh random
- *      string on each call; the host MUST cache the original observable
- *      result so replay returns the SAME string the original got — not
- *      the bytes a fresh call would return now.
+ * The `conformance-phase4-nondet-tool` fixture ships in the suite (added
+ * via the RFC 0041 Phase 4 fixtures commit). These assertions are now
+ * runnable capability-gated `it()` bodies — consistent with the sibling
+ * `replay-divergence-at-refusal.test.ts`, which is likewise active and
+ * soft-skips on the same gate. They light up the moment a host advertises
+ * the `version: 4` replay-determinism contract; against hosts that don't
+ * (incl. the reference workflow-engine, which has not yet wired the
+ * pure-replay observable-cache path), they soft-skip honestly.
  *
- * Driving the assertion requires a workflow fixture whose tool call is
- * pure-nondeterministic (different bytes on each call) but whose
- * observable result is what gets cached. Reference workflow-engine ships
- * `core.noop` + deterministic fixtures; the `version: 4` wiring needs a
- * nondeterministic-tool fixture (e.g., `conformance-phase4-nondet-tool`).
- * Until that lands, the cross-boundary assertion is surfaced as `it.todo`
- * so test reporters track the gap.
+ * RFC 0042 §B note: RFC 0041 §C is `Active` (not yet `Accepted`), so its
+ * wire shape MAY shift compatibly within v1.x — a host wiring this before
+ * RFC 0041 graduates SHOULD advertise `multiAgent.executionModel.tier:
+ * 'experimental'` + `experimentalUntil` per RFC 0042 §A.
  *
  * @see RFCS/0041-multi-agent-replay-under-nondeterminism.md §C
  * @see spec/v1/replay.md §"Observable-output-sequence determinism vs bit-equivalent execution (MAE-9 closure)"
  * @see spec/v1/multi-agent-execution.md §"Replay determinism under nondeterminism (RFC 0041)"
  */
 
-import { describe, it } from 'vitest';
-
-// Behavioral assertions in this file are currently `it.todo` placeholders;
-// the `conformance-phase4-nondet-tool` fixture hasn't shipped yet. When
-// it does, the `it.todo` calls flip back to runnable `it(...)` bodies
-// that read discovery (via `driver.get('/.well-known/openwop')`), gate
-// on `multiAgent.executionModel.version >= 4` AND
-// `replayDeterminism.supported: true`, and drive the workflow through
-// the fixture.
-
-describe('replay-observable-sequence-determinism: prefix byte-equivalence (RFC 0041 §C)', () => {
-  // Behavioral assertion drives a workflow with at least one node whose
-  // underlying tool call is nondeterministic (different bytes on each
-  // call). The assertion sequence:
-  //   1. POST /v1/runs { workflowId: 'conformance-phase4-nondet-tool' }
-  //      → runs to completion, capturing the original event log.
-  //   2. Capture original event-log prefix [0, N] where N is the index
-  //      after the nondeterministic-tool node fires.
-  //   3. POST /v1/runs/{runId}:fork { mode: 'replay', fromSeq: N }
-  //   4. Read replay event-log prefix [0, N].
-  //   5. Assert byte-equivalence modulo the carve-outs:
-  //      - per-region observedAt timestamps (RFC 0036 §E)
-  //      - ULID component-T entropy on newly-minted eventIds
-  //   6. Read original + replay RunSnapshot at index N; assert
-  //      variables + channels + status byte-equivalent.
-  // Surfaced as `todo` until the `conformance-phase4-nondet-tool`
-  // fixture ships in the suite — consistent with the sibling RFC 0041
-  // scenarios (`replay-divergence-at-refusal.test.ts`,
-  // `replay-llm-cache-key-portable.test.ts`).
-  // Marked out of stable profile via RFC 0042 §B (experimental tier):
-  // RFC 0041 §C remains Active, so its wire shape MAY shift compatibly
-  // within v1.x. Hosts that wire this assertion before RFC 0041 graduates
-  // to Accepted SHOULD advertise `multiAgent.executionModel.tier:
-  // 'experimental'` + `experimentalUntil` per RFC 0042 §A. Path-to-runnable
-  // requires: (a) host pure-replay observable-cache emission via the
-  // `:fork mode: replay` re-dispatch path and (b) the test seam endpoint
-  // contract for cache-hit-vs-fresh-call distinction (see
-  // `spec/v1/host-sample-test-seams.md` for the established seam pattern).
-  it.skip('original and replay event-log prefixes [0, fromSeq] MUST be byte-equivalent (modulo per-region clock + ULID-T entropy) — out of stable profile via RFC 0042');
-});
-
-describe('replay-observable-sequence-determinism: observable-result caching (RFC 0041 §C)', () => {
-  // The load-bearing assertion: a nondeterministic tool call's OBSERVABLE
-  // RESULT (return value + side-effects on workflow state + emitted events)
-  // is what gets cached, not the bytes-on-the-wire of the underlying call.
-  // The replay's reproduction of the observable sequence is what makes
-  // this a valid determinism contract — bit-equivalent execution would
-  // require unbounded caching (rejected per RFC 0041 §"Alternatives
-  // considered" #2).
-  // Marked out of stable profile via RFC 0042 §B (experimental tier):
-  // see the prefix-byte-equivalence comment above for the same routing.
-  // This is RFC 0041 §C's load-bearing assertion; it lands as a runnable
-  // `it()` when RFC 0041 graduates to Accepted on first non-steward host
-  // adoption.
-  it.skip('replay of a workflow containing a nondeterministic tool call reproduces the original observable result, NOT a fresh call — out of stable profile via RFC 0042');
-});
+import { describe, it, expect } from 'vitest';
+import { driver } from '../lib/driver.js';
+import { capabilityFamily } from '../lib/discovery-capabilities.js';
+
+const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
+const FIXTURE = 'conformance-phase4-nondet-tool';
+const NONDET_NODE_ID = 'nondet-tool';
+
+interface ExecutionModelCaps {
+  version?: unknown;
+  replayDeterminism?: { supported?: unknown };
+}
+interface DiscoveryDoc {
+  capabilities?: {
+    multiAgent?: { executionModel?: ExecutionModelCaps };
+  };
+}
+
+interface RunSnapshot {
+  status?: string;
+}
+interface RunEventDoc {
+  type: string;
+  nodeId?: string;
+  sequence?: number;
+  payload?: Record<string, unknown>;
+}
+
+async function readDiscovery(): Promise<DiscoveryDoc | null> {
+  try {
+    const res = await driver.get('/.well-known/openwop');
+    if (res.status !== 200) return null;
+    return res.json as DiscoveryDoc;
+  } catch {
+    return null;
+  }
+}
+
+/** Soft-skip unless the host advertises the RFC 0041 §C version-4 contract. */
+async function gateOnPhase4(ctx: { skip: () => void }): Promise<boolean> {
+  const d = await readDiscovery();
+  const em = capabilityFamily<{ executionModel?: ExecutionModelCaps }>(d, 'multiAgent')?.executionModel;
+  const version = typeof em?.version === 'number' ? em.version : 0;
+  if (em?.replayDeterminism?.supported !== true || version < 4) {
+    ctx.skip();
+    return false;
+  }
+  return true;
+}
+
+async function pollUntilTerminal(runId: string): Promise<RunSnapshot> {
+  for (let i = 0; i < 50; i++) {
+    const r = await driver.get(`/v1/runs/${encodeURIComponent(runId)}`);
+    const snap = r.json as RunSnapshot;
+    if (snap.status === 'completed' || snap.status === 'failed' || snap.status === 'cancelled') {
+      return snap;
+    }
+    await new Promise((resolve) => setTimeout(resolve, 100));
+  }
+  throw new Error(`run ${runId} did not reach terminal within 5s`);
+}
+
+async function readEvents(runId: string): Promise<RunEventDoc[]> {
+  const r = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/events`);
+  const body = r.json as { events?: RunEventDoc[] };
+  return body.events ?? [];
+}
+
+/**
+ * Volatile field names that differ legitimately between an original run and
+ * its replay: freshly-minted event ids/ULIDs, the run id, and per-region
+ * clock fields (RFC 0036 §E carve-out). Stripped wherever they appear —
+ * including NESTED inside payloads — so the byte-equivalence comparison
+ * tolerates only these carve-outs and flags any other divergence.
+ */
+const VOLATILE_KEYS = new Set([
+  'eventId',
+  'runId',
+  'observedAt',
+  'timestamp',
+  'occurredAt',
+  'emittedAt',
+  'id',
+]);
+
+/**
+ * Recursively strip {@link VOLATILE_KEYS} from an event so two runs of the
+ * same workflow are comparable. Recurses into nested objects + arrays (a
+ * host that buries a clock or ULID inside a payload is normalized too),
+ * leaving every non-volatile field intact for the equivalence assertion.
+ */
+function stripVolatile(ev: RunEventDoc): unknown {
+  const walk = (node: unknown): unknown => {
+    if (Array.isArray(node)) return node.map(walk);
+    if (node !== null && typeof node === 'object') {
+      const out: Record<string, unknown> = {};
+      for (const [k, v] of Object.entries(node as Record<string, unknown>)) {
+        if (VOLATILE_KEYS.has(k)) continue;
+        out[k] = walk(v);
+      }
+      return out;
+    }
+    return node;
+  };
+  return walk(JSON.parse(JSON.stringify(ev)));
+}
+
+/** Create the fixture run; returns null (with a skip) if it isn't advertised. */
+async function startFixtureRun(ctx: { skip: () => void }): Promise<string | null> {
+  const create = await driver.post('/v1/runs', { workflowId: FIXTURE });
+  if (create.status === 404 || create.status === 422) {
+    ctx.skip(); // fixture not advertised by this host
+    return null;
+  }
+  expect(create.status).toBe(201);
+  return (create.json as { runId: string }).runId;
+}
+
+describe.skipIf(HTTP_SKIP)(
+  'replay-observable-sequence-determinism: prefix byte-equivalence (RFC 0041 §C)',
+  () => {
+    it('original and replay event-log prefixes MUST be byte-equivalent (modulo per-event clock + ULID entropy)', async (ctx) => {
+      if (!(await gateOnPhase4(ctx))) return;
+
+      const sourceRunId = await startFixtureRun(ctx);
+      if (sourceRunId === null) return;
+      const sourceTerminal = await pollUntilTerminal(sourceRunId);
+      expect(sourceTerminal.status).toBe('completed');
+      const sourceEvents = await readEvents(sourceRunId);
+
+      const forkRes = await driver.post(`/v1/runs/${encodeURIComponent(sourceRunId)}:fork`, {
+        fromSeq: 0,
+        mode: 'replay',
+      });
+      expect(forkRes.status).toBe(201);
+      const replayRunId = (forkRes.json as { runId: string }).runId;
+      await pollUntilTerminal(replayRunId);
+      const replayEvents = await readEvents(replayRunId);
+
+      const sourceNorm = sourceEvents.map(stripVolatile);
+      const replayNorm = replayEvents.map(stripVolatile);
+      expect(
+        replayNorm,
+        driver.describe(
+          'RFCS/0041-multi-agent-replay-under-nondeterminism.md §C',
+          'a mode:replay fork MUST reproduce the original observable event-log sequence byte-for-byte modulo volatile per-event fields (eventId/ULID entropy, per-region observedAt clock)',
+        ),
+      ).toEqual(sourceNorm);
+    });
+  },
+);
+
+describe.skipIf(HTTP_SKIP)(
+  'replay-observable-sequence-determinism: observable-result caching (RFC 0041 §C)',
+  () => {
+    it('replay of a nondeterministic tool node reproduces the ORIGINAL observable result, NOT a fresh call', async (ctx) => {
+      if (!(await gateOnPhase4(ctx))) return;
+
+      const sourceRunId = await startFixtureRun(ctx);
+      if (sourceRunId === null) return;
+      expect((await pollUntilTerminal(sourceRunId)).status).toBe('completed');
+      const sourceEvents = await readEvents(sourceRunId);
+
+      // The terminal event(s) for the nondeterministic node carry its
+      // observable result. Capture every event scoped to that node.
+      const sourceNodeEvents = sourceEvents.filter((e) => e.nodeId === NONDET_NODE_ID).map(stripVolatile);
+      expect(
+        sourceNodeEvents.length,
+        driver.describe(
+          'RFCS/0041-multi-agent-replay-under-nondeterminism.md §C',
+          `the fixture's nondeterministic node \`${NONDET_NODE_ID}\` MUST emit at least one observable event`,
+        ),
+      ).toBeGreaterThan(0);
+
+      const forkRes = await driver.post(`/v1/runs/${encodeURIComponent(sourceRunId)}:fork`, {
+        fromSeq: 0,
+        mode: 'replay',
+      });
+      expect(forkRes.status).toBe(201);
+      const replayRunId = (forkRes.json as { runId: string }).runId;
+      await pollUntilTerminal(replayRunId);
+      const replayEvents = await readEvents(replayRunId);
+      const replayNodeEvents = replayEvents.filter((e) => e.nodeId === NONDET_NODE_ID).map(stripVolatile);
+
+      expect(
+        replayNodeEvents,
+        driver.describe(
+          'RFCS/0041-multi-agent-replay-under-nondeterminism.md §C',
+          'the nondeterministic tool node MUST replay its ORIGINAL observable result (cached event-log entry) rather than re-executing — bit-equivalent re-execution would require unbounded caching, rejected per RFC 0041 §"Alternatives considered" #2',
+        ),
+      ).toEqual(sourceNodeEvents);
+    });
+  },
+);

package/src/scenarios/secret-leakage-otel-attribute.test.ts

@@ -56,6 +56,7 @@ import { driver } from '../lib/driver.js';
 import { pollUntilTerminal } from '../lib/polling.js';
 import { isFixtureAdvertised } from '../lib/fixtures.js';
 import { capabilityFamily } from '../lib/discovery-capabilities.js';
+import { getCollector, waitForRunSpans } from '../lib/otel-collector.js';
 
 const HTTP_SKIP = !process.env.OPENWOP_BASE_URL;
 const BYOK_WORKFLOW_ID = 'openwop-smoke-byok-roundtrip';
@@ -205,6 +206,57 @@ describe.skipIf(HTTP_SKIP || FIXTURE_SKIP)(
   },
 );
 
+describe.skipIf(HTTP_SKIP || FIXTURE_SKIP)(
+  'secret-leakage-otel-attribute: real OTLP export scrape (collector-side)',
+  () => {
+    // Distinct from the scrape-seam probe above: this asserts against what
+    // the host's OTLP exporter ACTUALLY shipped over the wire to the
+    // conformance collector, not what the host self-reports via its
+    // `/v1/host/sample/test/otel/spans` seam. A host could redact in its
+    // seam yet leak on the real export — only this catches that. Closes
+    // the `docs/KNOWN-LIMITS.md` "collector seam doesn't inspect span
+    // attributes" gap. Gated on the in-process collector being active
+    // (`OPENWOP_OTEL_COLLECTOR=true` + the host configured to export to it).
+    it('NO real-exported OTel span/metric attribute MUST contain the BYOK canary plaintext', async (ctx) => {
+      const collector = getCollector();
+      if (!collector || !CANARY_VALUE) {
+        ctx.skip();
+        return;
+      }
+      const d = await readDiscovery();
+      const secretsOk = capabilityFamily<{ supported?: unknown }>(d, 'secrets')?.supported === true;
+      const obsOk = capabilityFamily<unknown>(d, 'observability') !== undefined;
+      if (!secretsOk || !obsOk) {
+        ctx.skip();
+        return;
+      }
+
+      collector.reset();
+      const runId = await startByokRun();
+      if (runId === null) {
+        ctx.skip();
+        return;
+      }
+      const terminal = await pollUntilTerminal(runId);
+      expect(terminal.status).toBe('completed');
+
+      // Hosts export spans asynchronously after terminal; poll until the
+      // run's spans land (or the timeout elapses — an absent export is a
+      // separate coverage concern, not a leak).
+      await waitForRunSpans(runId, { timeoutMs: 8_000 });
+
+      const leaks = collector.findCanaryLeakage(CANARY_VALUE);
+      expect(
+        leaks,
+        driver.describe(
+          'SECURITY/invariants.yaml secret-leakage-otel-attribute',
+          `no real-exported OTel span/metric attribute may contain the BYOK canary plaintext. Leaking surfaces: ${JSON.stringify(leaks)}`,
+        ),
+      ).toEqual([]);
+    });
+  },
+);
+
 describe.skipIf(HTTP_SKIP || FIXTURE_SKIP)(
   'secret-leakage-otel-attribute: advertisement-shape probe (RFC 0034 §A)',
   () => {