@openwop/openwop-conformance 1.0.0

package/src/scenarios/replayDeterminism.test.ts

@@ -0,0 +1,171 @@
+/**
+ * Replay-determinism scenarios per spec/v1/replay.md.
+ *
+ * A `mode: 'replay'` fork re-executes a run from a chosen `fromSeq`
+ * point; per `replay.md` §"Replay determinism," the new run's events
+ * (modulo timestamps + IDs) MUST match the original run's events past
+ * the fork point.
+ *
+ * Profile gating: `openwop-replay-fork`. Hosts that don't advertise
+ * `replay.supported: true` skip-equivalent. Hosts that advertise but
+ * 501 on `mode: 'replay'` (e.g., OpenWOP as of 2026-05-01) ALSO
+ * skip-equivalent — the runtime check catches that case via the
+ * 501 response.
+ *
+ * @see spec/v1/replay.md
+ * @see lib/profiles.ts — openwop-replay-fork predicate
+ */
+
+import { describe, it, expect } from 'vitest';
+import { driver } from '../lib/driver.js';
+import { pollUntilTerminal } from '../lib/polling.js';
+import { isFixtureAdvertised } from '../lib/fixtures.js';
+
+const NOOP_WORKFLOW_ID = 'conformance-noop';
+const SKIP_NO_NOOP = !isFixtureAdvertised(NOOP_WORKFLOW_ID);
+
+interface DiscoveryReplay {
+  supported?: unknown;
+  modes?: unknown;
+}
+
+async function fetchReplayCapability(): Promise<DiscoveryReplay | null> {
+  const res = await driver.get('/.well-known/openwop', { authenticated: false });
+  if (res.status !== 200) return null;
+  const body = res.json as { replay?: DiscoveryReplay };
+  return body.replay ?? null;
+}
+
+interface RawEvent {
+  seq?: number;
+  sequence?: number;
+  type?: string;
+  nodeId?: string | null;
+  data?: unknown;
+  [key: string]: unknown;
+}
+
+function getSeq(e: RawEvent): number | null {
+  if (typeof e.sequence === 'number') return e.sequence;
+  if (typeof e.seq === 'number') return e.seq;
+  return null;
+}
+
+/**
+ * Strip non-deterministic fields so two runs can be compared
+ * structurally. Removed: timestamp, runId, eventId. Preserved: type,
+ * nodeId, data shape.
+ */
+function structuralShape(events: readonly RawEvent[]): Array<{ type: unknown; nodeId: unknown; data: unknown }> {
+  return events.map((e) => ({
+    type: e.type,
+    nodeId: e.nodeId ?? null,
+    data: e.data ?? null,
+  }));
+}
+
+describe('replay-determinism: openwop-replay-fork profile gate', () => {
+  it('host advertising replay.supported MUST also advertise replay.modes', async () => {
+    const replay = await fetchReplayCapability();
+    if (replay === null || replay.supported !== true) return; // skip-equivalent
+
+    expect(Array.isArray(replay.modes), driver.describe(
+      'spec/v1/replay.md',
+      'host advertising replay.supported MUST advertise replay.modes as an array',
+    )).toBe(true);
+    if (Array.isArray(replay.modes)) {
+      for (const m of replay.modes) {
+        expect(typeof m, driver.describe(
+          'spec/v1/replay.md',
+          'each replay.modes entry MUST be a string',
+        )).toBe('string');
+      }
+    }
+  });
+});
+
+describe.skipIf(SKIP_NO_NOOP)('replay-determinism: same fromSeq + same workflow yields identical event shape', () => {
+  it(
+    'two replay forks of the same point produce structurally-identical event lists',
+    async () => {
+      const replay = await fetchReplayCapability();
+      if (replay === null || replay.supported !== true) return; // host doesn't claim replay
+      if (!Array.isArray(replay.modes) || !replay.modes.includes('replay')) return; // mode not supported
+
+      // Phase 1: complete an original run.
+      const create = await driver.post('/v1/runs', { workflowId: NOOP_WORKFLOW_ID });
+      if (create.status !== 201) return;
+      const originalRunId = (create.json as { runId: string }).runId;
+      await pollUntilTerminal(originalRunId, { timeoutMs: 10_000 });
+
+      // Phase 2: fork in replay mode at fromSeq=0 (start of run).
+      const fork1 = await driver.post(`/v1/runs/${encodeURIComponent(originalRunId)}:fork`, {
+        mode: 'replay',
+        fromSeq: 0,
+      });
+      if (fork1.status === 501) return; // mode not implemented; skip-equivalent
+      expect(fork1.status, driver.describe(
+        'spec/v1/replay.md',
+        'POST /v1/runs/{runId}:fork with mode=replay MUST return 201',
+      )).toBe(201);
+      const fork1Id = (fork1.json as { runId: string }).runId;
+      await pollUntilTerminal(fork1Id, { timeoutMs: 10_000 });
+
+      // Phase 3: fork the SAME original run again from fromSeq=0.
+      const fork2 = await driver.post(`/v1/runs/${encodeURIComponent(originalRunId)}:fork`, {
+        mode: 'replay',
+        fromSeq: 0,
+      });
+      if (fork2.status === 501) return;
+      expect(fork2.status).toBe(201);
+      const fork2Id = (fork2.json as { runId: string }).runId;
+      await pollUntilTerminal(fork2Id, { timeoutMs: 10_000 });
+
+      // Phase 4: fetch both fork event streams.
+      const fork1Events = await driver.get(`/v1/runs/${encodeURIComponent(fork1Id)}/events/poll`);
+      const fork2Events = await driver.get(`/v1/runs/${encodeURIComponent(fork2Id)}/events/poll`);
+      if (fork1Events.status !== 200 || fork2Events.status !== 200) return;
+
+      const fork1Body = fork1Events.json as { events?: RawEvent[] };
+      const fork2Body = fork2Events.json as { events?: RawEvent[] };
+      if (!fork1Body.events || !fork2Body.events) return;
+
+      // Phase 5: assert structural identity (modulo timestamps + IDs).
+      expect(fork1Body.events.length, driver.describe(
+        'spec/v1/replay.md §"Replay determinism"',
+        'two replay forks MUST produce the same number of events',
+      )).toBe(fork2Body.events.length);
+
+      const shape1 = structuralShape(fork1Body.events);
+      const shape2 = structuralShape(fork2Body.events);
+      expect(shape1, driver.describe(
+        'spec/v1/replay.md §"Replay determinism"',
+        'event sequence (type/nodeId/data) MUST be identical across two replay forks of the same point',
+      )).toEqual(shape2);
+    },
+    60_000,
+  );
+});
+
+describe.skipIf(SKIP_NO_NOOP)('replay-determinism: branch-mode is permitted to diverge', () => {
+  it('branch mode does NOT need to produce identical event sequences (negative-control)', async () => {
+    const replay = await fetchReplayCapability();
+    if (replay === null || replay.supported !== true) return;
+    if (!Array.isArray(replay.modes) || !replay.modes.includes('branch')) return;
+
+    // Self-test on the spec interpretation: branch and replay are
+    // SEMANTICALLY DIFFERENT modes per spec/v1/replay.md. Branch may
+    // diverge by design (variable overlay, runOptionsOverlay). This
+    // assertion just pins the interpretation; no actual round-trip
+    // needed.
+    expect((replay.modes as string[]).includes('branch'), driver.describe(
+      'spec/v1/replay.md',
+      'branch mode is documented; this self-test ensures the suite does not assume branch determinism',
+    )).toBe(true);
+
+    // Note: getSeq is exported here only so future scenarios can reuse
+    // it; we don't actually call it.
+    const dummy: RawEvent = { sequence: 0, type: 'noop' };
+    expect(getSeq(dummy)).toBe(0);
+  });
+});

package/src/scenarios/route-coverage.test.ts

@@ -0,0 +1,129 @@
+/**
+ * Route coverage scenarios — direct probes for OpenAPI operations that are
+ * otherwise only indirectly exercised by fixture flows.
+ *
+ * These tests intentionally stay small: they assert route existence, status
+ * class, and canonical error-envelope shape for edge cases that every host can
+ * answer without special fixtures.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { driver } from '../lib/driver.js';
+import { isFixtureAdvertised } from '../lib/fixtures.js';
+
+interface ErrorEnvelope {
+  error?: unknown;
+  message?: unknown;
+  details?: unknown;
+  [key: string]: unknown;
+}
+
+const NOOP_WORKFLOW_ID = 'conformance-noop';
+const SKIP_NO_NOOP = !isFixtureAdvertised(NOOP_WORKFLOW_ID);
+
+function assertCanonicalErrorEnvelope(body: unknown, specSection: string): void {
+  expect(typeof body, driver.describe(specSection, 'error response MUST be a JSON object')).toBe(
+    'object',
+  );
+  expect(body, driver.describe(specSection, 'error response MUST NOT be null')).not.toBeNull();
+
+  const env = body as ErrorEnvelope;
+  expect(typeof env.error, driver.describe(
+    specSection,
+    'error envelope MUST include machine-readable string `error`',
+  )).toBe('string');
+  expect(typeof env.message, driver.describe(
+    specSection,
+    'error envelope MUST include human-readable string `message`',
+  )).toBe('string');
+
+  if (env.details !== undefined) {
+    expect(typeof env.details, driver.describe(
+      specSection,
+      'error envelope `details`, when present, MUST be an object',
+    )).toBe('object');
+    expect(env.details, driver.describe(
+      specSection,
+      'error envelope `details`, when present, MUST NOT be null',
+    )).not.toBeNull();
+  }
+
+  const allowedKeys = new Set(['error', 'message', 'details']);
+  const extras = Object.keys(env).filter((key) => !allowedKeys.has(key));
+  expect(extras, driver.describe(
+    'schemas/error-envelope.schema.json',
+    'error envelope MUST NOT contain top-level keys outside {error,message,details}',
+  )).toEqual([]);
+}
+
+describe.skipIf(SKIP_NO_NOOP)('route coverage: GET /v1/workflows/{workflowId}', () => {
+  it('returns the seeded workflow definition for an advertised fixture workflow', async () => {
+    const res = await driver.get(`/v1/workflows/${encodeURIComponent(NOOP_WORKFLOW_ID)}`);
+
+    expect(res.status, driver.describe(
+      'api/openapi.yaml operationId=getWorkflow',
+      'GET /v1/workflows/{workflowId} MUST return 200 for a known workflow',
+    )).toBe(200);
+
+    const body = res.json as { id?: unknown; nodes?: unknown } | undefined;
+    expect(body?.id, driver.describe(
+      'schemas/workflow-definition.schema.json',
+      'workflow definition MUST echo its id',
+    )).toBe(NOOP_WORKFLOW_ID);
+    expect(Array.isArray(body?.nodes), driver.describe(
+      'schemas/workflow-definition.schema.json',
+      'workflow definition MUST include a nodes array',
+    )).toBe(true);
+  });
+});
+
+describe('route coverage: negative operation probes', () => {
+  it('GET /v1/workflows/{unknownWorkflowId} returns a canonical 404 or 403 envelope', async () => {
+    const res = await driver.get('/v1/workflows/openwop-conformance-missing-workflow');
+
+    expect([403, 404].includes(res.status), driver.describe(
+      'api/openapi.yaml operationId=getWorkflow',
+      'unknown workflow MUST return 404 or 403 if existence is protected',
+    )).toBe(true);
+    assertCanonicalErrorEnvelope(res.json, 'rest-endpoints.md error envelope');
+  });
+
+  it('GET /v1/runs/{runId}/artifacts/{artifactId} for an unknown artifact returns a canonical 404 or 403 envelope', async () => {
+    const res = await driver.get(
+      '/v1/runs/openwop-conformance-missing-run/artifacts/openwop-conformance-missing-artifact',
+    );
+
+    expect([403, 404].includes(res.status), driver.describe(
+      'api/openapi.yaml operationId=getArtifact',
+      'unknown artifact MUST return 404 or 403 if existence is protected',
+    )).toBe(true);
+    assertCanonicalErrorEnvelope(res.json, 'rest-endpoints.md error envelope');
+  });
+
+  it('POST /v1/webhooks with an invalid URL returns a canonical validation envelope', async () => {
+    const res = await driver.post('/v1/webhooks', {
+      url: 'not-a-valid-url',
+      events: ['run.completed'],
+    });
+
+    expect(res.status, driver.describe(
+      'api/openapi.yaml operationId=registerWebhook',
+      'invalid webhook registration MUST return a 4xx validation response',
+    )).toBeGreaterThanOrEqual(400);
+    expect(res.status).toBeLessThan(500);
+    assertCanonicalErrorEnvelope(res.json, 'rest-endpoints.md error envelope');
+  });
+
+  it('DELETE /v1/webhooks/{webhookId} for an unknown subscription returns 204, 404, or 403', async () => {
+    const res = await driver.delete('/v1/webhooks/openwop-conformance-missing-webhook');
+
+    expect([204, 403, 404].includes(res.status), driver.describe(
+      'api/openapi.yaml operationId=unregisterWebhook',
+      'unknown webhook unregister MUST be idempotent 204 or return 404/403',
+    )).toBe(true);
+
+    if (res.status !== 204) {
+      assertCanonicalErrorEnvelope(res.json, 'rest-endpoints.md error envelope');
+    }
+  });
+});

package/src/scenarios/runs-lifecycle.test.ts

@@ -0,0 +1,65 @@
+/**
+ * Run-lifecycle scenarios — exercises POST /v1/runs + terminal status visibility.
+ *
+ * Uses the `conformance-noop` fixture from `../../fixtures/openwop-conformance-noop.json`.
+ * Server MUST have seeded that fixture before this test runs (see fixtures.md).
+ *
+ * The suite assumes synchronous-or-fast completion. For servers that take
+ * >10s on a noop, bump OPENWOP_LIFECYCLE_TIMEOUT_MS in the environment.
+ * Polling cadence is 250ms.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { driver } from '../lib/driver.js';
+import { pollUntilTerminal } from '../lib/polling.js';
+import { isFixtureAdvertised } from '../lib/fixtures.js';
+
+const NOOP_WORKFLOW_ID = 'conformance-noop';
+const SKIP_NO_NOOP = !isFixtureAdvertised(NOOP_WORKFLOW_ID);
+
+describe.skipIf(SKIP_NO_NOOP)('run lifecycle: conformance-noop fixture', () => {
+  it('POST /v1/runs returns 201 with runId per rest-endpoints.md', async () => {
+    const res = await driver.post('/v1/runs', { workflowId: NOOP_WORKFLOW_ID });
+
+    expect(res.status, driver.describe(
+      'rest-endpoints.md',
+      'POST /v1/runs MUST return 201 on accepted run',
+    )).toBe(201);
+
+    const body = res.json as { runId?: unknown; status?: unknown } | undefined;
+    expect(typeof body?.runId, driver.describe(
+      'rest-endpoints.md',
+      'POST /v1/runs response body MUST include `runId` string',
+    )).toBe('string');
+    expect(typeof body?.status, driver.describe(
+      'rest-endpoints.md',
+      'POST /v1/runs response body MUST include `status` string',
+    )).toBe('string');
+  });
+
+  it('reaches terminal `completed` within bounded time per fixtures.md noop spec', async () => {
+    const create = await driver.post('/v1/runs', { workflowId: NOOP_WORKFLOW_ID });
+    expect(create.status).toBe(201);
+    const runId = (create.json as { runId: string }).runId;
+
+    const terminal = await pollUntilTerminal(runId);
+
+    expect(terminal.status, driver.describe(
+      'fixtures.md conformance-noop §Terminal status',
+      'fixture MUST reach terminal status `completed`',
+    )).toBe('completed');
+
+    expect(terminal.runId, driver.describe(
+      'rest-endpoints.md RunSnapshot',
+      'GET /v1/runs/{runId} MUST echo runId',
+    )).toBe(runId);
+  });
+
+  it('GET /v1/runs/{nonexistentId} returns 404 (or 403) per rest-endpoints.md', async () => {
+    const res = await driver.get('/v1/runs/openwop-conformance-this-run-id-does-not-exist');
+    expect(
+      [403, 404].includes(res.status),
+      driver.describe('rest-endpoints.md', 'unknown run MUST return 404 or 403'),
+    ).toBe(true);
+  });
+});

package/src/scenarios/runtime-capabilities.test.ts

@@ -0,0 +1,118 @@
+/**
+ * Runtime capabilities scenarios — optional v1 extension coverage.
+ *
+ * Runtime Capability Declarations let a NodeModule declare host facilities
+ * it depends on via `requires: ['chat.sendPrompt']` and let the host
+ * advertise registered providers via `runtimeCapabilities: string[]` in the
+ * `/.well-known/openwop` response. The field is optional in v1; clients MUST
+ * tolerate its absence.
+ *
+ * Scenarios in this file:
+ *
+ *   1. Forward-compat shape check — IF `runtimeCapabilities` is present
+ *      in the discovery response, it MUST be a string array of unique,
+ *      non-empty entries.
+ *   2. End-to-end dispatch refusal — when a workflow uses a
+ *      NodeModule that declares `requires: ['<unsupported>']`, the run
+ *      MUST terminate with `RunSnapshot.error.code =
+ *      'capability_not_provided'` and MUST NOT execute the node.
+ *
+ * The E2E path needs a fixture node that declares a `requires` entry
+ * the conformance host is guaranteed not to provide. Hosts that do not
+ * advertise that fixture skip-equivalent.
+ *
+ * Spec references:
+ *   - capabilities.md §"Runtime capabilities"
+ */
+
+import { describe, it, expect } from 'vitest';
+import { driver } from '../lib/driver.js';
+import { pollUntilTerminal } from '../lib/polling.js';
+import { isFixtureAdvertised } from '../lib/fixtures.js';
+
+const CAPABILITY_MISSING_WORKFLOW_ID = 'conformance-capability-missing';
+const SKIP_NO_FIXTURE = !isFixtureAdvertised(CAPABILITY_MISSING_WORKFLOW_ID);
+
+describe('runtime-capabilities: /.well-known/openwop forward-compat shape', () => {
+  it('IF runtimeCapabilities is present, it MUST be a string[] of unique non-empty entries', async () => {
+    const res = await driver.get('/.well-known/openwop', { authenticated: false });
+
+    expect(res.status, driver.describe(
+      'capabilities.md §2',
+      'discovery endpoint MUST return 200',
+    )).toBe(200);
+
+    const body = res.json as { runtimeCapabilities?: unknown } | undefined;
+    const caps = body?.runtimeCapabilities;
+
+    if (caps === undefined) {
+      // Spec-allowed — runtimeCapabilities is optional. The vast majority
+      // of v1 hosts will omit it. Assertion passes trivially; don't
+      // force a value on a host that doesn't advertise any.
+      expect(caps).toBeUndefined();
+      return;
+    }
+
+    expect(Array.isArray(caps), driver.describe(
+      'capabilities.md §"Runtime capabilities"',
+      'runtimeCapabilities MUST be an array when present',
+    )).toBe(true);
+
+    const arr = caps as unknown[];
+    for (const entry of arr) {
+      expect(typeof entry, driver.describe(
+        'capabilities.md §"Runtime capabilities"',
+        'every runtimeCapabilities entry MUST be a string',
+      )).toBe('string');
+      expect((entry as string).length, driver.describe(
+        'capabilities.md §"Runtime capabilities"',
+        'every runtimeCapabilities entry MUST be non-empty',
+      )).toBeGreaterThan(0);
+    }
+
+    const unique = new Set(arr as string[]);
+    expect(unique.size, driver.describe(
+      'capabilities.md §"Runtime capabilities"',
+      'runtimeCapabilities entries MUST be unique',
+    )).toBe(arr.length);
+  });
+
+});
+
+// ── E2E dispatch refusal ─────────────────────────────────────────────────
+//
+// Requires the host to have registered the `conformance.requiresMissing`
+// fixture node + seeded `conformance-capability-missing` workflow. The
+// Hosts that don't expose the optional fixture surface skip this describe
+// block via fixture advertisement.
+
+describe.skipIf(SKIP_NO_FIXTURE)('runtime-capabilities: dispatch refusal on unsatisfied requires', () => {
+  it('terminates the run with error.code = capability_not_provided', async () => {
+    const create = await driver.post('/v1/runs', {
+      workflowId: 'conformance-capability-missing',
+    });
+    expect(create.status, driver.describe(
+      'capabilities.md §"Runtime capabilities"',
+      'POST /v1/runs MUST accept the run; refusal is engine-side at dispatch time, not request-validation',
+    )).toBe(201);
+    const runId = (create.json as { runId: string }).runId;
+
+    const terminal = await pollUntilTerminal(runId);
+
+    expect(terminal.status, driver.describe(
+      'capabilities.md §"Runtime capabilities"',
+      'a node with unsatisfied requires MUST cause the run to terminate as failed',
+    )).toBe('failed');
+
+    const error = (terminal as { error?: { code?: string; message?: string } }).error;
+    expect(error?.code, driver.describe(
+      'rest-endpoints.md §"Common error codes"',
+      'terminal RunSnapshot.error.code MUST be "capability_not_provided"',
+    )).toBe('capability_not_provided');
+
+    expect(error?.message, driver.describe(
+      'capabilities.md §"Runtime capabilities"',
+      'error.message MUST name the missing capability id verbatim so operators can act without grepping logs',
+    )).toContain('conformance.never-provided');
+  });
+});