@openwop/openwop-conformance 1.0.0

package/src/scenarios/debugBundle.test.ts

@@ -0,0 +1,222 @@
+/**
+ * Debug-bundle scenarios per `spec/v1/debug-bundle.md`.
+ *
+ * GET /v1/runs/{runId}/debug-bundle returns a portable JSON snapshot
+ * of a single run's diagnostic state — run snapshot + events + spans
+ * + metrics + redaction state.
+ *
+ * Profile gating: hosts that don't advertise
+ * `capabilities.debugBundle.supported: true` skip-equivalent.
+ *
+ * What this scenario verifies:
+ *
+ *   1. **Schema validity** — the response validates against
+ *      `schemas/debug-bundle.schema.json`.
+ *   2. **Event-count invariant** — `metrics.eventCount` equals
+ *      `events.length` (per debug-bundle.md §"Field reference").
+ *   3. **Bundle/event-stream agreement** — the events in the bundle
+ *      match the events from `/events/poll` for the same run.
+ *   4. **Redaction marker validity** — `redactionApplied: true` MUST
+ *      NOT coexist with `redactionMode: passthrough` (malformed shape
+ *      per debug-bundle.md §"Redaction guarantees").
+ *   5. **Canary safety** — bundles MUST inherit redaction. A canary
+ *      injected through workflow inputs MUST NOT echo verbatim in the
+ *      bundle response.
+ *
+ * Cross-references SECURITY/invariants.yaml `secret-leakage-debug-bundle`.
+ *
+ * @see spec/v1/debug-bundle.md
+ * @see schemas/debug-bundle.schema.json
+ * @see SECURITY/threat-model-secret-leakage.md
+ */
+
+import { describe, it, expect } from 'vitest';
+import { driver } from '../lib/driver.js';
+import { pollUntilTerminal } from '../lib/polling.js';
+import { CANARY_MARKER, getCanary } from '../lib/canaries.js';
+import { isFixtureAdvertised } from '../lib/fixtures.js';
+
+const NOOP_WORKFLOW_ID = 'conformance-noop';
+const SKIP_NO_NOOP = !isFixtureAdvertised(NOOP_WORKFLOW_ID);
+
+interface DebugBundleShape {
+  bundleVersion?: unknown;
+  generatedAt?: unknown;
+  host?: { name?: unknown; version?: unknown };
+  run?: { runId?: unknown; status?: unknown };
+  events?: unknown[];
+  spans?: unknown[];
+  metrics?: { eventCount?: unknown; nodeCount?: unknown };
+  redactionApplied?: unknown;
+  redactionMode?: unknown;
+  truncated?: unknown;
+}
+
+async function isAdvertised(): Promise<boolean> {
+  const res = await driver.get('/.well-known/openwop', { authenticated: false });
+  if (res.status !== 200) return false;
+  const body = res.json as { debugBundle?: { supported?: unknown } };
+  return body.debugBundle?.supported === true;
+}
+
+describe.skipIf(SKIP_NO_NOOP)('debug-bundle: GET /v1/runs/{runId}/debug-bundle response shape', () => {
+  it('host advertising capabilities.debugBundle.supported returns 200 with valid bundle', async () => {
+    if (!(await isAdvertised())) return; // skip-equivalent
+
+    const create = await driver.post('/v1/runs', { workflowId: NOOP_WORKFLOW_ID });
+    expect(create.status).toBe(201);
+    const runId = (create.json as { runId: string }).runId;
+    await pollUntilTerminal(runId);
+
+    const res = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/debug-bundle`);
+    expect(res.status, driver.describe(
+      'spec/v1/debug-bundle.md §Endpoint',
+      'host advertising debugBundle.supported MUST return 200 on /debug-bundle',
+    )).toBe(200);
+
+    const bundle = res.json as DebugBundleShape | undefined;
+    expect(bundle, driver.describe(
+      'spec/v1/debug-bundle.md',
+      'response MUST be JSON',
+    )).toBeDefined();
+
+    expect(typeof bundle?.bundleVersion, driver.describe(
+      'debug-bundle.md §Field reference',
+      'bundleVersion MUST be a string',
+    )).toBe('string');
+    expect(typeof bundle?.generatedAt, driver.describe(
+      'debug-bundle.md',
+      'generatedAt MUST be a string',
+    )).toBe('string');
+    expect(typeof bundle?.host?.name, driver.describe(
+      'debug-bundle.md',
+      'host.name MUST be a string',
+    )).toBe('string');
+    expect(typeof bundle?.host?.version, driver.describe(
+      'debug-bundle.md',
+      'host.version MUST be a string',
+    )).toBe('string');
+    expect(typeof bundle?.run?.runId, driver.describe(
+      'debug-bundle.md',
+      'run.runId MUST be a string',
+    )).toBe('string');
+    expect(Array.isArray(bundle?.events), driver.describe(
+      'debug-bundle.md',
+      'events MUST be an array',
+    )).toBe(true);
+    expect(typeof bundle?.redactionApplied, driver.describe(
+      'debug-bundle.md',
+      'redactionApplied MUST be a boolean',
+    )).toBe('boolean');
+    expect(typeof bundle?.redactionMode, driver.describe(
+      'debug-bundle.md',
+      'redactionMode MUST be a string',
+    )).toBe('string');
+  });
+
+  it('hosts not advertising debugBundle return 404 on the endpoint', async () => {
+    if (await isAdvertised()) return; // skip-equivalent for hosts that DO advertise
+
+    // Use any runId — even a synthetic one — since the host should 404
+    // on the endpoint regardless of run existence.
+    const res = await driver.get('/v1/runs/openwop-conformance-no-such-run/debug-bundle');
+    expect(res.status, driver.describe(
+      'debug-bundle.md §Endpoint',
+      'host NOT advertising debugBundle.supported MUST return 404',
+    )).toBe(404);
+  });
+});
+
+describe.skipIf(SKIP_NO_NOOP)('debug-bundle: invariants per debug-bundle.md', () => {
+  it('metrics.eventCount equals events.length', async () => {
+    if (!(await isAdvertised())) return;
+
+    const create = await driver.post('/v1/runs', { workflowId: NOOP_WORKFLOW_ID });
+    expect(create.status).toBe(201);
+    const runId = (create.json as { runId: string }).runId;
+    await pollUntilTerminal(runId);
+
+    const res = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/debug-bundle`);
+    expect(res.status).toBe(200);
+    const bundle = res.json as DebugBundleShape;
+
+    if (bundle.metrics?.eventCount !== undefined) {
+      expect(bundle.metrics.eventCount, driver.describe(
+        'debug-bundle.md §"Field reference"',
+        'metrics.eventCount MUST equal events.length',
+      )).toBe(bundle.events?.length ?? 0);
+    }
+  });
+
+  it('redactionApplied=true is incompatible with redactionMode=passthrough', async () => {
+    if (!(await isAdvertised())) return;
+
+    const create = await driver.post('/v1/runs', { workflowId: NOOP_WORKFLOW_ID });
+    expect(create.status).toBe(201);
+    const runId = (create.json as { runId: string }).runId;
+    await pollUntilTerminal(runId);
+
+    const res = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/debug-bundle`);
+    expect(res.status).toBe(200);
+    const bundle = res.json as DebugBundleShape;
+
+    if (bundle.redactionApplied === true) {
+      expect(bundle.redactionMode, driver.describe(
+        'debug-bundle.md §"Redaction guarantees"',
+        'redactionApplied=true MUST NOT coexist with redactionMode=passthrough — that combination is malformed',
+      )).not.toBe('passthrough');
+    }
+  });
+
+  it('bundle events agree with /events/poll for the same run', async () => {
+    if (!(await isAdvertised())) return;
+
+    const create = await driver.post('/v1/runs', { workflowId: NOOP_WORKFLOW_ID });
+    expect(create.status).toBe(201);
+    const runId = (create.json as { runId: string }).runId;
+    await pollUntilTerminal(runId);
+
+    const bundleRes = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/debug-bundle`);
+    const eventsRes = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/events/poll`);
+    if (eventsRes.status !== 200) return; // host without polling
+
+    const bundle = bundleRes.json as DebugBundleShape;
+    const polledEvents = (eventsRes.json as { events?: unknown[] }).events ?? [];
+
+    expect(bundle.events?.length, driver.describe(
+      'debug-bundle.md',
+      'bundle event count MUST agree with /events/poll for the same run',
+    )).toBe(polledEvents.length);
+  });
+});
+
+describe.skipIf(SKIP_NO_NOOP)('debug-bundle: redaction inheritance per SECURITY/invariants.yaml secret-leakage-debug-bundle', () => {
+  it('canary in workflow inputs MUST NOT appear verbatim in the bundle response', async () => {
+    if (!(await isAdvertised())) return;
+
+    const canary = getCanary('byok-credential-ref').value;
+
+    const create = await driver.post('/v1/runs', {
+      workflowId: NOOP_WORKFLOW_ID,
+      inputs: {
+        userField: canary,
+      },
+    });
+    expect(create.status).toBe(201);
+    const runId = (create.json as { runId: string }).runId;
+    await pollUntilTerminal(runId);
+
+    const res = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/debug-bundle`);
+    expect(res.status).toBe(200);
+
+    const bundleText = res.text;
+    expect(bundleText.includes(canary), driver.describe(
+      'SECURITY/invariants.yaml secret-leakage-debug-bundle',
+      'BYOK-shaped canary submitted as workflow input MUST NOT appear verbatim in the debug bundle',
+    )).toBe(false);
+    expect(bundleText.includes(CANARY_MARKER), driver.describe(
+      'SECURITY/invariants.yaml secret-leakage-debug-bundle',
+      'canary marker MUST NOT appear in the debug bundle',
+    )).toBe(false);
+  });
+});

package/src/scenarios/discovery.test.ts

@@ -0,0 +1,147 @@
+/**
+ * Discovery scenarios — `/.well-known/openwop` and `/v1/openapi.json`.
+ *
+ * These are the only two endpoints that MUST work without authentication
+ * (per `auth.md` §2 + `rest-endpoints.md`). They're the cheapest cross-
+ * implementation contracts to verify.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { driver } from '../lib/driver.js';
+
+describe('discovery: /.well-known/openwop', () => {
+  it('returns 200 with required Capabilities fields per capabilities.md §2', async () => {
+    const res = await driver.get('/.well-known/openwop', { authenticated: false });
+
+    expect(res.status, driver.describe(
+      'capabilities.md §2',
+      'discovery endpoint MUST be reachable without auth and return 200',
+    )).toBe(200);
+
+    const body = res.json as Record<string, unknown> | undefined;
+    expect(body, driver.describe('capabilities.md §2', 'response MUST be JSON')).toBeDefined();
+
+    // Per capabilities.md §3 (in-package shape), these 4 fields are REQUIRED.
+    for (const required of ['protocolVersion', 'supportedEnvelopes', 'schemaVersions', 'limits']) {
+      expect(body?.[required], driver.describe(
+        'capabilities.md §3',
+        `Capabilities.${required} MUST be present`,
+      )).toBeDefined();
+    }
+  });
+
+  it('serves Cache-Control per capabilities.md §4 (caching guidance)', async () => {
+    const res = await driver.get('/.well-known/openwop', { authenticated: false });
+    const cacheControl = res.headers.get('cache-control');
+
+    expect(cacheControl, driver.describe(
+      'capabilities.md §4',
+      'response SHOULD carry a Cache-Control header to allow client caching',
+    )).toBeTruthy();
+  });
+
+  it('IF Capabilities-Etag is present, it is non-empty and stable within the cache window', async () => {
+    const first = await driver.get('/.well-known/openwop', { authenticated: false });
+    const firstEtag = first.headers.get('capabilities-etag');
+
+    if (firstEtag === null) {
+      expect(firstEtag, driver.describe(
+        'capabilities-change-detection.md §Capabilities-Etag',
+        'Capabilities-Etag is optional; hosts that omit it remain conformant',
+      )).toBeNull();
+      return;
+    }
+
+    expect(firstEtag.trim().length, driver.describe(
+      'capabilities-change-detection.md §Capabilities-Etag',
+      'Capabilities-Etag, when present, MUST be a non-empty opaque string',
+    )).toBeGreaterThan(0);
+
+    const second = await driver.get('/.well-known/openwop', { authenticated: false });
+    const secondEtag = second.headers.get('capabilities-etag');
+
+    expect(secondEtag, driver.describe(
+      'capabilities-change-detection.md §Conformance expectations',
+      'repeated discovery calls without host changes SHOULD return the same Capabilities-Etag within the cache window',
+    )).toBe(firstEtag);
+  });
+
+  it('declares non-zero limits per capabilities.md §3 (CapabilityLimiter shape)', async () => {
+    const res = await driver.get('/.well-known/openwop', { authenticated: false });
+    const limits = (res.json as { limits?: Record<string, number> } | undefined)?.limits;
+
+    expect(limits, driver.describe(
+      'capabilities.md §3',
+      'Capabilities.limits MUST be present',
+    )).toBeDefined();
+
+    for (const k of ['clarificationRounds', 'schemaRounds', 'envelopesPerTurn']) {
+      const v = limits?.[k];
+      expect(typeof v, driver.describe(
+        'capabilities.md §3',
+        `limits.${k} MUST be a non-negative integer`,
+      )).toBe('number');
+      expect(v ?? -1, driver.describe(
+        'capabilities.md §3',
+        `limits.${k} MUST be >= 0`,
+      )).toBeGreaterThanOrEqual(0);
+    }
+  });
+});
+
+describe('discovery: /.well-known/openwop fixtures field shape per RFC 0003', () => {
+  it('IF fixtures is present, it MUST be a string[] of unique non-empty entries', async () => {
+    const res = await driver.get('/.well-known/openwop', { authenticated: false });
+    expect(res.status).toBe(200);
+
+    const body = res.json as { fixtures?: unknown } | undefined;
+    const fixtures = body?.fixtures;
+
+    if (fixtures === undefined) {
+      // RFC 0003 makes the field OPTIONAL — pre-RFC hosts and hosts
+      // that opt out advertise nothing. Assertion passes trivially.
+      expect(fixtures).toBeUndefined();
+      return;
+    }
+
+    expect(Array.isArray(fixtures), driver.describe(
+      'capabilities.md §`fixtures` (RFC 0003)',
+      'fixtures MUST be an array when present',
+    )).toBe(true);
+
+    const arr = fixtures as unknown[];
+    for (const entry of arr) {
+      expect(typeof entry, driver.describe(
+        'capabilities.md §`fixtures` (RFC 0003)',
+        'every fixtures entry MUST be a string',
+      )).toBe('string');
+      expect((entry as string).length, driver.describe(
+        'capabilities.md §`fixtures` (RFC 0003)',
+        'every fixtures entry MUST be non-empty',
+      )).toBeGreaterThan(0);
+    }
+
+    const unique = new Set(arr as string[]);
+    expect(unique.size, driver.describe(
+      'capabilities.md §`fixtures` (RFC 0003)',
+      'fixtures entries SHOULD be unique (consumers MUST tolerate duplicates by deduplicating)',
+    )).toBe(arr.length);
+  });
+});
+
+describe('discovery: /v1/openapi.json', () => {
+  it('returns 200 with a parseable OpenAPI 3.1 document', async () => {
+    const res = await driver.get('/v1/openapi.json', { authenticated: false });
+
+    expect(res.status, driver.describe(
+      'rest-endpoints.md',
+      'self-describing OpenAPI endpoint MUST return 200',
+    )).toBe(200);
+
+    const body = res.json as { openapi?: string } | undefined;
+    expect(body?.openapi, driver.describe(
+      'rest-endpoints.md',
+      'response MUST declare openapi >= 3.1',
+    )).toMatch(/^3\.[1-9]/);
+  });
+});

package/src/scenarios/dispatchLoop.test.ts

@@ -0,0 +1,52 @@
+/**
+ * Dispatch loop scenarios (Phase 6 / RFC 0007) — exercises `conformance-dispatch-loop`
+ * which uses `core.dispatch` to process orchestrator decisions and route to child workflows
+ * or terminate the loop.
+ *
+ * Verifies:
+ *   1. Dispatch loop handles `next-worker` by delegating to child run.
+ *   2. Dispatch loop handles `terminate` cleanly.
+ *
+ * Spec references:
+ *   - RFCS/0007-dispatch-loop.md
+ */
+
+import { describe, it, expect } from 'vitest';
+import { driver } from '../lib/driver.js';
+import { pollUntilTerminal } from '../lib/polling.js';
+import { isFixtureAdvertised } from '../lib/fixtures.js';
+
+const DISPATCH_LOOP_WORKFLOW_ID = 'conformance-dispatch-loop';
+const SKIP_NO_FIXTURE = !isFixtureAdvertised(DISPATCH_LOOP_WORKFLOW_ID);
+
+describe.skipIf(SKIP_NO_FIXTURE)('dispatchLoop: core.dispatch consumes OrchestratorDecision', () => {
+  it('host correctly processes orchestrator decisions and terminates', async () => {
+    // 1. Create the run
+    const create = await driver.post('/v1/runs', { workflowId: DISPATCH_LOOP_WORKFLOW_ID });
+    expect(create.status).toBe(201);
+    const runId = (create.json as { runId: string }).runId;
+
+    // Hosts may drive orchestrator decisions internally or through a conformance
+    // mock. The black-box contract here is terminal completion.
+    
+    const terminal = await pollUntilTerminal(runId);
+    expect(terminal.status, driver.describe(
+      'RFCS/0007-dispatch-loop.md §H',
+      'run MUST reach terminal `completed` after dispatch processes terminate decision',
+    )).toBe('completed');
+
+    // Verify the event log has the expected orchestrator decisions
+    const eventsRes = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/events`);
+    expect(eventsRes.status).toBe(200);
+    const events = (eventsRes.json as { events?: any[] })?.events ?? [];
+
+    const decisions = events
+      .filter((e) => e.type === 'runOrchestrator.decided')
+      .map((e) => e.payload?.decision?.kind);
+
+    expect(decisions, driver.describe(
+      'RFCS/0007-dispatch-loop.md §D',
+      'host MUST emit next-worker then terminate in a loop topology'
+    )).toEqual(['next-worker', 'terminate']);
+  });
+});

package/src/scenarios/errors.test.ts

@@ -0,0 +1,144 @@
+/**
+ * Error envelope shape — every non-2xx response MUST share the same
+ * `{error, message, details?}` JSON shape. Per rest-endpoints.md.
+ *
+ * We exercise a few intentional failure paths and verify each error
+ * response has the correct envelope.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { driver } from '../lib/driver.js';
+
+interface ErrorEnvelope {
+  error: unknown;
+  message: unknown;
+  details?: unknown;
+  [key: string]: unknown;
+}
+
+function assertErrorEnvelope(body: unknown, specSection: string): void {
+  expect(typeof body, driver.describe(specSection, 'error response MUST be a JSON object')).toBe(
+    'object',
+  );
+  const env = body as ErrorEnvelope;
+
+  expect(typeof env.error, driver.describe(
+    specSection,
+    'error envelope MUST include `error` (machine-readable string)',
+  )).toBe('string');
+
+  expect(typeof env.message, driver.describe(
+    specSection,
+    'error envelope MUST include `message` (human-readable string)',
+  )).toBe('string');
+
+  if (env.details !== undefined) {
+    expect(typeof env.details, driver.describe(
+      specSection,
+      'error envelope `details` (when present) MUST be a JSON object',
+    )).toBe('object');
+    expect(env.details, driver.describe(
+      specSection,
+      'error envelope `details` (when present) MUST NOT be null',
+    )).not.toBeNull();
+  }
+
+  // schemas/error-envelope.schema.json declares `additionalProperties: false`.
+  // Top-level body keys MUST be exactly some subset of {error, message, details}.
+  // Hosts that emit extras at the top level (correlationId, hint, requestId, etc.)
+  // violate the schema. The canonical home for contextual data is `details`.
+  const allowedKeys = new Set(['error', 'message', 'details']);
+  const extraneousKeys = Object.keys(env as Record<string, unknown>).filter(
+    (k) => !allowedKeys.has(k),
+  );
+  expect(extraneousKeys, driver.describe(
+    'schemas/error-envelope.schema.json (additionalProperties:false)',
+    `error envelope MUST NOT have keys outside {error, message, details}; extraneous: [${extraneousKeys.join(', ')}]`,
+  )).toEqual([]);
+}
+
+/**
+ * Assert correlationId convention per spec/v1/rest-endpoints.md §error-envelope.
+ * When a host issues a server-side trace ID for a 5xx, it goes under
+ * `details.correlationId` (the contextual-data slot), NEVER at the top level.
+ * RECOMMENDED, not REQUIRED — hosts that don't emit trace IDs are still
+ * conformant. This helper just pins the placement.
+ */
+function assertCorrelationIdShape(body: unknown, specSection: string): void {
+  const env = body as ErrorEnvelope;
+  // Top-level correlationId would be a spec violation; the assertErrorEnvelope
+  // additionalProperties check above catches it. This helper additionally
+  // pins the type when present under details.
+  if (env.details && typeof env.details === 'object') {
+    const det = env.details as Record<string, unknown>;
+    if (det.correlationId !== undefined) {
+      expect(typeof det.correlationId, driver.describe(
+        specSection,
+        'when present, details.correlationId MUST be a string',
+      )).toBe('string');
+      expect((det.correlationId as string).length, driver.describe(
+        specSection,
+        'details.correlationId MUST be a non-empty string',
+      )).toBeGreaterThan(0);
+    }
+  }
+}
+
+describe('errors: 404 envelope', () => {
+  it('GET /v1/runs/{nonexistentId} returns canonical envelope', async () => {
+    const res = await driver.get('/v1/runs/openwop-conformance-this-run-id-does-not-exist');
+
+    expect(
+      [403, 404].includes(res.status),
+      driver.describe('rest-endpoints.md', 'unknown run MUST return 404 (or 403 if leaking existence is forbidden)'),
+    ).toBe(true);
+
+    assertErrorEnvelope(res.json, 'rest-endpoints.md error envelope');
+    assertCorrelationIdShape(res.json, 'rest-endpoints.md §error-envelope');
+  });
+});
+
+describe('errors: 400 envelope (validation)', () => {
+  it('POST /v1/runs with empty body returns canonical envelope', async () => {
+    const res = await driver.post('/v1/runs', {});
+
+    // 400 is canonical, but some servers may return 422; accept either as
+    // long as the envelope is correct. The point of this test is the shape,
+    // not the status code.
+    expect(
+      res.status,
+      driver.describe('rest-endpoints.md', 'malformed POST /v1/runs MUST return 4xx'),
+    ).toBeGreaterThanOrEqual(400);
+    expect(res.status).toBeLessThan(500);
+
+    assertErrorEnvelope(res.json, 'rest-endpoints.md error envelope');
+    assertCorrelationIdShape(res.json, 'rest-endpoints.md §error-envelope');
+  });
+});
+
+describe('errors: details.correlationId placement convention', () => {
+  it('correlationId (when emitted on 5xx) MUST be in details, never top-level', async () => {
+    // We can't easily induce a 5xx from a black-box client, but we CAN
+    // verify the structural constraint on every error envelope returned
+    // throughout the suite: `correlationId` at the top level is a
+    // schema violation per `additionalProperties: false`. The
+    // additionalProperties check in assertErrorEnvelope catches that.
+    // This test provides a focused recap so the convention is
+    // explicitly named in the suite.
+    //
+    // Hosts MAY omit correlationId entirely (it's RECOMMENDED, not
+    // REQUIRED). This scenario passes against any host that conforms.
+    const res = await driver.get('/v1/runs/openwop-conformance-correlation-probe-' + Date.now());
+    expect(
+      [400, 403, 404].includes(res.status),
+      driver.describe('rest-endpoints.md', 'unknown run returns 4xx envelope'),
+    ).toBe(true);
+    const body = res.json as Record<string, unknown>;
+    expect(body, driver.describe('rest-endpoints.md', 'response MUST be a JSON object')).toBeDefined();
+    // The structural constraint: no top-level correlationId.
+    expect(body.correlationId, driver.describe(
+      'spec/v1/rest-endpoints.md §error-envelope (correlationId convention)',
+      'correlationId MUST NOT appear at the top level of the error envelope; canonical home is `details.correlationId`',
+    )).toBeUndefined();
+  });
+});