@openwop/openwop-conformance 1.0.0 → 1.1.0

package/src/scenarios/metric-emission.test.ts

@@ -0,0 +1,113 @@
+/**
+ * Track 11: metric-emission verification.
+ *
+ * Verifies that hosts claiming `capabilities.observability.metrics`
+ * emit the canonical `openwop.run.backlog`, `openwop.queue.depth`, and
+ * (after at least one completed run) `openwop.run.duration` metrics
+ * documented in `spec/v1/observability.md`.
+ *
+ * Operator contract (same as `otel-emission.test.ts`):
+ *   1. Start the conformance suite with `OPENWOP_OTEL_COLLECTOR=true`
+ *      and `OPENWOP_OTEL_COLLECTOR_PORT=<port>`.
+ *   2. Boot the host with `OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:<port>`.
+ *
+ * Skip conditions:
+ *   - Collector disabled (`OPENWOP_OTEL_COLLECTOR` unset / false).
+ *   - Host doesn't advertise `capabilities.observability.metrics.supported`.
+ *
+ * @see spec/v1/observability.md §"Metrics"
+ */
+
+import { describe, it, expect } from 'vitest';
+import { driver } from '../lib/driver.js';
+import { pollUntilTerminal } from '../lib/polling.js';
+import { isFixtureAdvertised } from '../lib/fixtures.js';
+import { getCollector } from '../lib/otel-collector.js';
+
+const FIXTURE = 'conformance-noop';
+
+interface MetricsCaps {
+  supported?: boolean;
+  names?: ReadonlyArray<string>;
+}
+
+async function metricsAdvertised(): Promise<MetricsCaps | null> {
+  try {
+    const disco = await driver.get('/.well-known/openwop');
+    const caps = (disco.json as {
+      capabilities?: { observability?: { metrics?: MetricsCaps } };
+    }).capabilities;
+    return caps?.observability?.metrics ?? null;
+  } catch {
+    return null;
+  }
+}
+
+async function waitForMetric(name: string, timeoutMs = 5_000): Promise<boolean> {
+  const collector = getCollector();
+  if (!collector) return false;
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    if (collector.metricByName(name)) return true;
+    await new Promise((r) => setTimeout(r, 100));
+  }
+  return false;
+}
+
+describe('metric-emission: canonical openwop.* metrics arrive at the collector', () => {
+  it('host emits openwop.run.backlog, openwop.queue.depth, and openwop.run.duration', async () => {
+    if (!getCollector()) {
+      // eslint-disable-next-line no-console
+      console.warn(
+        '[metric-emission] collector not started; set OPENWOP_OTEL_COLLECTOR=true to run',
+      );
+      return;
+    }
+    const metricsCaps = await metricsAdvertised();
+    if (!metricsCaps?.supported) {
+      // eslint-disable-next-line no-console
+      console.warn(
+        '[metric-emission] host does not advertise observability.metrics.supported; skipping',
+      );
+      return;
+    }
+    if (!isFixtureAdvertised(FIXTURE)) {
+      // eslint-disable-next-line no-console
+      console.warn(`[metric-emission] ${FIXTURE} not advertised; skipping`);
+      return;
+    }
+
+    const collector = getCollector()!;
+    collector.reset();
+
+    // Drive at least one completed run so openwop.run.duration has a sample.
+    const create = await driver.post('/v1/runs', { workflowId: FIXTURE });
+    expect(create.status).toBe(201);
+    const runId = (create.json as { runId: string }).runId;
+    await pollUntilTerminal(runId, { timeoutMs: 10_000 });
+
+    // Wait for the host's metric-emit tick to land at the collector.
+    const sawBacklog = await waitForMetric('openwop.run.backlog', 5_000);
+    expect(sawBacklog, driver.describe(
+      'observability.md §"Metrics"',
+      'host claiming metrics MUST emit openwop.run.backlog',
+    )).toBe(true);
+
+    const sawQueueDepth = await waitForMetric('openwop.queue.depth', 5_000);
+    expect(sawQueueDepth, driver.describe(
+      'observability.md §"Metrics"',
+      'host claiming metrics MUST emit openwop.queue.depth',
+    )).toBe(true);
+
+    const sawDuration = await waitForMetric('openwop.run.duration', 5_000);
+    expect(sawDuration, driver.describe(
+      'observability.md §"Metrics"',
+      'host claiming metrics MUST emit openwop.run.duration after a completed run',
+    )).toBe(true);
+
+    // Shape spot-check: backlog gauge data point has a numeric value.
+    const backlog = collector.metricByName('openwop.run.backlog')!;
+    expect(backlog.kind).toBe('gauge');
+    expect(typeof backlog.dataPoint.value).toBe('number');
+  });
+});

package/src/scenarios/orchestratorConservativePath.test.ts

@@ -1,5 +1,6 @@
 /**
  * Multi-Agent Shift Phase 5 — CP-1 conservative-path orchestrator suspend.
+ * Normative reference: RFCS/0006-orchestrator.md
  *
  * Verifies the CP-1 invariant: when a `core.orchestrator.supervisor`
  * would emit a decision with `confidence < escalationThreshold`, the

package/src/scenarios/orchestratorDispatch.test.ts

@@ -1,5 +1,6 @@
 /**
  * Multi-Agent Shift Phase 5 — orchestrator → dispatch → next-worker round-trip.
+ * Normative reference: RFCS/0006-orchestrator.md
  *
  * Verifies that a workflow with `core.orchestrator.supervisor` →
  * `core.dispatch` topology emits the canonical event sequence:

package/src/scenarios/orchestratorTermination.test.ts

@@ -1,5 +1,6 @@
 /**
  * Multi-Agent Shift Phase 5 — orchestrator terminate decision (CO-3).
+ * Normative reference: RFCS/0006-orchestrator.md
  *
  * Verifies that when an `core.orchestrator.supervisor` emits a decision
  * with `kind: 'terminate'`:

package/src/scenarios/otel-emission-grpc.test.ts

@@ -0,0 +1,98 @@
+/**
+ * Track 11: OTel span emission over OTLP/gRPC.
+ *
+ * Verifies that hosts advertising `capabilities.observability.otel.exportProtocols`
+ * including `"grpc"` can emit `openwop.*` spans over OTLP/gRPC to the
+ * in-suite collector and that the gRPC framing path captures the same
+ * `openwop.run` + `openwop.node.*` shape as the HTTP-JSON path.
+ *
+ * The gRPC collector is the parallel HTTP/2 server inside
+ * `OtelCollector` (started via `startGrpc()` in setup.ts when the
+ * `OPENWOP_OTEL_COLLECTOR=true` flag is set). The host points its
+ * exporter at the printed `:<grpcPort>` (h2c) with
+ * `OTEL_EXPORTER_OTLP_PROTOCOL=grpc`. Spans captured over gRPC land
+ * in the same store as HTTP — `getCollector().spans()` returns the
+ * union.
+ *
+ * Skip conditions:
+ *   - Collector disabled (`OPENWOP_OTEL_COLLECTOR` unset / false).
+ *   - Host does not advertise `capabilities.observability.otel.exportProtocols`
+ *     including `"grpc"` (presumed not configured for gRPC emission).
+ *   - Required fixture (`conformance-noop`) not advertised.
+ *
+ * @see spec/v1/observability.md §"Export protocols"
+ * @see conformance/src/lib/otel-collector.ts §_handleGrpcStream
+ * @see conformance/src/lib/grpc-framing.ts
+ */
+
+import { describe, it, expect } from 'vitest';
+import { driver } from '../lib/driver.js';
+import { pollUntilTerminal } from '../lib/polling.js';
+import { isFixtureAdvertised } from '../lib/fixtures.js';
+import { getCollector, waitForRunSpans } from '../lib/otel-collector.js';
+
+const FIXTURE = 'conformance-noop';
+
+async function advertisesGrpcExport(): Promise<boolean> {
+  try {
+    const disco = await driver.get('/.well-known/openwop');
+    const caps = (disco.json as {
+      capabilities?: {
+        observability?: { otel?: { exportProtocols?: unknown } };
+      };
+    }).capabilities;
+    const protocols = caps?.observability?.otel?.exportProtocols;
+    return Array.isArray(protocols) && protocols.includes('grpc');
+  } catch {
+    return false;
+  }
+}
+
+describe('otel-emission-grpc: OTLP/gRPC export path', () => {
+  it('host emits openwop.run spans over OTLP/gRPC; collector captures them via the shared store', async () => {
+    if (!getCollector()) {
+      // eslint-disable-next-line no-console
+      console.warn('[otel-emission-grpc] collector not started; set OPENWOP_OTEL_COLLECTOR=true to run');
+      return;
+    }
+    if (!isFixtureAdvertised(FIXTURE)) {
+      // eslint-disable-next-line no-console
+      console.warn(`[otel-emission-grpc] fixture ${FIXTURE} not advertised; skipping`);
+      return;
+    }
+    if (!(await advertisesGrpcExport())) {
+      // eslint-disable-next-line no-console
+      console.warn(
+        '[otel-emission-grpc] host does not advertise capabilities.observability.otel.exportProtocols including "grpc"; skipping. ' +
+          'Hosts MAY opt into gRPC export by emitting OTLP via the OTLP/gRPC transport and adding `"grpc"` to the array.',
+      );
+      return;
+    }
+
+    const collector = getCollector()!;
+    collector.reset();
+
+    const create = await driver.post('/v1/runs', { workflowId: FIXTURE });
+    expect(create.status).toBe(201);
+    const runId = (create.json as { runId: string }).runId;
+
+    await pollUntilTerminal(runId, { timeoutMs: 15_000 });
+
+    // gRPC and HTTP-JSON spans both land in `_spans`, so the existing
+    // span-query helpers work transparently. If the host emits over
+    // both transports, we capture both; the assertion only requires
+    // at least one openwop.run span correlated by runId.
+    const runSpans = await waitForRunSpans(runId, { timeoutMs: 5_000, minCount: 1 });
+
+    expect(runSpans.length, driver.describe(
+      'observability.md §"Export protocols" + RFC 0008/0009 Track 11',
+      'host advertising exportProtocols ∋ "grpc" MUST emit openwop.* spans over OTLP/gRPC',
+    )).toBeGreaterThan(0);
+
+    const runSpan = runSpans.find((s) => s.name === 'openwop.run');
+    expect(runSpan?.attributes.get('openwop.run_id'), driver.describe(
+      'observability.md §"Run-level attributes"',
+      'openwop.run span MUST carry openwop.run_id attribute',
+    )).toBe(runId);
+  });
+});

package/src/scenarios/pause-resume.test.ts

@@ -107,3 +107,122 @@ describe.skipIf(SKIP)('pause/resume: :resume on a non-paused run returns 409', (
     });
   });
 });
+
+describe.skipIf(SKIP)('pause/resume: pause is idempotent when already paused', () => {
+  it(':pause on an already-paused run is a no-op (200/202) — idempotent', async () => {
+    const create = await driver.post('/v1/runs', {
+      workflowId: FIXTURE!,
+      inputs: { delaySeconds: 30 },
+    });
+    expect(create.status).toBe(201);
+    const runId = (create.json as { runId: string }).runId;
+    await pollUntilStatus(runId, 'running', { timeoutMs: 10_000 });
+
+    const first = await driver.post(`/v1/runs/${encodeURIComponent(runId)}:pause`, {});
+    if (first.status === 404) {
+      await driver.post(`/v1/runs/${encodeURIComponent(runId)}/cancel`, {
+        reason: 'conformance-cleanup',
+      });
+      return;
+    }
+    expect([200, 202]).toContain(first.status);
+    await pollUntilStatus(runId, 'paused', { timeoutMs: 10_000 });
+
+    // Idempotent second :pause — MUST NOT 409 just because the run is
+    // already paused. 200/202 are both acceptable per the additive
+    // contract; 409 would force callers to read state before calling.
+    const second = await driver.post(`/v1/runs/${encodeURIComponent(runId)}:pause`, {});
+    expect(
+      [200, 202].includes(second.status),
+      driver.describe(
+        'rest-endpoints.md POST /v1/runs/{runId}:pause',
+        ':pause on an already-paused run MUST be idempotent (200/202), not 409',
+      ),
+    ).toBe(true);
+
+    await driver.post(`/v1/runs/${encodeURIComponent(runId)}/cancel`, {
+      reason: 'conformance-cleanup',
+    });
+  });
+});
+
+describe.skipIf(SKIP)('pause/resume: :pause on a terminal run returns 409', () => {
+  it(':pause on a completed/cancelled/failed run MUST return 409', async () => {
+    const create = await driver.post('/v1/runs', {
+      workflowId: 'conformance-noop',
+    });
+    if (create.status !== 201) return; // conformance-noop not seeded; skip cleanly
+    const runId = (create.json as { runId: string }).runId;
+    await pollUntilTerminal(runId, { timeoutMs: 10_000 });
+
+    const pause = await driver.post(`/v1/runs/${encodeURIComponent(runId)}:pause`, {});
+    if (pause.status === 404) return;
+    expect(pause.status, driver.describe(
+      'rest-endpoints.md POST /v1/runs/{runId}:pause',
+      ':pause on a terminal run MUST return 409',
+    )).toBe(409);
+
+    const body = pause.json as { error?: string; details?: { runStatus?: string } };
+    expect(body.error).toBe('conflict');
+    // Spec requires `details.runStatus` to disclose the terminal state so
+    // the caller can decide whether to retry or surface the conflict.
+    expect(['completed', 'failed', 'cancelled']).toContain(body.details?.runStatus);
+  });
+});
+
+describe.skipIf(SKIP)('pause/resume: :pause-during-suspend race', () => {
+  it(':pause MUST NOT silently override an active interrupt suspend', async () => {
+    // If the host seeds an approval fixture, drive a suspend then attempt
+    // :pause. The expected behavior is that :pause either (a) noops with
+    // 409 because the run is already waiting-approval (not in a pausable
+    // state), or (b) accepts and stacks pause atop the suspend with the
+    // run's terminal state still being waiting-approval. Either is
+    // acceptable; what's NOT acceptable is the host quietly flipping
+    // status to `paused` and discarding the suspended interrupt.
+    if (!isFixtureAdvertised('conformance-approval')) {
+      // eslint-disable-next-line no-console
+      console.warn(
+        '[pause-resume] conformance-approval not advertised; skipping :pause-during-suspend race subtest',
+      );
+      return;
+    }
+    const create = await driver.post('/v1/runs', { workflowId: 'conformance-approval' });
+    expect(create.status).toBe(201);
+    const runId = (create.json as { runId: string }).runId;
+    await pollUntilStatus(runId, 'waiting-approval', { timeoutMs: 10_000 });
+
+    const pause = await driver.post(`/v1/runs/${encodeURIComponent(runId)}:pause`, {
+      reason: 'race-test',
+    });
+    if (pause.status === 404) {
+      // Cleanup.
+      await driver.post(`/v1/runs/${encodeURIComponent(runId)}/cancel`, {
+        reason: 'conformance-cleanup',
+      });
+      return;
+    }
+
+    // Either rejection (preferred) or stacked-pause is OK; silent override is not.
+    if (pause.status === 409) {
+      const body = pause.json as { details?: { runStatus?: string } };
+      expect(body.details?.runStatus, driver.describe(
+        'rest-endpoints.md POST /v1/runs/{runId}:pause',
+        ':pause-during-suspend MUST surface the active waiting-* status in the conflict envelope',
+      )).toMatch(/^waiting-/);
+    } else {
+      // Stacked-pause accepted: verify the run's reported status still
+      // surfaces the underlying suspend — the host MUST NOT lose track
+      // of the interrupt waiting for resolution.
+      const snap = await driver.get(`/v1/runs/${encodeURIComponent(runId)}`);
+      const status = (snap.json as { status: string }).status;
+      expect(
+        status === 'paused' || status.startsWith('waiting-'),
+        ':pause-during-suspend MUST NOT silently discard the active interrupt',
+      ).toBe(true);
+    }
+
+    await driver.post(`/v1/runs/${encodeURIComponent(runId)}/cancel`, {
+      reason: 'conformance-cleanup',
+    });
+  });
+});