@openwop/openwop-conformance 1.0.0

package/src/scenarios/eventOrdering.test.ts

@@ -0,0 +1,144 @@
+/**
+ * Event-ordering scenarios per `spec/v1/observability.md` and
+ * `spec/v1/stream-modes.md`.
+ *
+ * Polling and streaming MUST yield events in the same monotonic order
+ * for a given run. Sequence numbers (or seq, depending on host event
+ * shape) are stable across reads; multiple polls return events in
+ * non-decreasing order.
+ *
+ * Profile gating: `openwop-stream-poll`. Hosts that don't expose
+ * `/v1/runs/{runId}/events/poll` skip-equivalent.
+ *
+ * @see spec/v1/observability.md
+ * @see spec/v1/stream-modes.md
+ * @see spec/v1/idempotency.md (companion event-shape work)
+ */
+
+import { describe, it, expect } from 'vitest';
+import { driver } from '../lib/driver.js';
+import { pollUntilTerminal } from '../lib/polling.js';
+import { isFixtureAdvertised } from '../lib/fixtures.js';
+
+const NOOP_WORKFLOW_ID = 'conformance-noop';
+const SKIP_NO_NOOP = !isFixtureAdvertised(NOOP_WORKFLOW_ID);
+
+interface RawEvent {
+  // Event shape may use `seq` or `sequence` depending on host's event-
+  // schema generation. Suite is permissive here; the canonical shape
+  // is `sequence` per run-event.schema.json. Either is acceptable
+  // until version-negotiation.test.ts converges hosts.
+  seq?: number;
+  sequence?: number;
+  type?: string;
+  [key: string]: unknown;
+}
+
+function getSeq(event: RawEvent): number | null {
+  if (typeof event.sequence === 'number') return event.sequence;
+  if (typeof event.seq === 'number') return event.seq;
+  return null;
+}
+
+describe.skipIf(SKIP_NO_NOOP)('event-ordering: polling returns events in monotonic order', () => {
+  it('events from a single poll have non-decreasing sequence numbers', async () => {
+    const create = await driver.post('/v1/runs', { workflowId: NOOP_WORKFLOW_ID });
+    expect(create.status).toBe(201);
+    const runId = (create.json as { runId: string }).runId;
+    await pollUntilTerminal(runId);
+
+    const res = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/events/poll`);
+    if (res.status !== 200) return;
+
+    const body = res.json as { events?: RawEvent[] } | undefined;
+    if (!body?.events) return;
+    if (body.events.length < 2) return; // single-event runs have no ordering to verify
+
+    const seqs = body.events.map(getSeq);
+    for (let i = 1; i < seqs.length; i++) {
+      const curr = seqs[i];
+      const prev = seqs[i - 1];
+      if (curr === null || prev === null) continue; // host without seq fields
+      expect(curr, driver.describe(
+        'observability.md §"Event ordering"',
+        `event[${i}].sequence (${curr}) MUST be >= event[${i - 1}].sequence (${prev})`,
+      )).toBeGreaterThanOrEqual(prev);
+    }
+  });
+
+  it('repeated polls of a terminal run yield identical event sequences', async () => {
+    const create = await driver.post('/v1/runs', { workflowId: NOOP_WORKFLOW_ID });
+    expect(create.status).toBe(201);
+    const runId = (create.json as { runId: string }).runId;
+    await pollUntilTerminal(runId);
+
+    const a = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/events/poll`);
+    const b = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/events/poll`);
+    if (a.status !== 200 || b.status !== 200) return;
+
+    const aBody = a.json as { events?: RawEvent[] } | undefined;
+    const bBody = b.json as { events?: RawEvent[] } | undefined;
+    if (!aBody?.events || !bBody?.events) return;
+
+    expect(aBody.events.length, driver.describe(
+      'observability.md',
+      'repeated polls of terminal run MUST return same number of events',
+    )).toBe(bBody.events.length);
+
+    for (let i = 0; i < aBody.events.length; i++) {
+      const aSeq = getSeq(aBody.events[i]!);
+      const bSeq = getSeq(bBody.events[i]!);
+      if (aSeq === null || bSeq === null) continue;
+      expect(aSeq, driver.describe(
+        'observability.md',
+        `event[${i}] sequence MUST be stable across repeated polls`,
+      )).toBe(bSeq);
+    }
+  });
+});
+
+describe.skipIf(SKIP_NO_NOOP)('event-ordering: terminal run has at most one terminal event', () => {
+  it('event stream contains exactly one of run.completed / run.failed / run.cancelled', async () => {
+    const create = await driver.post('/v1/runs', { workflowId: NOOP_WORKFLOW_ID });
+    expect(create.status).toBe(201);
+    const runId = (create.json as { runId: string }).runId;
+    await pollUntilTerminal(runId);
+
+    const res = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/events/poll`);
+    if (res.status !== 200) return;
+
+    const body = res.json as { events?: RawEvent[] } | undefined;
+    if (!body?.events) return;
+
+    const TERMINAL_TYPES = new Set(['run.completed', 'run.failed', 'run.cancelled']);
+    const terminalCount = body.events.filter((e) => typeof e.type === 'string' && TERMINAL_TYPES.has(e.type)).length;
+
+    expect(terminalCount, driver.describe(
+      'observability.md §"Run lifecycle events"',
+      'a run MUST emit exactly one terminal event (run.completed / run.failed / run.cancelled)',
+    )).toBe(1);
+  });
+
+  it('the terminal event is the LAST event in the stream', async () => {
+    const create = await driver.post('/v1/runs', { workflowId: NOOP_WORKFLOW_ID });
+    expect(create.status).toBe(201);
+    const runId = (create.json as { runId: string }).runId;
+    await pollUntilTerminal(runId);
+
+    const res = await driver.get(`/v1/runs/${encodeURIComponent(runId)}/events/poll`);
+    if (res.status !== 200) return;
+
+    const body = res.json as { events?: RawEvent[] } | undefined;
+    if (!body?.events || body.events.length === 0) return;
+
+    const TERMINAL_TYPES = new Set(['run.completed', 'run.failed', 'run.cancelled']);
+    const lastEvent = body.events[body.events.length - 1]!;
+    expect(
+      typeof lastEvent.type === 'string' && TERMINAL_TYPES.has(lastEvent.type),
+      driver.describe(
+        'observability.md §"Run lifecycle events"',
+        'terminal event MUST be the last event in the stream — no events after a terminal type',
+      ),
+    ).toBe(true);
+  });
+});

package/src/scenarios/failure-path.test.ts

@@ -0,0 +1,46 @@
+/**
+ * Failure-path scenarios — exercises the `conformance-failure` fixture
+ * which always throws. Verifies the terminal `failed` status surface
+ * and `RunSnapshot.error` shape per rest-endpoints.md.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { driver } from '../lib/driver.js';
+import { pollUntilTerminal } from '../lib/polling.js';
+import { isFixtureAdvertised } from '../lib/fixtures.js';
+
+const WORKFLOW_ID = 'conformance-failure';
+const SKIP_NO_FIXTURE = !isFixtureAdvertised(WORKFLOW_ID);
+
+describe.skipIf(SKIP_NO_FIXTURE)('failure: conformance-failure fixture reaches terminal `failed`', () => {
+  it('POST /v1/runs accepts the run and run terminates as failed with structured error', async () => {
+    const create = await driver.post('/v1/runs', { workflowId: WORKFLOW_ID });
+    expect(create.status, driver.describe(
+      'rest-endpoints.md',
+      'POST /v1/runs MUST return 201 even for fixtures that fail at runtime',
+    )).toBe(201);
+    const runId = (create.json as { runId: string }).runId;
+
+    const terminal = await pollUntilTerminal(runId);
+
+    expect(terminal.status, driver.describe(
+      'fixtures.md conformance-failure §Terminal status',
+      'fixture MUST reach terminal `failed`',
+    )).toBe('failed');
+
+    expect(typeof terminal.error, driver.describe(
+      'rest-endpoints.md RunSnapshot.error',
+      'RunSnapshot.error MUST be a structured object on terminal `failed`',
+    )).toBe('object');
+    expect(terminal.error, 'RunSnapshot.error MUST be non-null').not.toBeNull();
+
+    expect(typeof terminal.error?.code, driver.describe(
+      'rest-endpoints.md RunSnapshot.error.code',
+      'RunSnapshot.error.code MUST be a string',
+    )).toBe('string');
+    expect(typeof terminal.error?.message, driver.describe(
+      'rest-endpoints.md RunSnapshot.error.message',
+      'RunSnapshot.error.message MUST be a string',
+    )).toBe('string');
+  });
+});

package/src/scenarios/fixtures-gating.test.ts

@@ -0,0 +1,137 @@
+/**
+ * Fixture-gating helper unit tests (RFC 0003).
+ *
+ * Server-free. Verifies the in-memory cache and predicate behavior of
+ * `lib/fixtures.ts`. The full suite-init flow (top-level await in
+ * `setup.ts` reading the live host's discovery doc) is integration-
+ * level and exercised by every other fixture-dependent scenario.
+ *
+ * Critical invariants verified here:
+ *   1. Predicate returns false until the cache is populated.
+ *   2. Setting a non-array `fixtures` field collapses to "advertises
+ *      none" (resilience against host bugs).
+ *   3. Empty-string entries are filtered out.
+ *   4. Vendor-prefixed ids are passed through unchanged.
+ *   5. The cache is replaced (not merged) on subsequent calls.
+ *
+ * @see lib/fixtures.ts
+ * @see RFCS/0003-fixture-gating.md
+ */
+
+import { describe, it, expect, beforeEach } from 'vitest';
+import {
+  isFixtureAdvertised,
+  setAdvertisedFixtures,
+  getAdvertisedFixtures,
+  isFixtureCacheReady,
+  __resetForTests,
+} from '../lib/fixtures.js';
+
+beforeEach(() => {
+  __resetForTests();
+});
+
+describe('fixtures: cache lifecycle', () => {
+  it('returns false from isFixtureAdvertised before cache is populated', () => {
+    expect(isFixtureCacheReady()).toBe(false);
+    expect(isFixtureAdvertised('conformance-noop')).toBe(false);
+  });
+
+  it('returns null from getAdvertisedFixtures before cache is populated', () => {
+    expect(getAdvertisedFixtures()).toBe(null);
+  });
+
+  it('isFixtureCacheReady becomes true after setAdvertisedFixtures', () => {
+    setAdvertisedFixtures({ fixtures: [] });
+    expect(isFixtureCacheReady()).toBe(true);
+  });
+
+  it('returns empty set when called with null', () => {
+    setAdvertisedFixtures(null);
+    expect(getAdvertisedFixtures()?.size).toBe(0);
+    expect(isFixtureAdvertised('conformance-noop')).toBe(false);
+  });
+
+  it('returns empty set when called with undefined', () => {
+    setAdvertisedFixtures(undefined);
+    expect(getAdvertisedFixtures()?.size).toBe(0);
+  });
+});
+
+describe('fixtures: discovery-payload parsing', () => {
+  it('populates cache from a well-formed payload', () => {
+    setAdvertisedFixtures({ fixtures: ['conformance-noop', 'conformance-delay'] });
+    expect(isFixtureAdvertised('conformance-noop')).toBe(true);
+    expect(isFixtureAdvertised('conformance-delay')).toBe(true);
+    expect(isFixtureAdvertised('conformance-not-advertised')).toBe(false);
+  });
+
+  it('treats absent fixtures field as "advertises none"', () => {
+    setAdvertisedFixtures({ protocolVersion: '1.0' });
+    expect(getAdvertisedFixtures()?.size).toBe(0);
+    expect(isFixtureAdvertised('conformance-noop')).toBe(false);
+  });
+
+  it('treats non-array fixtures field as "advertises none"', () => {
+    setAdvertisedFixtures({ fixtures: 'conformance-noop' as unknown as string[] });
+    expect(getAdvertisedFixtures()?.size).toBe(0);
+  });
+
+  it('filters out empty-string entries', () => {
+    setAdvertisedFixtures({ fixtures: ['', 'conformance-noop', ''] });
+    const set = getAdvertisedFixtures();
+    expect(set?.size).toBe(1);
+    expect(isFixtureAdvertised('conformance-noop')).toBe(true);
+  });
+
+  it('filters out non-string entries', () => {
+    setAdvertisedFixtures({
+      fixtures: ['conformance-noop', 42, null, undefined] as unknown as string[],
+    });
+    expect(getAdvertisedFixtures()?.size).toBe(1);
+    expect(isFixtureAdvertised('conformance-noop')).toBe(true);
+  });
+
+  it('passes vendor-prefixed ids through unchanged', () => {
+    setAdvertisedFixtures({
+      fixtures: ['conformance-noop', 'openwop.smoke.byok', 'acme.fixture.foo'],
+    });
+    expect(isFixtureAdvertised('openwop.smoke.byok')).toBe(true);
+    expect(isFixtureAdvertised('acme.fixture.foo')).toBe(true);
+  });
+
+  it('deduplicates entries via Set semantics', () => {
+    setAdvertisedFixtures({
+      fixtures: ['conformance-noop', 'conformance-noop', 'conformance-noop'],
+    });
+    expect(getAdvertisedFixtures()?.size).toBe(1);
+  });
+});
+
+describe('fixtures: cache replacement (not merge)', () => {
+  it('a second setAdvertisedFixtures call replaces the cache, not merges', () => {
+    setAdvertisedFixtures({ fixtures: ['conformance-noop'] });
+    expect(isFixtureAdvertised('conformance-noop')).toBe(true);
+
+    setAdvertisedFixtures({ fixtures: ['conformance-delay'] });
+    expect(isFixtureAdvertised('conformance-noop')).toBe(false);
+    expect(isFixtureAdvertised('conformance-delay')).toBe(true);
+  });
+
+  it('replacing with empty array means no fixtures advertised', () => {
+    setAdvertisedFixtures({ fixtures: ['conformance-noop'] });
+    setAdvertisedFixtures({ fixtures: [] });
+    expect(getAdvertisedFixtures()?.size).toBe(0);
+    expect(isFixtureAdvertised('conformance-noop')).toBe(false);
+  });
+});
+
+describe('fixtures: __resetForTests', () => {
+  it('returns the cache to the pre-populated state', () => {
+    setAdvertisedFixtures({ fixtures: ['conformance-noop'] });
+    expect(isFixtureCacheReady()).toBe(true);
+    __resetForTests();
+    expect(isFixtureCacheReady()).toBe(false);
+    expect(getAdvertisedFixtures()).toBe(null);
+  });
+});

package/src/scenarios/fixtures-valid.test.ts

@@ -0,0 +1,140 @@
+/**
+ * Fixture-validity test — pure local check that every fixture JSON
+ * validates against the workflow-definition schema. Runs without a
+ * server target so it can gate the suite in CI before deployment.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { readFileSync, readdirSync } from 'node:fs';
+import { join } from 'node:path';
+import Ajv2020 from 'ajv/dist/2020.js';
+import addFormats from 'ajv-formats';
+import type { ErrorObject } from 'ajv';
+import { FIXTURES_DIR, SCHEMAS_DIR } from '../lib/paths.js';
+
+// Layout-aware paths — `lib/paths.ts` resolves these for both repo
+// checkouts (schemas one level above the conformance package) and the
+// published tarball (schemas vendored at the package root by `prepack`).
+const PACK_MANIFEST_FIXTURES_DIR = join(FIXTURES_DIR, 'pack-manifests');
+const SCHEMA_PATH = join(SCHEMAS_DIR, 'workflow-definition.schema.json');
+const PACK_MANIFEST_SCHEMA_PATH = join(SCHEMAS_DIR, 'node-pack-manifest.schema.json');
+
+describe('fixtures: workflow-definition schema validity', () => {
+  const ajv = new Ajv2020({ allErrors: true, strict: false });
+  addFormats(ajv);
+  // Pre-load the agent-ref peer schema so cross-schema `$ref` in
+  // workflow-definition (Phase 1 — `WorkflowNode.agent`) resolves.
+  // The relative file-name `agent-ref.schema.json` is how
+  // workflow-definition references it; register under that name so
+  // Ajv's $ref resolver finds it.
+  const agentRefPath = join(SCHEMAS_DIR, 'agent-ref.schema.json');
+  const agentRefSchema = JSON.parse(readFileSync(agentRefPath, 'utf8'));
+  ajv.addSchema(agentRefSchema, 'agent-ref.schema.json');
+  const schema = JSON.parse(readFileSync(SCHEMA_PATH, 'utf8'));
+  const validate = ajv.compile(schema);
+
+  const files = readdirSync(FIXTURES_DIR)
+    .filter((f) => f.endsWith('.json')) // top-level only — sub-dirs hold non-workflow fixtures
+    .sort();
+
+  it('finds at least one fixture file', () => {
+    expect(files.length).toBeGreaterThan(0);
+  });
+
+  for (const file of files) {
+    it(`${file} validates against workflow-definition.schema.json`, () => {
+      const data = JSON.parse(readFileSync(join(FIXTURES_DIR, file), 'utf8'));
+      const ok = validate(data);
+      const errors = (validate.errors ?? [])
+        .map((e: ErrorObject) => `${e.instancePath || '/'}: ${e.message}`)
+        .join('\n');
+      expect(ok, `Fixture ${file} fails workflow-definition schema:\n${errors}`).toBe(true);
+    });
+  }
+
+  it('every fixture id matches its filename', () => {
+    for (const file of files) {
+      const data = JSON.parse(readFileSync(join(FIXTURES_DIR, file), 'utf8')) as { id: string };
+      const expected = file.replace(/\.json$/, '');
+      expect(
+        data.id,
+        `Fixture file ${file} declares id "${data.id}" — MUST match filename`,
+      ).toBe(expected);
+    }
+  });
+
+  it('every fixture has a manual trigger so the conformance driver can start it', () => {
+    for (const file of files) {
+      const data = JSON.parse(readFileSync(join(FIXTURES_DIR, file), 'utf8')) as {
+        id: string;
+        triggers: Array<{ type: string }>;
+      };
+      const hasManual = data.triggers.some((t) => t.type === 'manual');
+      expect(
+        hasManual,
+        `Fixture ${data.id} MUST include a manual trigger per fixtures.md §Seeding contract`,
+      ).toBe(true);
+    }
+  });
+});
+
+describe('fixtures: node-pack-manifest schema validity', () => {
+  // Pack-manifest fixtures live in `fixtures/pack-manifests/` so the
+  // top-level workflow-definition validator above doesn't try to apply
+  // the wrong schema. They serve as schema-level proof points (e.g., the
+  // `private.<host>.*` scope is accepted by the canonical schema).
+  const ajv = new Ajv2020({ allErrors: true, strict: false });
+  addFormats(ajv);
+  // Pre-load the agent-manifest peer schema so the Phase 2 `agents[]`
+  // $ref in node-pack-manifest resolves under the same name the
+  // manifest schema uses.
+  const agentManifestPath = join(SCHEMAS_DIR, 'agent-manifest.schema.json');
+  ajv.addSchema(
+    JSON.parse(readFileSync(agentManifestPath, 'utf8')),
+    'agent-manifest.schema.json',
+  );
+  const schema = JSON.parse(readFileSync(PACK_MANIFEST_SCHEMA_PATH, 'utf8'));
+  const validate = ajv.compile(schema);
+
+  const files = readdirSync(PACK_MANIFEST_FIXTURES_DIR)
+    .filter((f) => f.endsWith('.json'))
+    .sort();
+
+  it('finds at least one pack-manifest fixture (private-scope coverage)', () => {
+    expect(files.length).toBeGreaterThan(0);
+  });
+
+  for (const file of files) {
+    it(`pack-manifests/${file} validates against node-pack-manifest.schema.json`, () => {
+      const data = JSON.parse(
+        readFileSync(join(PACK_MANIFEST_FIXTURES_DIR, file), 'utf8'),
+      );
+      const ok = validate(data);
+      const errors = (validate.errors ?? [])
+        .map((e: ErrorObject) => `${e.instancePath || '/'}: ${e.message}`)
+        .join('\n');
+      expect(
+        ok,
+        `Fixture pack-manifests/${file} fails node-pack-manifest schema:\n${errors}`,
+      ).toBe(true);
+    });
+  }
+
+  it('the private-scope fixture exercises the v1.0 private-pack pattern', () => {
+    // Regression pin: the pattern in node-pack-manifest.schema.json was
+    // widened from `(core|vendor|community)` to
+    // `(core|vendor|community|private)` in v1.0. If this test
+    // fails, either the schema regressed or the fixture got renamed —
+    // both are CHANGELOG-worthy.
+    const privateFixtures = files.filter((f) => {
+      const data = JSON.parse(
+        readFileSync(join(PACK_MANIFEST_FIXTURES_DIR, f), 'utf8'),
+      ) as { name?: string };
+      return typeof data.name === 'string' && data.name.startsWith('private.');
+    });
+    expect(
+      privateFixtures.length,
+      'Expected at least one pack-manifest fixture under the `private.<host>.*` scope to assert v1.0 private-scope pattern coverage',
+    ).toBeGreaterThan(0);
+  });
+});