@openwop/openwop-conformance 1.0.0

package/src/scenarios/pause-resume.test.ts

@@ -0,0 +1,109 @@
+/**
+ * Track 13: operator-driven pause/resume (rest-endpoints.md v1.1).
+ *
+ * Exercises the `POST /v1/runs/{runId}:pause` and `:resume` endpoints
+ * against a long-running fixture (`conformance-delay` or `conformance-cancellable`).
+ *
+ * Verifies:
+ *   1. :pause on a running run transitions to `paused` and emits `run.paused`.
+ *   2. :resume on a paused run transitions to `running` and emits `run.resumed`.
+ *   3. :pause on a terminal run returns 409 with details.runStatus.
+ *   4. :resume on a non-paused run returns 409 with details.runStatus.
+ *
+ * Capability gating: skips when the host doesn't advertise
+ * `capabilities.runs.pauseResume.supported: true` (when present) AND
+ * skips when no long-running fixture is advertised.
+ *
+ * @see spec/v1/rest-endpoints.md §pause/resume
+ */
+
+import { describe, it, expect } from 'vitest';
+import { driver } from '../lib/driver.js';
+import { pollUntilStatus, pollUntilTerminal } from '../lib/polling.js';
+import { isFixtureAdvertised } from '../lib/fixtures.js';
+
+const FIXTURE =
+  (isFixtureAdvertised('conformance-cancellable') && 'conformance-cancellable') ||
+  (isFixtureAdvertised('conformance-delay') && 'conformance-delay') ||
+  null;
+
+const SKIP = !FIXTURE;
+
+describe.skipIf(SKIP)('pause/resume: running → paused → running → terminal', () => {
+  it('pause transitions to paused; resume returns the run to running', async () => {
+    const create = await driver.post('/v1/runs', {
+      workflowId: FIXTURE!,
+      inputs: { delaySeconds: 30 },
+    });
+    expect(create.status).toBe(201);
+    const runId = (create.json as { runId: string }).runId;
+
+    await pollUntilStatus(runId, 'running', { timeoutMs: 10_000 });
+
+    const pause = await driver.post(`/v1/runs/${encodeURIComponent(runId)}:pause`, {
+      reason: 'conformance-test',
+      drainPolicy: 'drain-current-node',
+    });
+    if (pause.status === 404) {
+      // Pause endpoint not yet implemented by the host — surface the skip honestly.
+      // eslint-disable-next-line no-console
+      console.warn(
+        '[pause-resume] host returned 404 for :pause — endpoint not implemented; skipping rest',
+      );
+      await driver.post(`/v1/runs/${encodeURIComponent(runId)}/cancel`, {
+        reason: 'conformance-cleanup',
+      });
+      return;
+    }
+    expect(pause.status, driver.describe(
+      'rest-endpoints.md POST /v1/runs/{runId}:pause',
+      ':pause MUST return 202 on a pausable run',
+    )).toBe(202);
+
+    await pollUntilStatus(runId, 'paused', { timeoutMs: 10_000 });
+
+    const resume = await driver.post(`/v1/runs/${encodeURIComponent(runId)}:resume`, {
+      reason: 'conformance-test',
+    });
+    expect(resume.status, driver.describe(
+      'rest-endpoints.md POST /v1/runs/{runId}:resume',
+      ':resume MUST return 202 on a paused run',
+    )).toBe(202);
+
+    const terminal = await pollUntilTerminal(runId, { timeoutMs: 60_000 });
+    expect(['completed', 'cancelled']).toContain(terminal.status);
+  });
+});
+
+describe.skipIf(SKIP)('pause/resume: :resume on a non-paused run returns 409', () => {
+  it('resuming a running (not paused) run returns 409 with details.runStatus', async () => {
+    const create = await driver.post('/v1/runs', {
+      workflowId: FIXTURE!,
+      inputs: { delaySeconds: 30 },
+    });
+    expect(create.status).toBe(201);
+    const runId = (create.json as { runId: string }).runId;
+
+    await pollUntilStatus(runId, 'running', { timeoutMs: 10_000 });
+
+    const resume = await driver.post(`/v1/runs/${encodeURIComponent(runId)}:resume`, {});
+    if (resume.status === 404) {
+      await driver.post(`/v1/runs/${encodeURIComponent(runId)}/cancel`, {
+        reason: 'conformance-cleanup',
+      });
+      return;
+    }
+    expect(resume.status, driver.describe(
+      'rest-endpoints.md POST /v1/runs/{runId}:resume',
+      ':resume on a non-paused run MUST return 409',
+    )).toBe(409);
+
+    const body = resume.json as { error?: string; details?: { runStatus?: string } };
+    expect(body.error).toBe('conflict');
+    expect(typeof body.details?.runStatus).toBe('string');
+
+    await driver.post(`/v1/runs/${encodeURIComponent(runId)}/cancel`, {
+      reason: 'conformance-cleanup',
+    });
+  });
+});

package/src/scenarios/policies.test.ts

@@ -0,0 +1,162 @@
+/**
+ * Provider-policy scenarios — `capabilities.md` §"`aiProviders.policies`".
+ *
+ * Vendor-neutral discovery-shape contracts for the four-mode provider-
+ * policy taxonomy (`disabled` / `optional` / `required` / `restricted`)
+ * that hosts MAY advertise on `/.well-known/openwop`.
+ *
+ * Why these scenarios are discovery-shape only:
+ *
+ *   The four modes describe HOST-SIDE enforcement decisions. A round-
+ *   trip enforcement scenario (e.g., "configure a `disabled` policy for
+ *   anthropic, then assert that an anthropic run is rejected with
+ *   `provider_policy_denied`") requires a configured policy document
+ *   AND a real provider call AND admin write access — far outside the
+ *   black-box contract surface this suite asserts. Hosts MUST run their
+ *   own integration tests for enforcement; the in-tree reference impl
+ *   carries 31 such tests in `openwop-provider-policy-modes`.
+ *
+ *   What IS testable cross-implementation: the wire shape of the
+ *   capability advertisement and the documented denial-error contract.
+ *
+ * Scenario gating:
+ *
+ *   - **Discovery shape contract** runs against every host. It verifies
+ *     `aiProviders.policies` is well-formed when present and absent-
+ *     friendly when omitted (hosts MAY skip the field entirely).
+ *
+ *   - **Mode-enum contract** runs against hosts that advertise
+ *     `policies.modes`. Verifies every advertised mode is one of the
+ *     four canonical values per the spec section.
+ *
+ * @see capabilities.md §"`aiProviders.policies`"
+ * @see schemas/capabilities.schema.json — additive `policies` subtree
+ */
+
+import { describe, it, expect } from 'vitest';
+import { driver } from '../lib/driver.js';
+
+const CANONICAL_MODES = ['disabled', 'optional', 'required', 'restricted'] as const;
+
+interface PoliciesShape {
+  modes?: unknown;
+  scopes?: unknown;
+  errorCode?: unknown;
+}
+
+interface AiProvidersShape {
+  supported?: unknown;
+  byok?: unknown;
+  policies?: PoliciesShape;
+}
+
+async function fetchAiProviders(): Promise<AiProvidersShape | undefined> {
+  const res = await driver.get('/.well-known/openwop', { authenticated: false });
+  expect(res.status).toBe(200);
+  const body = res.json as { aiProviders?: AiProvidersShape } | undefined;
+  return body?.aiProviders;
+}
+
+describe('policies: /.well-known/openwop aiProviders.policies shape contract', () => {
+  it('aiProviders.policies is well-formed when present (or absent — both spec-allowed)', async () => {
+    const ap = await fetchAiProviders();
+    if (ap === undefined) return; // Optional v1 field — hosts MAY omit aiProviders entirely.
+
+    const policies = ap.policies;
+    if (policies === undefined) {
+      // Spec-allowed: omitting `policies` means the host implements no
+      // enforcement. Clients see only `optional` semantics. Nothing
+      // further to assert.
+      return;
+    }
+
+    expect(typeof policies, driver.describe(
+      'capabilities.md §"`aiProviders.policies`"',
+      'aiProviders.policies MUST be an object when present',
+    )).toBe('object');
+    expect(policies, driver.describe(
+      'capabilities.md §"`aiProviders.policies`"',
+      'aiProviders.policies MUST NOT be null when present',
+    )).not.toBeNull();
+  });
+
+  it('policies.modes — every advertised mode is one of the four canonical values', async () => {
+    const ap = await fetchAiProviders();
+    const modes = ap?.policies?.modes;
+    if (modes === undefined) return;
+
+    expect(Array.isArray(modes), driver.describe(
+      'capabilities.md §"`aiProviders.policies`"',
+      'policies.modes MUST be a string[] when present',
+    )).toBe(true);
+
+    const arr = modes as unknown[];
+    for (const mode of arr) {
+      expect(typeof mode, driver.describe(
+        'capabilities.md §"`aiProviders.policies`"',
+        'policies.modes entries MUST be strings',
+      )).toBe('string');
+      expect(CANONICAL_MODES, driver.describe(
+        'capabilities.md §"`aiProviders.policies`"',
+        `mode "${mode}" MUST be one of ${CANONICAL_MODES.join(', ')}`,
+      )).toContain(mode);
+    }
+  });
+
+  it('policies.modes — no duplicate entries (uniqueItems contract)', async () => {
+    const ap = await fetchAiProviders();
+    const modes = ap?.policies?.modes;
+    if (!Array.isArray(modes)) return;
+
+    const arr = modes as string[];
+    const set = new Set(arr);
+    expect(set.size, driver.describe(
+      'capabilities.schema.json — policies.modes uniqueItems: true',
+      'policies.modes MUST NOT contain duplicate entries',
+    )).toBe(arr.length);
+  });
+
+  it('policies.scopes — string[] of non-empty entries when present', async () => {
+    const ap = await fetchAiProviders();
+    const scopes = ap?.policies?.scopes;
+    if (scopes === undefined) return;
+
+    expect(Array.isArray(scopes), driver.describe(
+      'capabilities.md §"`aiProviders.policies`"',
+      'policies.scopes MUST be a string[] when present',
+    )).toBe(true);
+
+    const arr = scopes as unknown[];
+    for (const scope of arr) {
+      expect(typeof scope, driver.describe(
+        'capabilities.md §"`aiProviders.policies`"',
+        'policies.scopes entries MUST be strings',
+      )).toBe('string');
+      expect((scope as string).length, driver.describe(
+        'capabilities.md §"`aiProviders.policies`"',
+        'policies.scopes entries MUST be non-empty',
+      )).toBeGreaterThan(0);
+    }
+
+    const set = new Set(arr as string[]);
+    expect(set.size, driver.describe(
+      'capabilities.schema.json — policies.scopes uniqueItems: true',
+      'policies.scopes MUST NOT contain duplicate entries',
+    )).toBe(arr.length);
+  });
+
+  it('policies.errorCode — non-empty string when present (defaults to provider_policy_denied)', async () => {
+    const ap = await fetchAiProviders();
+    const errorCode = ap?.policies?.errorCode;
+    if (errorCode === undefined) return;
+
+    expect(typeof errorCode, driver.describe(
+      'capabilities.md §"`aiProviders.policies`"',
+      'policies.errorCode MUST be a string when present',
+    )).toBe('string');
+    expect((errorCode as string).length, driver.describe(
+      'capabilities.md §"`aiProviders.policies`"',
+      'policies.errorCode MUST be non-empty when present',
+    )).toBeGreaterThan(0);
+  });
+});

package/src/scenarios/profileDerivation.test.ts

@@ -0,0 +1,335 @@
+/**
+ * Profile-derivation scenarios — verify that `deriveProfiles()`
+ * produces the correct profile set for representative discovery
+ * payloads.
+ *
+ * Server-free. Runs against fixture payloads, not a live host. The
+ * derivation MUST be deterministic and pure (per spec/v1/profiles.md
+ * §"Derivation"); these scenarios are the proof of that property.
+ *
+ * A separate runtime check would derive the profile set from the live
+ * `/.well-known/openwop` response; that's covered piecemeal by
+ * `discovery.test.ts` and the per-profile runtime suites
+ * (`stream-modes*.test.ts`, `pack-registry*.test.ts`, etc.).
+ */
+
+import { describe, it, expect } from 'vitest';
+import {
+  deriveProfiles,
+  hasProfile,
+  isCore,
+  isInterrupts,
+  isSecrets,
+  isProviderPolicy,
+  isFixtures,
+  type DiscoveryPayload,
+  type ProfileName,
+} from '../lib/profiles.js';
+
+/**
+ * Minimum payload that satisfies `openwop-core`. Other fixtures extend this.
+ */
+const CORE_PAYLOAD: DiscoveryPayload = {
+  protocolVersion: '1.0',
+  supportedEnvelopes: ['prd.create'],
+  schemaVersions: { 'prd.create': 1 },
+  limits: {
+    clarificationRounds: 3,
+    schemaRounds: 2,
+    envelopesPerTurn: 5,
+  },
+};
+
+describe('profiles: openwop-core predicate per spec/v1/profiles.md §`openwop-core`', () => {
+  it('accepts the minimum-conforming payload', () => {
+    expect(isCore(CORE_PAYLOAD)).toBe(true);
+  });
+
+  it('rejects payload without protocolVersion', () => {
+    const { protocolVersion: _omit, ...rest } = CORE_PAYLOAD;
+    expect(isCore(rest as DiscoveryPayload)).toBe(false);
+  });
+
+  it('rejects v2.x protocolVersion', () => {
+    expect(isCore({ ...CORE_PAYLOAD, protocolVersion: '2.0.0' })).toBe(false);
+  });
+
+  it('rejects negative limits (RFC 2119 MUST: non-negative integers)', () => {
+    expect(
+      isCore({
+        ...CORE_PAYLOAD,
+        limits: { ...CORE_PAYLOAD.limits!, clarificationRounds: -1 },
+      }),
+    ).toBe(false);
+  });
+
+  it('rejects fractional limits', () => {
+    expect(
+      isCore({
+        ...CORE_PAYLOAD,
+        limits: { ...CORE_PAYLOAD.limits!, schemaRounds: 1.5 },
+      }),
+    ).toBe(false);
+  });
+
+  it('accepts empty supportedEnvelopes array (engine-only host)', () => {
+    expect(isCore({ ...CORE_PAYLOAD, supportedEnvelopes: [] })).toBe(true);
+  });
+
+  it('rejects non-array supportedEnvelopes', () => {
+    expect(
+      isCore({ ...CORE_PAYLOAD, supportedEnvelopes: 'prd.create' as unknown as string[] }),
+    ).toBe(false);
+  });
+});
+
+describe('profiles: openwop-interrupts predicate per spec/v1/profiles.md §`openwop-interrupts`', () => {
+  it('passes when clarification.request is in supportedEnvelopes', () => {
+    expect(
+      isInterrupts({
+        ...CORE_PAYLOAD,
+        supportedEnvelopes: ['prd.create', 'clarification.request'],
+      }),
+    ).toBe(true);
+  });
+
+  it('fails when clarification.request is absent (fire-and-forget host)', () => {
+    expect(isInterrupts(CORE_PAYLOAD)).toBe(false);
+  });
+
+  it('implies openwop-core', () => {
+    const broken = {
+      ...CORE_PAYLOAD,
+      protocolVersion: '2.0.0',
+      supportedEnvelopes: ['clarification.request'],
+    };
+    expect(isInterrupts(broken)).toBe(false);
+  });
+});
+
+describe('profiles: openwop-secrets predicate per spec/v1/profiles.md §`openwop-secrets`', () => {
+  it('passes when secrets.supported=true and scopes includes user', () => {
+    expect(
+      isSecrets({
+        ...CORE_PAYLOAD,
+        secrets: { supported: true, scopes: ['user'] },
+      }),
+    ).toBe(true);
+  });
+
+  it('passes with multiple scopes', () => {
+    expect(
+      isSecrets({
+        ...CORE_PAYLOAD,
+        secrets: { supported: true, scopes: ['user', 'tenant'] },
+      }),
+    ).toBe(true);
+  });
+
+  it('fails when scopes omits user', () => {
+    expect(
+      isSecrets({
+        ...CORE_PAYLOAD,
+        secrets: { supported: true, scopes: ['tenant'] },
+      }),
+    ).toBe(false);
+  });
+
+  it('fails when secrets.supported=false', () => {
+    expect(
+      isSecrets({
+        ...CORE_PAYLOAD,
+        secrets: { supported: false, scopes: ['user'] },
+      }),
+    ).toBe(false);
+  });
+
+  it('fails when secrets field is absent', () => {
+    expect(isSecrets(CORE_PAYLOAD)).toBe(false);
+  });
+});
+
+describe('profiles: openwop-provider-policy predicate per spec/v1/profiles.md §`openwop-provider-policy`', () => {
+  it('passes when policies.modes contains optional', () => {
+    expect(
+      isProviderPolicy({
+        ...CORE_PAYLOAD,
+        aiProviders: {
+          supported: ['anthropic'],
+          policies: { modes: ['optional', 'required'] },
+        },
+      }),
+    ).toBe(true);
+  });
+
+  it('fails when policies.modes is empty (per spec: empty {} not a valid third state)', () => {
+    expect(
+      isProviderPolicy({
+        ...CORE_PAYLOAD,
+        aiProviders: {
+          supported: ['anthropic'],
+          policies: { modes: [] },
+        },
+      }),
+    ).toBe(false);
+  });
+
+  it('fails when policies.modes omits optional (cannot satisfy default-no-restriction case)', () => {
+    expect(
+      isProviderPolicy({
+        ...CORE_PAYLOAD,
+        aiProviders: {
+          supported: ['anthropic'],
+          policies: { modes: ['required'] },
+        },
+      }),
+    ).toBe(false);
+  });
+
+  it('fails when policies field is absent', () => {
+    expect(
+      isProviderPolicy({
+        ...CORE_PAYLOAD,
+        aiProviders: { supported: ['anthropic'] },
+      }),
+    ).toBe(false);
+  });
+});
+
+describe('profiles: openwop-fixtures predicate per spec/v1/profiles.md §`openwop-fixtures` (RFC 0003)', () => {
+  it('rejects payloads missing `fixtures`', () => {
+    expect(isFixtures(CORE_PAYLOAD)).toBe(false);
+  });
+
+  it('rejects empty `fixtures` array', () => {
+    expect(isFixtures({ ...CORE_PAYLOAD, fixtures: [] })).toBe(false);
+  });
+
+  it('accepts non-empty string array', () => {
+    expect(
+      isFixtures({ ...CORE_PAYLOAD, fixtures: ['conformance-noop'] }),
+    ).toBe(true);
+  });
+
+  it('accepts vendor-prefixed fixture ids', () => {
+    expect(
+      isFixtures({
+        ...CORE_PAYLOAD,
+        fixtures: ['conformance-noop', 'openwop.smoke.byok'],
+      }),
+    ).toBe(true);
+  });
+
+  it('rejects array with empty-string entries', () => {
+    expect(isFixtures({ ...CORE_PAYLOAD, fixtures: ['', 'conformance-noop'] })).toBe(false);
+  });
+
+  it('rejects non-array `fixtures`', () => {
+    expect(isFixtures({ ...CORE_PAYLOAD, fixtures: 'conformance-noop' })).toBe(false);
+    expect(
+      isFixtures({ ...CORE_PAYLOAD, fixtures: { 'conformance-noop': true } }),
+    ).toBe(false);
+  });
+
+  it('rejects array with non-string entries', () => {
+    expect(
+      isFixtures({ ...CORE_PAYLOAD, fixtures: ['conformance-noop', 42] as unknown[] }),
+    ).toBe(false);
+  });
+
+  it('rejects when openwop-core fails (predicate is layered)', () => {
+    expect(isFixtures({ fixtures: ['conformance-noop'] })).toBe(false);
+  });
+});
+
+describe('profiles: deriveProfiles produces the full set', () => {
+  it('returns openwop-core + stream-* + node-packs for the minimum payload', () => {
+    // The minimum payload satisfies the structural profiles automatically:
+    // openwop-core (predicate trivially), openwop-stream-sse + openwop-stream-poll
+    // (no supportedTransports set => permitted), openwop-node-packs (discovery-
+    // only predicate is openwop-core).
+    const result = deriveProfiles(CORE_PAYLOAD);
+    expect(result).toContain('openwop-core');
+    expect(result).toContain('openwop-stream-sse');
+    expect(result).toContain('openwop-stream-poll');
+    expect(result).toContain('openwop-node-packs');
+    expect(result).not.toContain('openwop-interrupts');
+    expect(result).not.toContain('openwop-secrets');
+    expect(result).not.toContain('openwop-provider-policy');
+  });
+
+  it('returns the full set for a richly-advertised host', () => {
+    const rich: DiscoveryPayload = {
+      ...CORE_PAYLOAD,
+      supportedEnvelopes: ['prd.create', 'clarification.request'],
+      supportedTransports: ['rest', 'mcp'],
+      secrets: { supported: true, scopes: ['user', 'tenant'] },
+      aiProviders: {
+        supported: ['anthropic', 'openai'],
+        policies: { modes: ['optional', 'required', 'restricted'] },
+      },
+      fixtures: ['conformance-noop'],
+    };
+    const result = deriveProfiles(rich);
+    const expected: ProfileName[] = [
+      'openwop-core',
+      'openwop-interrupts',
+      'openwop-stream-sse',
+      'openwop-stream-poll',
+      'openwop-secrets',
+      'openwop-provider-policy',
+      'openwop-node-packs',
+      'openwop-fixtures',
+    ];
+    expect(result).toEqual(expected);
+  });
+
+  it('returns stable order matching PROFILE_NAMES', () => {
+    const rich: DiscoveryPayload = {
+      ...CORE_PAYLOAD,
+      supportedEnvelopes: ['clarification.request'],
+      secrets: { supported: true, scopes: ['user'] },
+    };
+    const first = deriveProfiles(rich);
+    const second = deriveProfiles(rich);
+    expect(first).toEqual(second);
+    // Specifically: openwop-interrupts before openwop-secrets even though openwop-secrets
+    // was added to the payload "second."
+    expect(first.indexOf('openwop-interrupts')).toBeLessThan(first.indexOf('openwop-secrets'));
+  });
+
+  it('returns empty for a non-conforming payload', () => {
+    const broken: DiscoveryPayload = { protocolVersion: '0.9.0' };
+    expect(deriveProfiles(broken)).toEqual([]);
+  });
+
+  it('is deterministic across calls (same input → same output)', () => {
+    const calls = Array.from({ length: 10 }, () => deriveProfiles(CORE_PAYLOAD));
+    for (let i = 1; i < calls.length; i++) {
+      expect(calls[i]).toEqual(calls[0]);
+    }
+  });
+});
+
+describe('profiles: hasProfile is consistent with deriveProfiles', () => {
+  it('membership matches the derived set for every profile', () => {
+    const rich: DiscoveryPayload = {
+      ...CORE_PAYLOAD,
+      supportedEnvelopes: ['clarification.request'],
+      secrets: { supported: true, scopes: ['user'] },
+    };
+    const derived = new Set(deriveProfiles(rich));
+    for (const p of [
+      'openwop-core',
+      'openwop-interrupts',
+      'openwop-stream-sse',
+      'openwop-stream-poll',
+      'openwop-secrets',
+      'openwop-provider-policy',
+      'openwop-node-packs',
+      'openwop-fixtures',
+    ] as const) {
+      expect(hasProfile(rich, p)).toBe(derived.has(p));
+    }
+  });
+});