@openwop/openwop-conformance 1.14.0 → 1.16.0

package/src/scenarios/budget-enforcement.test.ts

@@ -0,0 +1,152 @@
+/**
+ * Budget enforcement — the §C lifecycle + §D hard-stop (RFC 0084) — behavioral.
+ *
+ * Gated on `capabilities.budget.supported` (root-first per RFC 0073). Soft-skips
+ * when unadvertised (default) / hard-fails under `OPENWOP_REQUIRE_BEHAVIOR=true`.
+ * The always-on wire-shape coverage lives in `budget-policy-shape.test.ts`; this
+ * asserts host BEHAVIOR via the `POST /v1/host/sample/budget/run` seam + the test
+ * event-log seam:
+ *
+ *   1. HARD COST EXHAUST (§C/§D, requires `enforce:"hard"`) — a hard-cost run
+ *      accrues to exhaustion, emitting in strict sequence:
+ *      `budget.reserved` → `budget.consumed` → `budget.threshold.crossed{percent}`
+ *      → `budget.exhausted` → `cap.breached{kind:"budget-cost"}` →
+ *      `run.failed{error:"budget_exhausted"}`.
+ *   2. MODEL DENIED (§D model policy) — a run whose model violates the budget
+ *      allow/deny list is refused with `budget_model_denied` BEFORE the provider
+ *      call (no model call, fail-closed).
+ *   3. ADVISORY (§D, `enforce:"advisory"`) — the same accrual emits the
+ *      `budget.*` events but does NOT stop the run (no `cap.breached`, no
+ *      `run.failed{budget_exhausted}`).
+ *   4. CONTENT-FREE (SR-1 / `budget-no-pricing-leak`) — every `budget.*` payload
+ *      carries only dimension/limit/consumed/remaining/percent scalars, never a
+ *      provider pricing table or per-token rate.
+ *
+ * Spec references:
+ *   - https://github.com/openwop/openwop/blob/main/spec/v1/budget-policy.md (§C/§D)
+ *   - https://github.com/openwop/openwop/blob/main/RFCS/0084-budget-quota-and-cost-policy.md
+ *   - https://github.com/openwop/openwop/blob/main/SECURITY/invariants.yaml (budget-no-pricing-leak)
+ */
+
+import { describe, it, expect } from 'vitest';
+import { driver } from '../lib/driver.js';
+import { behaviorGate } from '../lib/behavior-gate.js';
+import { readBudgetCap, driveBudgetRun, BUDGET_CAP_KINDS, BUDGET_CONTENT_FORBIDDEN } from '../lib/budgetPolicy.js';
+import { queryTestEvents, isEventLogSeamAvailable, resetTestSeam } from '../lib/event-log-query.js';
+import type { TestEvent } from '../lib/event-log-query.js';
+
+function seq(events: TestEvent[], type: string): number {
+  const e = events.find((x) => x.type === type);
+  return e ? e.sequence : -1;
+}
+
+function expectContentFree(events: TestEvent[]): void {
+  for (const e of events.filter((x) => x.type.startsWith('budget.'))) {
+    for (const f of BUDGET_CONTENT_FORBIDDEN) {
+      expect(
+        !(f in e.payload),
+        driver.describe('RFC 0084 §F (SR-1) / budget-no-pricing-leak', `budget.* MUST be content-free (no ${f})`),
+      ).toBe(true);
+    }
+  }
+}
+
+describe('budget-enforcement (RFC 0084 §C/§D)', () => {
+  it('runs the reserved→consumed→threshold→exhausted→cap.breached→run.failed chain, refuses denied models, and honors advisory mode', async () => {
+    const cap = await readBudgetCap();
+    if (!behaviorGate('openwop-budget-enforcement', cap?.supported === true)) return;
+    if (!(await isEventLogSeamAvailable())) return; // event-log seam absent — soft-skip
+
+    // ---- Leg 1: hard cost exhaust (§C/§D) -------------------------------
+    const hard = await driveBudgetRun({ scenario: 'hard-cost-exhaust' });
+    if (hard === null) return; // budget seam absent — soft-skip the whole behavior
+    if (hard.runId) {
+      const q = await queryTestEvents(hard.runId);
+      if (q.ok) {
+        const ev = q.events.slice().sort((a, b) => a.sequence - b.sequence);
+        const reserved = seq(ev, 'budget.reserved');
+        const threshold = seq(ev, 'budget.threshold.crossed');
+        const exhausted = seq(ev, 'budget.exhausted');
+        const failed = seq(ev, 'run.failed');
+        const capBreached = ev.find((e) => e.type === 'cap.breached' && typeof e.payload.kind === 'string' && (e.payload.kind as string).startsWith('budget-'));
+
+        expect(
+          reserved >= 0 && exhausted >= 0,
+          driver.describe('budget-policy.md §C', 'a hard budget run MUST emit budget.reserved + budget.exhausted'),
+        ).toBe(true);
+        // §C ordering: reserved < threshold.crossed < exhausted < run.failed.
+        if (threshold >= 0) {
+          expect(
+            reserved < threshold && threshold < exhausted,
+            driver.describe('RFC 0084 §C', 'ordering MUST be reserved < threshold.crossed < exhausted'),
+          ).toBe(true);
+          const tc = ev.find((e) => e.type === 'budget.threshold.crossed');
+          expect(
+            typeof tc?.payload.percent === 'number',
+            driver.describe('run-event-payloads.schema.json#budgetThresholdCrossed', 'threshold.crossed MUST carry a numeric percent'),
+          ).toBe(true);
+        }
+        // §D hard-stop: exhausted → cap.breached{budget-*} → run.failed{budget_exhausted}.
+        expect(
+          capBreached !== undefined,
+          driver.describe('RFC 0084 §D', 'exhaustion MUST emit cap.breached with a budget-* kind'),
+        ).toBe(true);
+        if (capBreached) {
+          expect(
+            BUDGET_CAP_KINDS.includes(capBreached.payload.kind as string),
+            driver.describe('RFC 0084 §D', 'cap.breached.kind MUST be in the closed budget vocabulary'),
+          ).toBe(true);
+          expect(
+            exhausted <= capBreached.sequence && capBreached.sequence <= failed,
+            driver.describe('RFC 0084 §D', 'ordering MUST be exhausted ≤ cap.breached ≤ run.failed'),
+          ).toBe(true);
+        }
+        const failedEvt = ev.find((e) => e.type === 'run.failed');
+        expect(
+          failedEvt?.payload.error === 'budget_exhausted',
+          driver.describe('RFC 0084 §D', 'a hard-budget overrun MUST fail the run with error budget_exhausted'),
+        ).toBe(true);
+        expectContentFree(ev);
+      }
+    }
+
+    // ---- Leg 2: model denied (§D model policy, fail-closed) -------------
+    const denied = await driveBudgetRun({ scenario: 'model-denied' });
+    if (denied !== null) {
+      expect(
+        denied.error === 'budget_model_denied',
+        driver.describe('RFC 0084 §D', 'a model violating the budget allow/deny list MUST be refused with budget_model_denied'),
+      ).toBe(true);
+      expect(
+        denied.modelCalled !== true,
+        driver.describe('RFC 0084 §D', 'a denied model MUST be refused BEFORE the provider call (fail-closed)'),
+      ).toBe(true);
+    }
+
+    // ---- Leg 3: advisory mode emits events but never stops --------------
+    if (cap?.enforce === 'advisory' || cap?.enforce === undefined) {
+      const adv = await driveBudgetRun({ scenario: 'advisory' });
+      if (adv !== null && adv.runId) {
+        const q = await queryTestEvents(adv.runId);
+        if (q.ok) {
+          const ev = q.events;
+          const hasBudgetEvents = ev.some((e) => e.type.startsWith('budget.'));
+          const stopped = ev.some(
+            (e) =>
+              (e.type === 'cap.breached' && typeof e.payload.kind === 'string' && (e.payload.kind as string).startsWith('budget-')) ||
+              (e.type === 'run.failed' && e.payload.error === 'budget_exhausted'),
+          );
+          if (hasBudgetEvents) {
+            expect(
+              !stopped,
+              driver.describe('RFC 0084 §D', 'advisory enforcement MUST emit budget.* events without stopping the run'),
+            ).toBe(true);
+          }
+          expectContentFree(ev);
+        }
+      }
+    }
+
+    await resetTestSeam();
+  });
+});

package/src/scenarios/memory-degraded-projection.test.ts

@@ -0,0 +1,121 @@
+/**
+ * Memory-capability degraded projection (RFC 0080 §C) — behavioral.
+ *
+ * Gated on `capabilities.agents.manifestRuntime` + `capabilities.memory`
+ * (root-first per RFC 0073). Soft-skips when either is unadvertised (default) /
+ * hard-fails under `OPENWOP_REQUIRE_BEHAVIOR=true`. The always-on wire-shape
+ * coverage lives in `memory-capability-model-shape.test.ts` (the schema fields +
+ * the closed dimension enum); this asserts host BEHAVIOR on the NORMATIVE
+ * `GET /v1/agents` inventory:
+ *
+ *   §C iff-contract — for EVERY inventory entry, when the host cannot satisfy an
+ *   agent's requested `memoryShape` it MUST stamp `memoryDegraded: true` together
+ *   with a NON-EMPTY `degradedMemoryDimensions[]` whose members are the RFC 0080
+ *   §A dimension names (the CLOSED enum, NOT the `memoryShape` keys) and are
+ *   unique; a non-degraded entry MUST carry `memoryDegraded` absent or `false`
+ *   and MUST NOT carry a non-empty `degradedMemoryDimensions`.
+ *
+ *   Non-vacuity — the inventory MUST be non-empty (the cap is advertised + the
+ *   endpoint serves). When `OPENWOP_DEGRADED_AGENT_ID` names an agent the host
+ *   knows is degraded (an agent whose `memoryShape` exceeds host capability —
+ *   e.g. one requesting `longTerm` on a host without long-term durability), the
+ *   degraded branch is asserted NON-VACUOUSLY against that agent.
+ *
+ * Black-box on the normative path — no POST seam.
+ *
+ * Spec references:
+ *   - https://github.com/openwop/openwop/blob/main/spec/v1/agent-memory.md (§"Memory capability model")
+ *   - https://github.com/openwop/openwop/blob/main/RFCS/0080-agent-memory-capability-reconciliation.md
+ */
+
+import { describe, it, expect } from 'vitest';
+import { driver } from '../lib/driver.js';
+import { behaviorGate } from '../lib/behavior-gate.js';
+import { readCapabilityFamily } from '../lib/discovery-capabilities.js';
+import { readManifestRuntimeCap, listManifestAgents } from '../lib/agentRuntime.js';
+
+/** The CLOSED RFC 0080 §A dimension vocabulary (agent-inventory-response.schema.json
+ *  `degradedMemoryDimensions` enum). NOT the `memoryShape` keys. */
+const DIMENSIONS = [
+  'read',
+  'write',
+  'search',
+  'long-term-durability',
+  'compaction',
+  'attribution',
+  'replay-snapshot',
+  'retention',
+];
+
+interface InventoryEntry {
+  agentId?: string;
+  memoryDegraded?: unknown;
+  degradedMemoryDimensions?: unknown;
+  [k: string]: unknown;
+}
+
+describe('memory-degraded-projection (RFC 0080 §C)', () => {
+  it('stamps memoryDegraded + a closed-enum degradedMemoryDimensions on degraded agents and nothing on the rest', async () => {
+    const mr = await readManifestRuntimeCap();
+    const memory = await readCapabilityFamily<Record<string, unknown>>('memory');
+    const advertised = mr?.supported === true && !!memory && memory.supported === true;
+    if (!behaviorGate('openwop-memory-degraded', advertised)) return;
+
+    const inv = await listManifestAgents();
+    if (inv === null) return; // host advertises the cap but doesn't serve /v1/agents — soft-skip
+    const agents = (inv.agents ?? []) as InventoryEntry[];
+
+    // Non-vacuity: an advertising + serving host MUST expose its inventory.
+    expect(
+      agents.length >= 1,
+      driver.describe('agent-memory.md §"Memory capability model"', 'GET /v1/agents MUST return the installed manifest agents'),
+    ).toBe(true);
+
+    // §C iff-contract on EVERY entry.
+    for (const a of agents) {
+      const degraded = a.memoryDegraded === true;
+      const dims = a.degradedMemoryDimensions;
+
+      if (degraded) {
+        expect(
+          Array.isArray(dims) && dims.length >= 1,
+          driver.describe('RFC 0080 §C', `memoryDegraded:true MUST carry a non-empty degradedMemoryDimensions (agent ${a.agentId})`),
+        ).toBe(true);
+        if (Array.isArray(dims)) {
+          for (const d of dims) {
+            expect(
+              typeof d === 'string' && DIMENSIONS.includes(d),
+              driver.describe('agent-inventory-response.schema.json', `degradedMemoryDimensions members MUST be RFC 0080 §A dimension names (got ${String(d)})`),
+            ).toBe(true);
+          }
+          expect(
+            new Set(dims as string[]).size === dims.length,
+            driver.describe('RFC 0080 §C', 'degradedMemoryDimensions MUST be unique'),
+          ).toBe(true);
+        }
+      } else {
+        // Not degraded ⇒ no non-empty dimension list (absent or empty both pass).
+        expect(
+          dims === undefined || (Array.isArray(dims) && dims.length === 0),
+          driver.describe('RFC 0080 §C', `a non-degraded entry MUST NOT carry a non-empty degradedMemoryDimensions (agent ${a.agentId})`),
+        ).toBe(true);
+      }
+    }
+
+    // Non-vacuous degraded branch when the host names a known-degraded agent.
+    const degradedId = process.env.OPENWOP_DEGRADED_AGENT_ID;
+    if (degradedId) {
+      const target = agents.find((a) => a.agentId === degradedId);
+      expect(
+        target !== undefined,
+        driver.describe('RFC 0080 §C', `OPENWOP_DEGRADED_AGENT_ID=${degradedId} MUST appear in the inventory`),
+      ).toBe(true);
+      if (target) {
+        expect(
+          target.memoryDegraded === true && Array.isArray(target.degradedMemoryDimensions) && target.degradedMemoryDimensions.length >= 1,
+          driver.describe('RFC 0080 §C', 'the named degraded agent MUST project memoryDegraded:true + a non-empty degradedMemoryDimensions'),
+        ).toBe(true);
+      }
+    }
+  });
+});

package/src/scenarios/otel-collector-canary-inspection.test.ts

@@ -0,0 +1,261 @@
+/**
+ * otel-collector-canary-inspection — always-on proof that the conformance
+ * OTel collector inspects real OTLP span attributes for secret leakage.
+ *
+ * Context: `secret-leakage-otel-attribute.test.ts` proves a host doesn't
+ * leak a BYOK canary on its `GET /v1/host/sample/test/otel/spans` scrape
+ * seam. But the scrape seam reports what the host *says* it emitted; a
+ * host could redact there yet still ship the plaintext over the wire via
+ * its real OTLP exporter. `docs/KNOWN-LIMITS.md` tracked exactly this gap:
+ *
+ *   "The conformance OTel collector seam doesn't yet inspect span
+ *    attributes; a host could pass conformance while leaking BYOK
+ *    material on telemetry exports."
+ *
+ * `OtelCollector.findCanaryLeakage()` (added with this scenario) closes
+ * the harness half: it scans every captured span's name, attributes, and
+ * resource attributes — plus metric data-point attributes — for the
+ * canary, returning the offending surface. This file is the always-on,
+ * server-free proof that the inspector is NON-VACUOUS: it stands up a
+ * real collector, POSTs synthetic OTLP/HTTP-JSON payloads through the
+ * collector's actual ingest path, and asserts the inspector (a) catches a
+ * canary embedded in a span attribute / resource attribute / span name /
+ * metric attribute, and (b) reports zero hits on a redacted payload.
+ *
+ * Pairing this with the live, capability-gated collector scrape in
+ * `secret-leakage-otel-attribute.test.ts` means the invariant is proven
+ * end-to-end the moment a host exports OTLP to the collector — the
+ * inspector itself is no longer the missing piece.
+ *
+ * Runs unconditionally (no host, no network): it exercises the collector
+ * library directly, so it executes in CI on every run.
+ *
+ * @see conformance/src/lib/otel-collector.ts findCanaryLeakage
+ * @see SECURITY/invariants.yaml secret-leakage-otel-attribute
+ * @see docs/KNOWN-LIMITS.md "Behavior tests too coarse to fully prove an invariant"
+ */
+
+import { describe, it, expect, afterEach } from 'vitest';
+import { OtelCollector } from '../lib/otel-collector.js';
+
+const CANARY = 'sk-canary-DO-NOT-LEAK-0f3a9c';
+const REDACTED = '[REDACTED:openwop-conformance-canary-secret]';
+
+/** Build an OTLP/HTTP-JSON traces export carrying the given span+resource attrs. */
+function tracesPayload(opts: {
+  spanName: string;
+  spanAttrs: Record<string, string>;
+  resourceAttrs: Record<string, string>;
+}): unknown {
+  const toAttrs = (m: Record<string, string>) =>
+    Object.entries(m).map(([key, value]) => ({ key, value: { stringValue: value } }));
+  return {
+    resourceSpans: [
+      {
+        resource: { attributes: toAttrs(opts.resourceAttrs) },
+        scopeSpans: [
+          {
+            scope: { name: 'openwop' },
+            spans: [
+              {
+                traceId: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
+                spanId: 'bbbbbbbbbbbbbbbb',
+                name: opts.spanName,
+                startTimeUnixNano: '1',
+                endTimeUnixNano: '2',
+                attributes: toAttrs(opts.spanAttrs),
+              },
+            ],
+          },
+        ],
+      },
+    ],
+  };
+}
+
+/** Build an OTLP/HTTP-JSON metrics export with one sum data point carrying attrs. */
+function metricsPayload(metricName: string, attrs: Record<string, string>): unknown {
+  return {
+    resourceMetrics: [
+      {
+        scopeMetrics: [
+          {
+            scope: { name: 'openwop' },
+            metrics: [
+              {
+                name: metricName,
+                sum: {
+                  dataPoints: [
+                    {
+                      asInt: '1',
+                      attributes: Object.entries(attrs).map(([key, value]) => ({
+                        key,
+                        value: { stringValue: value },
+                      })),
+                    },
+                  ],
+                },
+              },
+            ],
+          },
+        ],
+      },
+    ],
+  };
+}
+
+// NOTE: assertions here intentionally use bare `expect(...)` rather than
+// `expect(..., driver.describe('spec.md §section', 'requirement'))`. This is a
+// HARNESS self-test — it verifies the conformance collector's own
+// `findCanaryLeakage()` inspector, not a host's compliance with a spec
+// requirement, so there is no spec section to cite (consistent with other
+// library-level tests, e.g. `sandbox-wasm-isolation.test.ts`). The
+// host-facing, spec-citing assertion lives in the collector-export block of
+// `secret-leakage-otel-attribute.test.ts`.
+/**
+ * Build a traces export with `spanCount` spans that all share ONE resource
+ * (hence one set of resource attributes). Used to prove resource-attribute
+ * leaks are deduped to a single hit rather than reported once per span.
+ */
+function multiSpanSharedResourcePayload(spanCount: number, resourceAttrs: Record<string, string>): unknown {
+  const toAttrs = (m: Record<string, string>) =>
+    Object.entries(m).map(([key, value]) => ({ key, value: { stringValue: value } }));
+  return {
+    resourceSpans: [
+      {
+        resource: { attributes: toAttrs(resourceAttrs) },
+        scopeSpans: [
+          {
+            scope: { name: 'openwop' },
+            spans: Array.from({ length: spanCount }, (_unused, i) => ({
+              traceId: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
+              spanId: `span${i}`.padEnd(16, '0'),
+              name: `openwop.node.execute.${i}`,
+              startTimeUnixNano: '1',
+              endTimeUnixNano: '2',
+              attributes: toAttrs({ 'openwop.node.id': `n${i}` }),
+            })),
+          },
+        ],
+      },
+    ],
+  };
+}
+
+describe('otel-collector-canary-inspection: collector inspects real OTLP exports', () => {
+  let collector: OtelCollector | null = null;
+
+  afterEach(async () => {
+    if (collector) {
+      await collector.stop();
+      collector = null;
+    }
+  });
+
+  async function postTraces(payload: unknown): Promise<void> {
+    const res = await fetch(`${collector!.endpoint()}/v1/traces`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify(payload),
+    });
+    expect(res.status).toBeLessThan(300);
+  }
+
+  async function postMetrics(payload: unknown): Promise<void> {
+    const res = await fetch(`${collector!.endpoint()}/v1/metrics`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify(payload),
+    });
+    expect(res.status).toBeLessThan(300);
+  }
+
+  it('catches a canary embedded in a span attribute value', async () => {
+    collector = new OtelCollector();
+    await collector.start();
+    await postTraces(
+      tracesPayload({
+        spanName: 'openwop.node.execute',
+        spanAttrs: { 'openwop.node.id': 'n1', 'http.request.header.authorization': `Bearer ${CANARY}` },
+        resourceAttrs: { 'service.name': 'host' },
+      }),
+    );
+
+    const leaks = collector.findCanaryLeakage(CANARY);
+    expect(leaks.length).toBeGreaterThan(0);
+    const attrLeak = leaks.find((l) => l.surface === 'span.attribute');
+    expect(attrLeak).toBeDefined();
+    expect(attrLeak!.key).toBe('http.request.header.authorization');
+    expect(attrLeak!.value).toContain(CANARY);
+  });
+
+  it('catches a canary in a resource attribute and in a span name', async () => {
+    collector = new OtelCollector();
+    await collector.start();
+    await postTraces(
+      tracesPayload({
+        spanName: `openwop.run ${CANARY}`,
+        spanAttrs: { 'openwop.run.id': 'r1' },
+        resourceAttrs: { 'service.name': 'host', 'deployment.token': CANARY },
+      }),
+    );
+
+    const leaks = collector.findCanaryLeakage(CANARY);
+    const surfaces = new Set(leaks.map((l) => l.surface));
+    expect(surfaces.has('span.name')).toBe(true);
+    expect(surfaces.has('span.resourceAttribute')).toBe(true);
+  });
+
+  it('catches a canary in a metric data-point attribute', async () => {
+    collector = new OtelCollector();
+    await collector.start();
+    await postMetrics(metricsPayload('openwop.node.duration', { 'secret.echo': CANARY }));
+
+    const leaks = collector.findCanaryLeakage(CANARY);
+    const metricLeak = leaks.find((l) => l.surface === 'metric.attribute');
+    expect(metricLeak).toBeDefined();
+    expect(metricLeak!.emitterName).toBe('openwop.node.duration');
+  });
+
+  it('dedups a resource-attribute leak to ONE hit even when shared across many spans', async () => {
+    collector = new OtelCollector();
+    await collector.start();
+    // 5 spans sharing one resource whose attribute leaks the canary. Without
+    // dedup this would report 5 identical resource-attribute hits.
+    await postTraces(multiSpanSharedResourcePayload(5, { 'service.name': 'host', 'deployment.token': CANARY }));
+
+    const leaks = collector.findCanaryLeakage(CANARY);
+    const resourceLeaks = leaks.filter((l) => l.surface === 'span.resourceAttribute' && l.key === 'deployment.token');
+    expect(resourceLeaks.length).toBe(1);
+  });
+
+  it('reports ZERO hits when the host redacts the canary before export (positive control)', async () => {
+    collector = new OtelCollector();
+    await collector.start();
+    await postTraces(
+      tracesPayload({
+        spanName: 'openwop.node.execute',
+        spanAttrs: { 'openwop.node.id': 'n1', 'http.request.header.authorization': `Bearer ${REDACTED}` },
+        resourceAttrs: { 'service.name': 'host', 'deployment.token': REDACTED },
+      }),
+    );
+    await postMetrics(metricsPayload('openwop.node.duration', { 'secret.echo': REDACTED }));
+
+    expect(collector.findCanaryLeakage(CANARY)).toEqual([]);
+  });
+
+  it('an empty or whitespace canary never produces a (vacuous) hit', async () => {
+    collector = new OtelCollector();
+    await collector.start();
+    await postTraces(
+      tracesPayload({
+        spanName: 'openwop.node.execute',
+        spanAttrs: { 'a': 'b' },
+        resourceAttrs: { 'service.name': 'host' },
+      }),
+    );
+
+    expect(collector.findCanaryLeakage('')).toEqual([]);
+    expect(collector.findCanaryLeakage('   ')).toEqual([]);
+  });
+});