@inetafrica/open-claudia 2.15.0 → 3.0.0

package/test-utility-provider-policy.js

@@ -0,0 +1,486 @@
+#!/usr/bin/env node
+
+"use strict";
+
+const assert = require("assert");
+const crypto = require("crypto");
+const fs = require("fs");
+const os = require("os");
+const path = require("path");
+
+const root = fs.mkdtempSync(path.join(os.tmpdir(), "open-claudia-utility-policy-"));
+process.env.OPEN_CLAUDIA_TEST = "1";
+process.env.OPEN_CLAUDIA_CONFIG_DIR = path.join(root, "config");
+process.env.WORKSPACE = root;
+
+const fixture = path.join(__dirname, "test-fixtures", "fake-agent-cli.js");
+const { createClaudeProvider } = require("./core/providers/claude");
+const { createCodexProvider } = require("./core/providers/codex");
+const {
+  UtilityPolicyError,
+  createUtilityPolicy,
+  parseProviderFallbacks,
+} = require("./core/providers/utility-policy");
+const {
+  UtilityAgentError,
+  createUtilityAgentService,
+} = require("./core/utility-agent");
+
+function wait(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function waitForFile(file, timeoutMs = 2000) {
+  const started = Date.now();
+  while (!fs.existsSync(file)) {
+    if (Date.now() - started > timeoutMs) throw new Error(`timed out waiting for ${path.basename(file)}`);
+    await wait(5);
+  }
+}
+
+async function waitForPidExit(pid, timeoutMs = 2000) {
+  const started = Date.now();
+  while (Date.now() - started <= timeoutMs) {
+    try { process.kill(pid, 0); }
+    catch (error) { if (error.code === "ESRCH") return; throw error; }
+    await wait(10);
+  }
+  throw new Error(`descendant pid ${pid} survived utility cleanup`);
+}
+
+function providerEnv(kind, controls = {}) {
+  return {
+    PATH: process.env.PATH || "",
+    HOME: root,
+    FAKE_AGENT_KIND: kind,
+    ...controls,
+  };
+}
+
+function providers(controls = {}) {
+  return {
+    claude: createClaudeProvider({
+      hookTransport: false,
+      resolveExecutable: () => fixture,
+      defaultModel: "claude-high-fixture",
+      buildEnv: () => providerEnv("claude", controls.claude),
+      getAuthStatus: () => ({ state: "authenticated", reason: null }),
+    }),
+    codex: createCodexProvider({
+      hookTransport: false,
+      resolveExecutable: () => fixture,
+      defaultModel: "codex-high-fixture",
+      buildEnv: () => providerEnv("codex", controls.codex),
+      getAuthStatus: () => ({ state: "authenticated", reason: null }),
+    }),
+  };
+}
+
+function registry(providerMap, states = {}) {
+  return {
+    getProvider(id) {
+      if (!providerMap[id]) throw Object.assign(new Error(`unknown provider ${id}`), { code: "UNKNOWN_PROVIDER" });
+      return providerMap[id];
+    },
+    listProviders: () => Object.values(providerMap),
+    providerStatus(id) {
+      const state = states[id] || {};
+      return {
+        id,
+        availability: state.availability || { state: "available", executable: fixture, reason: null },
+        compatibility: state.compatibility || { state: "compatible", reason: null },
+        auth: state.auth || { state: "authenticated", reason: null },
+      };
+    },
+  };
+}
+
+async function assertRejectsCode(promise, code) {
+  await assert.rejects(promise, (error) => {
+    assert.strictEqual(error.code, code);
+    return true;
+  });
+}
+
+async function main() {
+  assert.deepStrictEqual(parseProviderFallbacks(" codex,claude,codex "), ["codex", "claude"]);
+  assert.throws(() => parseProviderFallbacks("cursor"), (error) => error.code === "INVALID_UTILITY_PROVIDER");
+
+  const map = providers();
+  const allReady = registry(map);
+
+  const activePolicy = createUtilityPolicy({
+    registry: allReady,
+    env: { UTILITY_PROVIDER: "active" },
+    activeProvider: () => "codex",
+  });
+  const active = activePolicy.resolve({ purpose: "recall", tier: "low" });
+  assert.strictEqual(active.providerId, "codex", "active chat provider wins for active utility policy");
+  assert.strictEqual(active.model, map.codex.tierModel("low"), "provider owns tier-to-model mapping");
+  assert.strictEqual(active.fallbackUsed, false);
+
+  const inheritedActive = createUtilityPolicy({
+    registry: allReady,
+    env: { UTILITY_PROVIDER: "active", OC_PROVIDER: "codex" },
+  }).resolve({ purpose: "subagent", tier: "low" });
+  assert.strictEqual(inheritedActive.providerId, "codex", "utility CLI children inherit the admitted provider through OC_PROVIDER");
+
+  const globalPolicy = createUtilityPolicy({
+    registry: allReady,
+    env: { UTILITY_PROVIDER: "active", DEFAULT_PROVIDER: "claude" },
+    activeProvider: () => null,
+  });
+  assert.strictEqual(globalPolicy.resolve({ purpose: "dream", tier: "high" }).providerId, "claude",
+    "global work uses DEFAULT_PROVIDER when no chat is active");
+
+  const codexOnlyGlobal = createUtilityPolicy({
+    registry: registry(map, {
+      claude: { availability: { state: "missing", executable: null, reason: "Claude absent" } },
+    }),
+    env: { UTILITY_PROVIDER: "active" },
+    activeProvider: () => null,
+  });
+  assert.strictEqual(codexOnlyGlobal.resolve({ purpose: "dream", tier: "high" }).providerId, "codex",
+    "an unset global default deterministically selects the first runnable provider in Codex-only installs");
+
+  const purposePolicy = createUtilityPolicy({
+    registry: allReady,
+    env: { UTILITY_PROVIDER: "claude", DREAM_PROVIDER: "codex", PACK_REVIEW_MODEL_CODEX: "review-codex-exact" },
+    activeProvider: () => "claude",
+  });
+  const purpose = purposePolicy.resolve({ purpose: "dream", tier: "high" });
+  assert.strictEqual(purpose.providerId, "codex", "per-purpose provider override has highest configured priority");
+  assert.strictEqual(
+    createUtilityPolicy({
+      registry: allReady,
+      env: { UTILITY_PROVIDER: "active", RECALL_PROVIDER: "codex" },
+      activeProvider: () => "claude",
+    }).resolve({ purpose: "recall", activeProvider: "claude", tier: "low" }).providerId,
+    "codex",
+    "purpose override wins over the foreground provider captured at admission",
+  );
+  assert.strictEqual(
+    createUtilityPolicy({
+      registry: allReady,
+      env: { UTILITY_PROVIDER: "active" },
+      activeProvider: () => "claude",
+    }).resolve({ purpose: "recall", activeProvider: "codex", tier: "low" }).providerId,
+    "codex",
+    "active policy uses the immutable captured provider instead of mutable live state",
+  );
+  const reviewExact = purposePolicy.resolve({ purpose: "review", provider: "codex", tier: "medium" });
+  assert.strictEqual(reviewExact.model, "review-codex-exact", "provider-specific exact-model override is preserved");
+
+  const legacyModelPolicy = createUtilityPolicy({
+    registry: allReady,
+    env: { UTILITY_PROVIDER: "claude", PACK_REVIEW_MODEL: "legacy-claude-review" },
+  });
+  assert.strictEqual(legacyModelPolicy.resolve({ purpose: "review", tier: "medium" }).model, "legacy-claude-review");
+  assert.notStrictEqual(
+    createUtilityPolicy({ registry: allReady, env: { UTILITY_PROVIDER: "codex", PACK_REVIEW_MODEL: "legacy-claude-review" } })
+      .resolve({ purpose: "review", tier: "medium" }).model,
+    "legacy-claude-review",
+    "legacy Claude model aliases never leak into Codex",
+  );
+
+  const fallbackRegistry = registry(map, {
+    claude: { availability: { state: "missing", executable: null, reason: "fixture unavailable" } },
+  });
+  const fallbackPolicy = createUtilityPolicy({
+    registry: fallbackRegistry,
+    env: { UTILITY_PROVIDER: "claude", PROVIDER_FALLBACKS: "codex" },
+  });
+  const fallback = fallbackPolicy.resolve({ purpose: "recall", tier: "low" });
+  assert.strictEqual(fallback.providerId, "codex");
+  assert.strictEqual(fallback.preferredProviderId, "claude");
+  assert.strictEqual(fallback.fallbackUsed, true);
+
+  const authFallback = createUtilityPolicy({
+    registry: registry(map, { claude: { auth: { state: "unauthenticated", reason: "login required" } } }),
+    env: { ENFORCER_PROVIDER: "claude", PROVIDER_FALLBACKS: "codex" },
+  }).resolve({ purpose: "enforcer", tier: "medium" });
+  assert.strictEqual(authFallback.providerId, "codex", "configured enforcer fallback handles auth failure");
+  assert.strictEqual(authFallback.fallbackUsed, true, "enforcer fallback is explicit in metadata");
+
+  assert.throws(() => createUtilityPolicy({
+    registry: fallbackRegistry,
+    env: { UTILITY_PROVIDER: "claude" },
+  }).resolve({ purpose: "enforcer", tier: "medium" }), (error) => {
+    assert.ok(error instanceof UtilityPolicyError);
+    assert.strictEqual(error.code, "NO_AVAILABLE_UTILITY_PROVIDER");
+    return true;
+  }, "enforcer never invents an unconfigured fallback");
+
+  const claudeCapture = path.join(root, "claude-capture.json");
+  const codexCapture = path.join(root, "codex-capture.json");
+  const serviceProviders = providers({
+    claude: { FAKE_AGENT_CAPTURE: claudeCapture, FAKE_AGENT_TEXT: "utility Claude output" },
+    codex: { FAKE_AGENT_CAPTURE: codexCapture, FAKE_AGENT_TEXT: "utility Codex output" },
+  });
+  const serviceRegistry = registry(serviceProviders);
+  const service = createUtilityAgentService({
+    registry: serviceRegistry,
+    env: { UTILITY_PROVIDER: "claude", PROVIDER_FALLBACKS: "codex" },
+    maxGlobalConcurrency: 3,
+    maxPurposeConcurrency: 2,
+  });
+
+  const shellSentinel = "literal $(touch must-not-run) $HOME";
+  const claudeRun = await service.spawnUtilityAgent(shellSentinel, {
+    purpose: "subagent",
+    tier: "low",
+    systemPrompt: "UTILITY CORE SENTINEL",
+    readOnly: true,
+    allowedTools: [],
+    cwd: root,
+  });
+  assert.strictEqual(claudeRun.ok, true);
+  assert.strictEqual(claudeRun.provider, "claude");
+  assert.strictEqual(claudeRun.model, serviceProviders.claude.tierModel("low"));
+  assert.strictEqual(claudeRun.text, "utility Claude output");
+  assert.strictEqual(claudeRun.fallbackUsed, false);
+  assert.strictEqual(claudeRun.noProductSideEffects.requested, true);
+  assert.strictEqual(claudeRun.noProductSideEffects.support, "native");
+  assert.strictEqual(claudeRun.noTools.requested, true);
+  assert.strictEqual(claudeRun.noTools.verified, false, "tool suppression is never overstated without a provider probe");
+  const claudeArgv = JSON.parse(fs.readFileSync(claudeCapture, "utf8")).argv;
+  assert.ok(claudeArgv.includes("--no-session-persistence"), "utility calls are fresh and ephemeral");
+  assert.ok(!claudeArgv.includes("--resume"));
+  assert.ok(claudeArgv.includes("UTILITY CORE SENTINEL"));
+  assert.ok(!claudeArgv.includes(shellSentinel), "private utility prompt is absent from argv");
+  const claudeCaptureRecord = JSON.parse(fs.readFileSync(claudeCapture, "utf8"));
+  assert.deepStrictEqual(claudeCaptureRecord.stdin, {
+    present: true,
+    bytes: Buffer.byteLength(shellSentinel),
+    sha256: crypto.createHash("sha256").update(shellSentinel).digest("hex"),
+  }, "fixture records only protected-stdin size/hash, never prompt content");
+  assert.ok(!fs.readFileSync(claudeCapture, "utf8").includes(shellSentinel), "capture never persists utility prompt content");
+  assert.ok(!fs.existsSync(path.join(root, "must-not-run")), "utility prompt is never shell-evaluated");
+
+  const codexRun = await service.spawnUtilityAgent("codex utility", {
+    purpose: "recall",
+    provider: "codex",
+    tier: "low",
+    systemPrompt: "CODEX UTILITY CORE",
+    readOnly: true,
+    cwd: root,
+  });
+  assert.strictEqual(codexRun.ok, true);
+  assert.strictEqual(codexRun.provider, "codex");
+  assert.strictEqual(codexRun.text, "utility Codex output");
+  const codexArgv = JSON.parse(fs.readFileSync(codexCapture, "utf8")).argv;
+  assert.deepStrictEqual(codexArgv.slice(0, 1), ["exec"]);
+  assert.ok(codexArgv.includes("--ephemeral"));
+  assert.ok(codexArgv.includes("read-only"));
+
+  const runtimeFallbackService = createUtilityAgentService({
+    registry: registry(providers({
+      claude: { FAKE_AGENT_SCENARIO: "auth-failure" },
+      codex: { FAKE_AGENT_TEXT: "fallback judged safely" },
+    })),
+    env: { ENFORCER_PROVIDER: "claude", PROVIDER_FALLBACKS: "codex" },
+  });
+  const runtimeFallback = await runtimeFallbackService.spawnUtilityAgent("guard", {
+    purpose: "enforcer", tier: "medium", systemPrompt: "guard", readOnly: true, cwd: root,
+  });
+  assert.strictEqual(runtimeFallback.provider, "codex", "runtime auth failure advances to configured fallback");
+  assert.strictEqual(runtimeFallback.preferredProvider, "claude");
+  assert.strictEqual(runtimeFallback.fallbackUsed, true);
+  assert.strictEqual(runtimeFallback.text, "fallback judged safely");
+
+  const redactionService = createUtilityAgentService({
+    registry: registry(providers({
+      claude: { FAKE_AGENT_STDERR: "do not leak sk-proj-abcdefghijklmnopqrstuvwxyz", FAKE_AGENT_TEXT: "safe" },
+    })),
+    env: { UTILITY_PROVIDER: "claude" },
+    maxStderrBytes: 24,
+  });
+  const redacted = await redactionService.spawnUtilityAgent("redact", {
+    purpose: "subagent", systemPrompt: "redact", cwd: root,
+  });
+  assert.ok(!redacted.stderr.includes("sk-proj-"), "bounded stderr is redacted at the service boundary");
+  assert.strictEqual(redacted.stderrTruncated, true);
+
+  const schemaProviders = providers({
+    codex: { FAKE_AGENT_TEXT: '{"decision":"allow"}' },
+  });
+  const schemaService = createUtilityAgentService({
+    registry: registry(schemaProviders),
+    env: { UTILITY_PROVIDER: "codex" },
+  });
+  const schemaRun = await schemaService.spawnUtilityAgent("judge", {
+    purpose: "enforcer",
+    tier: "medium",
+    systemPrompt: "schema judge",
+    readOnly: true,
+    outputSchema: {
+      type: "object",
+      required: ["decision"],
+      properties: { decision: { type: "string", enum: ["allow", "block", "escalate"] } },
+    },
+    cwd: root,
+  });
+  assert.deepStrictEqual(schemaRun.json, { decision: "allow" });
+  assert.strictEqual(schemaRun.provider, "codex");
+
+  const structuredClaudeService = createUtilityAgentService({
+    registry: registry(providers({
+      claude: {
+        FAKE_AGENT_TEXT: '```json\n{"decision":"block","explanation":"presentation only"}\n```',
+        FAKE_AGENT_STRUCTURED_JSON: '{"decision":"allow"}',
+      },
+    })),
+    env: { UTILITY_PROVIDER: "claude" },
+    maxOutputBytes: 24,
+  });
+  const structuredClaude = await structuredClaudeService.spawnUtilityAgent("judge", {
+    purpose: "enforcer",
+    tier: "medium",
+    systemPrompt: "schema judge",
+    readOnly: true,
+    outputSchema: {
+      type: "object",
+      required: ["decision"],
+      properties: { decision: { type: "string", enum: ["allow", "block", "escalate"] } },
+    },
+    cwd: root,
+  });
+  assert.deepStrictEqual(structuredClaude.json, { decision: "allow" });
+  assert.strictEqual(structuredClaude.text, '{"decision":"allow"}');
+  assert.strictEqual(structuredClaude.truncated, false, "truncation follows the selected structured result");
+
+  await assertRejectsCode(schemaService.spawnUtilityAgent("judge", {
+    purpose: "enforcer",
+    tier: "medium",
+    systemPrompt: "schema judge",
+    readOnly: true,
+    outputSchema: {
+      type: "object",
+      required: ["missing"],
+      properties: { missing: { type: "string" } },
+    },
+    cwd: root,
+  }), "UTILITY_SCHEMA_INVALID");
+
+  const truncService = createUtilityAgentService({
+    registry: registry(providers({ claude: { FAKE_AGENT_TEXT: "0123456789abcdefghijklmnopqrstuvwxyz" } })),
+    env: { UTILITY_PROVIDER: "claude" },
+    maxOutputBytes: 12,
+  });
+  const truncated = await truncService.spawnUtilityAgent("truncate", {
+    purpose: "subagent", systemPrompt: "truncate", cwd: root,
+  });
+  assert.strictEqual(truncated.text, "0123456789ab");
+  assert.strictEqual(truncated.truncated, true);
+
+  const delayedCapture = path.join(root, "delayed-capture.json");
+  const limited = createUtilityAgentService({
+    registry: registry(providers({ claude: { FAKE_AGENT_CAPTURE: delayedCapture, FAKE_AGENT_CLOSE_DELAY_MS: "180" } })),
+    env: { UTILITY_PROVIDER: "claude" },
+    maxGlobalConcurrency: 1,
+    maxPurposeConcurrency: 1,
+  });
+  const first = limited.spawnUtilityAgent("first", { purpose: "subagent", systemPrompt: "limited", cwd: root });
+  await waitForFile(delayedCapture);
+  assert.deepStrictEqual(limited.stats(), { activeGlobal: 1, activeByPurpose: { subagent: 1 } });
+  await assertRejectsCode(
+    limited.spawnUtilityAgent("second", { purpose: "subagent", systemPrompt: "limited", cwd: root }),
+    "UTILITY_CONCURRENCY_LIMIT",
+  );
+  await first;
+  assert.deepStrictEqual(limited.stats(), { activeGlobal: 0, activeByPurpose: {} });
+
+  const recursive = createUtilityAgentService({
+    registry: serviceRegistry,
+    env: { UTILITY_PROVIDER: "claude" },
+    maxRecursionDepth: 1,
+  });
+  await recursive.withUtilityContext(async () => {
+    await assertRejectsCode(
+      recursive.spawnUtilityAgent("nested", { purpose: "subagent", systemPrompt: "nested", cwd: root }),
+      "UTILITY_RECURSION_LIMIT",
+    );
+  });
+
+  const signalCapture = path.join(root, "timeout-signal.json");
+  const descendantCapture = path.join(root, "timeout-descendant.json");
+  const timeoutService = createUtilityAgentService({
+    registry: registry(providers({
+      claude: {
+        FAKE_AGENT_SCENARIO: "forced-termination",
+        FAKE_AGENT_IGNORE_SIGTERM: "1",
+        FAKE_AGENT_SIGNAL_CAPTURE: signalCapture,
+        FAKE_AGENT_DESCENDANT_CAPTURE: descendantCapture,
+      },
+    })),
+    env: { UTILITY_PROVIDER: "claude" },
+    killGraceMs: 40,
+  });
+  const timeoutStarted = Date.now();
+  await assert.rejects(
+    timeoutService.spawnUtilityAgent("timeout", {
+      purpose: "subagent", systemPrompt: "timeout", timeoutMs: 180, cwd: root,
+    }),
+    (error) => {
+      assert.ok(error instanceof UtilityAgentError);
+      assert.strictEqual(error.code, "UTILITY_TIMEOUT");
+      assert.strictEqual(error.result.timedOut, true);
+      assert.strictEqual(error.result.provider, "claude");
+      return true;
+    },
+  );
+  assert.ok(Date.now() - timeoutStarted >= 180, "timeout settles only after its deadline and process close");
+  await waitForFile(signalCapture);
+  assert.strictEqual(JSON.parse(fs.readFileSync(signalCapture, "utf8")).signal, "SIGTERM");
+  await waitForFile(descendantCapture);
+  await waitForPidExit(JSON.parse(fs.readFileSync(descendantCapture, "utf8")).pid);
+  assert.strictEqual(timeoutService.stats().activeGlobal, 0, "timeout cleanup releases concurrency only after close");
+
+  const abortCapture = path.join(root, "abort-signal.json");
+  const abortService = createUtilityAgentService({
+    registry: registry(providers({
+      codex: { FAKE_AGENT_SCENARIO: "forced-termination", FAKE_AGENT_SIGNAL_CAPTURE: abortCapture },
+    })),
+    env: { UTILITY_PROVIDER: "codex" },
+  });
+  const controller = new AbortController();
+  const aborted = abortService.spawnUtilityAgent("abort", {
+    purpose: "recall", systemPrompt: "abort", signal: controller.signal, cwd: root,
+  });
+  await wait(100);
+  controller.abort();
+  await assertRejectsCode(aborted, "UTILITY_CANCELLED");
+  await waitForFile(abortCapture);
+
+  const drainCapture = path.join(root, "drain-signal.json");
+  const drainService = createUtilityAgentService({
+    registry: registry(providers({
+      claude: { FAKE_AGENT_SCENARIO: "forced-termination", FAKE_AGENT_SIGNAL_CAPTURE: drainCapture },
+    })),
+    env: { UTILITY_PROVIDER: "claude" },
+    killGraceMs: 40,
+  });
+  const drainingRun = drainService.spawnUtilityAgent("shutdown drain", {
+    purpose: "review", systemPrompt: "drain", cwd: root,
+  });
+  const drainingAssertion = assertRejectsCode(drainingRun, "UTILITY_CANCELLED");
+  await wait(100);
+  const drained = await drainService.cancelAll({ timeoutMs: 1000 });
+  await drainingAssertion;
+  assert.deepStrictEqual(drained, { drained: true, active: 0 }, "bounded shutdown cancellation waits for utility child cleanup");
+  await waitForFile(drainCapture);
+
+  const config = require("./core/config");
+  for (const key of ["DEFAULT_PROVIDER", "UTILITY_PROVIDER", "PROVIDER_FALLBACKS", "UTILITY_PURPOSE_PROVIDERS"]) {
+    assert.ok(Object.prototype.hasOwnProperty.call(config, key), `config exports ${key}`);
+  }
+
+  console.log("utility provider policy OK");
+}
+
+main().catch((error) => {
+  console.error(error.stack || error.message);
+  process.exitCode = 1;
+}).finally(() => {
+  fs.rmSync(root, { recursive: true, force: true });
+});