@inetafrica/open-claudia 2.15.0 → 3.0.0

package/test-provider-pack-review.js

@@ -0,0 +1,436 @@
+#!/usr/bin/env node
+
+"use strict";
+
+const assert = require("assert");
+const fs = require("fs");
+const os = require("os");
+const path = require("path");
+const { spawnSync } = require("child_process");
+
+const RESULT_PREFIX = "__PACK_REVIEW_RESULT__";
+
+function deferred() {
+  let resolve;
+  let reject;
+  const promise = new Promise((res, rej) => { resolve = res; reject = rej; });
+  return { promise, resolve, reject };
+}
+
+function wait(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); }
+
+function isolate(prefix) {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
+  process.env.OPEN_CLAUDIA_TEST = "1";
+  process.env.OPEN_CLAUDIA_CONFIG_DIR = path.join(root, "config");
+  process.env.PACKS_DIR = path.join(root, "packs");
+  process.env.ENTITIES_DIR = path.join(root, "entities");
+  process.env.LESSONS_FILE = path.join(root, "lessons.md");
+  process.env.RECALL_METRICS = "on";
+  process.env.HOME = root;
+  process.env.WORKSPACE = root;
+  fs.mkdirSync(process.env.OPEN_CLAUDIA_CONFIG_DIR, { recursive: true });
+  return root;
+}
+
+function providerRegistry(provider) {
+  return {
+    getProvider(id) {
+      if (id !== provider.id) throw Object.assign(new Error(`unknown provider ${id}`), { code: "UNKNOWN_PROVIDER" });
+      return provider;
+    },
+    listProviders: () => [provider],
+    providerStatus(id) {
+      if (id !== provider.id) throw Object.assign(new Error(`unknown provider ${id}`), { code: "UNKNOWN_PROVIDER" });
+      return {
+        id,
+        availability: { state: "available", executable: provider.executable(), reason: null },
+        compatibility: { state: "compatible", reason: null },
+        auth: { state: "authenticated", reason: null },
+      };
+    },
+  };
+}
+
+function semanticPack(pack) {
+  return {
+    dir: pack.dir,
+    name: pack.name,
+    description: pack.description,
+    tags: pack.tags,
+    kind: pack.kind,
+    sections: pack.sections,
+  };
+}
+
+function semanticEntity(entity) {
+  return {
+    slug: entity.slug,
+    name: entity.name,
+    type: entity.type,
+    aliases: entity.aliases,
+    description: entity.description,
+    relationship: entity.relationship,
+    sections: entity.sections,
+  };
+}
+
+async function providerProbe(kind) {
+  const root = isolate(`open-claudia-review-${kind}-`);
+  const fixture = path.join(__dirname, "test-fixtures", "fake-agent-cli.js");
+  const capture = path.join(root, `${kind}-capture.json`);
+  const secret = ["sk", "proj", "provider", "review", "secret", "fixture"].join("-");
+  const decision = {
+    actions: [
+      {
+        action: "update",
+        pack: "project",
+        state: "Provider-neutral review is complete.",
+        stance: "Keep the original project facts and use bounded utility review.",
+        procedure: "Run the reviewer read-only, then serialize its mutation batch.",
+        journal: "Verified provider-neutral memory review.",
+      },
+      {
+        action: "update",
+        pack: "unrelated",
+        state: "THIS BLIND REWRITE MUST NOT LAND",
+        journal: "Recorded a safe append-only observation outside active scope.",
+      },
+      { action: "create", dir: "over-limit", name: "Over Limit", state: "must not exist" },
+    ],
+    entities: [
+      { name: "Ada", type: "person", description: "Project owner", notes: "Owns the project.", log: "Confirmed ownership.", relationship: "owner", style: "Concise and technical." },
+      { name: "Nairobi", type: "place", description: "Delivery location", notes: "Primary delivery location.", log: "Used for delivery." },
+      { name: "Parity Harness", type: "system", description: "Provider parity fixture", notes: "Runs deterministic provider fixtures.", log: "Verified both adapters." },
+      { name: "Over Entity Limit", type: "thing", description: "must not exist", notes: "must not exist", log: "must not exist" },
+    ],
+    lessons: [
+      { text: "Always serialize memory mutation batches.", src: "project", trigger: "actually, serialize the whole batch" },
+      { text: "This second lesson must be clipped.", src: "project", trigger: "again" },
+    ],
+  };
+
+  const { createClaudeProvider } = require("./core/providers/claude");
+  const { createCodexProvider } = require("./core/providers/codex");
+  const createProvider = kind === "claude" ? createClaudeProvider : createCodexProvider;
+  const provider = createProvider({
+    resolveExecutable: () => fixture,
+    defaultModel: `${kind}-default-fixture`,
+    buildEnv: () => ({
+      PATH: process.env.PATH || "",
+      HOME: root,
+      FAKE_AGENT_KIND: kind,
+      FAKE_AGENT_CAPTURE: capture,
+      FAKE_AGENT_TEXT: JSON.stringify(decision),
+    }),
+    getAuthStatus: () => ({ state: "authenticated", reason: null }),
+  });
+  const model = `${kind}-review-exact`;
+  const { createUtilityAgentService } = require("./core/utility-agent");
+  const service = createUtilityAgentService({
+    registry: providerRegistry(provider),
+    env: {
+      UTILITY_PROVIDER: kind,
+      [`PACK_REVIEW_MODEL_${kind.toUpperCase()}`]: model,
+    },
+  });
+  const { MemoryMutationQueue } = require("./core/memory-mutation-queue");
+  const queue = new MemoryMutationQueue({
+    lockDir: path.join(root, "review.lock"),
+    journalFile: path.join(root, "review-intents.jsonl"),
+    lockPollMs: 5,
+  });
+  const packs = require("./core/packs");
+  const entities = require("./core/entities");
+  const lessons = require("./core/lessons");
+  const { createPackReviewer, REVIEW_OUTPUT_SCHEMA } = require("./core/pack-review");
+  packs.createPack({
+    dir: "project",
+    name: "Project",
+    description: "provider parity project",
+    stance: "Preserve this durable stance.",
+    procedure: "Preserve this durable procedure.",
+    state: "Review is pending.",
+  });
+  packs.createPack({
+    dir: "unrelated",
+    name: "Unrelated",
+    description: "unrelated durable memory",
+    state: "Unrelated current truth must survive.",
+  });
+
+  const callMetadata = [];
+  const reviewer = createPackReviewer({
+    mutationQueue: queue,
+    spawnUtilityAgent: (prompt, options) => {
+      callMetadata.push({
+        purpose: options.purpose,
+        activeProvider: options.activeProvider,
+        tier: options.tier,
+        readOnly: options.readOnly,
+        outputSchema: options.outputSchema,
+        secretRedacted: !prompt.includes(secret) && prompt.includes("[REDACTED_OPENAI_KEY]"),
+      });
+      return service.spawnUtilityAgent(prompt, options);
+    },
+    cancelUtilityAgents: service.cancelAll,
+  });
+  const announcements = [];
+  const result = await reviewer.reviewTurn({
+    userText: `Actually, serialize the whole batch while reviewing this provider parity project. ${secret} ${"durable context ".repeat(20)}`,
+    assistantText: `Implemented and verified the provider-neutral reviewer with active-pack protection. ${"verification detail ".repeat(20)}`,
+    activePacks: ["project"],
+    provider: kind,
+    runId: `${kind}-review-run`,
+    announce: async (text) => announcements.push(text),
+  });
+
+  assert.strictEqual(result.status, "applied");
+  assert.strictEqual(result.provider, kind);
+  assert.strictEqual(result.model, model);
+  assert.strictEqual(result.fallbackUsed, false);
+  assert.strictEqual(callMetadata.length, 1);
+  assert.deepStrictEqual(callMetadata[0], {
+    purpose: "review",
+    activeProvider: kind,
+    tier: "medium",
+    readOnly: true,
+    outputSchema: REVIEW_OUTPUT_SCHEMA,
+    secretRedacted: true,
+  });
+  assert.strictEqual(announcements.length, 1, "one complete reviewer batch produces one announcement");
+
+  const project = packs.readPack("project");
+  const unrelated = packs.readPack("unrelated");
+  assert.strictEqual(project.sections.State, "Provider-neutral review is complete.");
+  assert.match(project.sections.Journal, /Verified provider-neutral memory review/);
+  assert.strictEqual(unrelated.sections.State, "Unrelated current truth must survive.", "blind section rewrites stay blocked");
+  assert.match(unrelated.sections.Journal, /safe append-only observation/);
+  assert.strictEqual(packs.readPack("over-limit"), null, "pack action limit is preserved");
+  assert.strictEqual(entities.listEntities().length, 3, "entity action limit is preserved");
+  assert.strictEqual(entities.findEntity("Over Entity Limit"), null);
+  assert.strictEqual(lessons.listLessons().length, 1, "lesson action limit is preserved");
+  assert.strictEqual(lessons.listLessons()[0].text, "Always serialize memory mutation batches.");
+
+  const captureRecord = JSON.parse(fs.readFileSync(capture, "utf8"));
+  assert.ok(!fs.readFileSync(capture, "utf8").includes(secret), "private review text never enters argv or fixture logs");
+  assert.strictEqual(captureRecord.stdin.present, true);
+  assert.ok(captureRecord.argv.includes(model), "provider-specific exact reviewer model reaches the adapter");
+  if (kind === "claude") assert.ok(captureRecord.argv.includes("plan"), "Claude reviewer is read-only");
+  else {
+    assert.ok(captureRecord.argv.includes("read-only"), "Codex reviewer is read-only");
+    assert.ok(captureRecord.argv.includes("--output-schema"), "Codex receives the reviewer schema file");
+  }
+
+  const metrics = JSON.parse(fs.readFileSync(path.join(process.env.OPEN_CLAUDIA_CONFIG_DIR, "recall-metrics-summary.json"), "utf8"));
+  assert.strictEqual(metrics.reviewerByProviderModel[`${kind}:${model}`], 1);
+  const intents = fs.readFileSync(path.join(root, "review-intents.jsonl"), "utf8");
+  assert.ok(!intents.includes(secret), "intent journal contains metadata, never review content");
+
+  return {
+    provider: kind,
+    model,
+    mutations: {
+      packs: [semanticPack(project), semanticPack(unrelated)],
+      entities: entities.listEntities().map(semanticEntity),
+      lessons: lessons.listLessons().map(({ text, src }) => ({ text, src })),
+    },
+  };
+}
+
+async function lifecycleProbe() {
+  const root = isolate("open-claudia-review-lifecycle-");
+  const packs = require("./core/packs");
+  const { MemoryMutationQueue } = require("./core/memory-mutation-queue");
+  const { createPackReviewer } = require("./core/pack-review");
+  packs.createPack({ dir: "project", name: "Project", description: "lifecycle fixture", state: "initial state" });
+  const longTurn = {
+    userText: `Review the lifecycle behavior. ${"bounded durable context ".repeat(20)}`,
+    assistantText: `The lifecycle behavior is under deterministic test. ${"verified implementation detail ".repeat(20)}`,
+    activePacks: ["project"],
+  };
+
+  // A queued dream owns the shared mutation lane; a completed reviewer model
+  // cannot mutate the same stores until that whole dream batch releases it.
+  const serializationQueue = new MemoryMutationQueue({
+    lockDir: path.join(root, "serialization.lock"),
+    journalFile: path.join(root, "serialization.jsonl"),
+    lockPollMs: 5,
+  });
+  const dreamGate = deferred();
+  let dreamStarted = false;
+  const dream = serializationQueue.enqueue("dream", async () => {
+    dreamStarted = true;
+    await dreamGate.promise;
+  }, { purpose: "dream", trigger: "test" });
+  while (!dreamStarted) await wait(5);
+  const serializedReviewer = createPackReviewer({
+    mutationQueue: serializationQueue,
+    spawnUtilityAgent: async (_prompt, options) => ({
+      ok: true,
+      text: JSON.stringify({
+        actions: [{ action: "update", pack: "project", state: "serialized review state", journal: "Review followed the dream batch." }],
+        entities: [],
+        lessons: [],
+      }),
+      provider: options.activeProvider,
+      model: "serialized-review-model",
+      fallbackUsed: false,
+    }),
+  });
+  const serializedReview = serializedReviewer.reviewTurn({ ...longTurn, provider: "codex" });
+  await wait(30);
+  assert.strictEqual(packs.readPack("project").sections.State, "initial state", "review waits behind the active dream mutation batch");
+  dreamGate.resolve();
+  await Promise.all([dream, serializedReview]);
+  assert.strictEqual(packs.readPack("project").sections.State, "serialized review state");
+
+  // The memory version is captured before awaiting the model. A newer write
+  // wins; only the review's append-only journal observation is safe to retain.
+  const staleQueue = new MemoryMutationQueue({
+    lockDir: path.join(root, "stale.lock"),
+    journalFile: path.join(root, "stale.jsonl"),
+    lockPollMs: 5,
+  });
+  const staleGate = deferred();
+  let capturedOptions = null;
+  const staleReviewer = createPackReviewer({
+    mutationQueue: staleQueue,
+    spawnUtilityAgent: async (_prompt, options) => {
+      capturedOptions = options;
+      return staleGate.promise;
+    },
+  });
+  const input = { ...longTurn, provider: "codex", runId: "captured-run" };
+  const staleHandle = staleReviewer.reviewTurn(input);
+  input.provider = "claude";
+  assert.strictEqual(staleReviewer.activeReviewCount(), 1, "review handles remain tracked while model work is pending");
+  packs.updatePack("project", { state: "newer foreground state" }, "user");
+  staleGate.resolve({
+    ok: true,
+    text: JSON.stringify({
+      actions: [{ action: "update", pack: "project", state: "stale reviewer state", journal: "Stale review observation retained safely." }],
+      entities: [],
+      lessons: [],
+    }),
+    provider: "codex",
+    model: "captured-codex-model",
+    fallbackUsed: false,
+  });
+  const staleResult = await staleHandle;
+  assert.strictEqual(capturedOptions.activeProvider, "codex", "the admitted provider is captured before asynchronous review work");
+  assert.strictEqual(staleResult.status, "applied");
+  assert.strictEqual(packs.readPack("project").sections.State, "newer foreground state", "stale reviewer state never replaces a newer write");
+  assert.match(packs.readPack("project").sections.Journal, /Stale review observation retained safely/);
+  assert.strictEqual(staleReviewer.activeReviewCount(), 0);
+
+  // A store failure is a failed serialized batch, is journalled, and does not
+  // poison later queue work or emit a success announcement.
+  const failureQueue = new MemoryMutationQueue({
+    lockDir: path.join(root, "failure.lock"),
+    journalFile: path.join(root, "failure.jsonl"),
+    lockPollMs: 5,
+  });
+  const failureReviewer = createPackReviewer({
+    mutationQueue: failureQueue,
+    spawnUtilityAgent: async () => ({
+      ok: true,
+      text: JSON.stringify({ actions: [{ action: "update", pack: "project", journal: "write must fail" }], entities: [], lessons: [] }),
+      provider: "claude",
+      model: "failure-model",
+      fallbackUsed: false,
+    }),
+  });
+  const originalUpdatePack = packs.updatePack;
+  packs.updatePack = () => { throw Object.assign(new Error("pack write fixture failed"), { code: "PACK_WRITE_FAILED" }); };
+  const failureAnnouncements = [];
+  let failedResult;
+  try {
+    failedResult = await failureReviewer.reviewTurn({ ...longTurn, provider: "claude", announce: async (text) => failureAnnouncements.push(text) });
+  } finally {
+    packs.updatePack = originalUpdatePack;
+  }
+  assert.strictEqual(failedResult.status, "failed");
+  assert.strictEqual(failureAnnouncements.length, 0);
+  assert.match(fs.readFileSync(path.join(root, "failure.jsonl"), "utf8"), /"status":"failed"/);
+  assert.strictEqual(await failureQueue.enqueue("after-failure", async () => "healthy"), "healthy");
+
+  // Shutdown stops admission, records the still-pending intent, invokes
+  // utility cancellation after the bounded drain, and settles the handle.
+  const shutdownQueue = new MemoryMutationQueue({
+    lockDir: path.join(root, "shutdown.lock"),
+    journalFile: path.join(root, "shutdown.jsonl"),
+    lockPollMs: 5,
+  });
+  const shutdownGate = deferred();
+  let cancelCalls = 0;
+  const shutdownReviewer = createPackReviewer({
+    mutationQueue: shutdownQueue,
+    spawnUtilityAgent: async () => shutdownGate.promise,
+    cancelUtilityAgents: async () => {
+      cancelCalls += 1;
+      shutdownGate.resolve({
+        ok: true,
+        text: JSON.stringify({ actions: [], entities: [], lessons: [] }),
+        provider: "codex",
+        model: "shutdown-model",
+        fallbackUsed: false,
+      });
+      return { drained: true, active: 0 };
+    },
+  });
+  const pendingReview = shutdownReviewer.reviewTurn({ ...longTurn, provider: "codex", runId: "pending-shutdown-run" });
+  assert.strictEqual(shutdownReviewer.activeReviewCount(), 1);
+  const drain = await shutdownReviewer.drainReviews({ timeoutMs: 25, cancelOnTimeout: true });
+  assert.strictEqual(drain.drained, false, "a bounded timeout is reported even when cancellation subsequently settles work");
+  assert.strictEqual(cancelCalls, 1);
+  assert.strictEqual((await pendingReview).status, "no_change");
+  assert.match(fs.readFileSync(path.join(root, "shutdown.jsonl"), "utf8"), /"status":"pending_on_shutdown"/);
+  const late = await shutdownReviewer.reviewTurn({ ...longTurn, provider: "claude" });
+  assert.strictEqual(late.status, "not_admitted");
+  assert.match(fs.readFileSync(path.join(root, "shutdown.jsonl"), "utf8"), /"status":"pending_intent_rejected"/);
+
+  return { serialized: true, staleProtected: true, failureRecorded: true, shutdownDrained: true };
+}
+
+function runChild(args) {
+  const child = spawnSync(process.execPath, [__filename, ...args], {
+    cwd: __dirname,
+    env: { ...process.env },
+    encoding: "utf8",
+    timeout: 30000,
+  });
+  if (child.status !== 0) {
+    throw new Error(`child ${args.join(" ")} failed (${child.status})\n${child.stdout}\n${child.stderr}`);
+  }
+  const line = child.stdout.split("\n").find((entry) => entry.startsWith(RESULT_PREFIX));
+  if (!line) throw new Error(`child ${args.join(" ")} returned no result\n${child.stdout}\n${child.stderr}`);
+  return JSON.parse(line.slice(RESULT_PREFIX.length));
+}
+
+async function main() {
+  if (process.argv[2] === "--provider-probe") {
+    const result = await providerProbe(process.argv[3]);
+    console.log(`${RESULT_PREFIX}${JSON.stringify(result)}`);
+    return;
+  }
+  if (process.argv[2] === "--lifecycle-probe") {
+    const result = await lifecycleProbe();
+    console.log(`${RESULT_PREFIX}${JSON.stringify(result)}`);
+    return;
+  }
+
+  const claude = runChild(["--provider-probe", "claude"]);
+  const codex = runChild(["--provider-probe", "codex"]);
+  assert.deepStrictEqual(codex.mutations, claude.mutations, "identical fixture decisions produce identical memory mutations under Claude and Codex");
+  assert.strictEqual(claude.provider, "claude");
+  assert.strictEqual(codex.provider, "codex");
+  const lifecycle = runChild(["--lifecycle-probe"]);
+  assert.deepStrictEqual(lifecycle, { serialized: true, staleProtected: true, failureRecorded: true, shutdownDrained: true });
+  console.log("provider pack review OK");
+}
+
+main().catch((error) => {
+  console.error(error.stack || error.message);
+  process.exitCode = 1;
+});