@inetafrica/open-claudia 2.14.8 → 3.0.0

package/core/recall/discoverer.js

@@ -6,7 +6,7 @@
 //      1-2 hops; nodes over the firing threshold join the candidate set. This
 //      auto-pulls governed-by concerns (theme, commit style) the query never
 //      named, which is the whole point of the graph.
-//   4. Walker (haiku) — reads the seed+activated candidates' live sections and
+//   4. Walker (provider-owned low tier) — reads the candidates' live sections and
 //      returns the truly-relevant set WITH a one-line "why" per node. Facts are
 //      quoted, not paraphrased. Fail-open: any error keeps the seeds (classic
 //      baseline), so the discoverer never recalls worse than keyword matching.
@@ -18,7 +18,8 @@
 const graph = require("./graph");
 const metrics = require("./metrics");
 const tuning = require("./tuning");
-const { spawnSubagent } = require("../subagent");
+const { spawnUtilityAgent } = require("../utility-agent");
+const { resolveUtilityProvider } = require("../providers/utility-policy");
 const warmWalker = require("./warm-walker");
 const toolsLib = require("../tools");
 // Tool-graph is the directed-usage source of truth (pack↔tool edges). The
@@ -90,7 +91,6 @@ function renderToolGaps(gaps) {
 let transcriptIndex = null;
 try { transcriptIndex = require("../transcript-index"); } catch (e) { /* old node */ }
 
-const WALKER_MODEL = process.env.RECALL_DISCOVERER_MODEL || "haiku";
 const WALKER_TIMEOUT_MS = Number(process.env.RECALL_DISCOVERER_TIMEOUT_MS || 25000);
 const WALKER_ENABLED = String(process.env.RECALL_DISCOVERER_WALKER || "on").toLowerCase() !== "off";
 // Sizing/tier knobs (walker candidate cap, episode limit, seeds-tier width,
@@ -222,23 +222,71 @@ const WALKER_SYSTEM = [
 // low latency; on any warm-path error fall back to a cold spawn — identical
 // model/prompt/contract, so quality is unchanged and recall never silently
 // degrades to the classic engine. Returns { text, costUsd, source }.
-async function runWalker(prompt) {
-  const opts = { model: WALKER_MODEL, systemPrompt: WALKER_SYSTEM, timeoutMs: WALKER_TIMEOUT_MS };
-  if (warmWalker.isEnabled()) {
+async function runWalker(prompt, options = {}) {
+  const resolveSelection = options.resolveUtilityProvider || resolveUtilityProvider;
+  const runUtility = options.spawnUtilityAgent || spawnUtilityAgent;
+  const warm = options.warmWalker || warmWalker;
+  const selection = resolveSelection({
+    purpose: "recall",
+    activeProvider: options.provider || null,
+    tier: "low",
+    model: options.model || null,
+    modelEnvPrefix: "RECALL_DISCOVERER_MODEL",
+    legacyClaudeModelEnv: "RECALL_DISCOVERER_MODEL",
+  });
+  const opts = {
+    provider: selection.provider,
+    model: selection.model,
+    tier: "low",
+    systemPrompt: WALKER_SYSTEM,
+    timeoutMs: WALKER_TIMEOUT_MS,
+  };
+  if (warm.isEnabled(selection.provider)) {
     try {
-      const r = await warmWalker.walkWarm(prompt, opts);
-      return { text: r.text, costUsd: r.costUsd, source: "warm" };
+      const result = await warm.walkWarm(prompt, opts);
+      return {
+        text: result.text,
+        costUsd: result.costUsd,
+        source: "warm",
+        provider: selection.providerId,
+        model: selection.model,
+      };
     } catch (e) { /* fall back to cold spawn below */ }
   }
-  const { text } = await spawnSubagent(prompt, opts);
-  return { text, costUsd: null, source: "cold" };
+  const result = await runUtility(prompt, {
+    purpose: "recall",
+    provider: selection.providerId,
+    tier: "low",
+    model: selection.model,
+    modelEnvPrefix: "RECALL_DISCOVERER_MODEL",
+    legacyClaudeModelEnv: "RECALL_DISCOVERER_MODEL",
+    systemPrompt: WALKER_SYSTEM,
+    readOnly: true,
+    outputSchema: {
+      type: "array",
+      items: {
+        type: "object",
+        required: ["id"],
+        properties: { id: { type: "string" }, why: { type: "string" } },
+      },
+    },
+    timeoutMs: WALKER_TIMEOUT_MS,
+    cwd: options.cwd || process.cwd(),
+  });
+  return {
+    text: result.text,
+    costUsd: null,
+    source: "cold",
+    provider: result.provider || selection.providerId,
+    model: result.model || selection.model,
+  };
 }
 
 // Judge candidates. Returns { kept: Map|null, costUsd, source, walkerMs } —
 // kept null means fail-open (walker off/failed/unparseable).
-async function walk(userText, contextText, candidates) {
+async function walk(userText, contextText, candidates, options = {}) {
   if (!WALKER_ENABLED || candidates.length === 0) {
-    return { kept: null, costUsd: null, source: "off", walkerMs: 0 };
+    return { kept: null, costUsd: null, source: "off", walkerMs: 0, provider: options.provider || null, model: null };
   }
   const lines = candidates.map((c) => {
     const tag = c.activated ? ` [linked via ${c.via || "graph"}]` : "";
@@ -256,12 +304,12 @@ async function walk(userText, contextText, candidates) {
   ].join("\n");
   const t0 = Date.now();
   try {
-    const { text, costUsd, source } = await runWalker(prompt);
+    const { text, costUsd, source, provider, model } = await runWalker(prompt, options);
     const walkerMs = Date.now() - t0;
     const match = String(text || "").match(/\[[\s\S]*\]/);
-    if (!match) return { kept: null, costUsd, source, walkerMs };
+    if (!match) return { kept: null, costUsd, source, walkerMs, provider, model };
     const arr = JSON.parse(match[0]);
-    if (!Array.isArray(arr)) return { kept: null, costUsd, source, walkerMs };
+    if (!Array.isArray(arr)) return { kept: null, costUsd, source, walkerMs, provider, model };
     const known = new Set(candidates.map((c) => c.id));
     const out = new Map();
     for (const item of arr) {
@@ -269,14 +317,14 @@ async function walk(userText, contextText, candidates) {
         out.set(item.id, String(item.why || "").slice(0, 120));
       }
     }
-    return { kept: out, costUsd, source, walkerMs };
+    return { kept: out, costUsd, source, walkerMs, provider, model };
   } catch (e) {
-    return { kept: null, costUsd: null, source: "failed", walkerMs: Date.now() - t0 };
+    return { kept: null, costUsd: null, source: "failed", walkerMs: Date.now() - t0, provider: options.provider || null, model: null };
   }
 }
 
 async function run(ctx) {
-  const { userText, contextText, fullContext, packLimit, budget, helpers } = ctx;
+  const { userText, contextText, fullContext, packLimit, budget, helpers, provider } = ctx;
   const { packsLib, entitiesLib, mergeMatches, buildPackBlock, buildEntityBlock } = helpers;
   const started = Date.now();
 
@@ -314,7 +362,7 @@ async function run(ctx) {
   // 1: tiered pre-gate.
   const tier = recallTier(userText, seedCount);
   if (tier === "skip") {
-    metrics.logTurn({ engine: "discoverer", model: WALKER_MODEL, query: userText, gated: true, tier, latencyMs: Date.now() - started });
+    metrics.logTurn({ engine: "discoverer", provider: provider || "", model: "", query: userText, gated: true, tier, latencyMs: Date.now() - started });
     return { packBlock: "", entityBlock: "", toolBlock: "", episodeBlock: "", packMatches: [], entityMatches: [], toolMatches: [], toolGaps: [], episodeMatches: [], why: {}, gated: true, tier };
   }
 
@@ -453,7 +501,7 @@ async function run(ctx) {
     }
   }
 
-  const walkRes = await walk(userText, contextText || fullContext, judgeCands);
+  const walkRes = await walk(userText, contextText || fullContext, judgeCands, { provider: provider || null });
   const whyById = walkRes.kept;
 
   let shadowChecks = 0, shadowAgree = 0;
@@ -576,7 +624,8 @@ async function run(ctx) {
 
   metrics.logTurn({
     engine: "discoverer",
-    model: WALKER_MODEL,
+    provider: walkRes.provider || provider || "",
+    model: walkRes.model || "",
     query: userText,
     tier,
     seeds: seedNodes,
@@ -602,4 +651,4 @@ async function run(ctx) {
   };
 }
 
-module.exports = { name: "discoverer", run, needsRecall, recallTier, walk, cascadeVerdict, scoutToolGaps, renderToolGaps, episodesAllowedForSpeaker };
+module.exports = { name: "discoverer", run, runWalker, needsRecall, recallTier, walk, cascadeVerdict, scoutToolGaps, renderToolGaps, episodesAllowedForSpeaker };

package/core/recall/metrics.js

@@ -33,6 +33,7 @@ function logTurn(entry) {
     const rec = {
       ts: new Date().toISOString(),
       v: VERSION,
+      provider: entry.provider || "",
       model: entry.model || "",
       engine: entry.engine || "discoverer",
       query: String(entry.query || "").slice(0, 200),
@@ -106,6 +107,31 @@ function logReviewerSkip() {
   } catch (e) {}
 }
 
+function logReviewerRun(entry = {}) {
+  if (!enabled()) return;
+  try {
+    const rec = {
+      ts: new Date().toISOString(),
+      v: VERSION,
+      reviewer: {
+        provider: String(entry.provider || ""),
+        model: String(entry.model || ""),
+        status: String(entry.status || "unknown"),
+        fallbackUsed: !!entry.fallbackUsed,
+        ...(entry.code ? { code: String(entry.code).slice(0, 80) } : {}),
+      },
+    };
+    fs.appendFileSync(LOG_FILE, JSON.stringify(rec) + "\n");
+    const summary = readSummary();
+    summary.reviewerRuns = (summary.reviewerRuns || 0) + 1;
+    const key = `${rec.reviewer.provider || "unknown"}:${rec.reviewer.model || "default"}`;
+    summary.reviewerByProviderModel = summary.reviewerByProviderModel || {};
+    summary.reviewerByProviderModel[key] = (summary.reviewerByProviderModel[key] || 0) + 1;
+    if (rec.reviewer.status === "failed") summary.reviewerFailures = (summary.reviewerFailures || 0) + 1;
+    writeSummary(summary);
+  } catch (_) {}
+}
+
 function round(n) { return Math.round((Number(n) || 0) * 1000) / 1000; }
 function today() { return new Date().toISOString().slice(0, 10); }
 
@@ -311,6 +337,6 @@ function _resetForTest() {
 
 module.exports = {
   LOG_FILE, SUMMARY_FILE, NODE_STATS_FILE, KPI_FILE, VERSION, enabled,
-  logTurn, logUse, logShadow, logReviewerSkip, appendKpi, readKpi,
+  logTurn, logUse, logShadow, logReviewerSkip, logReviewerRun, appendKpi, readKpi,
   summary, nodeStats, evidence, _resetForTest,
 };

package/core/recall/tuning.js

@@ -113,7 +113,10 @@ function lastChange() {
 function history() { return readFile().history || []; }
 
 function write(data) {
-  try { fs.writeFileSync(TUNING_FILE, JSON.stringify(data, null, 2)); } catch (e) {}
+  try {
+    fs.mkdirSync(path.dirname(TUNING_FILE), { recursive: true });
+    fs.writeFileSync(TUNING_FILE, JSON.stringify(data, null, 2));
+  } catch (e) {}
   _cache = null; _cacheMtime = -1;
 }
 

package/core/recall/warm-walker.js

@@ -1,162 +1,205 @@
-// Warm walker: one long-lived `claude` process the discoverer reuses across
-// turns, instead of cold-spawning a fresh CLI for every recall.
-//
-// Why this exists: the Claude CLI initialises lazily on the FIRST message, so
-// a cold spawn pays ~3-16s of boot+init every turn. Measured: on a persistent
-// stream-json process the first message costs ~3.4s but every message after is
-// ~1.4s — init is paid once and amortised. Same model, same system prompt,
-// same JSON contract as the cold path, so recall quality is unchanged; only
-// the transport differs.
-//
-// Statelessness: stream-json keeps one growing session, so we recycle the
-// process on a small message/char budget to keep accumulated history tiny and
-// each walk effectively independent.
-//
-// Fail-open: any error here throws, and the caller (discoverer.walk) falls
-// back to the cold spawnSubagent path — never worse than before, and never a
-// drop to the classic engine.
+"use strict";
+
+// Optional low-latency transport for providers that explicitly advertise a
+// persistent utility stream. The current native implementation is Claude's
+// stream-json input protocol; Codex declares this capability unsupported and
+// therefore always uses the identical cold utility prompt/contract.
 
 const { spawn } = require("child_process");
-const { CLAUDE_PATH, botSubprocessEnv } = require("../config");
-const { claudeSubprocessEnv } = require("../auth-flow");
+const { StringDecoder } = require("string_decoder");
+const { killProcessTree } = require("../process-tree");
 const { redactSensitive } = require("../redact");
 
 const MAX_MSGS = Number(process.env.RECALL_WARM_WALKER_MAX_MSGS || 12);
 const MAX_CHARS = Number(process.env.RECALL_WARM_WALKER_MAX_CHARS || 24000);
 const DEFAULT_TIMEOUT_MS = 25000;
+const KILL_GRACE_MS = Number(process.env.RECALL_WARM_WALKER_KILL_GRACE_MS || 250);
+const MAX_BUFFER_BYTES = Number(process.env.RECALL_WARM_WALKER_MAX_BUFFER_BYTES || 1024 * 1024);
+
+let child = null;
+let childProviderId = null;
+let pending = null;
+let chain = Promise.resolve();
+let msgCount = 0;
+let charCount = 0;
+let costTotal = 0;
+let stopPromise = null;
 
-let child = null;          // current process, or null when none is alive
-let pending = null;        // { resolve, reject, timer } for the in-flight walk
-let chain = Promise.resolve(); // serialises walks (one message in flight)
-let msgCount = 0;          // messages sent to the current process
-let charCount = 0;         // prompt chars sent to the current process
-let costTotal = 0;         // cumulative total_cost_usd reported by the session
+function isEnabled(provider) {
+  if (String(process.env.RECALL_WARM_WALKER || "on").toLowerCase() === "off") return false;
+  return provider?.id === "claude"
+    && provider.capabilities?.persistentUtilityStream?.support === "native";
+}
 
-function isEnabled() {
-  return String(process.env.RECALL_WARM_WALKER || "on").toLowerCase() !== "off";
+function settle(kind, value) {
+  const current = pending;
+  pending = null;
+  if (!current) return;
+  clearTimeout(current.timer);
+  if (kind === "resolve") current.resolve(value);
+  else current.reject(value);
 }
 
-function killChild() {
+function stopChild(reason = "warm walker stopped") {
+  if (stopPromise) return stopPromise;
   const proc = child;
   child = null;
-  if (proc) {
-    try { proc.stdin.end(); } catch (e) {}
-    try { proc.kill("SIGTERM"); } catch (e) {}
-  }
+  childProviderId = null;
+  if (!proc) return Promise.resolve();
+  stopPromise = new Promise((resolve) => {
+    let done = false;
+    let killTimer = null;
+    const finish = () => {
+      if (done) return;
+      done = true;
+      if (killTimer) clearTimeout(killTimer);
+      stopPromise = null;
+      resolve();
+    };
+    proc.once("close", finish);
+    try { proc.stdin.end(); } catch (_) {}
+    try { killProcessTree(proc.pid, "SIGTERM"); } catch (_) { try { proc.kill("SIGTERM"); } catch (_) {} }
+    killTimer = setTimeout(() => {
+      try { killProcessTree(proc.pid, "SIGKILL"); } catch (_) { try { proc.kill("SIGKILL"); } catch (_) {} }
+    }, Math.max(1, KILL_GRACE_MS));
+    if (proc.exitCode !== null || proc.signalCode !== null) finish();
+  });
+  settle("reject", new Error(reason));
+  return stopPromise;
+}
+
+function persistentInvocation(cfg) {
+  const provider = cfg.provider;
+  if (!isEnabled(provider)) throw new Error(`${provider?.id || "provider"} has no enabled persistent utility stream`);
+  const invocation = provider.buildUtilityInvocation({
+    purpose: "recall",
+    tier: "low",
+    model: cfg.model || provider.tierModel("low"),
+    coreInstructions: cfg.systemPrompt || "You are Open Claudia's recall walker.",
+    dynamicContext: "",
+    userPrompt: "",
+    readOnly: true,
+    fresh: true,
+  });
+  const args = [...invocation.args];
+  const printAt = args.indexOf("-p");
+  args.splice(printAt >= 0 ? printAt + 1 : 0, 0, "--input-format", "stream-json");
+  return { ...invocation, args, stdin: null };
 }
 
 function spawnChild(cfg) {
-  const args = [
-    "-p",
-    "--input-format", "stream-json",
-    "--output-format", "stream-json",
-    "--verbose",
-    "--model", cfg.model || "haiku",
-    "--no-session-persistence",
-    "--append-system-prompt", cfg.systemPrompt || "",
-    "--dangerously-skip-permissions",
-  ];
-  const env = { ...botSubprocessEnv(), ...claudeSubprocessEnv() };
-  const proc = spawn(CLAUDE_PATH, args, { cwd: process.cwd(), env, stdio: ["pipe", "pipe", "pipe"] });
+  const invocation = persistentInvocation(cfg);
+  const proc = spawn(invocation.binary, invocation.args, {
+    cwd: cfg.cwd || process.cwd(),
+    env: {
+      ...invocation.env,
+      OC_PROVIDER: cfg.provider.id,
+      OC_UTILITY_PURPOSE: "recall",
+    },
+    stdio: ["pipe", "pipe", "pipe"],
+    detached: process.platform !== "win32",
+    shell: false,
+  });
   child = proc;
+  childProviderId = cfg.provider.id;
   msgCount = 0;
   charCount = 0;
   costTotal = 0;
-  let buf = "";
-  let asstText = "";
-
-  const settle = (fn, arg) => {
-    const p = pending;
-    pending = null;
-    if (!p) return;
-    clearTimeout(p.timer);
-    fn === "resolve" ? p.resolve(arg) : p.reject(arg);
-  };
-
-  proc.stdout.on("data", (d) => {
+  let buffer = "";
+  let assistantText = "";
+  const stringDecoder = new StringDecoder("utf8");
+
+  proc.stdout.on("data", (data) => {
     if (child !== proc) return;
-    buf += d.toString();
-    let nl;
-    while ((nl = buf.indexOf("\n")) >= 0) {
-      const line = buf.slice(0, nl).trim();
-      buf = buf.slice(nl + 1);
+    buffer += stringDecoder.write(data);
+    if (Buffer.byteLength(buffer, "utf8") > MAX_BUFFER_BYTES) {
+      stopChild("warm walker output exceeded its buffer limit").catch(() => {});
+      return;
+    }
+    let newline;
+    while ((newline = buffer.indexOf("\n")) >= 0) {
+      const line = buffer.slice(0, newline).trim();
+      buffer = buffer.slice(newline + 1);
       if (!line) continue;
-      let evt;
-      try { evt = JSON.parse(line); } catch (e) { continue; }
-      if (evt.type === "assistant" && evt.message && Array.isArray(evt.message.content)) {
-        for (const b of evt.message.content) {
-          if (b && b.type === "text" && typeof b.text === "string") asstText += b.text;
+      let event;
+      try { event = JSON.parse(line); } catch (_) { continue; }
+      if (event.type === "assistant" && Array.isArray(event.message?.content)) {
+        for (const block of event.message.content) {
+          if (block?.type === "text" && typeof block.text === "string") assistantText += block.text;
         }
-      } else if (evt.type === "result") {
-        const text = (typeof evt.result === "string" && evt.result) ? evt.result : asstText;
-        asstText = "";
-        // total_cost_usd is cumulative for the stream-json session; the delta
-        // from the previous result is this walk's cost.
+      } else if (event.type === "result") {
+        const text = typeof event.result === "string" && event.result ? event.result : assistantText;
+        assistantText = "";
         let costUsd = null;
-        if (typeof evt.total_cost_usd === "number") {
-          costUsd = Math.max(0, evt.total_cost_usd - costTotal);
-          costTotal = evt.total_cost_usd;
+        if (typeof event.total_cost_usd === "number") {
+          costUsd = Math.max(0, event.total_cost_usd - costTotal);
+          costTotal = event.total_cost_usd;
         }
-        if (text && !evt.is_error) settle("resolve", { text: redactSensitive(String(text).trim()), costUsd });
-        else settle("reject", new Error("warm walker: empty/error result"));
+        if (text && !event.is_error) settle("resolve", { text: redactSensitive(String(text).trim()), costUsd });
+        else settle("reject", new Error("warm walker returned an empty or error result"));
       }
     }
   });
-  proc.stderr.on("data", () => {}); // surfaced via result/close, not needed here
-  proc.on("error", (e) => {
-    if (child === proc) child = null;
-    settle("reject", e);
+  proc.stderr.on("data", () => {});
+  proc.stdin.on("error", (error) => settle("reject", error));
+  proc.on("error", (error) => {
+    if (child === proc) { child = null; childProviderId = null; }
+    settle("reject", error);
   });
   proc.on("close", () => {
-    if (child === proc) child = null;
-    settle("reject", new Error("warm walker: process closed"));
+    if (child === proc) { child = null; childProviderId = null; }
+    settle("reject", new Error("warm walker process closed"));
   });
   return proc;
 }
 
-function doWalk(promptText, opts) {
-  // Recycle before sending if the current process is over its budget, so each
-  // walk runs against a small, near-stateless context.
-  if (child && (msgCount >= MAX_MSGS || charCount >= MAX_CHARS)) killChild();
-  if (!child) spawnChild({ systemPrompt: opts.systemPrompt, model: opts.model });
+async function doWalk(promptText, opts) {
+  if (!isEnabled(opts.provider)) throw new Error(`${opts.provider?.id || "provider"} persistent recall is unavailable`);
+  if (child && (childProviderId !== opts.provider.id || msgCount >= MAX_MSGS || charCount >= MAX_CHARS)) {
+    await stopChild("warm walker recycled");
+  }
+  if (!child) spawnChild(opts);
   const proc = child;
   const timeoutMs = opts.timeoutMs || DEFAULT_TIMEOUT_MS;
-
   return new Promise((resolve, reject) => {
-    const timer = setTimeout(() => {
+    const timer = setTimeout(async () => {
+      if (pending?.proc !== proc) return;
+      const current = pending;
       pending = null;
-      if (child === proc) killChild(); // process is wedged — drop it
-      reject(new Error("warm walker: timeout"));
+      await stopChild("warm walker timed out");
+      current.reject(new Error("warm walker timed out"));
     }, timeoutMs);
-    pending = { resolve, reject, timer };
-    msgCount++;
+    pending = { resolve, reject, timer, proc };
+    msgCount += 1;
     charCount += promptText.length;
     const payload = JSON.stringify({ type: "user", message: { role: "user", content: promptText } }) + "\n";
-    try {
-      proc.stdin.write(payload);
-    } catch (e) {
+    try { proc.stdin.write(payload); }
+    catch (error) {
       clearTimeout(timer);
       pending = null;
-      killChild();
-      reject(e);
+      stopChild("warm walker write failed").finally(() => reject(error));
     }
   });
 }
 
-// Serialise: one message in flight at a time. A failed walk must not poison the
-// queue, so the chain swallows outcomes while callers still see their result.
-// Resolves { text, costUsd } — costUsd is the per-walk delta of the session's
-// cumulative total_cost_usd (null when the CLI doesn't report it).
 function walkWarm(promptText, opts = {}) {
   const run = () => doWalk(promptText, opts);
-  const p = chain.then(run, run);
-  chain = p.then(() => {}, () => {});
-  return p;
+  const result = chain.then(run, run);
+  chain = result.then(() => {}, () => {});
+  return result;
+}
+
+async function shutdown() {
+  await stopChild("warm walker shutdown");
 }
 
-function shutdown() { killChild(); }
-function stats() { return { alive: !!child, msgCount, charCount, MAX_MSGS, MAX_CHARS }; }
+function stats() {
+  return { alive: !!child, providerId: childProviderId, msgCount, charCount, MAX_MSGS, MAX_CHARS };
+}
 
-process.on("exit", () => { try { killChild(); } catch (e) {} });
+process.on("exit", () => {
+  if (!child) return;
+  try { killProcessTree(child.pid, "SIGKILL"); } catch (_) {}
+});
 
 module.exports = { walkWarm, isEnabled, shutdown, stats };