@inetafrica/open-claudia 2.15.0 → 3.0.0

package/core/dream.js

@@ -12,9 +12,11 @@
 const fs = require("fs");
 const path = require("path");
 const cron = require("node-cron");
+const { atomicWriteFileSync } = require("./fsutil");
+const { memoryMutationQueue } = require("./memory-mutation-queue");
 
 const CONFIG_DIR = require("../config-dir");
-const { config, DEFAULT_CLAUDE_MODEL } = require("./config");
+const { config } = require("./config");
 const packs = require("./packs");
 const entities = require("./entities");
 const lessons = require("./lessons");
@@ -22,27 +24,49 @@ const ideas = require("./ideas");
 const daySeeds = require("./day-seeds");
 const packGuard = require("./pack-guard");
 const persona = require("./persona");
-const { spawnSubagent } = require("./subagent");
+const { spawnUtilityAgent } = require("./utility-agent");
 const recallMetrics = require("./recall/metrics");
 const recallTuning = require("./recall/tuning");
 const recallGraphLib = require("./recall/graph");
 
-// Graph surgery + threshold tuning is the highest-judgment work in the recall
-// system, so the dream runs on the best available high tier. Honour an explicit
-// DREAM_MODEL override; otherwise prefer opus (strongest reliably-available
-// Claude tier) with sonnet as graceful fallback. Provider-specific high tiers
-// (e.g. a connected Codex 5.5) can extend pickDreamModel later.
-function pickDreamModel() {
-  if (process.env.DREAM_MODEL) return process.env.DREAM_MODEL;
-  const tier = String(process.env.DREAM_TIER || "high").toLowerCase();
-  if (tier === "low") return "haiku";
-  if (tier === "medium") return "sonnet";
-  return DEFAULT_CLAUDE_MODEL; // high → the bot's top model (opus 4.8); tracks future upgrades
+// Graph surgery + threshold tuning is the highest-judgment recall work, so the
+// utility policy chooses a provider-owned model tier. The old DREAM_MODEL key
+// remains a Claude-only compatibility override; provider-specific pins are
+// DREAM_MODEL_CLAUDE and DREAM_MODEL_CODEX.
+function dreamTier() {
+  const tier = String(process.env.DREAM_TIER || "high").trim().toLowerCase();
+  return new Set(["low", "medium", "high"]).has(tier) ? tier : "high";
 }
-const DREAM_MODEL = pickDreamModel();
 // The dream is the foundation of long-term memory quality, so it runs at max
 // reasoning effort by default. Overridable via DREAM_EFFORT (low|medium|high|xhigh|max).
 const DREAM_EFFORT = process.env.DREAM_EFFORT || "max";
+const DREAM_OUTPUT_SCHEMA = Object.freeze({
+  type: "object",
+  properties: {
+    merges: { type: "array", items: { type: "object" } },
+    umbrellas: { type: "array", items: { type: "object" } },
+    parents: { type: "array", items: { type: "object" } },
+    retag: { type: "array", items: { type: "object" } },
+    entity_merges: { type: "array", items: { type: "object" } },
+    entity_notes: { type: "array", items: { type: "object" } },
+    archive: { type: "array", items: { type: "object" } },
+    lessons: { type: "array", items: { type: "object" } },
+    persona: {},
+    tuning: {},
+    report: { type: "string" },
+  },
+});
+const INTROSPECTION_OUTPUT_SCHEMA = Object.freeze({
+  type: "object",
+  properties: {
+    lessons_add: { type: "array", items: { type: "object" } },
+    doc_edits: { type: "array", items: { type: "object" } },
+    entity_edits: { type: "array", items: { type: "object" } },
+    ideas: { type: "array", items: { type: "object" } },
+    persona: {},
+    report: { type: "string" },
+  },
+});
 const DREAM_CRON = process.env.DREAM_CRON || "0 4 * * *";
 // Consolidation model timeout. The model weighs the WHOLE corpus in one pass,
 // so a fixed budget that was fine at 30 packs starves at 130 (the timeout that
@@ -74,8 +98,6 @@ const PROMOTE_MIN_PROJECTS = Number(process.env.DREAM_PROMOTE_MIN_PROJECTS || 3)
 const ARCHIVE_IDLE_DAYS = Number(process.env.DREAM_ARCHIVE_IDLE_DAYS || 30);
 const ARCHIVE_MAX_USAGE = Number(process.env.DREAM_ARCHIVE_MAX_USAGE || 3);
 
-let _dreaming = false;
-
 function enabled() {
   return (process.env.DREAM || "on").toLowerCase() !== "off";
 }
@@ -119,7 +141,7 @@ function readDreamState() {
   try { return JSON.parse(fs.readFileSync(DREAM_STATE_FILE, "utf8")) || {}; } catch (e) { return {}; }
 }
 function writeDreamState(s) {
-  try { fs.writeFileSync(DREAM_STATE_FILE, JSON.stringify(s, null, 2)); } catch (e) {}
+  try { atomicWriteFileSync(DREAM_STATE_FILE, JSON.stringify(s, null, 2), { mode: 0o600, backup: true }); } catch (e) {}
 }
 
 // Windowed metric deltas between two summary snapshots (counters only grow).
@@ -660,8 +682,8 @@ function applyDream(decision, backupRoot) {
 // A SECOND dream pass that runs with read-only tools. It reviews the day's
 // compaction seeds (what was actually worked on), reflects on its current self
 // with that self injected, may read its own code to understand how it works
-// (read-only — it has no tools to edit code or soul.md, enforced by the
-// --allowedTools whitelist in subagent.js), then proposes improvements to its
+// (provider-native read-only mode prevents edits to code or soul.md), then
+// proposes improvements to its
 // LEARNED MEMORY only: lessons promoted from missed corrections, sharpened
 // pack/entity notes, and self-improvement ideas. All writes happen here in
 // Node, bounded to memory docs, guard-scanned, and backed up first.
@@ -851,9 +873,15 @@ function writeDreamReport(data) {
     fs.mkdirSync(DREAMS_DIR, { recursive: true, mode: 0o700 });
     const stamp = new Date().toISOString();
     const file = path.join(DREAMS_DIR, `${stamp.slice(0, 10)}.md`);
+    const consolidationIntelligence = data.intelligence || {};
+    const introspectionIntelligence = data.introspectionIntelligence || {};
+    const provider = consolidationIntelligence.provider || data.provider || "unknown";
+    const model = consolidationIntelligence.model || data.model || "unknown";
+    const effort = consolidationIntelligence.effort || data.effort || "provider default";
     const sec = [
       `# Dream — ${stamp}`,
-      `Model: ${data.model} · effort: ${data.effort} · trigger: ${data.trigger || "?"}`,
+      `Provider: ${provider} · Model: ${model} · effort: ${effort} · trigger: ${data.trigger || "?"}`,
+      `Introspection: ${introspectionIntelligence.provider || "not run"} · Model: ${introspectionIntelligence.model || "not run"} · status: ${introspectionIntelligence.status || "unknown"}`,
       "",
       "## Consolidation",
       data.consolidation.report || "(no report)",
@@ -873,7 +901,7 @@ function writeDreamReport(data) {
     const block = sec.join("\n") + "\n\n---\n\n";
     let existing = "";
     try { existing = fs.readFileSync(file, "utf-8"); } catch (e) {}
-    fs.writeFileSync(file, existing + block, { mode: 0o600 });
+    atomicWriteFileSync(file, existing + block, { mode: 0o600, backup: true });
     return file;
   } catch (e) { return null; }
 }
@@ -990,16 +1018,66 @@ function negativeHebbianPhase(ev, { cap = 12 } = {}) {
   return { lines, nodes, weakened, pruned };
 }
 
-async function runDream({ trigger = "manual" } = {}) {
-  if (!enabled()) return { skipped: "dream is disabled (DREAM=off)" };
-  if (_dreaming) return { skipped: "a dream is already in progress" };
+function dreamUtilityOptions(purpose, { provider = null, timeoutMs, outputSchema, systemPrompt, cwd = process.cwd() } = {}) {
+  return {
+    purpose,
+    provider,
+    tier: dreamTier(),
+    modelEnvPrefix: "DREAM_MODEL",
+    legacyClaudeModelEnv: "DREAM_MODEL",
+    readOnly: true,
+    outputSchema,
+    effort: DREAM_EFFORT,
+    timeoutMs,
+    cwd,
+    systemPrompt,
+  };
+}
+
+function utilityPayload(result) {
+  return result?.json ? JSON.stringify(result.json) : String(result?.text || "");
+}
+
+function intelligenceMetadata(result, status = "succeeded") {
+  return {
+    provider: result?.provider || null,
+    model: result?.model || null,
+    effort: result?.effort || null,
+    fallbackUsed: !!result?.fallbackUsed,
+    status,
+  };
+}
+
+function failedIntelligenceMetadata(error) {
+  const result = error?.result || null;
+  return {
+    ...intelligenceMetadata(result, "failed"),
+    code: String(error?.code || result?.error?.code || "DREAM_UTILITY_FAILED").slice(0, 80),
+  };
+}
+
+async function previewDreamWith(runUtility, { provider = null } = {}) {
+  const result = await runUtility(buildDreamPrompt(), dreamUtilityOptions("dream", {
+    provider,
+    timeoutMs: 8 * 60 * 1000,
+    outputSchema: DREAM_OUTPUT_SCHEMA,
+    systemPrompt: "You are a background memory consolidation process. Reply with ONLY the requested JSON object. No prose, no markdown, no tool use.",
+  }));
+  const decision = parseDream(utilityPayload(result));
+  if (!decision) {
+    const error = new Error("Dream utility returned unreadable output");
+    error.code = "DREAM_OUTPUT_INVALID";
+    throw error;
+  }
+  return { decision, ...intelligenceMetadata(result) };
+}
+
+async function runDreamMutation({ trigger = "manual", provider = null, runUtility = spawnUtilityAgent } = {}) {
   const packCount = packs.listPacks().length;
   const entityCount = entities.listEntities().length;
   if (packCount === 0 && entityCount === 0) return { skipped: "no memory to consolidate yet" };
 
-  _dreaming = true;
-  try {
-    const nowISO = new Date().toISOString();
+  const nowISO = new Date().toISOString();
     const dreamState = readDreamState();
     const fullSweep = !dreamState.lastFullSweep ||
       (Date.now() - Date.parse(dreamState.lastFullSweep)) / 86400000 >= FULL_SWEEP_DAYS;
@@ -1050,16 +1128,25 @@ async function runDream({ trigger = "manual" } = {}) {
     // decision (applyDream no-ops), note it for the report, and carry on.
     let decision = null;
     let modelFailNote = "";
+    let consolidationIntelligence = intelligenceMetadata(null, "not_run");
     try {
-      const { text } = await spawnSubagent(buildDreamPrompt({ sinceTs: dreamState.lastRun || null, fullSweep, clusters, evidence: ev, windowDelta }), {
-        model: DREAM_MODEL,
-        effort: DREAM_EFFORT,
-        timeoutMs: dreamTimeoutMs(packCount),
-        systemPrompt: "You are a background memory consolidation process. Reply with ONLY the requested JSON object. No prose, no markdown, no tool use.",
-      });
-      decision = parseDream(text);
-      if (!decision) modelFailNote = "the AI consolidation step returned unreadable output";
+      const dreamRun = await runUtility(
+        buildDreamPrompt({ sinceTs: dreamState.lastRun || null, fullSweep, clusters, evidence: ev, windowDelta }),
+        dreamUtilityOptions("dream", {
+          provider,
+          timeoutMs: dreamTimeoutMs(packCount),
+          outputSchema: DREAM_OUTPUT_SCHEMA,
+          systemPrompt: "You are a background memory consolidation process. Reply with ONLY the requested JSON object. No prose, no markdown, no tool use.",
+        }),
+      );
+      consolidationIntelligence = intelligenceMetadata(dreamRun);
+      decision = parseDream(utilityPayload(dreamRun));
+      if (!decision) {
+        consolidationIntelligence.status = "invalid_output";
+        modelFailNote = "the AI consolidation step returned unreadable output";
+      }
     } catch (e) {
+      consolidationIntelligence = failedIntelligenceMetadata(e);
       modelFailNote = `the AI consolidation step didn't finish (${e.message})`;
     }
     if (!decision) decision = { report: "" };
@@ -1102,37 +1189,36 @@ async function runDream({ trigger = "manual" } = {}) {
     }
 
     // Phase 2: self-improvement introspection. Reviews the day's seeds, reads
-    // its own code/memory READ-ONLY (allowedTools whitelist), promotes lessons
+    // its own code/memory through provider-native read-only mode, promotes lessons
     // from missed corrections, refreshes docs, captures ideas. Bounded to memory
     // docs; never code or soul. Applied unless DREAM_SELF_APPLY=off, in which
     // case the proposal is reported for morning approval instead.
     let introReport = "";
     let introApplied = [];
     let introProposed = null;
+    let introspectionIntelligence = intelligenceMetadata(null, introspectEnabled() ? "not_run" : "disabled");
     if (introspectEnabled()) {
       try {
-        const introRun = await spawnSubagent(buildIntrospectionPrompt(), {
-          model: DREAM_MODEL,
-          effort: DREAM_EFFORT,
+        const introRun = await runUtility(buildIntrospectionPrompt(), dreamUtilityOptions("introspection", {
+          provider,
           cwd: CODE_ROOT,
           timeoutMs: 12 * 60 * 1000,
-          // Plan mode is the load-bearing read-only guarantee. Verified: a tool
-          // whitelist alone does NOT restrict, because the bot's default
-          // --dangerously-skip-permissions overrides --allowedTools. Plan mode
-          // (without skip-permissions) is the only setting that actually blocks
-          // writes/Bash while still allowing Read/Glob/Grep — so introspection
-          // can read its own code + memory but cannot edit code or soul.
-          permissionMode: "plan",
-          allowedTools: ["Read", "Glob", "Grep"],
-          systemPrompt: "You are Open Claudia's nightly self-introspection. You may READ files (your code, memory, soul) to understand yourself, but you have no tools to write or run anything. After investigating, reply with ONLY the requested JSON object — no prose, no code fences.",
-        });
-        const intro = parseIntrospection(introRun.text);
+          outputSchema: INTROSPECTION_OUTPUT_SCHEMA,
+          systemPrompt: "You are Open Claudia's nightly self-introspection. You may READ files (your code, memory, soul) to understand yourself, but read-only mode forbids product side effects. After investigating, reply with ONLY the requested JSON object — no prose, no code fences.",
+        }));
+        introspectionIntelligence = intelligenceMetadata(introRun);
+        const intro = parseIntrospection(utilityPayload(introRun));
         if (intro) {
           introReport = intro.report || "";
           if (selfApplyEnabled()) introApplied = applyIntrospection(intro, backupRoot, { personaAlreadyChanged: !!decision.persona });
           else introProposed = intro;
+        } else {
+          introspectionIntelligence.status = "invalid_output";
         }
-      } catch (e) { console.warn(`[dream] introspection phase failed: ${e.message}`); }
+      } catch (e) {
+        introspectionIntelligence = failedIntelligenceMetadata(e);
+        console.warn(`[dream] introspection phase failed: ${e.message}`);
+      }
     }
 
     // Recall-graph maintenance: refresh structural edges, decay reinforced
@@ -1271,10 +1357,15 @@ async function runDream({ trigger = "manual" } = {}) {
     try {
       recallMetrics.appendKpi({
         trigger, fullSweep,
+        providers: {
+          dream: consolidationIntelligence.provider,
+          introspection: introspectionIntelligence.provider,
+        },
         models: {
-          walker: process.env.RECALL_DISCOVERER_MODEL || "haiku",
-          reviewer: process.env.PACK_REVIEW_MODEL || "claude-sonnet-4-6",
-          dream: DREAM_MODEL,
+          walker: process.env.RECALL_DISCOVERER_MODEL || "provider tier:low",
+          reviewer: process.env.PACK_REVIEW_MODEL || "provider tier:medium",
+          dream: consolidationIntelligence.model,
+          introspection: introspectionIntelligence.model,
         },
         window: windowDelta || null,
         health: windowDelta ? healthOf(windowDelta) : null,
@@ -1287,11 +1378,22 @@ async function runDream({ trigger = "manual" } = {}) {
       dreamState.lastRun = nowISO;
       if (fullSweep) dreamState.lastFullSweep = nowISO;
       if (ev?.summary) dreamState.summarySnapshot = ev.summary;
+      dreamState.provider = consolidationIntelligence.provider;
+      dreamState.model = consolidationIntelligence.model;
+      dreamState.effort = consolidationIntelligence.effort;
+      dreamState.fallbackUsed = consolidationIntelligence.fallbackUsed;
+      dreamState.lastRunStatus = consolidationIntelligence.status;
+      dreamState.introspection = introspectionIntelligence;
       writeDreamState(dreamState);
     } catch (e) { console.warn(`[dream] state write failed: ${e.message}`); }
 
     const reportPath = writeDreamReport({
-      model: DREAM_MODEL, effort: DREAM_EFFORT, trigger,
+      provider: consolidationIntelligence.provider,
+      model: consolidationIntelligence.model,
+      effort: consolidationIntelligence.effort || DREAM_EFFORT,
+      intelligence: consolidationIntelligence,
+      introspectionIntelligence,
+      trigger,
       consolidation: { report: decision.report, lines: consolidationLines },
       introspection: { report: introReport, lines: introApplied, proposed: introProposed },
       healthNote, graphNote, episodicNote, toolNote, staleNote, backupRoot,
@@ -1312,16 +1414,70 @@ async function runDream({ trigger = "manual" } = {}) {
     if (episodicNote) parts.push(episodicNote);
     if (toolNote) parts.push(toolNote);
     if (staleNote) parts.push(staleNote);
+    if (consolidationIntelligence.provider || consolidationIntelligence.model) {
+      const introEngine = introspectionIntelligence.provider
+        ? ` · introspection ${introspectionIntelligence.provider}/${introspectionIntelligence.model || "default"}`
+        : "";
+      parts.push(`⚙️ Dream intelligence: ${consolidationIntelligence.provider || "unknown"}/${consolidationIntelligence.model || "default"}${introEngine}`);
+    }
     if (dreamLines.length) parts.push(`🗄 Anything changed or merged away is backed up under ${backupRoot}`);
     if (reportPath) parts.push(`📔 Full dream log: ${reportPath}`);
     const message = parts.length ? parts.join("\n\n") : "";
 
-    return { applied: dreamLines, report, introReport, message, healthNote, staleNote, graphNote, episodicNote, toolNote, reportPath, trigger, fullSweep };
-  } finally {
-    _dreaming = false;
+  return {
+    applied: dreamLines,
+    report,
+    introReport,
+    message,
+    healthNote,
+    staleNote,
+    graphNote,
+    episodicNote,
+    toolNote,
+    reportPath,
+    trigger,
+    fullSweep,
+    provider: consolidationIntelligence.provider,
+    model: consolidationIntelligence.model,
+    effort: consolidationIntelligence.effort,
+    fallbackUsed: consolidationIntelligence.fallbackUsed,
+    introspectionProvider: introspectionIntelligence.provider,
+    introspectionModel: introspectionIntelligence.model,
+    introspectionFallbackUsed: introspectionIntelligence.fallbackUsed,
+  };
+}
+
+function createDreamService(dependencies = {}) {
+  const runUtility = dependencies.spawnUtilityAgent || spawnUtilityAgent;
+  const queue = dependencies.mutationQueue || memoryMutationQueue;
+  let dreaming = false;
+
+  async function runDream({ trigger = "manual", provider = null } = {}) {
+    if (!enabled()) return { skipped: "dream is disabled (DREAM=off)" };
+    if (dreaming) return { skipped: "a dream is already in progress" };
+    dreaming = true;
+    try {
+      return await queue.enqueue("dream", () => runDreamMutation({ trigger, provider, runUtility }), {
+        purpose: "dream",
+        trigger,
+        provider,
+      });
+    } finally {
+      dreaming = false;
+    }
   }
+
+  return {
+    isDreaming: () => dreaming,
+    previewDream: (options) => previewDreamWith(runUtility, options),
+    runDream,
+  };
 }
 
+const defaultDreamService = createDreamService();
+const runDream = defaultDreamService.runDream;
+const previewDream = defaultDreamService.previewDream;
+
 // In-bot scheduler: one quiet-hours cron, reporting to the owner's
 // Telegram chat. When the dream summary is on (default) it posts a short
 // note after every run — including the charming "all tidy" line when
@@ -1346,14 +1502,15 @@ function initDream(adapters) {
       console.error("dream: run failed:", e.message);
     }
   });
-  console.log(`dream: scheduled (${DREAM_CRON}, model ${DREAM_MODEL}, effort ${DREAM_EFFORT}, introspect ${introspectEnabled() ? "on" : "off"})`);
+  console.log(`dream: scheduled (${DREAM_CRON}, provider policy tier ${dreamTier()}, effort ${DREAM_EFFORT}, introspect ${introspectEnabled() ? "on" : "off"})`);
 }
 
 module.exports = {
-  runDream, initDream, buildDreamPrompt, parseDream, applyDream, manageAbilityTiers, nestPrefixFamilies, dreamTimeoutMs,
+  createDreamService, previewDream, runDream, initDream, buildDreamPrompt, parseDream, applyDream, manageAbilityTiers, nestPrefixFamilies, dreamTimeoutMs,
   buildIntrospectionPrompt, parseIntrospection, applyIntrospection, writeDreamReport,
   summaryDelta, healthOf, mergeClusters, selectDumpDirs, journalBackfillDedupe,
   coRescueEdgesPhase, negativeHebbianPhase, readDreamState, writeDreamState,
-  enabled, summaryEnabled, introspectEnabled, selfApplyEnabled,
-  DREAM_CRON, DREAM_MODEL, DREAM_EFFORT, PROMOTE_MIN_PROJECTS, DREAM_STATE_FILE, FULL_SWEEP_DAYS,
+  enabled, summaryEnabled, introspectEnabled, selfApplyEnabled, dreamTier,
+  DREAM_CRON, DREAM_EFFORT, DREAM_OUTPUT_SCHEMA, INTROSPECTION_OUTPUT_SCHEMA,
+  PROMOTE_MIN_PROJECTS, DREAM_STATE_FILE, FULL_SWEEP_DAYS,
 };

package/core/entities.js

@@ -18,6 +18,7 @@
 
 const fs = require("fs");
 const path = require("path");
+const { atomicWriteFileSync } = require("./fsutil");
 
 let DatabaseSync = null;
 try { ({ DatabaseSync } = require("node:sqlite")); } catch (e) { /* old node — matching disabled */ }
@@ -168,7 +169,7 @@ function findEntity(nameOrSlug) {
 
 function writeEntity(ent) {
   ensureDir();
-  fs.writeFileSync(entityFile(ent.slug), serialize(ent), { mode: 0o600 });
+  atomicWriteFileSync(entityFile(ent.slug), serialize(ent), { mode: 0o600, backup: true });
   markIndexDirty();
   return ent.slug;
 }

package/core/fsutil.js

@@ -9,10 +9,11 @@
 const fs = require("fs");
 const path = require("path");
 
-function atomicWriteFileSync(file, data, { backup = false } = {}) {
+function atomicWriteFileSync(file, data, { backup = false, mode = 0o600 } = {}) {
   fs.mkdirSync(path.dirname(file), { recursive: true });
   const tmp = `${file}.${process.pid}.${Date.now()}.tmp`;
-  const fd = fs.openSync(tmp, "w");
+  const fd = fs.openSync(tmp, "w", mode);
+  let renamed = false;
   try {
     fs.writeSync(fd, data);
     fs.fsyncSync(fd);
@@ -22,9 +23,25 @@ function atomicWriteFileSync(file, data, { backup = false } = {}) {
   // Snapshot the previous good version before we overwrite, so a corrupt or
   // wrong write is still recoverable via readJsonWithFallback.
   if (backup) {
-    try { if (fs.existsSync(file)) fs.copyFileSync(file, `${file}.bak`); } catch (e) {}
+    try {
+      if (fs.existsSync(file)) {
+        fs.copyFileSync(file, `${file}.bak`);
+        fs.chmodSync(`${file}.bak`, mode);
+      }
+    } catch (e) {}
+  }
+  try {
+    fs.renameSync(tmp, file);
+    renamed = true;
+    try {
+      const dirFd = fs.openSync(path.dirname(file), "r");
+      try { fs.fsyncSync(dirFd); } finally { fs.closeSync(dirFd); }
+    } catch (_) { /* directory fsync is not supported on every platform */ }
+  } finally {
+    if (!renamed) {
+      try { fs.unlinkSync(tmp); } catch (_) {}
+    }
   }
-  fs.renameSync(tmp, file);
 }
 
 // Parse JSON from `file`, transparently falling back to `<file>.bak` if the