niahere 0.2.61 → 0.2.63

package/src/core/consolidator.ts

@@ -1,20 +1,13 @@
 /**
- * Memory consolidator — "hippocampal replay" for Nia.
+ * Memory consolidator — stage 1 of the two-stage memory pipeline.
  *
- * After a chat session goes idle or a job completes, this module reviews
- * what happened and saves memories worth keeping.
+ * After a chat session goes idle, reflects on the transcript and appends
+ * CANDIDATE memories to ~/.niahere/self/staging.md. The nightly
+ * memory-promoter job handles promotion from staging to memory.md/rules.md.
+ * The write-path restriction is enforced by the consolidator prompt, not
+ * by tool sandboxing.
  *
- * This decouples memory formation from task execution: during a conversation,
- * the agent focuses on the task. Afterward, a background pass extracts what's
- * worth remembering — just like the brain consolidates memories during sleep.
- *
- * The consolidator uses the same agent loop as cron jobs — full Nia system
- * prompt, full tool access, same runner. It's just a specialized job.
- *
- * Research basis:
- * - LangChain: "background" memory formation avoids latency + competing optimization pressures
- * - Mem0: LLM-driven extraction with ADD/UPDATE/NOOP decisions against existing memories
- * - Cognitive science: hippocampal replay consolidates experiences after the fact, not during
+ * See AGENTS.md > "Two-stage memory" for the full architecture.
  */
 
 import { Message } from "../db/models";
@@ -40,71 +33,114 @@ function shouldSkip(room: string): boolean {
 function formatTranscript(messages: SessionMessage[]): string {
   const recent = messages.slice(-MAX_TRANSCRIPT_MESSAGES);
   const skipped = messages.length - recent.length;
-  const prefix =
-    skipped > 0 ? `[...${skipped} earlier messages omitted]\n\n` : "";
-
-  return (
-    prefix +
-    recent
-      .map((m) => `[${m.sender}] (${m.createdAt}): ${m.content.slice(0, 2000)}`)
-      .join("\n\n")
-  );
+  const prefix = skipped > 0 ? `[...${skipped} earlier messages omitted]\n\n` : "";
+
+  return prefix + recent.map((m) => `[${m.sender}] (${m.createdAt}): ${m.content.slice(0, 2000)}`).join("\n\n");
 }
 
-/** Build the extraction prompt from a conversation transcript. */
+/** Build the consolidation prompt from a conversation transcript. */
 function buildConsolidationPrompt(transcript: string, source: string): string {
+  const today = new Date().toISOString().slice(0, 10);
   return `Job: memory-consolidation (triggered by ${source})
 
-You just finished a session. It has gone idle.
-Your task: review the transcript below and save anything worth keeping for future sessions.
+A chat session has gone idle. Your task is to reflect on it and update
+the memory staging log — NOT the durable memory files.
+
+## Context
+
+Nia uses a two-stage memory architecture. You are stage 1.
+
+- Stage 1 (you): append candidates to \`~/.niahere/self/staging.md\`. Never
+  write to \`memory.md\` or \`rules.md\` directly.
+- Stage 2: the nightly \`memory-promoter\` job reviews candidates with
+  \`count >= 2\` and promotes qualifying ones to durable memory. Entries with
+  \`count < 2\` expire after 14 days.
+
+Your persona includes guidance to "save proactively" — that guidance applies
+to LIVE chat, where you act on immediate user instruction. In THIS
+consolidation pass, your default action is to do nothing. You only act when
+you can point to a specific user turn that taught you something durable.
 
 ## Transcript
+
 ${transcript}
 
-## Instructions
-1. First, read your existing memories (read_memory tool) and rules (read rules.md) to avoid duplicates
-2. Review the transcript for things worth persisting. Use the RIGHT tool for each:
-
-   **Use add_memory for FACTS** (nouns — things that are true):
-   - People: names, roles, orgs, relationships
-   - Decisions: what was decided or agreed on
-   - Technical facts: system details, API quirks, config gotchas
-   - Patterns: recurring issues, user behaviors, workflow tendencies
-   - Events: travel, deadlines, incidents, milestones with dates
-
-   **Use add_rule for INSTRUCTIONS** (verbs — how to behave):
-   - User corrected your tone, format, or approach
-   - User said "from now on" / "always" / "never" / "stop doing X"
-   - User expressed a preference about how you communicate or work
-
-3. Skip anything already in existing memories or rules (no duplicates)
-4. Skip small talk, greetings, conversational filler
-5. Skip transient state ("currently working on X")
-6. Quality over quantity — saving nothing is fine if the conversation was trivial
-7. If existing memories are outdated based on new info, note what should be updated
-
-Do NOT message the user about this. Save silently and report a brief summary of what you saved.`;
+## Step 1 — Read existing state
+
+Read these files in full before doing anything else. You need to know what
+already exists so you can dedupe and reinforce rather than duplicate.
+
+- \`~/.niahere/self/memory.md\` — durable facts already saved
+- \`~/.niahere/self/rules.md\` — behavioral rules already in effect
+- \`~/.niahere/self/staging.md\` — candidates already staged (including the
+  file's header, which documents the staging format)
+
+## Step 2 — Reflect
+
+Answer these questions silently. If the answer to all of them is "nothing",
+stop here and do not write anything.
+
+1. What did the user correct, clarify, or teach you in this session?
+2. What NEW fact about the user, their projects, or their systems do you
+   now know that you did not at session start?
+3. What decision was made that will constrain future work?
+
+Trivial small talk, greetings, task-execution chatter, and status updates
+are NOT answers. If you cannot quote a specific user turn that produced the
+learning, you are fishing — stop.
+
+## Step 3 — Update staging.md
+
+For each substantive answer:
+
+1. Check \`memory.md\` and \`rules.md\`. If the learning is already covered
+   there, do nothing — it is already durable.
+2. Check \`staging.md\`. If there is a near-match (same subject, same intent,
+   even if worded differently):
+   - Use the Edit tool to bump the count: \`[1×]\` → \`[2×]\`, \`[2×]\` → \`[3×]\`
+   - Update the \`last_seen\` date to \`${today}\`
+   - Do NOT append a new line
+3. If genuinely new AND durable AND fits one of the four types, append a
+   new line to staging.md using this exact format:
+
+   \`- [1×] [type] content :: ${today} → ${today}\`
+
+   Where \`type\` is exactly one of:
+   - \`persona\`    — facts about the user (role, habits, preferences)
+   - \`project\`    — active work decisions, architecture, stakeholders
+   - \`reference\`  — pointers to external systems (dashboards, repos)
+   - \`correction\` — behavioral preference for how Nia should work
+
+   If the learning does not fit one of these four types, do not stage it.
+
+## Hard constraints
+
+- Do NOT write to \`memory.md\` or \`rules.md\`. Only the promoter job can.
+- Do NOT use \`add_memory\` or \`add_rule\` MCP tools. Edit staging.md directly.
+- Do NOT message the user.
+- Default action is to do nothing. Most sessions have nothing to stage.
+
+Report a one-line summary of what you did: "staged N new / reinforced M /
+skipped (trivial session)". No preamble.`;
 }
 
-/** Run the consolidation agent loop. */
-async function runConsolidation(
-  transcript: string,
-  source: string,
-): Promise<void> {
-  await runTask({
+async function runConsolidation(transcript: string, source: string): Promise<void> {
+  const output = await runTask({
     name: "consolidator",
     prompt: buildConsolidationPrompt(transcript, source),
   });
+  // runTask returns {error} on failure instead of throwing; escalate so
+  // consolidateSession doesn't mark the session processed on a failed run.
+  if (output.error) {
+    throw new Error(`consolidator task failed: ${output.error}`);
+  }
 }
 
 /**
  * Consolidate a chat session's conversation into memories.
  * Called when a chat engine goes idle or is explicitly closed.
  */
-export async function consolidateSession(
-  sessionId: string,
-  room: string,
-): Promise<void> {
+export async function consolidateSession(sessionId: string, room: string): Promise<void> {
   if (shouldSkip(room)) return;
   if (inFlight.has(sessionId)) return;
 
@@ -117,10 +153,7 @@ export async function consolidateSession(
 
     inFlight.add(sessionId);
 
-    log.info(
-      { sessionId, room, messageCount: messages.length },
-      "consolidator: extracting memories from chat",
-    );
+    log.info({ sessionId, room, messageCount: messages.length }, "consolidator: extracting memories from chat");
 
     const transcript = formatTranscript(messages);
     await runConsolidation(transcript, `chat session idle — ${room}`);
@@ -135,35 +168,8 @@ export async function consolidateSession(
     }
   } catch (err) {
     log.error({ err, sessionId, room }, "consolidator: chat extraction failed");
+    throw err;
   } finally {
     inFlight.delete(sessionId);
   }
 }
-
-/**
- * Consolidate a job run's output into memories.
- * Called after a job completes in the runner.
- */
-export async function consolidateJobRun(
-  jobName: string,
-  jobPrompt: string,
-  result: string,
-): Promise<void> {
-  // Skip if the job itself is the consolidator (prevent infinite loop)
-  if (jobName === "memory-consolidation") return;
-
-  const transcript = `[job-prompt]: ${jobPrompt}\n\n[job-result]: ${result}`;
-
-  // Skip trivial results
-  if (result.length < 50) return;
-
-  try {
-    log.info(
-      { jobName, resultChars: result.length },
-      "consolidator: extracting memories from job",
-    );
-    await runConsolidation(transcript, `job run — ${jobName}`);
-  } catch (err) {
-    log.error({ err, jobName }, "consolidator: job extraction failed");
-  }
-}

package/src/core/daemon.ts

@@ -1,9 +1,11 @@
-import { closeSync, existsSync, mkdirSync, openSync, readFileSync, unlinkSync, writeFileSync } from "fs";
-import { dirname } from "path";
+import { closeSync, existsSync, mkdirSync, openSync, readFileSync, writeFileSync } from "fs";
+import { dirname, resolve as pathResolve } from "path";
+import { fileURLToPath } from "url";
 import { getPaths } from "../utils/paths";
 import { getConfig, resetConfig } from "../utils/config";
 import { log } from "../utils/log";
-import { ActiveEngine } from "../db/models";
+import { isRunning, readPid, removePid, writePid } from "../utils/pid";
+import { ActiveEngine, Job } from "../db/models";
 import { runMigrations } from "../db/migrate";
 import { closeDb, getSql } from "../db/connection";
 import { registerAllChannels, startChannels, stopChannels, getStarted } from "../channels";
@@ -14,45 +16,7 @@ import { createNiaMcpServer } from "../mcp/server";
 import { setMcpFactory } from "../mcp";
 import { processPending, cleanupOldRequests } from "./finalizer";
 
-export function writePid(pid: number): void {
-  const { pid: pidPath } = getPaths();
-  mkdirSync(dirname(pidPath), { recursive: true });
-  writeFileSync(pidPath, String(pid));
-}
-
-export function readPid(): number | null {
-  const { pid: pidPath } = getPaths();
-  if (!existsSync(pidPath)) return null;
-
-  try {
-    return parseInt(readFileSync(pidPath, "utf8").trim(), 10);
-  } catch {
-    return null;
-  }
-}
-
-export function removePid(): void {
-  const { pid: pidPath } = getPaths();
-  try {
-    unlinkSync(pidPath);
-  } catch {
-    // Already gone
-  }
-}
-
-export function isRunning(): boolean {
-  const pid = readPid();
-  if (pid === null) return false;
-
-  try {
-    process.kill(pid, 0);
-    return true;
-  } catch {
-    log.warn({ stalePid: pid }, "removing stale pid file (process not running)");
-    removePid();
-    return false;
-  }
-}
+export { isRunning, readPid, removePid, writePid };
 
 export function startDaemon(): number {
   const { daemonLog } = getPaths();
@@ -124,7 +88,7 @@ function waitForExit(timeoutMs: number): void {
 /** Return PIDs of running daemon processes (excluding ourselves). */
 export function findDaemonPids(): number[] {
   try {
-    const result = Bun.spawnSync(["pgrep", "-f", "src/cli\\.ts run$"]);
+    const result = Bun.spawnSync(["pgrep", "-f", "src/cli/index\\.ts run$"]);
     const stdout = new TextDecoder().decode(result.stdout).trim();
     if (!stdout) return [];
     return stdout
@@ -149,6 +113,44 @@ function killAllDaemons(knownPid?: number | null): number {
   return toKill.size;
 }
 
+// ---------------------------------------------------------------------------
+// System jobs — auto-installed on daemon startup
+// ---------------------------------------------------------------------------
+
+/**
+ * Ensure system jobs exist. Uses create-if-not-exists semantics so user
+ * customizations (via `nia job update/disable`) are preserved across restarts.
+ * Currently manages: memory-promoter.
+ */
+async function bootstrapSystemJobs(): Promise<void> {
+  const here = dirname(fileURLToPath(import.meta.url));
+
+  const systemJobs = [
+    {
+      name: "memory-promoter",
+      schedule: "0 3 * * *",
+      scheduleType: "cron" as const,
+      always: true,
+      stateless: true,
+      promptPath: pathResolve(here, "../../defaults/memory-promoter.md"),
+    },
+  ];
+
+  for (const j of systemJobs) {
+    const existing = await Job.get(j.name);
+    if (existing) continue;
+
+    if (!existsSync(j.promptPath)) {
+      log.warn({ job: j.name, promptPath: j.promptPath }, "system job prompt missing, skipping bootstrap");
+      continue;
+    }
+
+    const prompt = readFileSync(j.promptPath, "utf8");
+    await Job.create(j.name, j.schedule, prompt, j.always, j.scheduleType, undefined, undefined, j.stateless);
+    log.info({ job: j.name, schedule: j.schedule }, "bootstrapped system job");
+  }
+}
+
 export async function runDaemon(): Promise<void> {
   // Ensure we never pass nested-session env vars to SDK subprocesses,
   // regardless of how the daemon was launched (nia start, nia run, etc.)
@@ -210,6 +212,24 @@ export async function runDaemon(): Promise<void> {
     log.warn({ err }, "failed to ensure watches dir");
   }
 
+  // Ensure staging.md exists — used by the memory consolidator to stage
+  // candidate memories before the nightly promoter reviews them. Existing
+  // installs that pre-date two-stage memory get a seed file on next start.
+  try {
+    const stagingPath = `${getPaths().selfDir}/staging.md`;
+    if (!existsSync(stagingPath)) {
+      const here = dirname(fileURLToPath(import.meta.url));
+      const seed = pathResolve(here, "../../defaults/self/staging.md");
+      if (existsSync(seed)) {
+        mkdirSync(getPaths().selfDir, { recursive: true });
+        writeFileSync(stagingPath, readFileSync(seed, "utf8"));
+        log.info({ stagingPath }, "seeded staging.md from defaults");
+      }
+    }
+  } catch (err) {
+    log.warn({ err }, "failed to ensure staging.md");
+  }
+
   // Startup recovery
   try {
     await runMigrations();
@@ -219,6 +239,14 @@ export async function runDaemon(): Promise<void> {
     log.warn({ err }, "startup recovery: postgres unavailable, skipping");
   }
 
+  // Seed system jobs (create-if-not-exists). Currently: memory-promoter.
+  // Users can disable or customize these via `nia job disable/update`.
+  try {
+    await bootstrapSystemJobs();
+  } catch (err) {
+    log.warn({ err }, "failed to bootstrap system jobs");
+  }
+
   // Clear stale "running" job states
   const { readState, writeState } = await import("../utils/logger");
   const state = readState();

package/src/core/finalizer.ts

@@ -78,15 +78,32 @@ async function processOne(sessionId: string, room: string, messageCount: number)
   const requestId = claimed[0].id;
 
   try {
-    await Promise.allSettled([consolidateSession(sessionId, room), summarizeSession(sessionId, room)]);
+    const [consolidateResult, summarizeResult] = await Promise.allSettled([
+      consolidateSession(sessionId, room),
+      summarizeSession(sessionId, room),
+    ]);
+
+    const errors: string[] = [];
+    if (consolidateResult.status === "rejected") {
+      errors.push(`consolidate: ${formatRejection(consolidateResult.reason)}`);
+    }
+    if (summarizeResult.status === "rejected") {
+      errors.push(`summarize: ${formatRejection(summarizeResult.reason)}`);
+    }
+
+    const finalStatus = errors.length === 0 ? "done" : "failed";
 
     await sql`
       UPDATE finalization_requests
-      SET status = 'done', updated_at = NOW()
+      SET status = ${finalStatus}, updated_at = NOW()
       WHERE id = ${requestId}
     `;
 
-    log.info({ sessionId, room, messageCount }, "finalizer: completed");
+    if (errors.length === 0) {
+      log.info({ sessionId, room, messageCount }, "finalizer: completed");
+    } else {
+      log.error({ sessionId, room, messageCount, errors }, "finalizer: completed with task failures");
+    }
   } catch (err) {
     await sql`
       UPDATE finalization_requests
@@ -98,6 +115,17 @@ async function processOne(sessionId: string, room: string, messageCount: number)
   }
 }
 
+/** Normalize a Promise rejection reason into a loggable string. */
+function formatRejection(reason: unknown): string {
+  if (reason instanceof Error) return reason.message;
+  if (typeof reason === "string") return reason;
+  try {
+    return JSON.stringify(reason);
+  } catch {
+    return String(reason);
+  }
+}
+
 /** Drain all pending finalization requests. Called by daemon on startup and on NOTIFY. */
 export async function processPending(): Promise<void> {
   const sql = getSql();

package/src/core/health.ts

@@ -2,7 +2,7 @@ import { existsSync, statSync } from "fs";
 import { join } from "path";
 import { getConfig, readRawConfig } from "../utils/config";
 import { getPaths } from "../utils/paths";
-import { isRunning, readPid } from "./daemon";
+import { isRunning, readPid } from "../utils/pid";
 import { errMsg } from "../utils/errors";
 import { localTime } from "../utils/time";
 import { withRetry } from "../utils/retry";
@@ -123,11 +123,7 @@ export async function runHealthChecks(): Promise<Check[]> {
           }),
         );
         const data = (await resp.json()) as { ok: boolean; error?: string };
-        results.push(
-          data.ok
-            ? "slack: connected"
-            : `slack: ${data.error || "auth failed"}`,
-        );
+        results.push(data.ok ? "slack: connected" : `slack: ${data.error || "auth failed"}`);
         if (!data.ok)
           checks.push({
             name: "slack",
@@ -159,10 +155,7 @@ export async function runHealthChecks(): Promise<Check[]> {
   // API keys
   const geminiKey = config.gemini_api_key;
   const rawConfig = readRawConfig();
-  const openaiKey =
-    typeof rawConfig.openai_api_key === "string"
-      ? rawConfig.openai_api_key
-      : null;
+  const openaiKey = typeof rawConfig.openai_api_key === "string" ? rawConfig.openai_api_key : null;
   const apiKeys: string[] = [];
   if (geminiKey) apiKeys.push("gemini");
   if (openaiKey) apiKeys.push("openai");
@@ -174,16 +167,11 @@ export async function runHealthChecks(): Promise<Check[]> {
 
   // Persona files
   const personaFiles = ["identity.md", "owner.md", "soul.md"];
-  const missing = personaFiles.filter(
-    (f) => !existsSync(join(paths.selfDir, f)),
-  );
+  const missing = personaFiles.filter((f) => !existsSync(join(paths.selfDir, f)));
   checks.push({
     name: "persona",
     status: missing.length === 0 ? "ok" : "warn",
-    detail:
-      missing.length === 0
-        ? "all files present"
-        : "missing: " + missing.join(", "),
+    detail: missing.length === 0 ? "all files present" : "missing: " + missing.join(", "),
   });
 
   // Daemon log

package/src/core/runner.ts

@@ -33,11 +33,7 @@ function resolveCodexPath(): string {
   return candidates.find((p) => existsSync(p)) || "codex";
 }
 
-async function runJobWithCodex(
-  fullPrompt: string,
-  cwd: string,
-  model: string,
-): Promise<RunnerOutput> {
+async function runJobWithCodex(fullPrompt: string, cwd: string, model: string): Promise<RunnerOutput> {
   const codexPath = resolveCodexPath();
   const args = [
     codexPath,
@@ -72,10 +68,7 @@ async function runJobWithCodex(
       if (event.type === "thread.started" && event.thread_id) {
         sessionId = event.thread_id;
       }
-      if (
-        event.type === "item.completed" &&
-        event.item?.type === "agent_message"
-      ) {
+      if (event.type === "item.completed" && event.item?.type === "agent_message") {
         agentText = event.item.text || "";
       }
     } catch {}
@@ -153,10 +146,7 @@ export async function runJobWithClaude(
 
         if (message.type === "stream_event") {
           const event = msg.event;
-          if (
-            event?.type === "content_block_start" &&
-            event.content_block?.type === "thinking"
-          ) {
+          if (event?.type === "content_block_start" && event.content_block?.type === "thinking") {
             accumulatedThinking = "";
             lastThinkingLine = "";
             onActivity("thinking...");
@@ -253,10 +243,7 @@ export async function runTask(opts: TaskOptions): Promise<RunnerOutput> {
     if (output.error) {
       log.error({ task: opts.name, error: output.error }, "task failed");
     } else {
-      log.info(
-        { task: opts.name, resultChars: output.agentText.length },
-        "task completed",
-      );
+      log.info({ task: opts.name, resultChars: output.agentText.length }, "task completed");
     }
     return output;
   } finally {
@@ -269,10 +256,7 @@ export async function runTask(opts: TaskOptions): Promise<RunnerOutput> {
 // ---------------------------------------------------------------------------
 
 /** Build the working memory block for a stateful job. Returns empty string for stateless jobs. */
-export function buildWorkingMemory(
-  jobName: string,
-  stateless?: boolean,
-): string {
+export function buildWorkingMemory(jobName: string, stateless?: boolean): string {
   if (stateless) return "";
 
   const jobDir = join(getPaths().jobsDir, jobName);
@@ -287,9 +271,7 @@ export function buildWorkingMemory(
     }
   }
 
-  const stateBlock = stateContent
-    ? `\n${stateContent}\n`
-    : "(first run — no prior state)";
+  const stateBlock = stateContent ? `\n${stateContent}\n` : "(first run — no prior state)";
 
   return `
 
@@ -307,10 +289,7 @@ Before finishing, update \`state.md\` with: what you did this run, what you noti
 // Public API
 // ---------------------------------------------------------------------------
 
-export async function runJob(
-  job: JobInput,
-  onActivity?: ActivityCallback,
-): Promise<JobResult> {
+export async function runJob(job: JobInput, onActivity?: ActivityCallback): Promise<JobResult> {
   const config = getConfig();
   const timestamp = new Date().toISOString();
   const startMs = performance.now();
@@ -354,13 +333,7 @@ export async function runJob(
       const fullPrompt = `${systemPrompt}\n\n---\n\n${jobPrompt}`;
       output = await runJobWithCodex(fullPrompt, cwd, resolvedModel);
     } else {
-      output = await runJobWithClaude(
-        systemPrompt,
-        jobPrompt,
-        cwd,
-        onActivity,
-        resolvedModel,
-      );
+      output = await runJobWithClaude(systemPrompt, jobPrompt, cwd, onActivity, resolvedModel);
     }
 
     const duration_ms = Math.round(performance.now() - startMs);
@@ -399,15 +372,6 @@ export async function runJob(
     };
     writeState(freshState);
 
-    // Memory consolidation — review what the job learned (fire-and-forget)
-    if (ok && result.result) {
-      import("./consolidator")
-        .then(({ consolidateJobRun }) => {
-          consolidateJobRun(job.name, jobPrompt, result.result).catch(() => {});
-        })
-        .catch(() => {});
-    }
-
     return result;
   } catch (err) {
     const duration_ms = Math.round(performance.now() - startMs);