niahere 0.5.8 → 0.5.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "niahere",
-  "version": "0.5.8",
+  "version": "0.5.9",
   "description": "A personal AI assistant daemon — chat, scheduled jobs, persona system, extensible via skills.",
   "type": "module",
   "scripts": {

package/src/chat/engine.ts

@@ -13,6 +13,7 @@ import { log } from "../utils/log";
 import { asError, errMsg, ignore } from "../utils/errors";
 import { registerActiveHandle, unregisterActiveHandle } from "../core/active-handles";
 import { resolveJobPrompt } from "../core/job-prompt";
+import { initialDeliveryStatus } from "../core/delivery";
 import { truncate } from "../utils/format-activity";
 import { resolveChain, ChainCursor, describeEntry, providerHealth, type AgentSession, type FailoverScope } from "../agent";
 import { scopeOf, parseFailure } from "../agent/failure";
@@ -337,7 +338,7 @@ export async function createChatEngine(opts: EngineOptions): Promise<ChatEngine>
                     sender: "nia",
                     content: ev.text,
                     isFromAgent: true,
-                    deliveryStatus: "pending" as const,
+                    deliveryStatus: initialDeliveryStatus(channel),
                     metadata: ev.metadata,
                   };
                   try {

package/src/chat/identity.ts

@@ -7,6 +7,7 @@ import { getAgentsSummary } from "../core/agents";
 import { getEmployeesSummary } from "../core/employees";
 import { Session } from "../db/models";
 import type { Mode } from "../types";
+import { splitMemory } from "../utils/memory-window";
 
 export { type SkillInfo } from "../core/skills";
 
@@ -18,11 +19,18 @@ function loadFile(dir: string, name: string): string {
 
 export function loadIdentity(): string {
   const { selfDir } = getPaths();
-  const files = ["identity.md", "owner.md", "soul.md", "rules.md", "memory.md"];
-  return files
-    .map((f) => loadFile(selfDir, f))
-    .filter(Boolean)
-    .join("\n\n");
+  // Rules load whole: they are verbs, and an unloaded rule is simply not
+  // followed. Memory is nouns — the newest and the curated sections load, the
+  // rest is one `search_memory` call away.
+  const always = ["identity.md", "owner.md", "soul.md", "rules.md"];
+  const parts = always.map((f) => loadFile(selfDir, f)).filter(Boolean);
+
+  const memory = loadFile(selfDir, "memory.md");
+  if (memory) {
+    const { loaded, pointer } = splitMemory(memory);
+    parts.push(loaded + pointer);
+  }
+  return parts.filter(Boolean).join("\n\n");
 }
 
 export function buildSystemPrompt(mode: Mode = "chat", channel: string = "terminal"): string {

package/src/core/delivery.ts

@@ -0,0 +1,60 @@
+import type { Check } from "../types/health";
+
+/**
+ * Whether a reply has a delivery to confirm.
+ *
+ * Every agent reply was written as `pending` and only channel code ever cleared
+ * it — so `nia run`, the REPL and jobs, which print to stdout and are done,
+ * left rows pending forever. 23 of the 30 stuck rows on the mini were that,
+ * oldest from March, which made the genuinely undelivered ones invisible among
+ * them.
+ */
+const DELIVERING_CHANNELS = new Set(["slack", "telegram", "sms", "whatsapp", "phone"]);
+
+/** Channels whose replies go nowhere but the screen. Unknown names are assumed
+ *  to deliver: a false alarm is cheaper than a message quietly lost. */
+const LOCAL_CHANNELS = new Set(["terminal", "system", "test"]);
+
+export function awaitsDelivery(channel: string): boolean {
+  if (DELIVERING_CHANNELS.has(channel)) return true;
+  return !LOCAL_CHANNELS.has(channel);
+}
+
+export function initialDeliveryStatus(channel: string): "pending" | "sent" {
+  return awaitsDelivery(channel) ? "pending" : "sent";
+}
+
+/** Longer than any real send. Slack/Twilio calls finish in seconds. */
+export const STUCK_AFTER_MS = 10 * 60 * 1000;
+
+export interface PendingRow {
+  room: string;
+  createdAt: string;
+}
+
+/**
+ * Report replies that were written but never confirmed.
+ *
+ * Deliberately reports rather than resends. The row says a send was *started*;
+ * it cannot say whether the channel API completed before the process died. A
+ * retry would double-send to a real person on every crash that happened after
+ * delivery — so this surfaces the problem and leaves the judgement to someone
+ * who can check.
+ */
+export function auditDelivery(rows: PendingRow[], now: number = Date.now()): Check {
+  const stuck = rows.filter((r) => {
+    const t = Date.parse(r.createdAt);
+    return Number.isFinite(t) && now - t > STUCK_AFTER_MS;
+  });
+  if (stuck.length === 0) {
+    return { name: "delivery", status: "ok", detail: "no messages awaiting confirmation" };
+  }
+  const oldest = stuck.reduce((a, b) => (Date.parse(a.createdAt) <= Date.parse(b.createdAt) ? a : b));
+  const days = Math.floor((now - Date.parse(oldest.createdAt)) / 86_400_000);
+  const age = days >= 1 ? `${days}d` : `${Math.round((now - Date.parse(oldest.createdAt)) / 3_600_000)}h`;
+  return {
+    name: "delivery",
+    status: "warn",
+    detail: `${stuck.length} repl${stuck.length === 1 ? "y" : "ies"} never confirmed sent, oldest ${age} in ${oldest.room}`,
+  };
+}

package/src/core/health.ts

@@ -9,10 +9,11 @@ import { withRetry } from "../utils/retry";
 import { codexAvailable, codexModelSlugs } from "../agent/catalog";
 import { providerHealth } from "../agent/health";
 import { claudeAuthStatus, codexAuthStatus, type AuthStatus } from "../agent/auth";
+import { auditDelivery } from "./delivery";
 import { IMPLEMENTED, describeRef, planChain, resolveModel, type ModelRef } from "../agent/models";
 
-export type CheckStatus = "ok" | "warn" | "fail";
-export type Check = { name: string; status: CheckStatus; detail: string };
+export type { Check, CheckStatus } from "../types/health";
+import type { Check } from "../types/health";
 
 /** Past this, the chain is not falling back — it has moved. */
 export const FAILOVER_INCIDENT_MS = 60 * 60 * 1000;
@@ -268,6 +269,15 @@ export async function runHealthChecks(): Promise<Check[]> {
   const primary = auth.find((a) => a.provider === plan[0]?.provider);
   checks.push(auditFailover(providerHealth.fallbackStreakMs(), providerHealth.lastServer(), primary));
 
+  // Replies written but never confirmed sent. 25 code paths write this column
+  // and, until now, nothing read it.
+  try {
+    const { Message } = await import("../db/models");
+    checks.push(auditDelivery(await Message.listPendingDeliveries()));
+  } catch (err) {
+    checks.push({ name: "delivery", status: "warn", detail: errMsg(err) });
+  }
+
   // API keys
   const geminiKey = config.gemini_api_key;
   const rawConfig = readRawConfig();

package/src/db/migrations/018_resolve_local_pending.ts

@@ -0,0 +1,20 @@
+import type postgres from "postgres";
+
+export const name = "018_resolve_local_pending";
+
+/**
+ * Replies from `nia run`, the REPL and jobs were written as pending and never
+ * confirmed, because those paths print to stdout and have no delivery to
+ * confirm. They are not undelivered; they were delivered to a terminal.
+ *
+ * Resolving them is what makes the genuinely stuck rows visible — they were
+ * hidden among 23 artifacts going back to March.
+ */
+export async function up(sql: postgres.Sql): Promise<void> {
+  await sql`
+    UPDATE messages SET delivery_status = 'sent'
+    WHERE is_from_agent
+      AND delivery_status = 'pending'
+      AND (room = 'terminal' OR room LIKE 'cli-run%' OR room LIKE '_system/%' OR room LIKE 'debug-%' OR room LIKE '%-debug-%')
+  `;
+}

package/src/db/models/message.ts

@@ -154,3 +154,14 @@ export async function getRoomStats(): Promise<RoomStats[]> {
     lastActivity: r.last_activity ? String(r.last_activity) : null,
   }));
 }
+
+/** Replies still marked pending — written, never confirmed sent. */
+export async function listPendingDeliveries(limit = 200): Promise<{ room: string; createdAt: string }[]> {
+  const sql = getSql();
+  const rows = await sql`
+    SELECT room, created_at FROM messages
+    WHERE is_from_agent AND delivery_status = 'pending'
+    ORDER BY created_at ASC LIMIT ${limit}
+  `;
+  return rows.map((r) => ({ room: String(r.room), createdAt: String(r.created_at) }));
+}

package/src/mcp/tools/misc.ts

@@ -4,17 +4,23 @@ import { join } from "path";
 import { getPaths } from "../../utils/paths";
 import { scanAgents } from "../../core/agents";
 import { listEmployeesForMcp } from "../../core/employees";
-import { readMemory as readMemoryUtil, addMemory as addMemoryUtil } from "../../utils/memory";
+import { readMemory as readMemoryUtil, addMemory as addMemoryUtil, searchMemory as searchMemoryUtil } from "../../utils/memory";
+import { findSecret } from "../../utils/secrets";
 
 export function addRule(rule: string): string {
+  const trimmed = rule.trim();
+  if (!trimmed) return "Rejected: empty rule.";
+  const secret = findSecret(trimmed);
+  if (secret) return `Rejected: looks like a ${secret}. Rules load into every session's prompt — put credentials in config.yaml.`;
   const { selfDir } = getPaths();
   const rulesPath = join(selfDir, "rules.md");
-  const line = `\n- ${rule}\n`;
+  const line = `\n- ${trimmed}\n`;
   appendFileSync(rulesPath, line, "utf8");
   return `Rule added to rules.md. Takes effect on next new session.`;
 }
 
 export const readMemory = readMemoryUtil;
+export const searchMemory = searchMemoryUtil;
 export const addMemory = addMemoryUtil;
 
 export function listAgents(): string {

package/src/mcp/tools/table.ts

@@ -210,6 +210,16 @@ export const NIA_TOOLS: NiaTool[] = [
     schema: { rule: z.string().describe("The rule to add (e.g. 'stamp updates: 1-2 lines max, no preamble')") },
     handler: (args) => handlers.addRule(args.rule),
   },
+  {
+    name: "search_memory",
+    description:
+      "Search all durable memory, including older entries that are not in your prompt. Only the newest and the curated sections are loaded each session, so check here before concluding something was never recorded.",
+    schema: {
+      query: z.string().describe("Text to look for in saved memories"),
+      limit: z.number().optional().describe("Max matches to return (default 20)"),
+    },
+    handler: (args) => handlers.searchMemory(args.query, args.limit),
+  },
   {
     name: "read_memory",
     description:

package/src/types/health.ts

@@ -0,0 +1,9 @@
+export type CheckStatus = "ok" | "warn" | "fail";
+
+/** One health finding. Lives here rather than beside the checks so a module can
+ *  return a Check without importing the aggregator that calls it. */
+export interface Check {
+  name: string;
+  status: CheckStatus;
+  detail: string;
+}

package/src/utils/memory-window.ts

@@ -0,0 +1,121 @@
+/**
+ * Load the memory that earns its place in every prompt, and make the rest
+ * reachable.
+ *
+ * `memory.md` reached 37.7 KB and, with `rules.md`, 54% of the chat system
+ * prompt — paid on every turn, on every channel, forever. But the file is not
+ * uniform: hand-organised topical sections (`## Personal`, `## Nia
+ * Architecture`) are curated and durable, while `## Promoted <date>` sections
+ * are an append log where recency is the best available proxy for relevance.
+ *
+ * So the topical sections always load, the newest dated ones load until a byte
+ * budget is spent, and everything else stays one `search_memory` call away.
+ * Rules are deliberately untouched: rules are verbs and an unloaded rule is
+ * simply not followed, where memory is nouns and can be fetched on demand.
+ */
+
+/** Roughly a third of what memory.md had grown to. */
+export const MEMORY_BUDGET_BYTES = 12_000;
+
+const DATED = /\d{4}-\d{2}-\d{2}/;
+
+interface Section {
+  heading: string;
+  body: string;
+  dated: boolean;
+}
+
+function parseSections(text: string): { preamble: string; sections: Section[] } {
+  const lines = text.split("\n");
+  const preamble: string[] = [];
+  const sections: Section[] = [];
+  let current: Section | null = null;
+
+  for (const line of lines) {
+    const m = /^##\s+(.*)$/.exec(line);
+    if (m) {
+      if (current) sections.push(current);
+      current = { heading: m[1]!.trim(), body: "", dated: DATED.test(m[1]!) };
+      continue;
+    }
+    if (current) current.body += line + "\n";
+    else preamble.push(line);
+  }
+  if (current) sections.push(current);
+  return { preamble: preamble.join("\n").trimEnd(), sections };
+}
+
+const render = (s: Section) => `## ${s.heading}\n${s.body}`.trimEnd();
+
+export interface MemorySplit {
+  /** What goes into the system prompt. */
+  loaded: string;
+  /** What was held back. */
+  deferred: string;
+  deferredSections: number;
+  /** A line telling the model the rest exists. Empty when nothing was held. */
+  pointer: string;
+}
+
+export function splitMemory(text: string, budgetBytes: number = MEMORY_BUDGET_BYTES): MemorySplit {
+  const { preamble, sections } = parseSections(text);
+  const topical = sections.filter((s) => !s.dated);
+  const datedNewestFirst = sections.filter((s) => s.dated).reverse();
+
+  const head = [preamble, ...topical.map(render)].filter(Boolean).join("\n\n");
+  let used = head.length;
+  const keep: Section[] = [];
+  for (const s of datedNewestFirst) {
+    const cost = render(s).length + 2;
+    if (used + cost > budgetBytes && keep.length > 0) break;
+    keep.push(s);
+    used += cost;
+  }
+
+  const keptSet = new Set(keep);
+  const held = sections.filter((s) => s.dated && !keptSet.has(s));
+  // Restore document order for whatever is loaded.
+  const loadedDated = sections.filter((s) => keptSet.has(s));
+
+  const loaded = [head, ...loadedDated.map(render)].filter(Boolean).join("\n\n");
+  const deferred = held.map(render).join("\n\n");
+  const entries = held.reduce((n, s) => n + (s.body.match(/^- /gm)?.length ?? 0), 0);
+
+  return {
+    loaded,
+    deferred,
+    deferredSections: held.length,
+    pointer: held.length
+      ? `\n\n_(${entries} older memories from ${held.length} earlier dates are not shown. Use the \`search_memory\` tool to look them up before assuming something is not recorded.)_`
+      : "",
+  };
+}
+
+export interface MemoryHit {
+  section: string;
+  entry: string;
+}
+
+/** Plain substring search. At this size an exact scan beats an index that has
+ *  to be kept in sync, and it cannot hallucinate a near-match. */
+export function searchMemoryText(text: string, query: string, limit = 20): MemoryHit[] {
+  const needle = query.trim().toLowerCase();
+  if (!needle) return [];
+  const { preamble, sections } = parseSections(text);
+  const scan: { section: string; body: string }[] = [
+    { section: "(top)", body: preamble },
+    ...sections.map((s) => ({ section: s.heading, body: s.body })),
+  ];
+
+  const hits: MemoryHit[] = [];
+  for (const { section, body } of scan) {
+    for (const line of body.split("\n")) {
+      if (!line.trim().startsWith("- ")) continue;
+      if (line.toLowerCase().includes(needle)) {
+        hits.push({ section, entry: line.trim() });
+        if (hits.length >= limit) return hits;
+      }
+    }
+  }
+  return hits;
+}

package/src/utils/memory.ts

@@ -8,6 +8,8 @@
 import { existsSync, readFileSync, appendFileSync, writeFileSync } from "fs";
 import { join } from "path";
 import { getPaths } from "./paths";
+import { findSecret } from "./secrets";
+import { searchMemoryText } from "./memory-window";
 
 export function readMemory(): string {
   const { selfDir } = getPaths();
@@ -31,6 +33,8 @@ export function addMemory(entry: string): string {
   if (trimmed.includes("[Thread context]") || trimmed.includes("[Current messag"))
     return "Rejected: no raw conversation transcripts.";
   if (trimmed.split("\n").length > 5) return "Rejected: too many lines. One concise insight per memory.";
+  const secret = findSecret(trimmed);
+  if (secret) return `Rejected: looks like a ${secret}. Durable memory loads into every session's prompt — put credentials in config.yaml.`;
 
   const { selfDir } = getPaths();
   const memoryPath = join(selfDir, "memory.md");
@@ -47,3 +51,13 @@ export function addMemory(entry: string): string {
   }
   return `Memory saved.`;
 }
+
+/** Search all durable memory, including entries not loaded into this prompt. */
+export function searchMemory(query: string, limit = 20): string {
+  const { selfDir } = getPaths();
+  const path = join(selfDir, "memory.md");
+  if (!existsSync(path)) return "No memory file yet.";
+  const hits = searchMemoryText(readFileSync(path, "utf8"), query, limit);
+  if (hits.length === 0) return `No memory matches "${query}".`;
+  return hits.map((h) => `[${h.section}] ${h.entry}`).join("\n");
+}

package/src/utils/secrets.ts

@@ -0,0 +1,41 @@
+/**
+ * Credential detection for anything about to be written into durable memory.
+ *
+ * `memory.md` and `rules.md` load into every session's system prompt, so a key
+ * that lands there has permanent, maximum-exposure blast radius — it is handed
+ * to every model, on every backend, on every turn, until someone notices and
+ * edits the file. `addMemory()` checked length and line count; `addRule()`
+ * validated nothing at all and appended straight to disk.
+ *
+ * Patterns are anchored on the issuer's own format rather than on words like
+ * "key" or "token", so discussing credentials stays possible and pasting one
+ * does not.
+ */
+
+export interface SecretPattern {
+  name: string;
+  pattern: RegExp;
+}
+
+export const SECRET_PATTERNS: SecretPattern[] = [
+  { name: "Anthropic API key", pattern: /\bsk-ant-api\d{2}-[A-Za-z0-9_-]{20,}/ },
+  { name: "Claude OAuth token", pattern: /\bsk-ant-oat\d{2}-[A-Za-z0-9_-]{20,}/ },
+  { name: "OpenAI key", pattern: /\bsk-(?:proj|svcacct|admin)?-?[A-Za-z0-9_-]{32,}/ },
+  { name: "Slack bot/user token", pattern: /\bxox[abpsr]-[A-Za-z0-9-]{10,}/ },
+  { name: "Slack app token", pattern: /\bxapp-\d-[A-Za-z0-9-]{10,}/ },
+  { name: "GitHub token", pattern: /\bgh[pousr]_[A-Za-z0-9]{20,}/ },
+  { name: "AWS access key id", pattern: /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/ },
+  { name: "database URL with password", pattern: /\b[a-z][a-z0-9+.-]*:\/\/[^\s:@/]+:[^\s@/]+@/i },
+  { name: "bearer credential", pattern: /\bBearer\s+[A-Za-z0-9._~+/-]{20,}/ },
+  { name: "private key block", pattern: /-----BEGIN (?:[A-Z ]+ )?PRIVATE KEY-----/ },
+  { name: "Google API key", pattern: /\bAIza[A-Za-z0-9_-]{30,}/ },
+  { name: "Twilio auth", pattern: /\bSK[a-f0-9]{32}\b/ },
+];
+
+/** The name of the first credential found, or null. */
+export function findSecret(text: string): string | null {
+  for (const { name, pattern } of SECRET_PATTERNS) {
+    if (pattern.test(text)) return name;
+  }
+  return null;
+}