@inetafrica/open-claudia 2.4.2 → 2.6.0

package/core/entities.js

@@ -0,0 +1,329 @@
+// Entity memory: short living notes on the people, places, projects,
+// orgs and systems that come up in conversation (a Honcho-style store,
+// our own version). Each entity is ~/.open-claudia/entities/<slug>.md
+// with frontmatter and two sections:
+//   ## Notes — current truth about the entity (replaces wholesale)
+//   ## Log   — dated one-line observations, newest last
+// The pre-turn router injects matched entities alongside context packs;
+// the post-turn reviewer extracts/updates them after each turn.
+
+const fs = require("fs");
+const path = require("path");
+
+let DatabaseSync = null;
+try { ({ DatabaseSync } = require("node:sqlite")); } catch (e) { /* old node — matching disabled */ }
+
+const CONFIG_DIR = require("../config-dir");
+const ENTITIES_DIR = process.env.ENTITIES_DIR ? path.resolve(process.env.ENTITIES_DIR) : path.join(CONFIG_DIR, "entities");
+
+const TYPES = ["person", "place", "project", "org", "system", "thing"];
+const MAX_LOG_ENTRIES = 40;
+
+function ensureDir() {
+  fs.mkdirSync(ENTITIES_DIR, { recursive: true, mode: 0o700 });
+}
+
+function slugify(name) {
+  return String(name || "").toLowerCase().trim()
+    .replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60);
+}
+
+function normalizeType(type) {
+  const t = String(type || "").toLowerCase().trim();
+  return TYPES.includes(t) ? t : "thing";
+}
+
+function parseFrontmatter(content) {
+  const m = String(content || "").match(/^---\n([\s\S]*?)\n---/);
+  if (!m) return { fm: {}, body: String(content || "") };
+  const fm = {};
+  for (const line of m[1].split("\n")) {
+    const kv = line.match(/^([a-z_]+):\s*(.*)$/);
+    if (kv) fm[kv[1]] = kv[2].trim().replace(/^["']|["']$/g, "");
+  }
+  return { fm, body: content.slice(m[0].length).replace(/^\n+/, "") };
+}
+
+function parseSections(body) {
+  const out = { Notes: "", Log: "" };
+  let current = null;
+  for (const line of String(body || "").split("\n")) {
+    const h = line.match(/^##\s+(Notes|Log)\s*$/i);
+    if (h) {
+      current = h[1].toLowerCase() === "notes" ? "Notes" : "Log";
+      continue;
+    }
+    if (current) out[current] += line + "\n";
+  }
+  out.Notes = out.Notes.trim();
+  out.Log = out.Log.trim();
+  return out;
+}
+
+function serialize(ent) {
+  return [
+    "---",
+    `name: ${ent.name}`,
+    `type: ${ent.type}`,
+    `aliases: ${(ent.aliases || []).join(", ")}`,
+    `description: ${ent.description || ""}`,
+    `created: ${ent.created || new Date().toISOString()}`,
+    `updated: ${ent.updated || new Date().toISOString()}`,
+    `last_seen: ${ent.last_seen || ""}`,
+    "---",
+    "",
+    `## Notes`,
+    "",
+    (ent.sections?.Notes || "").trim(),
+    "",
+    `## Log`,
+    "",
+    (ent.sections?.Log || "").trim(),
+    "",
+  ].join("\n");
+}
+
+function entityFile(slug) {
+  return path.join(ENTITIES_DIR, slug + ".md");
+}
+
+function readEntity(slug) {
+  let content;
+  try { content = fs.readFileSync(entityFile(slug), "utf-8"); }
+  catch (e) { return null; }
+  const { fm, body } = parseFrontmatter(content);
+  return {
+    slug,
+    name: fm.name || slug,
+    type: normalizeType(fm.type),
+    aliases: (fm.aliases || "").split(",").map((a) => a.trim()).filter(Boolean),
+    description: fm.description || "",
+    created: fm.created || "",
+    updated: fm.updated || "",
+    last_seen: fm.last_seen || "",
+    sections: parseSections(body),
+  };
+}
+
+function listEntities() {
+  let names;
+  try { names = fs.readdirSync(ENTITIES_DIR); } catch (e) { return []; }
+  const out = [];
+  for (const f of names) {
+    if (!f.endsWith(".md") || f.startsWith(".")) continue;
+    const e = readEntity(f.slice(0, -3));
+    if (e) out.push(e);
+  }
+  out.sort((a, b) => a.slug.localeCompare(b.slug));
+  return out;
+}
+
+function findEntity(nameOrSlug) {
+  const needle = String(nameOrSlug || "").trim().toLowerCase();
+  if (!needle) return null;
+  const direct = readEntity(slugify(needle));
+  if (direct) return direct;
+  return listEntities().find((e) =>
+    e.slug === needle ||
+    e.name.toLowerCase() === needle ||
+    e.aliases.some((a) => a.toLowerCase() === needle)
+  ) || null;
+}
+
+function writeEntity(ent) {
+  ensureDir();
+  fs.writeFileSync(entityFile(ent.slug), serialize(ent), { mode: 0o600 });
+  markIndexDirty();
+  return ent.slug;
+}
+
+function today() {
+  return new Date().toISOString().slice(0, 10);
+}
+
+// Create-or-merge. Notes replace (current truth); log lines append
+// (capped); aliases union; description/type update when supplied.
+function upsertEntity({ name, type, aliases, description, notes, log } = {}) {
+  const cleanName = String(name || "").trim();
+  if (!cleanName) throw new Error("entity needs a name");
+  const existing = findEntity(cleanName);
+  const ent = existing || {
+    slug: slugify(cleanName),
+    name: cleanName,
+    type: normalizeType(type),
+    aliases: [],
+    description: "",
+    created: new Date().toISOString(),
+    sections: { Notes: "", Log: "" },
+  };
+  if (!ent.slug) throw new Error("entity needs a name");
+  ent.updated = new Date().toISOString();
+  if (type) ent.type = normalizeType(type);
+  if (description) ent.description = String(description).trim();
+  for (const a of [].concat(aliases || [])) {
+    const alias = String(a).trim();
+    if (alias && alias.toLowerCase() !== ent.name.toLowerCase() &&
+        !ent.aliases.some((x) => x.toLowerCase() === alias.toLowerCase())) {
+      ent.aliases.push(alias);
+    }
+  }
+  ent.aliases = ent.aliases.slice(0, 8);
+  if (typeof notes === "string" && notes.trim()) ent.sections.Notes = notes.trim();
+  if (typeof log === "string" && log.trim()) {
+    const entries = ent.sections.Log.split("\n").filter((l) => l.trim());
+    entries.push(`- [${today()}] ${log.trim().replace(/\n+/g, " ")}`);
+    ent.sections.Log = entries.slice(-MAX_LOG_ENTRIES).join("\n");
+  }
+  writeEntity(ent);
+  return { entity: ent, created: !existing };
+}
+
+function removeEntity(nameOrSlug) {
+  const ent = findEntity(nameOrSlug);
+  if (!ent) return null;
+  fs.rmSync(entityFile(ent.slug), { force: true });
+  markIndexDirty();
+  return ent;
+}
+
+function touchSeen(slugs) {
+  const now = new Date().toISOString();
+  for (const slug of [].concat(slugs || [])) {
+    const ent = readEntity(slug);
+    if (!ent) continue;
+    ent.last_seen = now;
+    try { writeEntity(ent); } catch (e) {}
+  }
+}
+
+// Recognise a Write/Edit aimed at an entity file (for chat announcements).
+function entityNameFromPath(filePath) {
+  if (!filePath) return null;
+  const resolved = path.resolve(String(filePath));
+  const rel = path.relative(ENTITIES_DIR, resolved);
+  if (rel.startsWith("..") || path.isAbsolute(rel)) return null;
+  if (rel.includes(path.sep) || !rel.endsWith(".md")) return null;
+  return rel.slice(0, -3);
+}
+
+// ---------------------------------------------------------------------------
+// FTS matching — same field-weighted per-term scoring as packs (BM25 is
+// unusable on a small corpus): a hit on name/aliases counts 2, anything
+// else 1. Threshold 2 means a single name mention is enough to inject,
+// but one stray body word is not.
+
+let _db = null;
+let _indexedAt = 0;
+let _dirty = true;
+
+function markIndexDirty() { _dirty = true; }
+
+function openDb() {
+  if (!DatabaseSync) return null;
+  if (_db) return _db;
+  try {
+    ensureDir();
+    const dbPath = path.join(ENTITIES_DIR, "index.db");
+    const db = new DatabaseSync(dbPath);
+    try { fs.chmodSync(dbPath, 0o600); } catch (e) {}
+    db.exec("PRAGMA journal_mode=WAL");
+    db.exec("PRAGMA busy_timeout=3000");
+    db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS entities USING fts5(
+      name, aliases, description, content, slug UNINDEXED,
+      tokenize='porter unicode61'
+    )`);
+    _db = db;
+    return db;
+  } catch (e) {
+    return null;
+  }
+}
+
+function latestMtime() {
+  let latest = 0;
+  let names = [];
+  try { names = fs.readdirSync(ENTITIES_DIR); } catch (e) { return 0; }
+  for (const f of names) {
+    if (!f.endsWith(".md")) continue;
+    try {
+      const m = fs.statSync(path.join(ENTITIES_DIR, f)).mtimeMs;
+      if (m > latest) latest = m;
+    } catch (e) {}
+  }
+  return latest;
+}
+
+function reindex() {
+  const db = openDb();
+  if (!db) return false;
+  const all = listEntities();
+  db.exec("BEGIN");
+  try {
+    db.exec("DELETE FROM entities");
+    const insert = db.prepare("INSERT INTO entities (name, aliases, description, content, slug) VALUES (?, ?, ?, ?, ?)");
+    for (const e of all) {
+      insert.run(e.name, e.aliases.join(" "), e.description, [e.sections.Notes, e.sections.Log].join("\n"), e.slug);
+    }
+    db.exec("COMMIT");
+  } catch (e) {
+    try { db.exec("ROLLBACK"); } catch (e2) {}
+    return false;
+  }
+  _dirty = false;
+  _indexedAt = Date.now();
+  return true;
+}
+
+function ensureIndex() {
+  if (_dirty || latestMtime() > _indexedAt) reindex();
+}
+
+const STOPWORDS = new Set(("a an and are as at be but by can did do for from had has have how i if in is it its " +
+  "me my no not of on or our so that the their then there this to up us was we what when where which who why " +
+  "will with you your yes yeah ok okay please thanks just like dont don't im i'm its it's").split(" "));
+
+function queryTerms(text) {
+  const seen = new Set();
+  const terms = [];
+  for (const raw of String(text || "").toLowerCase().split(/[^a-z0-9_.-]+/)) {
+    const t = raw.replace(/^[.-]+|[.-]+$/g, "");
+    if (t.length < 3 || STOPWORDS.has(t) || seen.has(t)) continue;
+    seen.add(t);
+    terms.push(t);
+    if (terms.length >= 40) break;
+  }
+  return terms;
+}
+
+function matchEntities(text, { limit = 4, threshold = null } = {}) {
+  const db = openDb();
+  if (!db) return [];
+  ensureIndex();
+  const terms = queryTerms(text);
+  if (terms.length === 0) return [];
+  const minScore = threshold ?? Number(process.env.ENTITY_MATCH_THRESHOLD || 2);
+  const scores = new Map();
+  let stmt;
+  try { stmt = db.prepare("SELECT slug FROM entities WHERE entities MATCH ?"); } catch (e) { return []; }
+  for (const t of terms) {
+    const quoted = `"${t.replace(/"/g, '""')}"`;
+    let strong = [];
+    let any = [];
+    try { strong = stmt.all(`{name aliases} : ${quoted}`); } catch (e) {}
+    try { any = stmt.all(quoted); } catch (e) {}
+    const strongSet = new Set(strong.map((r) => r.slug));
+    for (const r of any) scores.set(r.slug, (scores.get(r.slug) || 0) + (strongSet.has(r.slug) ? 2 : 1));
+  }
+  const names = new Map(listEntities().map((e) => [e.slug, e.name]));
+  return [...scores.entries()]
+    .filter(([, s]) => s >= minScore)
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, Math.max(1, Math.min(10, limit)))
+    .map(([slug, score]) => ({ slug, name: names.get(slug) || slug, score }));
+}
+
+module.exports = {
+  ENTITIES_DIR, TYPES, slugify,
+  listEntities, findEntity, readEntity, writeEntity, upsertEntity, removeEntity,
+  touchSeen, entityNameFromPath, matchEntities, reindex, markIndexDirty,
+};

package/core/pack-review.js

@@ -0,0 +1,224 @@
+// Post-turn pack reviewer (Hermes-inspired, selective-but-active).
+// After each substantial turn a cheap model reviews the exchange and
+// decides whether a context pack should be created or mutated. The model
+// only RETURNS a JSON decision — all file writes are applied here by bot
+// code, so the reviewer needs no tools and no permissions. Every applied
+// mutation is announced in chat (one line), per the no-silent-learning
+// rule.
+
+const { spawnSubagent } = require("./subagent");
+const packs = require("./packs");
+const entities = require("./entities");
+const { redactSensitive } = require("./redact");
+
+const MIN_TURN_CHARS = 400;
+const MAX_TEXT_CHARS = 7000;
+const REVIEW_MODEL = process.env.PACK_REVIEW_MODEL || "haiku";
+const MAX_ACTIONS = 2;
+const MAX_ENTITY_ACTIONS = 3;
+
+function enabled() {
+  return (process.env.PACK_REVIEW || "on").toLowerCase() !== "off";
+}
+
+function clip(text, n = MAX_TEXT_CHARS) {
+  const s = String(text || "");
+  return s.length > n ? s.slice(0, n) + "\n…[truncated]" : s;
+}
+
+// Word-boundary clip for chat announcements — never cuts mid-word.
+function clipWords(text, max = 180) {
+  const s = String(text || "").replace(/\s+/g, " ").trim();
+  if (s.length <= max) return s;
+  const cut = s.slice(0, max);
+  const sp = cut.lastIndexOf(" ");
+  return (sp > max * 0.6 ? cut.slice(0, sp) : cut).replace(/[,;:.\-—]+$/, "") + "…";
+}
+
+const ENTITY_EMOJI = { person: "👤", place: "📍", project: "🚀", org: "🏢", system: "🖥️", thing: "🔖" };
+
+const ANNOUNCE_HEADERS = [
+  "🧠 Took some notes while we worked:",
+  "🧠 Filed a few things away:",
+  "🧠 Jotted this down for next time:",
+  "🧠 Memory updated:",
+];
+
+function formatAnnouncement(lines) {
+  if (lines.length === 0) return "";
+  const header = ANNOUNCE_HEADERS[Math.floor(Math.random() * ANNOUNCE_HEADERS.length)];
+  return [header, ...lines].join("\n\n");
+}
+
+function buildReviewPrompt(userText, assistantText) {
+  const index = packs.listPacks().map((p) =>
+    `- ${p.dir}: ${p.name} — ${p.description}${p.tags.length ? ` [${p.tags.join(", ")}]` : ""}`
+  ).join("\n") || "(none yet)";
+
+  const entityIndex = entities.listEntities().map((e) =>
+    `- ${e.slug}: ${e.name} (${e.type})${e.aliases.length ? ` aka ${e.aliases.join(", ")}` : ""} — ${e.description}`
+  ).join("\n") || "(none yet)";
+
+  return `You are the memory reviewer for a personal AI assistant. After each conversation turn you decide whether the assistant's long-term "context packs" and "entity notes" should change.
+
+A context pack is a living document about ONE topic (a project, a system, a recurring task, a domain). It has four sections:
+- Stance: how to think about the topic — preferences, framing, hard rules the user has expressed.
+- Procedure: verified how-to steps and commands.
+- State: where work on the topic stands NOW — decisions made, open questions, next steps. Replaces wholesale.
+- Journal: one-line dated log of what happened each session.
+
+An entity note is a short memory file about ONE specific named entity — a person, place, project, org, or system. It has:
+- Notes: current truth about the entity (who/what it is, role, preferences, relationships). Replaces wholesale.
+- Log: one-line dated observations, appended.
+
+Existing packs:
+${index}
+
+Known entities:
+${entityIndex}
+
+The turn to review:
+
+<user_message>
+${clip(userText)}
+</user_message>
+
+<assistant_reply>
+${clip(assistantText)}
+</assistant_reply>
+
+Decide. Rules:
+- Bias toward action: if the turn did real work on an identifiable topic (a named system, project, server, app, person, or domain), record it — at minimum a journal line. Most working turns deserve one. Reserve empty actions for small talk, pure status checks, and turns that contain nothing new.
+- UPDATE an existing pack when the turn touched that topic: append a journal line, and rewrite State if where-things-stand changed. Update Stance when the user expressed a lasting preference or rule; Procedure when a verified working method emerged.
+- CREATE a pack when the turn worked on a durable topic no existing pack covers. Topics recur more than you expect — a named system or project is durable by default. Do not create packs for one-off trivia, and prefer update when an existing pack fits.
+- Never store secrets, tokens, passwords, or credentials.
+- State should be concise (under 150 words). Journal entries one sentence. Never start a journal or log sentence with a date — dates are prepended automatically.
+- At most ${MAX_ACTIONS} pack actions.
+- Entities: add an item when the turn revealed something durable about a specific named person, place, project, org, or system — their role, status, preferences, or relationship to other work. Use the existing entity name when one matches (check aliases). Skip entities mentioned only in passing with nothing learned. Notes under 100 words; "notes" null means leave Notes unchanged. At most ${MAX_ENTITY_ACTIONS} entity items.
+- Packs and entities are independent — a turn can update both, either, or neither. Do not duplicate the same fact in a pack AND an entity unless it genuinely belongs to both.
+
+Reply with ONLY a JSON object, no prose, no code fences:
+{"actions": [
+  {"action": "update", "pack": "<existing dir>", "journal": "<one sentence>", "state": "<full new State or null>", "stance": null, "procedure": null}
+  | {"action": "create", "dir": "<kebab-slug>", "name": "<title>", "description": "<one line: when this pack is relevant>", "tags": ["..."], "stance": "<or empty>", "procedure": "<or empty>", "state": "<where things stand>", "journal": "<one sentence>"}
+],
+"entities": [
+  {"name": "<canonical name>", "type": "person|place|project|org|system|thing", "aliases": ["..."], "description": "<one line: who/what this is>", "notes": "<full new Notes or null>", "log": "<one sentence observation>"}
+]}
+or {"actions": [], "entities": []}`;
+}
+
+function parseDecision(text) {
+  const raw = String(text || "").trim().replace(/^```(?:json)?\s*/i, "").replace(/```\s*$/, "");
+  const start = raw.indexOf("{");
+  const end = raw.lastIndexOf("}");
+  if (start === -1 || end <= start) return null;
+  try {
+    const obj = JSON.parse(raw.slice(start, end + 1));
+    const actions = Array.isArray(obj.actions) ? obj.actions.slice(0, MAX_ACTIONS) : (obj.action ? [obj] : []);
+    const entityActions = Array.isArray(obj.entities) ? obj.entities.slice(0, MAX_ENTITY_ACTIONS) : [];
+    return { actions, entities: entityActions };
+  } catch (e) {
+    return null;
+  }
+}
+
+function applyAction(a) {
+  if (!a || typeof a !== "object") return null;
+  if (a.action === "update" && a.pack) {
+    const existing = packs.readPack(a.pack);
+    if (!existing) return null;
+    packs.updatePack(a.pack, {
+      journal: a.journal || "",
+      state: typeof a.state === "string" ? a.state : "",
+      stance: typeof a.stance === "string" ? a.stance : "",
+      procedure: typeof a.procedure === "string" ? a.procedure : "",
+    });
+    return { kind: "update", dir: a.pack, name: existing.name, note: a.journal || "state updated" };
+  }
+  if (a.action === "create" && (a.dir || a.name)) {
+    const dir = packs.slugify(a.dir || a.name);
+    if (packs.readPack(dir)) {
+      packs.updatePack(dir, { journal: a.journal || "", state: a.state || "" });
+      return { kind: "update", dir, name: a.name || dir, note: a.journal || "state updated" };
+    }
+    const pack = packs.createPack({
+      dir,
+      name: a.name,
+      description: a.description,
+      tags: Array.isArray(a.tags) ? a.tags.slice(0, 6) : [],
+      stance: a.stance || "",
+      procedure: a.procedure || "",
+      state: a.state || "",
+      journal: a.journal || "",
+    });
+    return { kind: "create", dir: pack.dir, name: pack.name, note: a.description || "" };
+  }
+  return null;
+}
+
+function applyEntityAction(e) {
+  if (!e || typeof e !== "object" || !String(e.name || "").trim()) return null;
+  const { entity, created } = entities.upsertEntity({
+    name: e.name,
+    type: e.type,
+    aliases: Array.isArray(e.aliases) ? e.aliases.slice(0, 5) : [],
+    description: e.description,
+    notes: typeof e.notes === "string" ? e.notes : "",
+    log: e.log || "",
+  });
+  return { kind: created ? "create" : "update", slug: entity.slug, name: entity.name, type: entity.type, note: e.log || e.description || "" };
+}
+
+// Fire-and-forget. `announce` is an async (text) => void bound to the
+// originating channel; failures are logged, never thrown into the turn.
+function reviewTurn({ userText, assistantText, channelId, announce }) {
+  if (!enabled()) return;
+  const combined = (String(userText || "") + String(assistantText || "")).trim();
+  if (combined.length < MIN_TURN_CHARS) return;
+
+  const prompt = buildReviewPrompt(redactSensitive(String(userText || "")), redactSensitive(String(assistantText || "")));
+
+  spawnSubagent(prompt, {
+    model: REVIEW_MODEL,
+    channelId,
+    timeoutMs: 3 * 60 * 1000,
+    systemPrompt: "You are a background memory reviewer. Reply with ONLY the requested JSON object. No prose, no markdown, no tool use.",
+  }).then(({ text }) => {
+    const decision = parseDecision(text);
+    if (!decision || (decision.actions.length === 0 && decision.entities.length === 0)) return;
+    const lines = [];
+    for (const a of decision.actions) {
+      try {
+        const r = applyAction(a);
+        if (r) {
+          lines.push(r.kind === "create"
+            ? `📦 New pack: ${r.name}\n${clipWords(r.note, 180)}\n↳ open-claudia pack show ${r.dir}`
+            : `✏️ ${r.name} — ${clipWords(r.note, 180)}`);
+        }
+      } catch (e) {
+        console.warn(`[pack-review] apply failed: ${e.message}`);
+      }
+    }
+    for (const ea of decision.entities) {
+      try {
+        const r = applyEntityAction(ea);
+        if (r) {
+          const emoji = ENTITY_EMOJI[r.type] || ENTITY_EMOJI.thing;
+          lines.push(r.kind === "create"
+            ? `${emoji} Now tracking ${r.name} (${r.type}) — ${clipWords(r.note, 160)}`
+            : `${emoji} ${r.name} — ${clipWords(r.note, 160)}`);
+        }
+      } catch (e) {
+        console.warn(`[pack-review] entity apply failed: ${e.message}`);
+      }
+    }
+    if (lines.length > 0 && typeof announce === "function") {
+      announce(formatAnnouncement(lines)).catch(() => {});
+    }
+  }).catch((e) => {
+    console.warn(`[pack-review] reviewer failed: ${e.message}`);
+  });
+}
+
+module.exports = { reviewTurn, parseDecision, applyAction, applyEntityAction, buildReviewPrompt, clipWords, ENTITY_EMOJI };