@vib795/agent-memory 0.1.0

package/src/compact.js

@@ -0,0 +1,242 @@
+import { readFileSync, existsSync } from 'node:fs';
+import { loadConfig, paths } from './config.js';
+import { listNotes, archiveNote, contentHash, nowIso, serializeNote } from './store.js';
+import { atomicWrite } from './atomic.js';
+import { openDb, reindex } from './index-db.js';
+import { buildDigest, buildTree, renderTree } from './digest.js';
+
+/**
+ * Compaction is pure code. No model is involved, and none should be.
+ *
+ * Everything here is a rule an operator can predict and check by hand: identical
+ * content merges, superseded knowledge steps aside, unread and unreferenced notes
+ * move to the archive. Because it needs no model, it never needs a schedule and
+ * never costs a premium request.
+ */
+
+function ageDays(iso, now) {
+  if (!iso) return Infinity;
+  const t = Date.parse(iso);
+  return Number.isFinite(t) ? (now - t) / 86400000 : Infinity;
+}
+
+function unionEdges(...lists) {
+  const seen = new Map();
+  for (const list of lists) {
+    for (const e of list || []) {
+      if (e?.rel && e?.dst) seen.set(`${e.rel} ${e.dst}`, { rel: e.rel, dst: e.dst });
+    }
+  }
+  return [...seen.values()].sort((a, b) =>
+    a.rel < b.rel ? -1 : a.rel > b.rel ? 1 : a.dst < b.dst ? -1 : 1,
+  );
+}
+
+/** Rewrite a note in place, preserving `updated`. Graph repair, not an edit. */
+function rewrite(node) {
+  atomicWrite(node.path, serializeNote(node));
+}
+
+/**
+ * Merge notes whose content is identical.
+ *
+ * The keeper is the earliest `created`, because the first capture is the one other
+ * notes are most likely to already point at. Timestamps are second-precision, so two
+ * notes captured in the same turn tie routinely; the tie goes to whichever node more
+ * things already reference, which is the same rationale stated directly rather than
+ * approximated by a timestamp. Id is the last resort, purely for determinism.
+ *
+ * Repos and edges are unioned so nothing a duplicate knew is lost, and inbound edges
+ * are repointed at the keeper so the merge leaves no dangling reference behind.
+ */
+function dedupe(active) {
+  const inbound = new Map();
+  for (const n of active) {
+    for (const e of n.edges || []) inbound.set(e.dst, (inbound.get(e.dst) || 0) + 1);
+  }
+
+  const groups = new Map();
+  for (const n of active) {
+    const h = contentHash(n);
+    if (!groups.has(h)) groups.set(h, []);
+    groups.get(h).push(n);
+  }
+
+  const merged = [];
+  const remap = new Map();
+  for (const group of groups.values()) {
+    if (group.length < 2) continue;
+    group.sort(
+      (a, b) =>
+        (a.created || '').localeCompare(b.created || '') ||
+        (inbound.get(b.id) || 0) - (inbound.get(a.id) || 0) ||
+        a.id.localeCompare(b.id),
+    );
+    const [keeper, ...dups] = group;
+    keeper.repos = [
+      ...new Set([...(keeper.repos || []), ...dups.flatMap((d) => d.repos || [])]),
+    ].sort();
+    keeper.edges = unionEdges(keeper.edges, ...dups.map((d) => d.edges))
+      // A merged node must not end up pointing at an id it just absorbed.
+      .filter((e) => !dups.some((d) => d.id === e.dst));
+    rewrite(keeper);
+
+    for (const d of dups) {
+      archiveNote(d.type, d.id);
+      remap.set(d.id, keeper.id);
+      merged.push({ id: d.id, into: keeper.id });
+    }
+  }
+
+  // Repoint every edge that referenced an absorbed id.
+  if (remap.size) {
+    for (const n of active) {
+      if (remap.has(n.id)) continue;
+      const before = JSON.stringify(n.edges || []);
+      n.edges = unionEdges(
+        (n.edges || []).map((e) => ({ rel: e.rel, dst: remap.get(e.dst) || e.dst })),
+      ).filter((e) => e.dst !== n.id);
+      if (JSON.stringify(n.edges) !== before) rewrite(n);
+    }
+  }
+  return merged;
+}
+
+/**
+ * A node that supersedes another archives it.
+ *
+ * The superseded note keeps its edges and stays reachable through
+ * `get --include-archived`, because "we used to do it this way and stopped" is
+ * often the exact thing someone needs six months later.
+ */
+function collapseSupersedes(active) {
+  const byId = new Map(active.map((n) => [n.id, n]));
+  const archived = [];
+  for (const n of active) {
+    if (!n.supersedes) continue;
+    const old = byId.get(n.supersedes);
+    if (!old || old.archived) continue;
+    if (archiveNote(old.type, old.id)) {
+      old.archived = 1;
+      archived.push({ id: old.id, by: n.id });
+    }
+  }
+  return archived;
+}
+
+/**
+ * Archive notes nobody points at and nobody has read in `decayDays`.
+ *
+ * Inbound edges are the exemption: a note other notes depend on is load-bearing
+ * whether or not anyone has opened it recently.
+ */
+function decay(active, cfg, now) {
+  const referenced = new Set(active.flatMap((n) => (n.edges || []).map((e) => e.dst)));
+  const archived = [];
+  for (const n of active) {
+    if (n.archived || referenced.has(n.id)) continue;
+    const last = n.accessed || n.updated || n.created;
+    if (ageDays(last, now) < cfg.decayDays) continue;
+    if (archiveNote(n.type, n.id)) archived.push({ id: n.id, lastSeen: last });
+  }
+  return archived;
+}
+
+/**
+ * Rewrite the `description:` line of an installed skill.
+ *
+ * This is the whole Tier-1 mechanism. An agent decides whether to invoke a skill by
+ * reading its description, so regenerating that line is how the store advertises
+ * what it now knows without costing anything at chat time.
+ */
+export function writeSkillDescription(skillPath, description) {
+  if (!existsSync(skillPath)) return false;
+  const src = readFileSync(skillPath, 'utf8');
+  if (!src.startsWith('---')) return false;
+  const end = src.indexOf('\n---', 3);
+  if (end === -1) return false;
+
+  const head = src.slice(0, end);
+  const rest = src.slice(end);
+  // JSON quoting is valid YAML double-quoting, and the digest contains colons and
+  // commas that would otherwise break the frontmatter.
+  const line = `description: ${JSON.stringify(description)}`;
+  const updated = /^description:.*$/m.test(head)
+    ? head.replace(/^description:.*$/m, line)
+    : `${head}\n${line}`;
+
+  atomicWrite(skillPath, updated + rest);
+  return true;
+}
+
+/** Regenerate everything derived: ROUTING.md and each installed skill description. */
+function regenerate(db, cfg) {
+  const digest = buildDigest(db, { cfg });
+  const tree = buildTree(db, { all: true, cfg });
+
+  const routing = [
+    '<!-- Generated by `agent-memory compact`. Edits here are overwritten. -->',
+    '',
+    digest,
+    '',
+    renderTree(tree),
+    '',
+  ].join('\n');
+  atomicWrite(paths.routing, routing);
+
+  const skills = [];
+  for (const p of cfg.skillPaths || []) {
+    if (writeSkillDescription(p, digest)) skills.push(p);
+  }
+  return { digest, digestChars: digest.length, routing: paths.routing, skills };
+}
+
+/**
+ * Dedup, collapse, decay, reindex, regenerate.
+ *
+ * Order matters: dedup before supersede collapse, because a merge can be what makes
+ * two supersede chains agree; decay last, because both steps before it change which
+ * nodes have inbound edges.
+ */
+export function compact({ cfg = loadConfig(), now = Date.now(), db: existing = null } = {}) {
+  const db = existing || openDb({ reindexOnCreate: false });
+  const all = listNotes();
+  const malformed = all.filter((n) => n.__error).map((n) => ({ path: n.path, error: n.__error }));
+  const active = all.filter((n) => !n.__error && !n.archived && n.id);
+
+  const merged = dedupe(active);
+  const mergedIds = new Set(merged.map((m) => m.id));
+  const remaining = active.filter((n) => !mergedIds.has(n.id));
+
+  const superseded = collapseSupersedes(remaining);
+  const supersededIds = new Set(superseded.map((s) => s.id));
+
+  const decayed = decay(remaining.filter((n) => !supersededIds.has(n.id)), cfg, now);
+
+  const indexed = reindex(db);
+  const derived = regenerate(db, cfg);
+  if (!existing) db.close();
+
+  return {
+    at: nowIso(),
+    merged,
+    superseded,
+    decayed,
+    malformed,
+    indexed: indexed.indexed,
+    duplicateIds: indexed.duplicates,
+    ...derived,
+  };
+}
+
+/**
+ * Compact automatically when the store has changed enough to be worth it.
+ *
+ * Runs on install and after a write that moves the node count past the threshold,
+ * so compaction never needs a scheduler, a daemon, or a model. Anything that needed
+ * one of those would not survive a locked-down desktop.
+ */
+export function maybeCompact(db, before, after, cfg = loadConfig()) {
+  if (Math.abs(after - before) < cfg.compactThreshold) return null;
+  return compact({ cfg, db });
+}

package/src/config.js

@@ -0,0 +1,91 @@
+import { readFileSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { homedir } from 'node:os';
+import { atomicWrite } from './atomic.js';
+
+// The store lives outside every repository on purpose. That is what makes a note
+// written in one window readable from a window opened on a different project.
+export const STORE_ROOT =
+  process.env.AGENT_MEMORY_HOME ||
+  join(process.env.USERPROFILE || homedir(), '.agents', 'memory');
+
+export const NOTE_TYPES = ['system', 'decision', 'convention', 'constraint'];
+
+export const paths = {
+  root: STORE_ROOT,
+  notes: join(STORE_ROOT, 'notes'),
+  archive: join(STORE_ROOT, 'notes', 'archive'),
+  db: join(STORE_ROOT, 'index.db'),
+  routing: join(STORE_ROOT, 'ROUTING.md'),
+  config: join(STORE_ROOT, 'config.json'),
+  typeDir: (type) => join(STORE_ROOT, 'notes', type),
+  archiveTypeDir: (type) => join(STORE_ROOT, 'notes', 'archive', type),
+};
+
+// Every cap in the spec is tunable. These defaults are judgment calls, not limits
+// derived from anything, so they belong in config rather than frozen in code.
+export const DEFAULTS = {
+  digestChars: 400, // standing context cost, loaded on every chat
+  treeLines: 80, // per-invocation cost, paid only when recall fires
+  getBudgetBytes: 8192, // ceiling on note bodies returned by a single get
+  decayDays: 90, // archive threshold for unreferenced, unread notes
+  staleAnnotateCommits: 10, // below this, staleness is not worth mentioning
+  staleReviewCommits: 100, // above this, doctor flags it for review
+  compactThreshold: 10, // node-count delta that triggers an automatic compact
+};
+
+// Written by the installer: every SKILL.md whose description compact regenerates.
+// The store cannot discover these on its own, because a skill may be symlinked
+// from a checkout or copied into place, and both are legitimate installs.
+export const LIST_KEYS = ['skillPaths'];
+
+export function loadConfig() {
+  if (!existsSync(paths.config)) return { ...DEFAULTS, skillPaths: [] };
+  try {
+    const raw = JSON.parse(readFileSync(paths.config, 'utf8'));
+    const merged = { ...DEFAULTS, skillPaths: [] };
+    for (const [k, v] of Object.entries(raw)) {
+      if (k in DEFAULTS && typeof v === 'number' && Number.isFinite(v) && v > 0) merged[k] = v;
+      if (LIST_KEYS.includes(k) && Array.isArray(v)) {
+        merged[k] = v.filter((s) => typeof s === 'string' && s.trim());
+      }
+    }
+    return merged;
+  } catch {
+    // A malformed config must not take the store down. Defaults are always valid.
+    return { ...DEFAULTS, skillPaths: [] };
+  }
+}
+
+/**
+ * Merge a patch into config.json.
+ *
+ * Lives here rather than in the install scripts so that bash and PowerShell do not
+ * each grow their own JSON merge, which is exactly the kind of duplication that
+ * drifts and only shows up on the machine you cannot test from.
+ *
+ * Only keys that already mean something are persisted, so a typo in an installer
+ * cannot quietly become permanent state.
+ */
+export function saveConfig(patch) {
+  const current = existsSync(paths.config)
+    ? (() => {
+        try {
+          return JSON.parse(readFileSync(paths.config, 'utf8'));
+        } catch {
+          return {};
+        }
+      })()
+    : {};
+
+  const next = { ...current };
+  for (const [k, v] of Object.entries(patch || {})) {
+    if (k in DEFAULTS && typeof v === 'number' && Number.isFinite(v) && v > 0) next[k] = v;
+    if (LIST_KEYS.includes(k) && Array.isArray(v)) {
+      next[k] = [...new Set(v.filter((s) => typeof s === 'string' && s.trim()))].sort();
+    }
+  }
+
+  atomicWrite(paths.config, `${JSON.stringify(next, null, 2)}\n`);
+  return next;
+}

package/src/digest.js

@@ -0,0 +1,199 @@
+import { loadConfig } from './config.js';
+
+/**
+ * Two-tier routing.
+ *
+ * Tier 1 is the `recall` skill's description, which is loaded into every chat
+ * whether or not memory is ever used. It is standing cost, so it has to read like
+ * a description rather than a document.
+ *
+ * Tier 2 is the tree, printed only when `recall` actually fires. Per-invocation
+ * cost, paid once, and only when someone is already looking something up.
+ *
+ * Neither tier costs a premium request. A request is charged per prompt, not per
+ * tool call, so both of these ride inside a turn that was already paid for.
+ */
+
+// Never dropped from either tier. A constraint is what stops an agent from burning
+// a retry loop on an approach the org forbids, which is where requests actually go.
+// Everything else here is negotiable; this is not.
+const PRIVILEGED = 'constraint';
+
+const TYPE_ORDER = ['constraint', 'decision', 'convention', 'system'];
+
+const USE_WHEN =
+  'Use when you need to know how a system works, why a decision was made, ' +
+  'what convention applies, or what the environment forbids.';
+
+function typeRank(type) {
+  const i = TYPE_ORDER.indexOf(type);
+  return i === -1 ? TYPE_ORDER.length : i;
+}
+
+/** id -> total edges touching it, in one query rather than one per node. */
+function degreeMap(db) {
+  const map = new Map();
+  for (const row of db
+    .prepare(`
+      SELECT n.id AS id, COUNT(e.rowid) AS degree
+        FROM nodes n
+        LEFT JOIN edges e ON e.src = n.id OR e.dst = n.id
+       GROUP BY n.id
+    `)
+    .all()) {
+    map.set(row.id, row.degree);
+  }
+  return map;
+}
+
+/**
+ * Tier 1: the skill description.
+ *
+ * Composition order is constraint count, then repos by note count, then topics by
+ * edge degree. On overflow the lowest-degree topics go first, then repos. Two
+ * pieces are structural and never dropped: the constraint count, and the closing
+ * "use when" clause, which is the entire reason an agent decides to invoke at all.
+ * Cutting that to save characters would save the cost of a feature by deleting it.
+ */
+export function buildDigest(db, { cfg = loadConfig() } = {}) {
+  const cap = cfg.digestChars;
+  const total = db.prepare('SELECT COUNT(*) AS c FROM nodes WHERE archived = 0').get().c;
+  if (!total) {
+    return 'Durable project knowledge store, currently empty. Run /remember to capture the first note.';
+  }
+
+  const constraints = db
+    .prepare("SELECT COUNT(*) AS c FROM nodes WHERE archived = 0 AND type = 'constraint'")
+    .get().c;
+
+  const repos = db
+    .prepare(`
+      SELECT r.repo AS repo, COUNT(*) AS c
+        FROM node_repos r JOIN nodes n ON n.id = r.node_id
+       WHERE n.archived = 0
+       GROUP BY r.repo
+       ORDER BY c DESC, r.repo
+    `)
+    .all()
+    .map((r) => r.repo);
+
+  const topics = db
+    .prepare(`
+      SELECT n.title AS title, COUNT(e.rowid) AS degree
+        FROM nodes n
+        LEFT JOIN edges e ON e.src = n.id OR e.dst = n.id
+       WHERE n.archived = 0
+       GROUP BY n.id
+       ORDER BY degree DESC, n.title
+    `)
+    .all()
+    .map((r) => r.title.toLowerCase());
+
+  const head =
+    `Durable project knowledge: ${total} note${total === 1 ? '' : 's'}` +
+    (constraints ? `, ${constraints} constraint${constraints === 1 ? '' : 's'}` : '');
+
+  // Shed the elastic middle one whole item at a time. Cutting mid-word would leave
+  // the description looking corrupted, which is worse than saying less.
+  const useRepos = repos.slice();
+  const useTopics = topics.slice();
+  const compose = () => {
+    let s = head;
+    if (useRepos.length) s += ` across ${useRepos.join(', ')}`;
+    s += '.';
+    if (useTopics.length) s += ` Topics: ${useTopics.join(', ')}.`;
+    return `${s} ${USE_WHEN}`;
+  };
+
+  let out = compose();
+  while (out.length > cap && useTopics.length) {
+    useTopics.pop();
+    out = compose();
+  }
+  while (out.length > cap && useRepos.length) {
+    useRepos.pop();
+    out = compose();
+  }
+  // If the head plus the use-when clause alone exceed the cap, the cap is too small
+  // for a usable description. Return the structural minimum; `doctor` reports it.
+  return out;
+}
+
+/**
+ * Tier 2: the routing tree, scoped.
+ *
+ * A repo view returns that repo's nodes plus every `scope: global` node, because a
+ * global constraint applies here too and hiding it is exactly the failure this
+ * design cares about.
+ */
+export function buildTree(db, { repo = null, all = false, cfg = loadConfig() } = {}) {
+  const rows = repo
+    ? db
+        .prepare(`
+          SELECT DISTINCT n.id AS id, n.type AS type, n.title AS title,
+                          n.archived AS archived, n.scope AS scope
+            FROM nodes n
+            LEFT JOIN node_repos r ON r.node_id = n.id
+           WHERE n.scope = 'global' OR r.repo = ?
+        `)
+        .all(repo)
+    : db.prepare('SELECT id, type, title, archived, scope FROM nodes').all();
+
+  const degrees = degreeMap(db);
+  const entries = rows.map((r) => ({ ...r, degree: degrees.get(r.id) ?? 0 }));
+
+  entries.sort(
+    (a, b) =>
+      a.archived - b.archived ||
+      typeRank(a.type) - typeRank(b.type) ||
+      b.degree - a.degree ||
+      (a.id < b.id ? -1 : 1),
+  );
+
+  const total = entries.length;
+  // One line for the header, one for the truncation notice.
+  const cap = Math.max(1, cfg.treeLines - 2);
+  if (all || total <= cap) {
+    return { repo, total, shown: total, omitted: [], lines: entries, truncated: false };
+  }
+
+  // Drop archived first, then by ascending degree. Least connected means least
+  // likely to be the hub anyone needed. Constraints are exempt at any cap.
+  const droppable = entries
+    .filter((e) => e.type !== PRIVILEGED)
+    .sort((a, b) => b.archived - a.archived || a.degree - b.degree || (a.id < b.id ? 1 : -1));
+
+  const dropped = new Set();
+  for (const e of droppable) {
+    if (total - dropped.size <= cap) break;
+    dropped.add(e.id);
+  }
+
+  const kept = entries.filter((e) => !dropped.has(e.id));
+  return {
+    repo,
+    total,
+    shown: kept.length,
+    omitted: entries.filter((e) => dropped.has(e.id)).map((e) => ({ id: e.id, type: e.type })),
+    lines: kept,
+    truncated: true,
+    // True when constraints alone exceed the cap. Reported, not silently fixed.
+    overCap: kept.length > cap,
+  };
+}
+
+export function renderTree(result) {
+  const scope = result.repo ? result.repo : 'all repos';
+  const out = [`# memory: ${scope} — ${result.total} notes`];
+  const width = result.lines.reduce((w, e) => Math.max(w, e.id.length), 0);
+  for (const e of result.lines) {
+    const mark = e.archived ? ' (archived)' : '';
+    out.push(`${e.type.padEnd(10)} ${e.id.padEnd(width)}  ${e.title}${mark}`);
+  }
+  if (result.omitted.length) {
+    // Always printed. Silent truncation that reads as full coverage is the single
+    // most dangerous thing this tool could do, because it looks like an answer.
+    out.push(`${result.omitted.length} nodes not shown, run agent-memory tree --all`);
+  }
+  return out.join('\n');
+}

package/src/graph.js

@@ -0,0 +1,99 @@
+import { hydrate } from './index-db.js';
+
+/**
+ * SQLite's recursive CTE is the graph engine.
+ *
+ * Nothing is superimposed on top of it: no adjacency cache, no in-memory graph, no
+ * second traversal in JavaScript. One prepared statement returns the neighborhood.
+ */
+
+/**
+ * Every node within `depth` edges of the root, in one query.
+ *
+ * `UNION` rather than `UNION ALL` collapses repeated visits, and the depth bound
+ * terminates the recursion. Both matter: `contradicts` edges make cycles inevitable
+ * and a cycle without either guard would recurse until SQLite gave up.
+ *
+ * Edges are followed in both directions. A note that depends on the auth service is
+ * part of the auth service's neighborhood, and an operator asking about auth wants
+ * to hear about it.
+ */
+export function neighborhood(db, rootId, { depth = 1, includeArchived = false } = {}) {
+  const rows = db
+    .prepare(`
+      WITH RECURSIVE hood(id, depth) AS (
+        SELECT ?, 0
+        UNION
+        SELECT CASE WHEN e.src = h.id THEN e.dst ELSE e.src END, h.depth + 1
+          FROM edges e
+          JOIN hood h ON e.src = h.id OR e.dst = h.id
+         WHERE h.depth < ?
+      )
+      SELECT n.*,
+             MIN(hood.depth) AS depth,
+             (SELECT COUNT(*) FROM edges e2 WHERE e2.src = n.id OR e2.dst = n.id) AS degree
+        FROM nodes n
+        JOIN hood ON n.id = hood.id
+       WHERE (? = 1 OR n.archived = 0)
+       GROUP BY n.id
+       ORDER BY depth, n.type, n.id
+    `)
+    .all(rootId, depth, includeArchived ? 1 : 0);
+
+  // A node reachable at two depths appears once, at the shorter one, because MIN
+  // plus GROUP BY collapses it. Ordering by id last keeps output byte-stable across
+  // rebuilds, which is what makes the reindex-determinism check meaningful.
+  return rows.map((r) => hydrate(db, r));
+}
+
+/** Bytes of body text, which is what actually lands in the model's context. */
+function bodyBytes(node) {
+  return Buffer.byteLength(String(node.body ?? ''), 'utf8');
+}
+
+/**
+ * Enforce the retrieval budget.
+ *
+ * `get --depth 2` on a well-connected node can return fifty notes and blow the
+ * context this whole design exists to protect. Prune order: deepest first, then
+ * least connected, because depth is distance from what was asked about and low
+ * degree means the node is unlikely to be the hub anyone needed.
+ *
+ * Two things are never dropped: the root, and any `constraint`. A constraint is what
+ * stops an agent from spending a dozen requests on an approach the org forbids, so
+ * dropping one to save bytes trades the cheap thing away for the expensive one.
+ *
+ * Omitted ids are always returned. Silent truncation that reads as full coverage is
+ * the failure this budget is most likely to cause and the one worth guarding.
+ */
+export function applyBudget(nodes, budgetBytes) {
+  let total = nodes.reduce((sum, n) => sum + bodyBytes(n), 0);
+  if (total <= budgetBytes) {
+    return { kept: nodes, omitted: [], overBudget: false, bytes: total };
+  }
+
+  const removable = nodes
+    .filter((n) => n.depth > 0 && n.type !== 'constraint')
+    .sort(
+      (a, b) =>
+        b.depth - a.depth || a.degree - b.degree || (a.id < b.id ? 1 : a.id > b.id ? -1 : 0),
+    );
+
+  const dropped = new Set();
+  for (const n of removable) {
+    if (total <= budgetBytes) break;
+    dropped.add(n.id);
+    total -= bodyBytes(n);
+  }
+
+  return {
+    kept: nodes.filter((n) => !dropped.has(n.id)),
+    omitted: nodes
+      .filter((n) => dropped.has(n.id))
+      .map((n) => ({ id: n.id, type: n.type, title: n.title, depth: n.depth })),
+    // True when the root and the constraints alone exceed the budget. Reported
+    // rather than fixed, because the fix would be dropping what is worth keeping.
+    overBudget: total > budgetBytes,
+    bytes: total,
+  };
+}