@ijfw/memory-server 1.3.0

package/src/memory/reader.js

@@ -0,0 +1,300 @@
+/**
+ * Memory reader -- unified 5-tier reader.
+ * Tier 1: Claude auto-memory  ~/.claude/projects/<slug>/memory/**\/*.md
+ * Tier 2: Project .ijfw/memory  <cwd>/.ijfw/memory/**\/*.md
+ * Tier 3: Session records  <cwd>/.ijfw/sessions/*.md
+ * Tier 4: Global observations  ~/.ijfw/observations.jsonl (summarized)
+ * Tier 5: Global handoff  ~/.ijfw/HANDOFF.md if exists
+ *
+ * Each file gets a `tier` field: "Auto-memory" / "Project" / "Sessions" / "Global" / "Handoff"
+ * Zero deps: node:fs, node:path, node:os only.
+ */
+
+import { readdirSync, statSync, readFileSync, existsSync } from 'node:fs';
+import { join, relative, basename } from 'node:path';
+import { homedir } from 'node:os';
+
+const HOME         = homedir();
+const IJFW_DIR     = join(HOME, '.ijfw');
+const CLAUDE_PROJS = join(HOME, '.claude', 'projects');
+const PREVIEW_CHARS = 300;
+
+/** Parse YAML-style frontmatter (key: value lines between --- fences). */
+function parseFrontmatter(raw) {
+  const fm = { title: null, description: null, type: null };
+  const m  = raw.match(/^---\r?\n([\s\S]*?)\r?\n---/);
+  if (!m) return fm;
+  for (const line of m[1].split('\n')) {
+    const kv = line.match(/^(\w+):\s*(.+)/);
+    if (!kv) continue;
+    const key = kv[1].toLowerCase();
+    if (key === 'title')       fm.title       = kv[2].trim();
+    if (key === 'description') fm.description = kv[2].trim();
+    if (key === 'type')        fm.type        = kv[2].trim();
+  }
+  return fm;
+}
+
+/** Walk dir recursively collecting .md files, max depth 4. */
+function walkMd(dir, base, depth = 0) {
+  if (depth > 4) return [];
+  let entries;
+  try { entries = readdirSync(dir, { withFileTypes: true }); } catch { return []; }
+  const results = [];
+  for (const e of entries) {
+    const full = join(dir, e.name);
+    if (e.isDirectory()) {
+      results.push(...walkMd(full, base, depth + 1));
+    } else if (e.isFile() && e.name.endsWith('.md')) {
+      results.push({ full, rel: relative(base, full) });
+    }
+  }
+  return results;
+}
+
+/** Build a file entry from a path + tier label. */
+function buildEntry(full, rel, tier) {
+  try {
+    const st  = statSync(full);
+    const raw = readFileSync(full, 'utf8');
+    const fm  = parseFrontmatter(raw);
+
+    let title = fm.title;
+    if (!title) {
+      const hm = raw.match(/^#\s+(.+)/m);
+      title = hm ? hm[1].trim() : basename(full, '.md');
+    }
+
+    const body    = raw.replace(/^---[\s\S]*?---\r?\n/, '').trimStart();
+    const preview = body.slice(0, PREVIEW_CHARS).replace(/\s+/g, ' ').trim();
+
+    return {
+      path: full,
+      relpath: rel,
+      title,
+      description: fm.description || null,
+      type: fm.type || null,
+      preview,
+      last_modified: st.mtimeMs,
+      size: st.size,
+      tier,
+    };
+  } catch {
+    return null;
+  }
+}
+
+/** Map a project dir path to its Claude project slug. */
+function pathToSlug(projectPath) {
+  // Claude uses the absolute path with / replaced by -. On Windows the path
+  // is C:\\Users\\... -- strip the drive letter and replace both separator
+  // styles so the slug is a flat dir name that round-trips with mkdirSync.
+  return projectPath.replace(/^[A-Z]:/i, '').replace(/[\\/]/g, '-');
+}
+
+/** Find the Claude project slug for a repo root by matching path-based slug. */
+function findClaudeSlug(repoRoot) {
+  if (!repoRoot || !existsSync(CLAUDE_PROJS)) return null;
+  const slug = pathToSlug(repoRoot);
+  const candidate = join(CLAUDE_PROJS, slug);
+  if (existsSync(candidate)) return slug;
+  // Try reading all slugs for a suffix match
+  try {
+    const slugs = readdirSync(CLAUDE_PROJS, { withFileTypes: true })
+      .filter(e => e.isDirectory())
+      .map(e => e.name);
+    // Match by resolved path suffix
+    const normalized = repoRoot.replace(/\/$/, '');
+    for (const s of slugs) {
+      if (normalized.endsWith(s.replace(/-/g, '/').replace(/^\//, ''))) return s;
+    }
+    return slugs.find(s => s === slug) || null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Tier 1: Claude auto-memory files for the current project.
+ */
+function readTier1(repoRoot) {
+  const slug = findClaudeSlug(repoRoot);
+  if (!slug) return [];
+  const memDir = join(CLAUDE_PROJS, slug, 'memory');
+  if (!existsSync(memDir)) return [];
+  return walkMd(memDir, memDir)
+    .map(({ full, rel }) => buildEntry(full, rel, 'Auto-memory'))
+    .filter(Boolean);
+}
+
+/**
+ * Tier 2: Project .ijfw/memory files.
+ * Reads both ~/.ijfw/memory (global project memory) and <repoRoot>/.ijfw/memory.
+ */
+function readTier2(repoRoot) {
+  const dirs = [];
+  if (repoRoot) dirs.push(join(repoRoot, '.ijfw', 'memory'));
+  const globalMem = join(IJFW_DIR, 'memory');
+  if (existsSync(globalMem)) dirs.push(globalMem);
+
+  const files = [];
+  for (const dir of dirs) {
+    if (!existsSync(dir)) continue;
+    const entries = walkMd(dir, dir)
+      .map(({ full, rel }) => buildEntry(full, rel, 'Project'))
+      .filter(Boolean);
+    // Deduplicate by path
+    for (const e of entries) {
+      if (!files.find(f => f.path === e.path)) files.push(e);
+    }
+  }
+  return files;
+}
+
+/**
+ * Tier 3: Session records -- .md files in <repoRoot>/.ijfw/sessions/.
+ */
+function readTier3(repoRoot) {
+  if (!repoRoot) return [];
+  const sessDir = join(repoRoot, '.ijfw', 'sessions');
+  if (!existsSync(sessDir)) return [];
+  return walkMd(sessDir, sessDir)
+    .map(({ full, rel }) => buildEntry(full, rel, 'Sessions'))
+    .filter(Boolean);
+}
+
+/**
+ * Tier 4: Global observations -- summarizes ~/.ijfw/observations.jsonl.
+ * Returns synthetic entries grouped by platform.
+ */
+function readTier4() {
+  const obsPath = join(IJFW_DIR, 'observations.jsonl');
+  if (!existsSync(obsPath)) return [];
+  try {
+    const lines = readFileSync(obsPath, 'utf8').split('\n').filter(Boolean);
+    const total = lines.length;
+    if (!total) return [];
+    const st = statSync(obsPath);
+    // Count by platform for recall counts
+    const platformCounts = {};
+    for (const line of lines) {
+      try {
+        const obj = JSON.parse(line);
+        const p = obj.platform || 'unknown';
+        platformCounts[p] = (platformCounts[p] || 0) + 1;
+      } catch {}
+    }
+    const platformSummary = Object.entries(platformCounts)
+      .map(([p, c]) => `${p}: ${c}`)
+      .join(', ');
+    return [{
+      path: obsPath,
+      relpath: 'observations.jsonl',
+      title: `Global observations (${total} events)`,
+      description: platformSummary || null,
+      type: 'observations',
+      preview: `${total} observation events across ${Object.keys(platformCounts).length} platforms. ${platformSummary}`,
+      last_modified: st.mtimeMs,
+      size: st.size,
+      tier: 'Global',
+      count: total,
+    }];
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Tier 5: Global HANDOFF.md.
+ */
+function readTier5() {
+  const handoffPath = join(IJFW_DIR, 'HANDOFF.md');
+  if (!existsSync(handoffPath)) return [];
+  const entry = buildEntry(handoffPath, 'HANDOFF.md', 'Handoff');
+  return entry ? [entry] : [];
+}
+
+/**
+ * List all memory files across all 5 tiers.
+ * @param {string|null} repoRoot
+ * @param {string|null} tierFilter - filter to one tier label (optional)
+ * @returns {{ files: Array, total: number, root: string|null, tiers: Object }}
+ */
+export function listMemoryFiles(repoRoot, tierFilter = null) {
+  const t1 = readTier1(repoRoot);
+  const t2 = readTier2(repoRoot);
+  const t3 = readTier3(repoRoot);
+  const t4 = readTier4();
+  const t5 = readTier5();
+
+  const all = [...t1, ...t2, ...t3, ...t4, ...t5];
+
+  // Compute per-tier counts before filtering
+  const tiers = {
+    'Auto-memory': t1.length,
+    'Project':     t2.length,
+    'Sessions':    t3.length,
+    'Global':      t4.length,
+    'Handoff':     t5.length,
+  };
+
+  let files = all;
+  if (tierFilter) {
+    files = all.filter(f => f.tier === tierFilter);
+  }
+
+  // Sort by most recently modified within each tier grouping
+  files.sort((a, b) => b.last_modified - a.last_modified);
+
+  // Use the first non-null path as the security root for /api/memory/file
+  const root = repoRoot || IJFW_DIR;
+
+  return { files, total: files.length, root, tiers };
+}
+
+/** List all known projects by scanning ~/.claude/projects/. */
+export function listKnownProjects() {
+  if (!existsSync(CLAUDE_PROJS)) return [];
+  try {
+    const entries = readdirSync(CLAUDE_PROJS, { withFileTypes: true })
+      .filter(e => e.isDirectory());
+    return entries.map(e => {
+      const slug = e.name;
+      const memDir = join(CLAUDE_PROJS, slug, 'memory');
+      let lastActivity = null;
+      let memCount = 0;
+      try {
+        const st = statSync(join(CLAUDE_PROJS, slug));
+        lastActivity = new Date(st.mtimeMs).toISOString();
+      } catch {}
+      if (existsSync(memDir)) {
+        try { memCount = readdirSync(memDir).filter(n => n.endsWith('.md')).length; } catch {}
+      }
+      // Convert slug back to path (best-effort)
+      const projectPath = slug.replace(/-/g, '/').replace(/^\//, '/');
+      return { slug, projectPath, lastActivity, memCount };
+    }).sort((a, b) => (b.lastActivity || '') > (a.lastActivity || '') ? 1 : -1);
+  } catch {
+    return [];
+  }
+}
+
+/** Resolve a "legacy" single-root for backward compat with /api/memory/file security check. */
+export function resolveMemoryRoot(repoRoot) {
+  if (repoRoot) {
+    const local = join(repoRoot, '.ijfw', 'memory');
+    if (existsSync(local)) return local;
+  }
+  const global = join(IJFW_DIR, 'memory');
+  if (existsSync(global)) return global;
+  return null;
+}
+
+/** Read raw body of a single memory file. Returns null if unreadable. */
+export function readMemoryFile(filePath) {
+  try {
+    return readFileSync(filePath, 'utf8');
+  } catch {
+    return null;
+  }
+}

package/src/memory/recall-counter.js

@@ -0,0 +1,76 @@
+/**
+ * Recall counter -- counts per-file recall events from observations.jsonl.
+ * A "memory-recall" observation has {type:"memory-recall", file_path:<str>}.
+ * Also accepts legacy observations where tool_name === "ijfw_memory_recall"
+ * and files[0] is the memory file path.
+ */
+
+import { readFileSync, existsSync } from 'node:fs';
+
+const MS_PER_WEEK = 7 * 24 * 60 * 60 * 1000;
+
+/**
+ * @param {string} ledgerPath  path to observations.jsonl
+ * @returns {{ counts: Map<string,number>, totalThisWeek: number, allTime: Map<string,number> }}
+ */
+export function buildRecallCounts(ledgerPath) {
+  const counts      = new Map(); // all-time per file
+  const weekCounts  = new Map(); // this-week per file
+  let totalThisWeek = 0;
+
+  if (!existsSync(ledgerPath)) return { counts, weekCounts, totalThisWeek };
+
+  const cutoff = Date.now() - MS_PER_WEEK;
+  let raw;
+  try { raw = readFileSync(ledgerPath, 'utf8'); } catch { return { counts, weekCounts, totalThisWeek }; }
+
+  for (const line of raw.split('\n')) {
+    if (!line.trim()) continue;
+    let obs;
+    try { obs = JSON.parse(line); } catch { continue; }
+
+    // Detect memory-recall observation by type or tool_name
+    const isRecall = obs.type === 'memory-recall' ||
+                     (obs.tool_name === 'ijfw_memory_recall');
+    if (!isRecall) continue;
+
+    const fp = obs.file_path || (obs.files && obs.files[0]) || null;
+    if (!fp) { totalThisWeek++; continue; } // count global recalls without file
+
+    counts.set(fp, (counts.get(fp) || 0) + 1);
+
+    const ts = obs.ts ? new Date(obs.ts).getTime() : 0;
+    if (ts >= cutoff) {
+      weekCounts.set(fp, (weekCounts.get(fp) || 0) + 1);
+      totalThisWeek++;
+    }
+  }
+
+  return { counts, weekCounts, totalThisWeek };
+}
+
+/**
+ * Merge recall counts into file list.
+ * @param {Array} files  from listMemoryFiles()
+ * @param {Map<string,number>} counts
+ * @param {Map<string,number>} weekCounts
+ * @returns {Array} files with recall_count + recall_count_week added
+ */
+export function mergeRecallCounts(files, counts, weekCounts) {
+  return files.map(f => ({
+    ...f,
+    recall_count:      counts.get(f.path) || 0,
+    recall_count_week: weekCounts.get(f.path) || 0,
+  }));
+}
+
+/**
+ * Top N recalled files (all-time).
+ */
+export function topRecalled(files, n = 5) {
+  return [...files]
+    .filter(f => f.recall_count > 0)
+    .sort((a, b) => b.recall_count - a.recall_count)
+    .slice(0, n)
+    .map(f => ({ path: f.path, relpath: f.relpath, title: f.title, count: f.recall_count }));
+}

package/src/memory/schema.sql

@@ -0,0 +1,79 @@
+-- IJFW v1.3.0 -- D-Pillar / D0 memory FTS5 schema.
+-- Source of truth: .planning/1.3.0/D-PILLAR-SPEC.md (D0)
+--
+-- Memory tier of the dual-tier index. Compute tier already ships its own
+-- FTS5 schema in ../compute/schema.sql (V3-B4). This is the parallel
+-- structure for the warm tier of the memory pipeline:
+--
+--   Hot   -- markdown files on disk (existing readers/walkers, unchanged)
+--   Warm  -- this FTS5 index (rebuilt-on-demand from the hot tier)
+--   Cold  -- optional vector layer (deferred, future milestone)
+--
+-- Tokenizer: porter unicode61. Same as compute/schema.sql v2 -- morphological
+-- variants ("authenticate"/"authenticating"/"running") collapse to a shared
+-- stem so memory recall isn't brittle to verb form. Porter does NOT handle
+-- irregular verbs ("ran" stays "ran"); documented limitation.
+--
+-- Schema is ADD-ONLY. user_version starts at 1 (this file = migration 001).
+
+PRAGMA user_version = 1;
+
+-- Content table: indexed memory entries.
+--   id           -- autoinc primary key
+--   body         -- the indexed text (full markdown body or excerpt)
+--   source       -- provenance pointer: file path, "session:N", "tier:auto",
+--                   etc. Nullable for free-form inserts.
+--   session_id   -- caller's session correlation id. Nullable; populated when
+--                   indexEntry is called from a session-aware path.
+--   created_at   -- unix ms; set by indexEntry when row is written.
+CREATE TABLE IF NOT EXISTS memory_entries (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  body TEXT NOT NULL,
+  source TEXT,
+  session_id TEXT,
+  created_at INTEGER NOT NULL
+);
+CREATE INDEX IF NOT EXISTS memory_entries_source_idx
+  ON memory_entries(source) WHERE source IS NOT NULL;
+CREATE INDEX IF NOT EXISTS memory_entries_session_idx
+  ON memory_entries(session_id) WHERE session_id IS NOT NULL;
+CREATE INDEX IF NOT EXISTS memory_entries_created_idx
+  ON memory_entries(created_at);
+
+-- FTS5 mirror over the body column. External-content table so the FTS5
+-- index stays a thin view over memory_entries; updates flow through the
+-- AI/AD/AU triggers below.
+CREATE VIRTUAL TABLE IF NOT EXISTS memory_entries_fts USING fts5(
+  body,
+  content='memory_entries',
+  content_rowid='id',
+  tokenize='porter unicode61'
+);
+
+-- External-content sync triggers (FTS5 standard pattern).
+CREATE TRIGGER IF NOT EXISTS memory_entries_ai
+  AFTER INSERT ON memory_entries BEGIN
+    INSERT INTO memory_entries_fts(rowid, body) VALUES (new.id, new.body);
+  END;
+CREATE TRIGGER IF NOT EXISTS memory_entries_ad
+  AFTER DELETE ON memory_entries BEGIN
+    INSERT INTO memory_entries_fts(memory_entries_fts, rowid, body)
+      VALUES('delete', old.id, old.body);
+  END;
+CREATE TRIGGER IF NOT EXISTS memory_entries_au
+  AFTER UPDATE ON memory_entries BEGIN
+    INSERT INTO memory_entries_fts(memory_entries_fts, rowid, body)
+      VALUES('delete', old.id, old.body);
+    INSERT INTO memory_entries_fts(rowid, body) VALUES (new.id, new.body);
+  END;
+
+-- Schema version table -- mirrors compute/schema.sql shape so a future
+-- merger of the two dbs (if ever attempted) lines up.
+CREATE TABLE IF NOT EXISTS schema_meta (
+  version INTEGER PRIMARY KEY,
+  applied_at INTEGER NOT NULL,
+  description TEXT
+);
+INSERT OR IGNORE INTO schema_meta(version, applied_at, description)
+  VALUES (1, CAST(strftime('%s','now') AS INTEGER) * 1000,
+          'memory v1.3.0 -- memory_entries + memory_entries_fts (porter unicode61)');