@cohortapp/agent-sdk 2.3.0

package/lib/learning/session-index.mjs

@@ -0,0 +1,352 @@
+/**
+ * lib/learning/session-index.mjs — verbatim transcript FTS5 store (WS3).
+ *
+ * Complementary to collective memory cards: cards are the fast, budgeted gist
+ * injected at SessionStart; THIS is the on-demand, verbatim recall of the
+ * agent's own past sessions. A separate SQLite DB at state/learning/sessions.db
+ * (so it never collides with the RAG corpus index), mirroring the FTS5 usage in
+ * lib/rag/indexer.mjs.
+ *
+ * Schema:
+ *   turns(body, session_id UNINDEXED, role UNINDEXED, ts UNINDEXED, seq UNINDEXED)
+ *     — FTS5 virtual table, one row per transcript turn.
+ *   sessions(session_id PK, file, mtime_ms, turn_count, first_ts, last_ts, indexed_at)
+ *     — ingest bookkeeping (idempotent re-ingest via mtime).
+ *
+ * Search has four modes (ported from Hermes):
+ *   discovery — bm25 ranked hits + snippet + a per-session BOOKEND (the first &
+ *               last turn of each matched session, for orientation).
+ *   scroll    — a window of ±N turns around a given (session_id, seq).
+ *   read      — every turn of one session, in order.
+ *   browse    — the most-recent sessions (metadata), newest first.
+ *
+ * Reads are fail-open at the call boundary (a missing DB → empty results), but
+ * ingest surfaces hard FS/DB errors to its caller (the reflect worker swallows
+ * them) so a genuinely broken DB is debuggable.
+ *
+ * @module lib/learning/session-index
+ */
+
+"use strict";
+
+import { readFileSync, statSync, existsSync, readdirSync, mkdirSync } from "node:fs";
+import { join, resolve, basename } from "node:path";
+
+let _Database = null;
+async function loadDatabase() {
+  if (_Database) return _Database;
+  const mod = await import("better-sqlite3");
+  _Database = mod.default || mod;
+  return _Database;
+}
+
+const DB_REL = "state/learning/sessions.db";
+
+const SCHEMA = `
+CREATE VIRTUAL TABLE IF NOT EXISTS turns USING fts5(
+  body,
+  session_id UNINDEXED,
+  role UNINDEXED,
+  ts UNINDEXED,
+  seq UNINDEXED,
+  tokenize = "porter unicode61 remove_diacritics 2"
+);
+
+CREATE TABLE IF NOT EXISTS sessions (
+  session_id TEXT PRIMARY KEY,
+  file       TEXT NOT NULL,
+  mtime_ms   INTEGER NOT NULL,
+  turn_count INTEGER NOT NULL,
+  first_ts   TEXT,
+  last_ts    TEXT,
+  indexed_at INTEGER NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS sessions_last_ts ON sessions(last_ts);
+`;
+
+/** Absolute path of the sessions DB for an agent root. */
+export function dbPath(agentRoot) {
+  return join(resolve(agentRoot), DB_REL);
+}
+
+function openDb(Database, agentRoot, { readonly = false } = {}) {
+  const p = dbPath(agentRoot);
+  if (readonly) {
+    if (!existsSync(p)) return null;
+    return new Database(p, { readonly: true });
+  }
+  mkdirSync(join(resolve(agentRoot), "state", "learning"), { recursive: true });
+  const db = new Database(p);
+  db.pragma("journal_mode = WAL");
+  db.pragma("synchronous = NORMAL");
+  db.exec(SCHEMA);
+  return db;
+}
+
+/** Pull role-tagged content out of one transcript record (Claude Code JSONL). */
+function recordToTurn(obj) {
+  const role = obj.role || (obj.message && obj.message.role) || obj.type || "";
+  let content = obj.content ?? (obj.message && obj.message.content);
+  if (Array.isArray(content)) {
+    content = content
+      .map((part) => (typeof part === "string" ? part : part && part.text ? part.text : ""))
+      .filter(Boolean)
+      .join(" ");
+  } else if (typeof content !== "string") {
+    content = typeof obj.text === "string" ? obj.text : "";
+  }
+  const ts = obj.timestamp || obj.ts || obj.time || "";
+  return { role: String(role || ""), body: String(content || "").trim(), ts: String(ts || "") };
+}
+
+/**
+ * Derive the session id for a transcript file. Prefer an explicit session id in
+ * the records; fall back to the filename stem (Claude Code names files by
+ * session id, e.g. `<uuid>.jsonl`).
+ */
+function deriveSessionId(records, file) {
+  for (const r of records) {
+    const sid = r.sessionId || r.session_id || (r.message && r.message.session_id);
+    if (sid) return String(sid);
+  }
+  return basename(file).replace(/\.jsonl$/i, "");
+}
+
+/** Parse a JSONL transcript file into turn rows. */
+function parseTranscript(file) {
+  const raw = readFileSync(file, "utf8");
+  const records = [];
+  for (const line of raw.split("\n")) {
+    const t = line.trim();
+    if (!t.startsWith("{")) continue;
+    try { records.push(JSON.parse(t)); } catch { /* skip bad line */ }
+  }
+  const sessionId = deriveSessionId(records, file);
+  const turns = [];
+  let seq = 0;
+  for (const rec of records) {
+    const turn = recordToTurn(rec);
+    if (!turn.body) continue;
+    turns.push({ ...turn, seq: seq++ });
+  }
+  return { sessionId, turns };
+}
+
+/**
+ * Ingest one transcript file. Idempotent: a file whose mtime matches the
+ * recorded one is skipped; a re-ingest under a changed mtime replaces the
+ * session's turns wholesale. Returns a small status object.
+ *
+ * @param {object} o - { agentRoot, file, force?, _db? }
+ * @returns {Promise<{sessionId:?string, turns:number, skipped:boolean, reason:string}>}
+ */
+export async function ingestSession(o = {}) {
+  const { agentRoot, file } = o;
+  if (!agentRoot || !file) return { sessionId: null, turns: 0, skipped: true, reason: "bad-args" };
+  if (!existsSync(file)) return { sessionId: null, turns: 0, skipped: true, reason: "missing-file" };
+
+  const Database = await loadDatabase();
+  const db = o._db || openDb(Database, agentRoot);
+  const ownDb = !o._db;
+  try {
+    const st = statSync(file);
+    const mtime = Math.floor(st.mtimeMs);
+    const { sessionId, turns } = parseTranscript(file);
+    if (!sessionId) return { sessionId: null, turns: 0, skipped: true, reason: "no-session-id" };
+
+    const existing = db.prepare("SELECT mtime_ms FROM sessions WHERE session_id = ?").get(sessionId);
+    if (existing && existing.mtime_ms === mtime && !o.force) {
+      return { sessionId, turns: 0, skipped: true, reason: "unchanged" };
+    }
+
+    const insertTurn = db.prepare(
+      "INSERT INTO turns (body, session_id, role, ts, seq) VALUES (?, ?, ?, ?, ?)"
+    );
+    const deleteTurns = db.prepare("DELETE FROM turns WHERE session_id = ?");
+    const upsertSession = db.prepare(
+      "INSERT INTO sessions (session_id, file, mtime_ms, turn_count, first_ts, last_ts, indexed_at) " +
+      "VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(session_id) DO UPDATE SET " +
+      "file=excluded.file, mtime_ms=excluded.mtime_ms, turn_count=excluded.turn_count, " +
+      "first_ts=excluded.first_ts, last_ts=excluded.last_ts, indexed_at=excluded.indexed_at"
+    );
+
+    const firstTs = turns.length ? turns[0].ts : "";
+    const lastTs = turns.length ? turns[turns.length - 1].ts : "";
+    const txn = db.transaction(() => {
+      deleteTurns.run(sessionId);
+      for (const t of turns) insertTurn.run(t.body, sessionId, t.role, t.ts, t.seq);
+      upsertSession.run(sessionId, file, mtime, turns.length, firstTs, lastTs, Date.now());
+    });
+    txn();
+    return { sessionId, turns: turns.length, skipped: false, reason: existing ? "updated" : "added" };
+  } finally {
+    if (ownDb) db.close();
+  }
+}
+
+/**
+ * Ingest every `*.jsonl` under a logs directory (default logs/sessions). One DB
+ * connection for the batch. Idempotent. Fail-open per-file (a bad file is
+ * skipped, not fatal).
+ *
+ * @param {object} o - { agentRoot, logsDir?, force? }
+ * @returns {Promise<{added:number, updated:number, skipped:number, files:number}>}
+ */
+export async function ingestAll(o = {}) {
+  const { agentRoot } = o;
+  const result = { added: 0, updated: 0, skipped: 0, files: 0 };
+  if (!agentRoot) return result;
+  const logsDir = o.logsDir || join(resolve(agentRoot), "logs", "sessions");
+  if (!existsSync(logsDir)) return result;
+
+  let files;
+  try { files = readdirSync(logsDir).filter((f) => f.endsWith(".jsonl")); }
+  catch { return result; }
+
+  const Database = await loadDatabase();
+  const db = openDb(Database, agentRoot);
+  try {
+    for (const f of files) {
+      result.files++;
+      try {
+        const r = await ingestSession({ agentRoot, file: join(logsDir, f), force: o.force, _db: db });
+        if (r.reason === "added") result.added++;
+        else if (r.reason === "updated") result.updated++;
+        else result.skipped++;
+      } catch {
+        result.skipped++; // a single corrupt transcript must not abort the batch
+      }
+    }
+  } finally {
+    db.close();
+  }
+  return result;
+}
+
+// ---------------------------------------------------------------------------
+// Search — 4 modes
+// ---------------------------------------------------------------------------
+
+const STOPWORDS = new Set(["the", "and", "for", "are", "but", "not", "you", "all", "with", "this", "that", "from", "what", "when", "have", "was", "were"]);
+
+function tokenizeQuery(q) {
+  const all = String(q || "")
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}\s]/gu, " ")
+    .split(/\s+/)
+    .filter((t) => t.length >= 3 && !STOPWORDS.has(t));
+  if (!all.length) return [];
+  return Array.from(new Set(all)).sort((a, b) => b.length - a.length).slice(0, 16).map((t) => `"${t.replace(/"/g, "")}"`);
+}
+
+/** First & last turn of a session — the orientation "bookend" for a hit. */
+function bookend(db, sessionId) {
+  const first = db.prepare("SELECT role, body, seq, ts FROM turns WHERE session_id = ? ORDER BY seq ASC LIMIT 1").get(sessionId);
+  const last = db.prepare("SELECT role, body, seq, ts FROM turns WHERE session_id = ? ORDER BY seq DESC LIMIT 1").get(sessionId);
+  return { first: first || null, last: last || null };
+}
+
+/**
+ * Search the transcript store.
+ *
+ * @param {string} query                 free-text (discovery/scroll); ignored for read/browse
+ * @param {object} [opts]
+ * @param {string} [opts.agentRoot]
+ * @param {"discovery"|"scroll"|"read"|"browse"} [opts.mode="discovery"]
+ * @param {number} [opts.topN=8]
+ * @param {string} [opts.sessionId]       required for read; the anchor for scroll
+ * @param {number} [opts.seq]             scroll anchor turn
+ * @param {number} [opts.window=6]        scroll half-window (turns each side)
+ * @returns {Promise<object>}             shape varies by mode (see code)
+ */
+export async function search(query, opts = {}) {
+  const agentRoot = opts.agentRoot || process.env.AGENT_ROOT || process.cwd();
+  const mode = opts.mode || "discovery";
+  const Database = await loadDatabase();
+  const db = openDb(Database, agentRoot, { readonly: true });
+  if (!db) return emptyFor(mode);
+  try {
+    if (mode === "read") return readMode(db, opts);
+    if (mode === "browse") return browseMode(db, opts);
+    if (mode === "scroll") return scrollMode(db, opts);
+    return discoveryMode(db, query, opts);
+  } finally {
+    db.close();
+  }
+}
+
+function emptyFor(mode) {
+  if (mode === "read") return { mode, sessionId: null, turns: [] };
+  if (mode === "browse") return { mode, sessions: [] };
+  if (mode === "scroll") return { mode, sessionId: null, turns: [] };
+  return { mode: "discovery", hits: [] };
+}
+
+function discoveryMode(db, query, opts) {
+  const topN = Math.min(Math.max(parseInt(opts.topN, 10) || 8, 1), 50);
+  const tokens = tokenizeQuery(query);
+  if (!tokens.length) return { mode: "discovery", hits: [] };
+  const match = tokens.join(" OR ");
+  const rows = db.prepare(`
+    SELECT session_id, role, ts, seq, body,
+           bm25(turns) AS score,
+           snippet(turns, 0, '«', '»', '…', 18) AS snippet
+    FROM turns
+    WHERE turns MATCH ?
+    ORDER BY bm25(turns)
+    LIMIT ?
+  `).all(match, topN);
+
+  const seen = new Map();
+  const hits = [];
+  for (const r of rows) {
+    const hit = {
+      session_id: r.session_id,
+      seq: r.seq,
+      role: r.role,
+      ts: r.ts,
+      snippet: r.snippet,
+      score: r.score,
+    };
+    if (!seen.has(r.session_id)) {
+      seen.set(r.session_id, true);
+      hit.bookend = bookend(db, r.session_id);
+    }
+    hits.push(hit);
+  }
+  return { mode: "discovery", hits };
+}
+
+function scrollMode(db, opts) {
+  const sessionId = opts.sessionId;
+  if (!sessionId) return { mode: "scroll", sessionId: null, turns: [] };
+  const anchor = Number.isFinite(opts.seq) ? opts.seq : 0;
+  const half = Number.isFinite(opts.window) ? opts.window : 6;
+  const lo = anchor - half;
+  const hi = anchor + half;
+  const turns = db.prepare(
+    "SELECT role, body, seq, ts FROM turns WHERE session_id = ? AND seq BETWEEN ? AND ? ORDER BY seq ASC"
+  ).all(sessionId, lo, hi);
+  return { mode: "scroll", sessionId, anchor, window: half, turns };
+}
+
+function readMode(db, opts) {
+  const sessionId = opts.sessionId;
+  if (!sessionId) return { mode: "read", sessionId: null, turns: [] };
+  const turns = db.prepare(
+    "SELECT role, body, seq, ts FROM turns WHERE session_id = ? ORDER BY seq ASC"
+  ).all(sessionId);
+  const meta = db.prepare("SELECT * FROM sessions WHERE session_id = ?").get(sessionId) || null;
+  return { mode: "read", sessionId, meta, turns };
+}
+
+function browseMode(db, opts) {
+  const topN = Math.min(Math.max(parseInt(opts.topN, 10) || 15, 1), 200);
+  const sessions = db.prepare(
+    "SELECT session_id, turn_count, first_ts, last_ts, indexed_at FROM sessions ORDER BY last_ts DESC, indexed_at DESC LIMIT ?"
+  ).all(topN);
+  return { mode: "browse", sessions };
+}
+
+export default { ingestSession, ingestAll, search, dbPath };

package/lib/learning/session-index.test.mjs

@@ -0,0 +1,125 @@
+/**
+ * session-index.test.mjs — transcript FTS5 store tests (temp DB, no network).
+ * Run: node --test lib/learning/session-index.test.mjs
+ *
+ * Covers: ingest idempotency (mtime skip + force replace), discovery bm25 +
+ * bookend, scroll window, read full session, browse recency.
+ */
+"use strict";
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, mkdirSync, writeFileSync, utimesSync, existsSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { ingestSession, ingestAll, search, dbPath } from "./session-index.mjs";
+
+function tmpRoot() {
+  const root = mkdtempSync(join(tmpdir(), "maestro-sessidx-"));
+  mkdirSync(join(root, "logs", "sessions"), { recursive: true });
+  return root;
+}
+
+/** Write a JSONL transcript with explicit sessionId on each record. */
+function writeTranscript(root, name, sessionId, turns, mtimeSec) {
+  const file = join(root, "logs", "sessions", `${name}.jsonl`);
+  const lines = turns.map((t) => JSON.stringify({
+    sessionId, role: t.role, content: t.body, timestamp: t.ts || "",
+  }));
+  writeFileSync(file, lines.join("\n") + "\n");
+  if (mtimeSec) utimesSync(file, mtimeSec, mtimeSec);
+  return file;
+}
+
+const TURNS = [
+  { role: "user", body: "We need to reconcile the billing export with stripe", ts: "2026-06-01T09:00:00Z" },
+  { role: "assistant", body: "I will pull the export and diff it against the ledger", ts: "2026-06-01T09:01:00Z" },
+  { role: "user", body: "Great, focus on the duplicate charge anomaly", ts: "2026-06-01T09:02:00Z" },
+  { role: "assistant", body: "Found three duplicates; refund issued and reconciliation closed", ts: "2026-06-01T09:03:00Z" },
+];
+
+test("ingestSession: adds turns, idempotent on unchanged mtime, replaces on force", async () => {
+  const root = tmpRoot();
+  const file = writeTranscript(root, "sess-a", "SID-A", TURNS, 1_700_000_000);
+
+  const r1 = await ingestSession({ agentRoot: root, file });
+  assert.equal(r1.sessionId, "SID-A");
+  assert.equal(r1.turns, 4);
+  assert.equal(r1.reason, "added");
+  assert.ok(existsSync(dbPath(root)), "db created");
+
+  // Re-ingest, same mtime → skipped (idempotent).
+  const r2 = await ingestSession({ agentRoot: root, file });
+  assert.equal(r2.skipped, true);
+  assert.equal(r2.reason, "unchanged");
+
+  // Force → replace.
+  const r3 = await ingestSession({ agentRoot: root, file, force: true });
+  assert.equal(r3.skipped, false);
+  assert.equal(r3.turns, 4);
+});
+
+test("ingestAll: ingests a directory, then skips unchanged on a second pass", async () => {
+  const root = tmpRoot();
+  writeTranscript(root, "sess-a", "SID-A", TURNS, 1_700_000_000);
+  writeTranscript(root, "sess-b", "SID-B", TURNS.slice(0, 2), 1_700_000_100);
+
+  const first = await ingestAll({ agentRoot: root });
+  assert.equal(first.files, 2);
+  assert.equal(first.added, 2);
+
+  const second = await ingestAll({ agentRoot: root });
+  assert.equal(second.added, 0);
+  assert.equal(second.skipped, 2, "unchanged files skipped");
+});
+
+test("search discovery: bm25 hit + per-session bookend (first & last turn)", async () => {
+  const root = tmpRoot();
+  const file = writeTranscript(root, "sess-a", "SID-A", TURNS, 1_700_000_000);
+  await ingestSession({ agentRoot: root, file });
+
+  const res = await search("duplicate charge anomaly", { agentRoot: root, mode: "discovery", topN: 5 });
+  assert.equal(res.mode, "discovery");
+  assert.ok(res.hits.length >= 1, "found a matching turn");
+  const top = res.hits[0];
+  assert.equal(top.session_id, "SID-A");
+  assert.match(top.snippet, /duplicate|anomaly|charge/i);
+  // Bookend present on the first hit of the session.
+  assert.ok(top.bookend, "bookend attached");
+  assert.match(top.bookend.first.body, /reconcile the billing export/);
+  assert.match(top.bookend.last.body, /reconciliation closed/);
+});
+
+test("search scroll: window of turns around an anchor seq", async () => {
+  const root = tmpRoot();
+  const file = writeTranscript(root, "sess-a", "SID-A", TURNS, 1_700_000_000);
+  await ingestSession({ agentRoot: root, file });
+
+  const res = await search("", { agentRoot: root, mode: "scroll", sessionId: "SID-A", seq: 1, window: 1 });
+  assert.equal(res.mode, "scroll");
+  const seqs = res.turns.map((t) => t.seq);
+  assert.deepEqual(seqs, [0, 1, 2], "±1 around seq 1");
+});
+
+test("search read: full session in order; browse: recent sessions", async () => {
+  const root = tmpRoot();
+  writeTranscript(root, "sess-a", "SID-A", TURNS, 1_700_000_000);
+  writeTranscript(root, "sess-b", "SID-B", TURNS.slice(0, 2), 1_700_000_100);
+  await ingestAll({ agentRoot: root });
+
+  const read = await search("", { agentRoot: root, mode: "read", sessionId: "SID-A" });
+  assert.equal(read.turns.length, 4);
+  assert.deepEqual(read.turns.map((t) => t.seq), [0, 1, 2, 3]);
+
+  const browse = await search("", { agentRoot: root, mode: "browse" });
+  assert.equal(browse.mode, "browse");
+  const ids = browse.sessions.map((s) => s.session_id);
+  assert.ok(ids.includes("SID-A") && ids.includes("SID-B"));
+});
+
+test("search: missing DB fails open to empty per mode", async () => {
+  const root = tmpRoot(); // nothing ingested
+  assert.deepEqual((await search("x", { agentRoot: root, mode: "discovery" })).hits, []);
+  assert.deepEqual((await search("", { agentRoot: root, mode: "browse" })).sessions, []);
+});