@remnic/core 1.1.12 → 1.1.14

package/src/lcm/queue.test.ts

@@ -0,0 +1,178 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { LcmWorkQueue } from "./queue.js";
+
+function deferred<T = void>() {
+  let resolve!: (value: T | PromiseLike<T>) => void;
+  let reject!: (reason?: unknown) => void;
+  const promise = new Promise<T>((res, rej) => {
+    resolve = res;
+    reject = rej;
+  });
+  return { promise, resolve, reject };
+}
+
+test("LcmWorkQueue coalesces duplicate pending jobs per session", async () => {
+  const blocker = deferred();
+  const executionOrder: string[] = [];
+
+  const queue = new LcmWorkQueue({
+    concurrency: 1,
+    worker: async (sessionId) => {
+      executionOrder.push(sessionId);
+      if (sessionId === "blocked") {
+        await blocker.promise;
+      }
+    },
+  });
+
+  queue.enqueue("blocked", [{ role: "user", content: "hold" }]);
+  queue.enqueue("session-a", [{ role: "user", content: "one" }]);
+  queue.enqueue("session-a", [{ role: "assistant", content: "two" }]);
+
+  await Promise.resolve();
+
+  assert.equal(queue.inFlightCount, 1);
+  assert.equal(queue.depth, 1);
+
+  blocker.resolve();
+  await queue.whenIdle();
+
+  assert.deepEqual(executionOrder, ["blocked", "session-a"]);
+  assert.equal(queue.inFlightCount, 0);
+  assert.equal(queue.depth, 0);
+});
+
+test("LcmWorkQueue continues after a worker failure", async () => {
+  const seen: string[] = [];
+
+  const queue = new LcmWorkQueue({
+    concurrency: 1,
+    worker: async (sessionId) => {
+      seen.push(sessionId);
+      if (sessionId === "session-a") {
+        throw new Error("boom");
+      }
+    },
+  });
+
+  queue.enqueue("session-a", [{ role: "user", content: "one" }]);
+  queue.enqueue("session-b", [{ role: "assistant", content: "two" }]);
+
+  await queue.whenIdle();
+
+  assert.deepEqual(seen, ["session-a", "session-b"]);
+  assert.equal(queue.inFlightCount, 0);
+  assert.equal(queue.depth, 0);
+});
+
+test("LcmWorkQueue preserves first enqueue time when pending jobs coalesce", async () => {
+  const realNow = Date.now;
+  let now = 100;
+  const blocker = deferred();
+  const waits = new Map<string, number>();
+
+  Date.now = () => now;
+
+  try {
+    const queue = new LcmWorkQueue({
+      concurrency: 1,
+      worker: async (sessionId) => {
+        if (sessionId === "blocked") {
+          await blocker.promise;
+        }
+      },
+      hooks: {
+        onJobStart: ({ sessionId, waitMs }) => waits.set(sessionId, waitMs),
+      },
+    });
+
+    queue.enqueue("blocked", [{ role: "user", content: "hold" }]);
+    await Promise.resolve();
+
+    now = 110;
+    queue.enqueue("session-a", [{ role: "user", content: "one" }]);
+
+    now = 150;
+    queue.enqueue("session-a", [{ role: "assistant", content: "two" }]);
+
+    now = 200;
+    blocker.resolve();
+    await queue.whenIdle();
+
+    assert.equal(waits.get("session-a"), 90);
+  } finally {
+    Date.now = realNow;
+  }
+});
+
+test("LcmWorkQueue does not start a second job for a session already in flight", async () => {
+  const blocker = deferred();
+  const started: string[] = [];
+  let concurrentSameSession = false;
+  const activeSessions = new Set<string>();
+
+  const queue = new LcmWorkQueue({
+    concurrency: 2,
+    worker: async (sessionId) => {
+      if (activeSessions.has(sessionId)) {
+        concurrentSameSession = true;
+      }
+      activeSessions.add(sessionId);
+      started.push(sessionId);
+      if (sessionId === "session-a") {
+        await blocker.promise;
+      }
+      activeSessions.delete(sessionId);
+    },
+  });
+
+  queue.enqueue("session-a", [{ role: "user", content: "first" }]);
+  await Promise.resolve();
+  queue.enqueue("session-a", [{ role: "assistant", content: "second" }]);
+  queue.enqueue("session-b", [{ role: "user", content: "other" }]);
+
+  await Promise.resolve();
+  await Promise.resolve();
+
+  assert.equal(concurrentSameSession, false);
+  assert.deepEqual(started, ["session-a", "session-b"]);
+  assert.equal(queue.depth, 1);
+
+  blocker.resolve();
+  await queue.whenIdle();
+
+  assert.equal(concurrentSameSession, false);
+  assert.deepEqual(started, ["session-a", "session-b", "session-a"]);
+});
+
+test("LcmWorkQueue waits for a specific session without waiting for unrelated idle work", async () => {
+  const blocker = deferred();
+  let sessionIdleResolved = false;
+
+  const queue = new LcmWorkQueue({
+    concurrency: 1,
+    worker: async (sessionId) => {
+      if (sessionId === "blocked") {
+        await blocker.promise;
+      }
+    },
+  });
+
+  queue.enqueue("blocked", [{ role: "user", content: "hold" }]);
+  queue.enqueue("session-a", [{ role: "assistant", content: "queued" }]);
+
+  await Promise.resolve();
+  const sessionIdlePromise = queue.whenSessionIdle("session-a").then(() => {
+    sessionIdleResolved = true;
+  });
+
+  await Promise.resolve();
+  assert.equal(sessionIdleResolved, false);
+
+  blocker.resolve();
+  await sessionIdlePromise;
+
+  assert.equal(sessionIdleResolved, true);
+  await queue.whenIdle();
+});

package/src/lcm/queue.ts

@@ -0,0 +1,200 @@
+export interface LcmObserveMessage {
+  role: string;
+  content: string;
+  parts?: import("../message-parts/index.js").LcmMessagePartInput[];
+  rawContent?: unknown;
+  sourceFormat?: import("../message-parts/index.js").MessagePartSourceFormat;
+}
+
+export interface LcmWorkQueueHooks {
+  onJobStart?: (input: {
+    sessionId: string;
+    depth: number;
+    inFlight: number;
+    waitMs: number;
+  }) => void;
+  onJobFinish?: (input: {
+    sessionId: string;
+    depth: number;
+    inFlight: number;
+    waitMs: number;
+    runMs: number;
+    totalMs: number;
+    error?: unknown;
+  }) => void;
+}
+
+export interface LcmWorkQueueOptions {
+  concurrency?: number;
+  worker: (sessionId: string, messages: LcmObserveMessage[]) => Promise<void>;
+  hooks?: LcmWorkQueueHooks;
+}
+
+interface PendingJob {
+  sessionId: string;
+  messages: LcmObserveMessage[];
+  enqueuedAt: number;
+}
+
+/**
+ * Small keyed work queue for LCM observe processing.
+ *
+ * Jobs are coalesced per session while they are pending. A session can have
+ * at most one in-flight job at a time, and the queue processes jobs FIFO up to
+ * the configured concurrency limit.
+ */
+export class LcmWorkQueue {
+  private readonly concurrency: number;
+  private readonly worker: (
+    sessionId: string,
+    messages: LcmObserveMessage[],
+  ) => Promise<void>;
+  private readonly hooks: LcmWorkQueueHooks;
+  private readonly pending = new Map<string, PendingJob>();
+  private readonly inFlightSessions = new Set<string>();
+  private inFlight = 0;
+  private readonly idleWaiters: Array<() => void> = [];
+  private readonly sessionIdleWaiters = new Map<string, Array<() => void>>();
+  private drainScheduled = false;
+
+  constructor(options: LcmWorkQueueOptions) {
+    this.concurrency = Math.max(1, Math.floor(options.concurrency ?? 1));
+    this.worker = options.worker;
+    this.hooks = options.hooks ?? {};
+  }
+
+  enqueue(sessionId: string, messages: LcmObserveMessage[]): void {
+    if (messages.length === 0) return;
+
+    const now = Date.now();
+    const existing = this.pending.get(sessionId);
+    if (existing) {
+      existing.messages.push(...messages.map((message) => ({ ...message })));
+    } else {
+      this.pending.set(sessionId, {
+        sessionId,
+        messages: messages.map((message) => ({ ...message })),
+        enqueuedAt: now,
+      });
+    }
+
+    this.scheduleDrain();
+  }
+
+  get depth(): number {
+    return this.pending.size;
+  }
+
+  get inFlightCount(): number {
+    return this.inFlight;
+  }
+
+  async whenIdle(): Promise<void> {
+    if (this.depth === 0 && this.inFlight === 0) return;
+    await new Promise<void>((resolve) => {
+      this.idleWaiters.push(resolve);
+    });
+  }
+
+  async whenSessionIdle(sessionId: string): Promise<void> {
+    if (!this.pending.has(sessionId) && !this.inFlightSessions.has(sessionId))
+      return;
+    await new Promise<void>((resolve) => {
+      const waiters = this.sessionIdleWaiters.get(sessionId) ?? [];
+      waiters.push(resolve);
+      this.sessionIdleWaiters.set(sessionId, waiters);
+    });
+  }
+
+  private startAvailableJobs(): void {
+    while (this.inFlight < this.concurrency && this.pending.size > 0) {
+      const next = Array.from(this.pending.entries()).find(
+        ([sessionId]) => !this.inFlightSessions.has(sessionId),
+      );
+      if (!next) break;
+
+      const [sessionId, job] = next;
+      this.pending.delete(sessionId);
+      this.inFlight++;
+      this.inFlightSessions.add(sessionId);
+
+      const startedAt = Date.now();
+      const waitMs = startedAt - job.enqueuedAt;
+      this.hooks.onJobStart?.({
+        sessionId,
+        depth: this.depth,
+        inFlight: this.inFlight,
+        waitMs,
+      });
+
+      void this.runJob(job, startedAt, waitMs);
+    }
+
+    this.resolveIdleWaitersIfNeeded();
+  }
+
+  private scheduleDrain(): void {
+    if (this.drainScheduled) return;
+    this.drainScheduled = true;
+
+    queueMicrotask(() => {
+      this.drainScheduled = false;
+      this.startAvailableJobs();
+    });
+  }
+
+  private async runJob(
+    job: PendingJob,
+    startedAt: number,
+    waitMs: number,
+  ): Promise<void> {
+    let error: unknown;
+
+    try {
+      await this.worker(job.sessionId, job.messages);
+    } catch (err) {
+      error = err;
+    } finally {
+      const finishedAt = Date.now();
+      const runMs = finishedAt - startedAt;
+      const totalMs = finishedAt - job.enqueuedAt;
+
+      this.inFlight--;
+      this.inFlightSessions.delete(job.sessionId);
+      this.hooks.onJobFinish?.({
+        sessionId: job.sessionId,
+        depth: this.depth,
+        inFlight: this.inFlight,
+        waitMs,
+        runMs,
+        totalMs,
+        error,
+      });
+
+      if (this.pending.size > 0) {
+        this.scheduleDrain();
+      } else {
+        this.resolveIdleWaitersIfNeeded();
+      }
+      this.resolveSessionIdleWaitersIfNeeded(job.sessionId);
+    }
+  }
+
+  private resolveIdleWaitersIfNeeded(): void {
+    if (this.depth !== 0 || this.inFlight !== 0) return;
+
+    const waiters = this.idleWaiters.splice(0, this.idleWaiters.length);
+    for (const resolve of waiters) resolve();
+  }
+
+  private resolveSessionIdleWaitersIfNeeded(sessionId: string): void {
+    if (this.pending.has(sessionId) || this.inFlightSessions.has(sessionId))
+      return;
+
+    const waiters = this.sessionIdleWaiters.get(sessionId);
+    if (!waiters || waiters.length === 0) return;
+
+    this.sessionIdleWaiters.delete(sessionId);
+    for (const resolve of waiters) resolve();
+  }
+}

package/src/lcm/recall.ts

@@ -0,0 +1,117 @@
+import type { LcmDag, SummaryNode } from "./dag.js";
+import type { LcmArchive } from "./archive.js";
+import { log } from "../logger.js";
+
+export interface LcmRecallConfig {
+  freshTailTurns: number;
+  budgetChars: number;
+}
+
+/**
+ * Assemble a compressed session history section from the summary DAG.
+ *
+ * Strategy:
+ * 1. For the "fresh tail" (most recent N turns), use leaf-level summaries for maximum detail.
+ * 2. For older portions, use the deepest available summary nodes (most compressed).
+ * 3. Fill within budget, prioritizing recent content.
+ */
+export function assembleCompressedHistory(
+  dag: LcmDag,
+  archive: LcmArchive,
+  sessionId: string,
+  config: LcmRecallConfig,
+): string {
+  const maxTurn = archive.getMaxTurnIndex(sessionId);
+  if (maxTurn < 0) return "";
+
+  const allNodes = dag.getAllNodes(sessionId);
+  if (allNodes.length === 0) return "";
+
+  const freshTailStart = Math.max(0, maxTurn - config.freshTailTurns + 1);
+  const sections: string[] = [];
+  let usedChars = 0;
+
+  // Collect nodes covering the "old" portion (before fresh tail)
+  if (freshTailStart > 0) {
+    const oldNodes = selectBestCoverage(allNodes, 0, freshTailStart - 1);
+    for (const node of oldNodes) {
+      const label = formatRangeLabel(node, maxTurn);
+      const entry = `**${label}** (depth ${node.depth}):\n${node.summary_text}`;
+      if (usedChars + entry.length > config.budgetChars) break;
+      sections.push(entry);
+      usedChars += entry.length;
+    }
+  }
+
+  // Collect leaf nodes that overlap with the fresh tail region.
+  // The old section uses deep nodes (depth > 0), so leaf nodes included here
+  // won't duplicate old-section content. Use msg_end >= freshTailStart to
+  // include straddling leaf nodes that partially cover the fresh region.
+  const freshNodes = allNodes
+    .filter(
+      (n) => n.depth === 0 && n.msg_end >= freshTailStart && n.msg_end <= maxTurn,
+    )
+    .sort((a, b) => a.msg_start - b.msg_start);
+
+  for (const node of freshNodes) {
+    const label = `Recent (turns ${node.msg_start}-${node.msg_end})`;
+    const entry = `**${label}**:\n${node.summary_text}`;
+    if (usedChars + entry.length > config.budgetChars) break;
+    sections.push(entry);
+    usedChars += entry.length;
+  }
+
+  if (sections.length === 0) return "";
+
+  return `## Session History (Compressed)\n\n${sections.join("\n\n")}`;
+}
+
+/**
+ * Select the best coverage for a turn range.
+ * Prefers deeper (more compressed) nodes. Avoids overlapping coverage.
+ */
+function selectBestCoverage(
+  allNodes: SummaryNode[],
+  fromTurn: number,
+  toTurn: number,
+): SummaryNode[] {
+  // Filter to nodes that overlap with the requested range
+  const candidates = allNodes.filter(
+    (n) => n.msg_start <= toTurn && n.msg_end >= fromTurn,
+  );
+
+  if (candidates.length === 0) return [];
+
+  // Sort by msg_start ascending, then depth descending (prefer deeper at same position)
+  candidates.sort((a, b) => {
+    if (a.msg_start !== b.msg_start) return a.msg_start - b.msg_start;
+    return b.depth - a.depth;
+  });
+
+  // Greedily select non-overlapping nodes, preferring deeper coverage.
+  // Clamp msg_end to toTurn to prevent deep nodes from extending
+  // past the requested range (e.g., into the fresh tail region).
+  const selected: SummaryNode[] = [];
+  let coveredUpTo = fromTurn - 1;
+
+  for (const node of candidates) {
+    if (node.msg_start > coveredUpTo) {
+      selected.push(node);
+      coveredUpTo = Math.min(node.msg_end, toTurn);
+      if (coveredUpTo >= toTurn) break;
+    }
+  }
+
+  // Re-sort selected by msg_start for chronological output
+  selected.sort((a, b) => a.msg_start - b.msg_start);
+  return selected;
+}
+
+function formatRangeLabel(node: SummaryNode, maxTurn: number): string {
+  const startPct = Math.round((node.msg_start / Math.max(1, maxTurn)) * 100);
+  const endPct = Math.round((node.msg_end / Math.max(1, maxTurn)) * 100);
+
+  if (startPct < 33) return `Early session (turns ${node.msg_start}-${node.msg_end})`;
+  if (endPct < 66) return `Mid session (turns ${node.msg_start}-${node.msg_end})`;
+  return `Later session (turns ${node.msg_start}-${node.msg_end})`;
+}

package/src/lcm/schema.ts

@@ -0,0 +1,154 @@
+import path from "node:path";
+import { mkdir } from "node:fs/promises";
+import { log } from "../logger.js";
+import {
+  openBetterSqlite3,
+  type BetterSqlite3Database,
+} from "../runtime/better-sqlite.js";
+
+const LCM_SCHEMA_VERSION = 2;
+
+export function openLcmDatabase(memoryDir: string): BetterSqlite3Database {
+  const dbPath = path.join(memoryDir, "state", "lcm.sqlite");
+  const db = openBetterSqlite3(dbPath);
+  db.pragma("journal_mode = WAL");
+  db.pragma("busy_timeout = 5000");
+  db.pragma("synchronous = NORMAL");
+  applySchema(db);
+  return db;
+}
+
+export async function ensureLcmStateDir(memoryDir: string): Promise<void> {
+  await mkdir(path.join(memoryDir, "state"), { recursive: true });
+}
+
+/**
+ * Apply (or upgrade) the LCM schema on an already-open SQLite handle.
+ *
+ * Exposed so optional host packages — e.g. importers using an in-memory
+ * destination database for true read-only `--dry-run` execution — can
+ * bootstrap the schema without going through `openLcmDatabase()` (which
+ * always touches the filesystem).
+ */
+export function applyLcmSchema(db: BetterSqlite3Database): void {
+  applySchema(db);
+}
+
+function applySchema(db: BetterSqlite3Database): void {
+  const versionRow = db
+    .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='lcm_meta'")
+    .get() as { name: string } | undefined;
+
+  if (!versionRow) {
+    createTables(db);
+    return;
+  }
+
+  const meta = db
+    .prepare("SELECT value FROM lcm_meta WHERE key = 'schema_version'")
+    .get() as { value: string } | undefined;
+  const currentVersion = meta ? parseInt(meta.value, 10) : 0;
+
+  if (currentVersion < LCM_SCHEMA_VERSION) {
+    log.info(`LCM schema upgrade: v${currentVersion} → v${LCM_SCHEMA_VERSION}`);
+    createTables(db);
+  }
+}
+
+function createTables(db: BetterSqlite3Database): void {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS lcm_meta (
+      key   TEXT PRIMARY KEY,
+      value TEXT NOT NULL
+    );
+
+    CREATE TABLE IF NOT EXISTS lcm_messages (
+      id          INTEGER PRIMARY KEY AUTOINCREMENT,
+      session_id  TEXT NOT NULL,
+      turn_index  INTEGER NOT NULL,
+      role        TEXT NOT NULL,
+      content     TEXT NOT NULL,
+      token_count INTEGER NOT NULL,
+      created_at  TEXT NOT NULL,
+      metadata    TEXT
+    );
+    CREATE INDEX IF NOT EXISTS idx_lcm_messages_session
+      ON lcm_messages(session_id, turn_index);
+
+    CREATE TABLE IF NOT EXISTS lcm_message_parts (
+      id          INTEGER PRIMARY KEY AUTOINCREMENT,
+      message_id  INTEGER NOT NULL REFERENCES lcm_messages(id) ON DELETE CASCADE,
+      ordinal     INTEGER NOT NULL,
+      kind        TEXT NOT NULL,
+      payload     TEXT NOT NULL,
+      tool_name   TEXT,
+      file_path   TEXT,
+      created_at  TEXT NOT NULL
+    );
+    CREATE INDEX IF NOT EXISTS idx_lcm_message_parts_msg
+      ON lcm_message_parts(message_id, ordinal);
+    CREATE INDEX IF NOT EXISTS idx_lcm_message_parts_tool
+      ON lcm_message_parts(tool_name);
+    CREATE INDEX IF NOT EXISTS idx_lcm_message_parts_file
+      ON lcm_message_parts(file_path);
+
+    CREATE TABLE IF NOT EXISTS lcm_summary_nodes (
+      id            TEXT PRIMARY KEY,
+      session_id    TEXT NOT NULL,
+      depth         INTEGER NOT NULL,
+      parent_id     TEXT,
+      summary_text  TEXT NOT NULL,
+      token_count   INTEGER NOT NULL,
+      msg_start     INTEGER NOT NULL,
+      msg_end       INTEGER NOT NULL,
+      escalation    INTEGER NOT NULL DEFAULT 0,
+      created_at    TEXT NOT NULL,
+      FOREIGN KEY (parent_id) REFERENCES lcm_summary_nodes(id)
+    );
+    CREATE INDEX IF NOT EXISTS idx_lcm_summary_session
+      ON lcm_summary_nodes(session_id, depth);
+    CREATE INDEX IF NOT EXISTS idx_lcm_summary_range
+      ON lcm_summary_nodes(session_id, msg_start, msg_end);
+
+    CREATE TABLE IF NOT EXISTS lcm_compaction_events (
+      id            INTEGER PRIMARY KEY AUTOINCREMENT,
+      session_id    TEXT NOT NULL,
+      fired_at      TEXT NOT NULL,
+      msg_before    INTEGER NOT NULL,
+      tokens_before INTEGER NOT NULL,
+      tokens_after  INTEGER NOT NULL
+    );
+  `);
+
+  // FTS5 tables — created separately so IF NOT EXISTS works correctly
+  const hasFts = db
+    .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='lcm_messages_fts'")
+    .get();
+  if (!hasFts) {
+    db.exec(`
+      CREATE VIRTUAL TABLE lcm_messages_fts USING fts5(
+        content,
+        content=lcm_messages,
+        content_rowid=id
+      );
+    `);
+  }
+
+  const hasSummaryFts = db
+    .prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='lcm_summaries_fts'")
+    .get();
+  if (!hasSummaryFts) {
+    db.exec(`
+      CREATE VIRTUAL TABLE lcm_summaries_fts USING fts5(
+        summary_text,
+        content=lcm_summary_nodes,
+        content_rowid=rowid
+      );
+    `);
+  }
+
+  // Upsert meta version
+  db.prepare("INSERT OR REPLACE INTO lcm_meta (key, value) VALUES ('schema_version', ?)").run(
+    String(LCM_SCHEMA_VERSION),
+  );
+}