@chendpoc/pi-memory 0.1.11 → 0.1.13

package/src/consolidation/stage1/store.ts

@@ -0,0 +1,411 @@
+import { createRequire } from "node:module";
+
+import type {
+  PendingSession,
+  Stage1Output,
+  ConsolidationStatus,
+} from "../types.js";
+
+export interface SqliteDatabase {
+  pragma(sql: string): unknown;
+  exec(sql: string): void;
+  prepare(sql: string): {
+    run(...args: unknown[]): unknown;
+    get(...args: unknown[]): unknown;
+    all(...args: unknown[]): unknown[];
+  };
+  transaction<T>(fn: () => T): () => T;
+  close(): void;
+}
+
+type SqliteConstructor = new (path: string) => SqliteDatabase;
+
+function loadSqlite(): SqliteConstructor | null {
+  try {
+    const require = createRequire(import.meta.url);
+    const mod = require("better-sqlite3") as SqliteConstructor;
+    return mod;
+  } catch {
+    return null;
+  }
+}
+
+const SCHEMA_VERSION = 1;
+
+const SCHEMA_SQL = `
+CREATE TABLE IF NOT EXISTS pending_sessions (
+  session_id TEXT PRIMARY KEY,
+  session_file TEXT NOT NULL,
+  cwd TEXT NOT NULL,
+  git_root TEXT,
+  project_hash TEXT,
+  parent_session_id TEXT,
+  parent_session_file TEXT,
+  user_turn_count INTEGER NOT NULL,
+  ended_at TEXT NOT NULL,
+  status TEXT NOT NULL CHECK (status IN ('pending', 'processing', 'done', 'skipped', 'failed')),
+  error_message TEXT,
+  created_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_pending_sessions_status ON pending_sessions(status);
+CREATE TABLE IF NOT EXISTS stage1_outputs (
+  session_id TEXT PRIMARY KEY,
+  session_file TEXT NOT NULL,
+  source_mtime_ms INTEGER NOT NULL,
+  generated_at TEXT NOT NULL,
+  raw_memory TEXT NOT NULL,
+  rollout_summary TEXT NOT NULL,
+  scope TEXT NOT NULL,
+  status TEXT NOT NULL CHECK (status IN ('pending', 'processing', 'done', 'skipped', 'failed')),
+  selected_for_phase2 INTEGER NOT NULL DEFAULT 0,
+  usage_count INTEGER NOT NULL DEFAULT 0,
+  last_usage TEXT,
+  error_message TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_stage1_outputs_status ON stage1_outputs(status);
+`;
+
+const UPSERT_PENDING_SQL = `
+INSERT INTO pending_sessions (
+  session_id,
+  session_file,
+  cwd,
+  git_root,
+  project_hash,
+  parent_session_id,
+  parent_session_file,
+  user_turn_count,
+  ended_at,
+  status,
+  error_message,
+  created_at,
+  updated_at
+) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ON CONFLICT(session_id) DO UPDATE SET
+  session_file = excluded.session_file,
+  cwd = excluded.cwd,
+  git_root = excluded.git_root,
+  project_hash = excluded.project_hash,
+  parent_session_id = excluded.parent_session_id,
+  parent_session_file = excluded.parent_session_file,
+  user_turn_count = excluded.user_turn_count,
+  ended_at = excluded.ended_at,
+  status = excluded.status,
+  error_message = excluded.error_message,
+  updated_at = excluded.updated_at;
+`;
+
+const UPSERT_STAGE1_SQL = `
+INSERT INTO stage1_outputs (
+  session_id,
+  session_file,
+  source_mtime_ms,
+  generated_at,
+  raw_memory,
+  rollout_summary,
+  scope,
+  status,
+  selected_for_phase2,
+  usage_count,
+  last_usage,
+  error_message
+) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ON CONFLICT(session_id) DO UPDATE SET
+  session_file = excluded.session_file,
+  source_mtime_ms = excluded.source_mtime_ms,
+  generated_at = excluded.generated_at,
+  raw_memory = excluded.raw_memory,
+  rollout_summary = excluded.rollout_summary,
+  scope = excluded.scope,
+  status = excluded.status,
+  selected_for_phase2 = excluded.selected_for_phase2,
+  usage_count = excluded.usage_count,
+  last_usage = excluded.last_usage,
+  error_message = excluded.error_message;
+`;
+
+export interface ConsolidationStore {
+  upsertPendingSession(session: Omit<PendingSession, "created_at" | "updated_at"> & {
+    created_at: string;
+    updated_at: string;
+  }): PendingSession;
+  upsertStage1Output(output: Stage1Output): void;
+  getPendingSession(sessionId: string): PendingSession | null;
+  listPendingSessions(limit?: number): PendingSession[];
+  setPendingSessionStatus(
+    sessionId: string,
+    status: ConsolidationStatus,
+    errorMessage?: string | null,
+    updatedAt?: string,
+  ): void;
+  getStage1Output(sessionId: string): Stage1Output | null;
+  listUnselectedStage1(limit: number): Stage1Output[];
+  listStage1OlderThan(cutoffIso: string, limit?: number): Stage1Output[];
+  markStage1Selected(sessionIds: string[]): void;
+  incrementStage1Usage(sessionId: string, usedAt?: string): void;
+  countPendingSessions(): number;
+  getConsolidationStatusCounts(): {
+    pending: number;
+    processing: number;
+    done: number;
+    skipped: number;
+    failed: number;
+  };
+  getStage1Count(): number;
+  getLastGeneratedAt(): string | null;
+  getLastUpdatedAt(): string | null;
+  close(): void;
+}
+
+function toIsoTimestamp(value: Date | string | undefined = new Date()): string {
+  const dt = typeof value === "string" ? new Date(value) : value;
+  return dt.toISOString();
+}
+
+function getUserVersion(db: SqliteDatabase): number {
+  const raw = db.pragma("user_version");
+  if (Array.isArray(raw) && raw[0] && typeof raw[0] === "object" && "user_version" in raw[0]) {
+    return Number((raw[0] as { user_version: number }).user_version);
+  }
+  if (typeof raw === "number") return raw;
+  return 0;
+}
+
+function setUserVersion(db: SqliteDatabase, version: number): void {
+  db.pragma(`user_version = ${version}`);
+}
+
+function normalizeSessionRow(row: Record<string, unknown> | undefined | null): PendingSession | null {
+  if (!row) return null;
+  return {
+    session_id: String(row.session_id ?? ""),
+    session_file: String(row.session_file ?? ""),
+    cwd: String(row.cwd ?? ""),
+    git_root: row.git_root == null ? null : String(row.git_root),
+    project_hash: row.project_hash == null ? null : String(row.project_hash),
+    parent_session_id: row.parent_session_id == null ? null : String(row.parent_session_id),
+    parent_session_file: row.parent_session_file == null ? null : String(row.parent_session_file),
+    user_turn_count: Number(row.user_turn_count ?? 0),
+    ended_at: String(row.ended_at ?? ""),
+    status: String(row.status ?? "pending") as ConsolidationStatus,
+    error_message: row.error_message == null ? null : String(row.error_message),
+    created_at: String(row.created_at ?? ""),
+    updated_at: String(row.updated_at ?? ""),
+  };
+}
+
+function normalizeStage1Row(row: Record<string, unknown> | undefined | null): Stage1Output | null {
+  if (!row) return null;
+  return {
+    session_id: String(row.session_id ?? ""),
+    session_file: String(row.session_file ?? ""),
+    source_mtime_ms: toNumber(row.source_mtime_ms, 0),
+    generated_at: String(row.generated_at ?? ""),
+    raw_memory: String(row.raw_memory ?? ""),
+    rollout_summary: String(row.rollout_summary ?? ""),
+    scope: String(row.scope ?? "global"),
+    status: String(row.status ?? "pending") as ConsolidationStatus,
+    selected_for_phase2: Number(row.selected_for_phase2 ?? 0) === 1,
+    usage_count: toNumber(row.usage_count, 0),
+    last_usage: row.last_usage == null ? null : String(row.last_usage),
+    error_message: row.error_message == null ? null : String(row.error_message),
+  };
+}
+
+function toBoolInt(value: boolean): number {
+  return value ? 1 : 0;
+}
+
+function toNumber(value: unknown, fallback = 0): number {
+  if (typeof value === "number" && Number.isFinite(value)) return value;
+  if (typeof value === "string") {
+    const parsed = Number(value);
+    return Number.isFinite(parsed) ? parsed : fallback;
+  }
+  return fallback;
+}
+
+/**
+ * Lazy-open a sqlite database and ensure the consolidation schema exists.
+ * Returns null when better-sqlite3 is unavailable and no injected DB is provided.
+ */
+export function openConsolidationStore(
+  dbPath: string,
+  injectedDb?: SqliteDatabase,
+): ConsolidationStore | null {
+  let db: SqliteDatabase;
+  if (injectedDb) {
+    db = injectedDb;
+  } else {
+    const Sqlite = loadSqlite();
+    if (!Sqlite) return null;
+    db = new Sqlite(dbPath);
+  }
+
+  db.pragma("journal_mode = WAL");
+  const current = getUserVersion(db);
+  if (current !== SCHEMA_VERSION) {
+    db.exec(SCHEMA_SQL);
+    setUserVersion(db, SCHEMA_VERSION);
+  }
+
+  const upsertPendingStmt = db.prepare(UPSERT_PENDING_SQL);
+  const upsertStage1Stmt = db.prepare(UPSERT_STAGE1_SQL);
+  const selectPendingByIdStmt = db.prepare("SELECT * FROM pending_sessions WHERE session_id = ?");
+  const listPendingStmt = db.prepare(
+    "SELECT * FROM pending_sessions WHERE status IN ('pending', 'failed') ORDER BY ended_at ASC LIMIT ?",
+  );
+  const updatePendingStatusStmt = db.prepare(
+    "UPDATE pending_sessions SET status = ?, error_message = ?, updated_at = ? WHERE session_id = ?",
+  );
+  const selectStage1ByIdStmt = db.prepare("SELECT * FROM stage1_outputs WHERE session_id = ?");
+  const listUnselectedStage1Stmt = db.prepare(
+    "SELECT * FROM stage1_outputs WHERE selected_for_phase2 = 0 AND status = 'done' ORDER BY generated_at DESC LIMIT ?",
+  );
+  const listStage1OlderThanStmt = db.prepare(
+    "SELECT * FROM stage1_outputs WHERE last_usage IS NOT NULL AND last_usage < ? ORDER BY last_usage ASC LIMIT ?",
+  );
+  const markStage1SelectedStmt = db.prepare(
+    "UPDATE stage1_outputs SET selected_for_phase2 = 1 WHERE session_id = ?",
+  );
+  const incrementStage1UsageStmt = db.prepare(
+    "UPDATE stage1_outputs SET usage_count = usage_count + 1, last_usage = ? WHERE session_id = ?",
+  );
+  const countPendingAllStmt = db.prepare("SELECT COUNT(*) AS count FROM pending_sessions");
+  const countPendingByStatusStmt = db.prepare(
+    "SELECT status, COUNT(*) AS count FROM pending_sessions GROUP BY status",
+  );
+  const countStage1Stmt = db.prepare("SELECT COUNT(*) AS count FROM stage1_outputs");
+  const lastGeneratedStmt = db.prepare("SELECT MAX(generated_at) AS generated_at FROM stage1_outputs");
+  const lastUpdatedStmt = db.prepare("SELECT MAX(updated_at) AS updated_at FROM pending_sessions");
+
+  return {
+    upsertPendingSession(session): PendingSession {
+      const now = toIsoTimestamp(session.updated_at);
+      const payload = {
+        ...session,
+        created_at: session.created_at ?? now,
+        updated_at: now,
+      };
+      upsertPendingStmt.run(
+        payload.session_id,
+        payload.session_file,
+        payload.cwd,
+        payload.git_root,
+        payload.project_hash,
+        payload.parent_session_id,
+        payload.parent_session_file,
+        payload.user_turn_count,
+        payload.ended_at,
+        payload.status,
+        payload.error_message,
+        payload.created_at,
+        payload.updated_at,
+      );
+      return payload;
+    },
+    upsertStage1Output(output): void {
+      upsertStage1Stmt.run(
+        output.session_id,
+        output.session_file,
+        output.source_mtime_ms,
+        output.generated_at,
+        output.raw_memory,
+        output.rollout_summary,
+        output.scope,
+        output.status,
+        toBoolInt(output.selected_for_phase2),
+        output.usage_count,
+        output.last_usage,
+        output.error_message,
+      );
+    },
+    getPendingSession(sessionId: string): PendingSession | null {
+      return normalizeSessionRow(
+        selectPendingByIdStmt.get(sessionId) as Record<string, unknown> | undefined,
+      );
+    },
+    listPendingSessions(limit = 100): PendingSession[] {
+      const rows = listPendingStmt.all(limit) as Array<Record<string, unknown>>;
+      return rows
+        .map((row) => normalizeSessionRow(row))
+        .filter((row): row is PendingSession => row != null);
+    },
+    setPendingSessionStatus(
+      sessionId: string,
+      status: ConsolidationStatus,
+      errorMessage: string | null = null,
+      updatedAt = new Date().toISOString(),
+    ): void {
+      updatePendingStatusStmt.run(status, errorMessage, updatedAt, sessionId);
+    },
+    getStage1Output(sessionId: string): Stage1Output | null {
+      return normalizeStage1Row(
+        selectStage1ByIdStmt.get(sessionId) as Record<string, unknown> | undefined,
+      );
+    },
+    listUnselectedStage1(limit: number): Stage1Output[] {
+      const rows = listUnselectedStage1Stmt.all(limit) as Array<Record<string, unknown>>;
+      return rows
+        .map((row) => normalizeStage1Row(row))
+        .filter((row): row is Stage1Output => row != null);
+    },
+    listStage1OlderThan(cutoffIso: string, limit = 100): Stage1Output[] {
+      const rows = listStage1OlderThanStmt.all(cutoffIso, limit) as Array<Record<string, unknown>>;
+      return rows
+        .map((row) => normalizeStage1Row(row))
+        .filter((row): row is Stage1Output => row != null);
+    },
+    markStage1Selected(sessionIds: string[]): void {
+      const tx = db.transaction(() => {
+        for (const sessionId of sessionIds) markStage1SelectedStmt.run(sessionId);
+      });
+      tx();
+    },
+    incrementStage1Usage(sessionId: string, usedAt = new Date().toISOString()): void {
+      incrementStage1UsageStmt.run(usedAt, sessionId);
+    },
+    countPendingSessions(): number {
+      const row = countPendingAllStmt.get() as { count?: number | string } | undefined;
+      return toNumber(row?.count, 0);
+    },
+    getConsolidationStatusCounts() {
+      const rows = countPendingByStatusStmt.all() as Array<{
+        status: ConsolidationStatus;
+        count: number | string;
+      }>;
+      const counts = {
+        pending: 0,
+        processing: 0,
+        done: 0,
+        skipped: 0,
+        failed: 0,
+      };
+      for (const row of rows) {
+        const status = row.status;
+        if (status in counts) {
+          counts[status] = toNumber(row.count, 0);
+        }
+      }
+      return counts;
+    },
+    getStage1Count(): number {
+      const row = countStage1Stmt.get() as { count?: number | string } | undefined;
+      return toNumber(row?.count, 0);
+    },
+    getLastGeneratedAt(): string | null {
+      const row = lastGeneratedStmt.get() as { generated_at?: string } | undefined;
+      if (!row?.generated_at) return null;
+      return String(row.generated_at);
+    },
+    getLastUpdatedAt(): string | null {
+      const row = lastUpdatedStmt.get() as { updated_at?: string } | undefined;
+      if (!row?.updated_at) return null;
+      return String(row.updated_at);
+    },
+    close(): void {
+      db.close();
+    },
+  };
+}

package/src/consolidation/types.ts

@@ -0,0 +1,52 @@
+export type ConsolidationStatus = "pending" | "processing" | "done" | "skipped" | "failed";
+
+export interface PendingSession {
+  session_id: string;
+  session_file: string;
+  cwd: string;
+  git_root: string | null;
+  project_hash: string | null;
+  parent_session_id: string | null;
+  parent_session_file: string | null;
+  user_turn_count: number;
+  ended_at: string;
+  status: ConsolidationStatus;
+  error_message: string | null;
+  created_at: string;
+  updated_at: string;
+}
+
+export interface Stage1Output {
+  session_id: string;
+  session_file: string;
+  source_mtime_ms: number;
+  generated_at: string;
+  raw_memory: string;
+  rollout_summary: string;
+  scope: string;
+  status: ConsolidationStatus;
+  selected_for_phase2: boolean;
+  usage_count: number;
+  last_usage: string | null;
+  error_message: string | null;
+}
+
+export interface JobReport {
+  session_id: string;
+  enqueued: boolean;
+  action: "created" | "updated" | "skipped";
+  status: ConsolidationStatus;
+  reason?: string;
+  now: string;
+}
+
+export interface ConsolidationStatusReport {
+  pending: number;
+  processing: number;
+  done: number;
+  skipped: number;
+  failed: number;
+  stage1Count: number;
+  lastGeneratedAt: string | null;
+  lastUpdatedAt: string | null;
+}

package/src/fallback/llmRerank.ts

@@ -1,5 +1,6 @@
 import type { LLMClient } from "../trainer/llmExtractor.js";
 import type { SessionSearchHit } from "./sessionSearch.js";
+import { cacheKeyForRerank, rerankCache } from "../cache/memoryCaches.js";
 
 export interface RerankOptions {
   client: LLMClient;
@@ -79,11 +80,17 @@ export async function rerankWithLLM(
   const maxCandidates = opts.maxCandidates ?? DEFAULT_MAX_CANDIDATES;
   const truncated = hits.slice(0, maxCandidates);
 
+  const cacheKey = cacheKeyForRerank(query, truncated);
+  const cached = rerankCache.get(cacheKey);
+  if (cached) return cached;
+
   const prompt = buildRerankPrompt(query, truncated);
 
   try {
     const response = await opts.client.complete(prompt);
-    return parseRerankResponse(response, truncated.length);
+    const results = parseRerankResponse(response, truncated.length);
+    if (results) rerankCache.set(cacheKey, results);
+    return results;
   } catch {
     return null;
   }

package/src/fallback/sessionIndex.ts

@@ -3,6 +3,7 @@ import fs from "node:fs/promises";
 import { createRequire } from "node:module";
 import path from "node:path";
 import type { SessionSearchHit } from "./sessionSearch.js";
+import { messageText, parseJsonlSession as parseActiveBranchJsonl } from "../session/activeBranch.js";
 
 export interface SqliteDatabase {
   pragma(sql: string): unknown;
@@ -44,21 +45,6 @@ interface PiSessionFile {
   messages?: PiSessionMessage[];
 }
 
-function messageText(content: unknown): string {
-  if (typeof content === "string") return content;
-  if (!Array.isArray(content)) return "";
-  const parts: string[] = [];
-  for (const block of content) {
-    if (typeof block === "string") { parts.push(block); continue; }
-    if (block && typeof block === "object") {
-      const b = block as Record<string, unknown>;
-      if (typeof b.text === "string") parts.push(b.text);
-      else if (typeof b.content === "string") parts.push(b.content);
-    }
-  }
-  return parts.join("\n");
-}
-
 export interface SessionIndex {
   /** Full rebuild — drop and re-populate from all session files. */
   rebuildIndex(sessionsDir: string): Promise<{ indexed: number }>;
@@ -167,40 +153,6 @@ export function openSessionIndex(dbPath: string, injectedDb?: SqliteDatabase): S
     return files;
   }
 
-  function parseJsonlMessages(raw: string, filePath: string): {
-    id: string; title: string; createdAt: string;
-    messages: Array<{ role: string; content: string; index: number }>;
-  } | null {
-    const lines = raw.split("\n").filter((l) => l.trim());
-    if (lines.length === 0) return null;
-    let id = path.basename(filePath, path.extname(filePath));
-    let title = "";
-    let createdAt = "";
-    const messages: Array<{ role: string; content: string; index: number }> = [];
-    let idx = 0;
-    for (const line of lines) {
-      let obj: Record<string, unknown>;
-      try { obj = JSON.parse(line) as Record<string, unknown>; } catch { continue; }
-      if (obj.type === "session") {
-        id = (obj.id as string) ?? id;
-        title = (obj.title as string) ?? "";
-        createdAt = (obj.timestamp as string) ?? "";
-        continue;
-      }
-      if (obj.type === "message") {
-        const msg = (obj as { message?: PiSessionMessage }).message;
-        if (!msg?.role || !msg.content) continue;
-        if (msg.role !== "user" && msg.role !== "assistant") continue;
-        const text = messageText(msg.content);
-        if (text.trim()) {
-          messages.push({ role: msg.role, content: text, index: idx++ });
-        }
-      }
-    }
-    if (messages.length === 0) return null;
-    return { id, title, createdAt, messages };
-  }
-
   async function loadSessionFiles(sessionsDir: string, modifiedAfter?: Date | null): Promise<Array<{
     id: string;
     title: string;
@@ -223,8 +175,19 @@ export function openSessionIndex(dbPath: string, injectedDb?: SqliteDatabase): S
       try { raw = await fs.readFile(filePath, "utf8"); } catch { continue; }
 
       if (filePath.endsWith(".jsonl")) {
-        const parsed = parseJsonlMessages(raw, filePath);
-        if (parsed) results.push(parsed);
+        const parsed = parseActiveBranchJsonl(raw, filePath);
+        if (!parsed) continue;
+        const messages = parsed.turns.map((turn) => ({
+          role: turn.role,
+          content: turn.content,
+          index: turn.turnIndex,
+        }));
+        results.push({
+          id: parsed.id,
+          title: parsed.title,
+          createdAt: parsed.createdAt,
+          messages,
+        });
       } else {
         let session: PiSessionFile;
         try { session = JSON.parse(raw) as PiSessionFile; } catch { continue; }

package/src/index.ts

@@ -19,8 +19,11 @@ export type {
 } from "./types.js";
 
 export {
+  defaultConsolidationConfig,
   defaultMemoryConfig,
   normalizeMemoryConfig,
+  type ConsolidationConfig,
+  type ConsolidationScheduleConfig,
   type MemoryConfig,
   type MemoryProvider,
 } from "./config.js";
@@ -97,10 +100,12 @@ export {
 
 export {
   PRIVATE_MEMORY_BODY_BYTE_CAP,
+  SEMANTIC_FALLBACK_CANDIDATES,
   renderFallbackPrivateMemory,
   renderPrivateMemoryContext,
   sanitizeUserBlock,
   truncatePrivateMemoryBody,
+  type FallbackRenderOptions,
   type PreflightQueryResult,
 } from "./preflight/render.js";
 
@@ -156,6 +161,7 @@ export {
 
 export {
   trainBundle,
+  loadSessionFile,
   loadSessions,
   extractFacts,
   extractFactsFromSessions,
@@ -237,3 +243,5 @@ export {
   type RerankOptions,
   type RankedResult,
 } from "./fallback/llmRerank.js";
+
+export * from "./consolidation/index.js";

package/src/local/graphQuery.ts

@@ -47,6 +47,7 @@ export class LocalGraphQuerier {
   private entityById = new Map<string, BundleEntity>();
   private entityByLabel = new Map<string, BundleEntity>();
   private loaded = false;
+  private graphMtime = 0;
 
   constructor(private readonly bundleRoot: string) {}
 
@@ -68,6 +69,7 @@ export class LocalGraphQuerier {
           this.entityByLabel.set(alias.toLowerCase(), e);
         }
       }
+      this.graphMtime = fs.statSync(graphPath).mtimeMs;
       this.loaded = true;
       return true;
     } catch {
@@ -79,6 +81,28 @@ export class LocalGraphQuerier {
     return this.loaded;
   }
 
+  /**
+   * Returns true when the on-disk graph.json has changed since the last load().
+   * Always returns false when not yet loaded.
+   */
+  isStale(): boolean {
+    if (!this.loaded) return false;
+    const graphPath = path.join(this.bundleRoot, "current", "graph.json");
+    try {
+      return fs.statSync(graphPath).mtimeMs !== this.graphMtime;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Re-loads from disk if isStale(). Returns true when a reload actually happened.
+   */
+  reloadIfStale(): boolean {
+    if (!this.isStale()) return false;
+    return this.load();
+  }
+
   private findEntity(mention: string): BundleEntity | null {
     const key = mention.toLowerCase().trim();
     return this.entityByLabel.get(key) ?? null;