@pi-unipi/compactor 0.1.7 → 0.2.2

package/src/session/analytics.ts

@@ -0,0 +1,198 @@
+/**
+ * AnalyticsEngine — Runtime savings + session continuity reporting.
+ *
+ * Ported from context-mode's AnalyticsEngine, trimmed to budget-focused stats.
+ * Omits: formatReport(), categoryLabels, categoryHints, ThinkInCodeComparison,
+ * SandboxIO, dataBar(), visual formatting helpers.
+ *
+ * Usage:
+ *   const engine = new AnalyticsEngine(sessionDb);
+ *   const report = engine.queryAll(runtimeStats);
+ */
+
+// ─────────────────────────────────────────────────────────
+// Types
+// ─────────────────────────────────────────────────────────
+
+/** Database adapter — anything with a prepare() method (better-sqlite3, bun:sqlite, etc.) */
+export interface DatabaseAdapter {
+  prepare(sql: string): {
+    run(...params: unknown[]): unknown;
+    get(...params: unknown[]): unknown;
+    all(...params: unknown[]): unknown[];
+  };
+}
+
+/** Context savings result */
+export interface ContextSavings {
+  rawBytes: number;
+  contextBytes: number;
+  savedBytes: number;
+  savedPercent: number;
+}
+
+/** Runtime stats tracked during a live session. */
+export interface RuntimeStats {
+  bytesReturned: Record<string, number>;
+  bytesIndexed: number;
+  bytesSandboxed: number;
+  calls: Record<string, number>;
+  sessionStart: number;
+  cacheHits: number;
+  cacheBytesSaved: number;
+}
+
+/** Unified report combining runtime stats, DB analytics, and continuity data. */
+export interface FullReport {
+  /** Runtime context savings */
+  savings: {
+    processed_kb: number;
+    entered_kb: number;
+    saved_kb: number;
+    pct: number;
+    savings_ratio: number;
+    by_tool: Array<{ tool: string; calls: number; context_kb: number; tokens: number }>;
+    total_calls: number;
+    total_bytes_returned: number;
+    kept_out: number;
+    total_processed: number;
+  };
+  /** Session metadata from SessionDB */
+  session: {
+    id: string;
+    uptime_min: string;
+  };
+  /** Session continuity data */
+  continuity: {
+    total_events: number;
+    compact_count: number;
+    resume_ready: boolean;
+  };
+  /** Persistent project memory — all events across all sessions */
+  projectMemory: {
+    total_events: number;
+    session_count: number;
+  };
+}
+
+// ─────────────────────────────────────────────────────────
+// AnalyticsEngine
+// ─────────────────────────────────────────────────────────
+
+export class AnalyticsEngine {
+  private readonly db: DatabaseAdapter;
+
+  constructor(db: DatabaseAdapter) {
+    this.db = db;
+  }
+
+  /**
+   * Build a FullReport by merging runtime stats (passed in)
+   * with continuity data from the DB.
+   */
+  queryAll(runtimeStats: RuntimeStats): FullReport {
+    // ── Resolve latest session ID ──
+    const latestSession = this.db.prepare(
+      "SELECT session_id FROM session_meta ORDER BY started_at DESC LIMIT 1",
+    ).get() as { session_id: string } | undefined;
+    const sid = latestSession?.session_id ?? "";
+
+    // ── Runtime savings ──
+    const totalBytesReturned = Object.values(runtimeStats.bytesReturned).reduce(
+      (sum, b) => sum + b, 0,
+    );
+    const totalCalls = Object.values(runtimeStats.calls).reduce(
+      (sum, c) => sum + c, 0,
+    );
+    const keptOut = runtimeStats.bytesIndexed + runtimeStats.bytesSandboxed;
+    const totalProcessed = keptOut + totalBytesReturned;
+    const savingsRatio = totalProcessed / Math.max(totalBytesReturned, 1);
+    const reductionPct = totalProcessed > 0
+      ? Math.round((1 - totalBytesReturned / totalProcessed) * 100)
+      : 0;
+
+    const toolNames = new Set([
+      ...Object.keys(runtimeStats.calls),
+      ...Object.keys(runtimeStats.bytesReturned),
+    ]);
+    const byTool = Array.from(toolNames).sort().map((tool) => ({
+      tool,
+      calls: runtimeStats.calls[tool] || 0,
+      context_kb: Math.round((runtimeStats.bytesReturned[tool] || 0) / 1024 * 10) / 10,
+      tokens: Math.round((runtimeStats.bytesReturned[tool] || 0) / 4),
+    }));
+
+    const uptimeMs = Date.now() - runtimeStats.sessionStart;
+    const uptimeMin = (uptimeMs / 60_000).toFixed(1);
+
+    // ── Continuity data (scoped to current session) ──
+    const eventTotal = (this.db.prepare(
+      "SELECT COUNT(*) as cnt FROM session_events WHERE session_id = ?",
+    ).get(sid) as { cnt: number }).cnt;
+
+    const meta = this.db.prepare(
+      "SELECT compact_count FROM session_meta WHERE session_id = ?",
+    ).get(sid) as { compact_count: number } | undefined;
+    const compactCount = meta?.compact_count ?? 0;
+
+    const resume = this.db.prepare(
+      "SELECT event_count, consumed FROM session_resume WHERE session_id = ? ORDER BY created_at DESC LIMIT 1",
+    ).get(sid) as { event_count: number; consumed: number } | undefined;
+    const resumeReady = resume ? !resume.consumed : false;
+
+    // ── Project-wide persistent memory (all sessions, no session_id filter) ──
+    const projectTotals = this.db.prepare(
+      "SELECT COUNT(*) as cnt, COUNT(DISTINCT session_id) as sessions FROM session_events",
+    ).get() as { cnt: number; sessions: number };
+
+    return {
+      savings: {
+        processed_kb: Math.round(totalProcessed / 1024 * 10) / 10,
+        entered_kb: Math.round(totalBytesReturned / 1024 * 10) / 10,
+        saved_kb: Math.round(keptOut / 1024 * 10) / 10,
+        pct: reductionPct,
+        savings_ratio: Math.round(savingsRatio * 10) / 10,
+        by_tool: byTool,
+        total_calls: totalCalls,
+        total_bytes_returned: totalBytesReturned,
+        kept_out: keptOut,
+        total_processed: totalProcessed,
+      },
+      session: {
+        id: sid,
+        uptime_min: uptimeMin,
+      },
+      continuity: {
+        total_events: eventTotal,
+        compact_count: compactCount,
+        resume_ready: resumeReady,
+      },
+      projectMemory: {
+        total_events: projectTotals.cnt,
+        session_count: projectTotals.sessions,
+      },
+    };
+  }
+}
+
+// ─────────────────────────────────────────────────────────
+// createMinimalDb — in-memory SQLite fallback
+// ─────────────────────────────────────────────────────────
+
+/**
+ * Create a minimal in-memory DatabaseAdapter for when SessionDB is unavailable.
+ * Returns zeroed/empty results for all queries.
+ */
+export function createMinimalDb(): DatabaseAdapter {
+  // Use an in-memory SQLite database with the expected schema
+  // so AnalyticsEngine queries don't fail.
+  const emptyStmt = {
+    run: (..._params: unknown[]) => {},
+    get: (..._params: unknown[]) => ({ cnt: 0, sessions: 0, compact_count: 0, session_id: "", event_count: 0, consumed: 1 }),
+    all: (..._params: unknown[]) => [] as unknown[],
+  };
+
+  return {
+    prepare: (_sql: string) => emptyStmt,
+  };
+}

package/src/session/auto-inject.ts

@@ -0,0 +1,60 @@
+/**
+ * Auto-injection builder — builds minimal behavioral state injection
+ * after compaction. Budget: 150 tokens max.
+ *
+ * Only includes:
+ * - behavioral_directive (role event) — never dropped
+ * - session_mode (intent event) — only if budget remains
+ *
+ * Rules and active_skills are dropped from auto-injection (findable via session_recall).
+ */
+
+import type { StoredEvent } from "../types.js";
+
+const MAX_TOKENS = 150;
+
+function estimateTokens(text: string): number {
+  // Rough: ~4 chars per token
+  return Math.ceil(text.length / 4);
+}
+
+export interface AutoInjection {
+  text: string;
+  tokens: number;
+}
+
+export function buildAutoInjection(events: StoredEvent[]): AutoInjection | null {
+  const parts: string[] = [];
+  let tokenBudget = MAX_TOKENS;
+
+  // 1. behavioral_directive (role) — critical, always included
+  const roleEvents = events.filter((e) => e.category === "rule");
+  if (roleEvents.length > 0) {
+    const directive = roleEvents[roleEvents.length - 1].data;
+    const directiveText = `[Role Directive]\n${directive}`;
+    const tokens = estimateTokens(directiveText);
+    if (tokens <= tokenBudget) {
+      parts.push(directiveText);
+      tokenBudget -= tokens;
+    }
+  }
+
+  // 2. session_mode (intent) — included if budget remains
+  if (tokenBudget > 80) {
+    const intentEvents = events.filter((e) => e.category === "intent");
+    if (intentEvents.length > 0) {
+      const mode = intentEvents[intentEvents.length - 1].data;
+      const modeText = `[Session Mode]\n${mode}`;
+      const tokens = estimateTokens(modeText);
+      if (tokens <= tokenBudget) {
+        parts.push(modeText);
+        tokenBudget -= tokens;
+      }
+    }
+  }
+
+  if (parts.length === 0) return null;
+
+  const text = parts.join("\n\n");
+  return { text, tokens: estimateTokens(text) };
+}

package/src/session/db.ts

@@ -69,7 +69,7 @@ const DEDUP_WINDOW = 5;
 
 export class SessionDB {
   private db: any;
-  private stmts: Map<string, PreparedStatement> = new Map();
+  private stmts: Map<string, PreparedStatement> | null = null;
   private dbPath: string;
 
   constructor(opts?: { dbPath?: string }) {
@@ -101,11 +101,8 @@ export class SessionDB {
         priority INTEGER NOT NULL DEFAULT 2,
         data TEXT NOT NULL,
         project_dir TEXT NOT NULL DEFAULT '',
-        attribution_source TEXT NOT NULL DEFAULT 'unknown',
-        attribution_confidence REAL NOT NULL DEFAULT 0,
         source_hook TEXT NOT NULL,
-        created_at TEXT NOT NULL DEFAULT (datetime('now')),
-        data_hash TEXT NOT NULL DEFAULT ''
+        created_at TEXT NOT NULL DEFAULT (datetime('now'))
       );
       CREATE INDEX IF NOT EXISTS idx_session_events_session ON session_events(session_id);
       CREATE INDEX IF NOT EXISTS idx_session_events_type ON session_events(session_id, type);
@@ -129,11 +126,42 @@ export class SessionDB {
         consumed INTEGER NOT NULL DEFAULT 0
       );
     `);
+
+    // Run version-gated schema migrations
+    this.runMigrations();
+  }
+
+  /** Run version-gated schema migrations using PRAGMA user_version. */
+  private runMigrations(): void {
+    const currentVersion = this.db.pragma("user_version", { simple: true }) as number;
+
+    if (currentVersion < 1) {
+      // V1: Add columns introduced by compactor gap analysis (2026-04-30)
+      // Each ALTER TABLE is wrapped individually — SQLite auto-commits DDL,
+      // so a partial failure from a prior run would leave some columns added
+      // and others not. We catch "duplicate column" to handle this safely.
+      const safeAddColumn = (table: string, col: string, def: string) => {
+        try {
+          this.db.exec(`ALTER TABLE ${table} ADD COLUMN ${col} ${def}`);
+        } catch (e: any) {
+          if (e?.message?.includes("duplicate column")) return;
+          throw e;
+        }
+      };
+      safeAddColumn("session_meta", "total_chars_before", "INTEGER NOT NULL DEFAULT 0");
+      safeAddColumn("session_meta", "total_chars_kept", "INTEGER NOT NULL DEFAULT 0");
+      safeAddColumn("session_meta", "total_messages_summarized", "INTEGER NOT NULL DEFAULT 0");
+      safeAddColumn("session_events", "attribution_source", "TEXT NOT NULL DEFAULT 'unknown'");
+      safeAddColumn("session_events", "attribution_confidence", "REAL NOT NULL DEFAULT 0");
+      safeAddColumn("session_events", "data_hash", "TEXT NOT NULL DEFAULT ''");
+      this.db.pragma("user_version = 1");
+    }
   }
 
   private prepareStatements(): void {
+    this.stmts = new Map();
     const p = (key: string, sql: string) => {
-      this.stmts.set(key, this.db.prepare(sql) as PreparedStatement);
+      this.stmts!.set(key, this.db.prepare(sql) as PreparedStatement);
     };
 
     p("insertEvent", `INSERT INTO session_events (session_id, type, category, priority, data, project_dir, attribution_source, attribution_confidence, source_hook, data_hash) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
@@ -143,8 +171,10 @@ export class SessionDB {
     p("evictLowestPriority", `DELETE FROM session_events WHERE id = (SELECT id FROM session_events WHERE session_id = ? ORDER BY priority ASC, id ASC LIMIT 1)`);
     p("updateMetaLastEvent", `UPDATE session_meta SET last_event_at = datetime('now'), event_count = event_count + 1 WHERE session_id = ?`);
     p("ensureSession", `INSERT OR IGNORE INTO session_meta (session_id, project_dir) VALUES (?, ?)`);
-    p("getSessionStats", `SELECT session_id, project_dir, started_at, last_event_at, event_count, compact_count FROM session_meta WHERE session_id = ?`);
+    p("getSessionStats", `SELECT session_id, project_dir, started_at, last_event_at, event_count, compact_count, total_chars_before, total_chars_kept, total_messages_summarized FROM session_meta WHERE session_id = ?`);
     p("incrementCompactCount", `UPDATE session_meta SET compact_count = compact_count + 1 WHERE session_id = ?`);
+    p("addCompactionStats", `UPDATE session_meta SET total_chars_before = total_chars_before + ?, total_chars_kept = total_chars_kept + ?, total_messages_summarized = total_messages_summarized + ? WHERE session_id = ?`);
+    p("getAllTimeStats", `SELECT COALESCE(SUM(total_chars_before), 0) AS all_chars_before, COALESCE(SUM(total_chars_kept), 0) AS all_chars_kept, COALESCE(SUM(total_messages_summarized), 0) AS all_messages_summarized, COALESCE(SUM(compact_count), 0) AS all_compactions FROM session_meta`);
     p("upsertResume", `INSERT INTO session_resume (session_id, snapshot, event_count) VALUES (?, ?, ?) ON CONFLICT(session_id) DO UPDATE SET snapshot = excluded.snapshot, event_count = excluded.event_count, created_at = datetime('now'), consumed = 0`);
     p("getResume", `SELECT snapshot, event_count, consumed FROM session_resume WHERE session_id = ?`);
     p("markResumeConsumed", `UPDATE session_resume SET consumed = 1 WHERE session_id = ?`);
@@ -155,10 +185,11 @@ export class SessionDB {
   }
 
   private stmt(key: string): PreparedStatement {
-    return this.stmts.get(key)!;
+    return this.stmts!.get(key)!;
   }
 
   insertEvent(sessionId: string, event: SessionEvent, sourceHook: string = "PostToolUse"): void {
+    if (!this.stmts) return;
     const dataHash = createHash("sha256").update(event.data).digest("hex").slice(0, 16).toUpperCase();
     const projectDir = String(event.project_dir ?? "").trim();
     const attributionSource = String(event.attribution_source ?? "unknown");
@@ -185,42 +216,67 @@ export class SessionDB {
   }
 
   getEvents(sessionId: string, opts?: { type?: string; minPriority?: number; limit?: number }): StoredEvent[] {
+    if (!this.stmts) return [];
     const limit = opts?.limit ?? 1000;
     return this.stmt("getEvents").all(sessionId, limit) as StoredEvent[];
   }
 
   getEventCount(sessionId: string): number {
+    if (!this.stmts) return 0;
     const row = this.stmt("getEventCount").get(sessionId) as { cnt: number };
     return row.cnt;
   }
 
   ensureSession(sessionId: string, projectDir: string): void {
+    if (!this.stmts) return;
     this.stmt("ensureSession").run(sessionId, projectDir);
   }
 
   getSessionStats(sessionId: string): SessionMeta | null {
+    if (!this.stmts) return null;
     const row = this.stmt("getSessionStats").get(sessionId) as SessionMeta | undefined;
     return row ?? null;
   }
 
   incrementCompactCount(sessionId: string): void {
+    if (!this.stmts) return;
     this.stmt("incrementCompactCount").run(sessionId);
   }
 
+  addCompactionStats(sessionId: string, charsBefore: number, charsKept: number, messagesSummarized: number): void {
+    if (!this.stmts) return;
+    this.stmt("addCompactionStats").run(charsBefore, charsKept, messagesSummarized, sessionId);
+  }
+
+  getAllTimeStats(): { allCharsBefore: number; allCharsKept: number; allMessagesSummarized: number; allCompactions: number } {
+    if (!this.stmts) return { allCharsBefore: 0, allCharsKept: 0, allMessagesSummarized: 0, allCompactions: 0 };
+    const row = this.stmt("getAllTimeStats").get() as { all_chars_before: number; all_chars_kept: number; all_messages_summarized: number; all_compactions: number };
+    return {
+      allCharsBefore: row?.all_chars_before ?? 0,
+      allCharsKept: row?.all_chars_kept ?? 0,
+      allMessagesSummarized: row?.all_messages_summarized ?? 0,
+      allCompactions: row?.all_compactions ?? 0,
+    };
+  }
+
   upsertResume(sessionId: string, snapshot: string, eventCount?: number): void {
+    if (!this.stmts) return;
     this.stmt("upsertResume").run(sessionId, snapshot, eventCount ?? 0);
   }
 
   getResume(sessionId: string): ResumeRow | null {
+    if (!this.stmts) return null;
     const row = this.stmt("getResume").get(sessionId) as ResumeRow | undefined;
     return row ?? null;
   }
 
   markResumeConsumed(sessionId: string): void {
+    if (!this.stmts) return;
     this.stmt("markResumeConsumed").run(sessionId);
   }
 
   deleteSession(sessionId: string): void {
+    if (!this.stmts) return;
     this.db.transaction(() => {
       this.stmt("deleteEvents").run(sessionId);
       this.stmt("deleteResume").run(sessionId);
@@ -229,6 +285,7 @@ export class SessionDB {
   }
 
   cleanupOldSessions(maxAgeDays: number = 7): number {
+    if (!this.stmts) return 0;
     const oldSessions = this.stmt("getOldSessions").all(`-${maxAgeDays}`) as Array<{ session_id: string }>;
     for (const { session_id } of oldSessions) {
       this.deleteSession(session_id);
@@ -236,6 +293,9 @@ export class SessionDB {
     return oldSessions.length;
   }
 
+  /** Expose the underlying db for AnalyticsEngine (read-only queries). Returns null if init failed. */
+  getDb(): any { return this.db ?? null; }
+
   close(): void {
     try { this.db.close(); } catch { /* ignore */ }
   }

package/src/session/resume-inject.ts

@@ -4,6 +4,8 @@
 
 import type { SessionDB } from "./db.js";
 import { buildResumeSnapshot } from "./snapshot.js";
+import { buildAutoInjection } from "./auto-inject.js";
+import { loadConfig } from "../config/manager.js";
 
 export async function injectResumeSnapshot(
   db: SessionDB,
@@ -20,6 +22,16 @@ export async function injectResumeSnapshot(
     searchTool: opts?.searchTool ?? "ctx_search",
   });
 
+  // Auto-injection: add behavioral state after compaction (if enabled)
+  const config = loadConfig();
+  let fullSnapshot = snapshot;
+  if (config.pipeline.autoInjection) {
+    const autoInjection = buildAutoInjection(events);
+    if (autoInjection) {
+      fullSnapshot = `${snapshot}\n\n${autoInjection}`;
+    }
+  }
+
   db.markResumeConsumed(sessionId);
-  return snapshot;
+  return fullSnapshot;
 }

package/src/store/db-base.ts

@@ -42,6 +42,17 @@ export async function loadSQLite() {
 export function applyWALPragmas(db: any): void {
   db.exec("PRAGMA journal_mode = WAL;");
   db.exec("PRAGMA synchronous = NORMAL;");
+  // Memory-map the DB file for read-heavy FTS5 search workloads (if enabled)
+  try {
+    const { loadConfig } = require("../config/manager.js");
+    const config = loadConfig();
+    if (config.pipeline?.mmapPragma !== false) {
+      db.exec("PRAGMA mmap_size = 268435456;"); // 256MB
+    }
+  } catch {
+    // Fallback: always apply mmap if config can't be loaded
+    try { db.exec("PRAGMA mmap_size = 268435456;"); } catch { /* unsupported runtime */ }
+  }
 }
 
 export function withRetry<T>(fn: () => T, maxRetries = 3): T {