@pi-unipi/compactor 0.2.2 → 2.0.0

package/src/compaction/hooks.ts

@@ -4,13 +4,20 @@
 
 import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
 import { convertToLlm } from "@mariozechner/pi-coding-agent";
+import type {
+  SessionEntry,
+  SessionMessageEntry,
+  SessionBeforeCompactEvent,
+  SessionCompactEvent,
+} from "@mariozechner/pi-coding-agent";
+import type { AgentMessage } from "@mariozechner/pi-agent-core";
 import { compile } from "./summarize.js";
 import { loadConfig } from "../config/manager.js";
-import { buildOwnCut } from "./cut.js";
+import { buildOwnCut, type OwnCutResult } from "./cut.js";
 import type { CompactionStats } from "../types.js";
 import type { SessionDB } from "../session/db.js";
 
-export const COMPACTOR_INSTRUCTION = "__compactor__";
+import { COMPACTOR_INSTRUCTION } from "@pi-unipi/core";
 
 let lastStats: CompactionStats | null = null;
 let lastCompactWasCompactor = false;
@@ -26,76 +33,105 @@ const dbg = (_debug: boolean, _event: string, _data?: Record<string, unknown>) =
   return;
 };
 
-const previewContent = (content: unknown): string => {
-  if (typeof content === "string") return content.slice(0, 300);
-  if (Array.isArray(content)) {
-    return content
-      .map((c: any) => {
-        if (c?.type === "text") return c.text ?? "";
-        if (c?.type === "toolCall") return `[toolCall:${c.name}]`;
-        if (c?.type === "thinking") return `[thinking]`;
-        if (c?.type === "image") return `[image:${c.mimeType}]`;
-        return `[${c?.type ?? "unknown"}]`;
-      })
-      .join("\n")
-      .slice(0, 300);
-  }
-  return "";
-};
-
 const REASON_MESSAGES: Record<import("./cut.js").OwnCutCancelReason, string> = {
   no_live_messages: "compactor: Nothing to compact (no live messages)",
   too_few_live_messages: "compactor: Too few messages to compact",
   no_user_message: "compactor: Cannot compact — no user message found",
 };
 
+/** Count chars in a content part array (TextContent, ToolCall, ToolResult, etc.) */
+function contentPartsChars(parts: Array<{ text?: string; name?: string; input?: unknown; content?: unknown }>): number {
+  return parts.reduce((s: number, p) => {
+    if (p.text) return s + p.text.length;
+    if (p.name) {
+      // ToolCall
+      const inputStr = typeof p.input === "string" ? p.input : JSON.stringify(p.input ?? "");
+      return s + p.name.length + inputStr.length;
+    }
+    if (p.content !== undefined) {
+      // ToolResult
+      const contentStr = typeof p.content === "string" ? p.content : JSON.stringify(p.content ?? "");
+      return s + contentStr.length;
+    }
+    return s;
+  }, 0);
+}
+
+/** Estimate char count for an AgentMessage (unwrapped — has role + content directly) */
+function messageChars(msg: AgentMessage): number {
+  const c = (msg as { content: unknown }).content;
+  if (typeof c === "string") return c.length;
+  if (Array.isArray(c)) return contentPartsChars(c as Array<{ text?: string; name?: string; input?: unknown; content?: unknown }>);
+  return 0;
+}
+
+/** Estimate char count for a SessionMessageEntry's message */
+function entryMessageChars(entry: SessionMessageEntry): number {
+  return messageChars(entry.message);
+}
+
+/** Filter entries to only SessionMessageEntry */
+function filterMessageEntries(entries: SessionEntry[]): SessionMessageEntry[] {
+  return entries.filter((e): e is SessionMessageEntry => e.type === "message");
+}
+
 export function registerCompactionHooks(
   pi: ExtensionAPI,
   deps?: { getSessionDB?: () => SessionDB | null; getSessionId?: () => string },
 ): void {
-  pi.on("session_before_compact", (event, ctx) => {
+  pi.on("session_before_compact", (event: SessionBeforeCompactEvent, ctx) => {
     const { preparation, branchEntries, customInstructions } = event;
     const config = loadConfig();
-    dbg(config.debug, "session_before_compact:enter", { entryCount: (branchEntries as any[])?.length, hasPrevSummary: !!preparation?.previousSummary, isCompactor: customInstructions === COMPACTOR_INSTRUCTION });
+    const isCompactor = customInstructions?.startsWith(COMPACTOR_INSTRUCTION) ?? false;
+    dbg(config.debug, "session_before_compact:enter", {
+      entryCount: branchEntries.length,
+      hasPrevSummary: !!preparation?.previousSummary,
+      isCompactor,
+    });
 
-    const isCompactor = customInstructions === COMPACTOR_INSTRUCTION;
     if (!isCompactor && !config.overrideDefaultCompaction) {
       dbg(config.debug, "session_before_compact:skip", { reason: "not_compactor_and_no_override" });
       return;
     }
 
-    const ownCut = buildOwnCut(branchEntries as any[]);
-    dbg(config.debug, "buildOwnCut", { ok: ownCut.ok, reason: !ownCut.ok ? (ownCut as any).reason : undefined });
+    const ownCut: OwnCutResult = buildOwnCut(branchEntries);
+    dbg(config.debug, "buildOwnCut", {
+      ok: ownCut.ok,
+      reason: !ownCut.ok ? (ownCut as { ok: false; reason: string }).reason : undefined,
+    });
     if (!ownCut.ok) {
       try {
-        ctx?.ui?.notify?.(REASON_MESSAGES[ownCut.reason], "warning");
+        ctx?.ui?.notify?.(REASON_MESSAGES[(ownCut as { ok: false; reason: import("./cut.js").OwnCutCancelReason }).reason], "warning");
       } catch {}
       return { cancel: true };
     }
 
-    const agentMessages = ownCut.messages;
-    const firstKeptEntryId = ownCut.firstKeptEntryId;
+    const { messages: agentMessages, firstKeptEntryId } = ownCut;
     const messages = convertToLlm(agentMessages);
 
-    const keptIdx = (branchEntries as any[]).findIndex((e: any) => e.id === firstKeptEntryId);
-    const keptEntries = keptIdx >= 0
-      ? (branchEntries as any[]).slice(keptIdx).filter((e: any) => e.type === "message")
+    // Find kept entries (from cut point onward)
+    const keptIdx = branchEntries.findIndex((e: SessionEntry) => e.id === firstKeptEntryId);
+    const keptMessageEntries: SessionMessageEntry[] = keptIdx >= 0
+      ? filterMessageEntries(branchEntries.slice(keptIdx))
       : [];
-    const keptChars = keptEntries.reduce((sum: number, e: any) => {
-      const c = e.message?.content;
-      if (typeof c === "string") return sum + c.length;
-      if (Array.isArray(c)) return sum + c.reduce((s: number, p: any) => {
-        if (p.text) return s + p.text.length;
-        if (p.type === "toolCall") return s + (p.name?.length ?? 0) + (typeof p.input === "string" ? p.input.length : JSON.stringify(p.input ?? "").length);
-        if (p.type === "toolResult") return s + (typeof p.content === "string" ? p.content.length : JSON.stringify(p.content ?? "").length);
-        return s;
-      }, 0);
-      return sum;
-    }, 0);
+
+    // Compute char estimates for proportional token estimation
+    const summarizedChars = agentMessages.reduce((sum, msg) => sum + messageChars(msg), 0);
+    const keptChars = keptMessageEntries.reduce((sum, e) => sum + entryMessageChars(e), 0);
+    const totalChars = summarizedChars + keptChars;
+
+    // Use Pi's real token count for "before", estimate "after" proportionally
+    const tokensBefore = preparation.tokensBefore;
+    const tokensAfterEst = totalChars > 0
+      ? Math.round(tokensBefore * keptChars / totalChars)
+      : 0;
+
     lastStats = {
       summarized: agentMessages.length,
-      kept: keptEntries.length,
-      keptTokensEst: Math.round(keptChars / 4),
+      kept: keptMessageEntries.length,
+      totalMessages: agentMessages.length + keptMessageEntries.length,
+      tokensBefore,
+      tokensAfterEst,
     };
 
     // Persist cumulative compaction stats
@@ -103,13 +139,7 @@ export function registerCompactionHooks(
     if (sessionDB && deps?.getSessionId) {
       try {
         const sessionId = deps.getSessionId();
-        const charsBefore = agentMessages.reduce((sum: number, msg: any) => {
-          const c = msg.message?.content;
-          if (typeof c === "string") return sum + c.length;
-          if (Array.isArray(c)) return sum + c.reduce((s: number, p: any) => s + (p.text?.length ?? 0), 0);
-          return sum;
-        }, 0);
-        sessionDB.addCompactionStats(sessionId, charsBefore, keptChars, agentMessages.length);
+        sessionDB.addCompactionStats(sessionId, summarizedChars, keptChars, agentMessages.length);
       } catch {
         // non-fatal
       }
@@ -154,7 +184,7 @@ export function registerCompactionHooks(
     };
   });
 
-  pi.on("session_compact", (event, ctx) => {
+  pi.on("session_compact", (event: SessionCompactEvent, ctx) => {
     const config = loadConfig();
     dbg(config.debug, "session_compact", { fromExtension: event.fromExtension, lastCompactWasCompactor });
     if (!event.fromExtension) return;
@@ -164,7 +194,7 @@ export function registerCompactionHooks(
     setTimeout(() => {
       try {
         ctx?.ui?.notify?.(
-          `compactor: ${stats.summarized} source entries processed; tail kept ${stats.kept} (~${formatTokens(stats.keptTokensEst)} tok).`,
+          `Compacted ${stats.totalMessages} messages (~${formatTokens(stats.tokensBefore)} tokens) → ${stats.kept} messages (~${formatTokens(stats.tokensAfterEst)} tokens)`,
           "info",
         );
       } catch {}

package/src/compaction/recall-scope.ts

@@ -12,7 +12,7 @@ export interface LineageRange {
  * the most recent compaction boundary.
  */
 export function getRecallScope(
-  branchEntries: any[],
+  branchEntries: Array<{ type: string; [key: string]: unknown }>,
   opts?: { expand?: boolean },
 ): LineageRange {
   let lastCompactionIdx = -1;

package/src/config/presets.ts

@@ -11,16 +11,16 @@ const preset = (
 ): CompactorConfig => ({
   ...structuredClone(DEFAULT_COMPACTOR_CONFIG),
   ...overrides,
-  sessionGoals: { ...DEFAULT_COMPACTOR_CONFIG.sessionGoals, ...(overrides.sessionGoals as any) },
-  filesAndChanges: { ...DEFAULT_COMPACTOR_CONFIG.filesAndChanges, ...(overrides.filesAndChanges as any) },
-  commits: { ...DEFAULT_COMPACTOR_CONFIG.commits, ...(overrides.commits as any) },
-  outstandingContext: { ...DEFAULT_COMPACTOR_CONFIG.outstandingContext, ...(overrides.outstandingContext as any) },
-  userPreferences: { ...DEFAULT_COMPACTOR_CONFIG.userPreferences, ...(overrides.userPreferences as any) },
-  briefTranscript: { ...DEFAULT_COMPACTOR_CONFIG.briefTranscript, ...(overrides.briefTranscript as any) },
-  sessionContinuity: { ...DEFAULT_COMPACTOR_CONFIG.sessionContinuity, ...(overrides.sessionContinuity as any) },
-  fts5Index: { ...DEFAULT_COMPACTOR_CONFIG.fts5Index, ...(overrides.fts5Index as any) },
-  sandboxExecution: { ...DEFAULT_COMPACTOR_CONFIG.sandboxExecution, ...(overrides.sandboxExecution as any) },
-  toolDisplay: { ...DEFAULT_COMPACTOR_CONFIG.toolDisplay, ...(overrides.toolDisplay as any) },
+  sessionGoals: { ...DEFAULT_COMPACTOR_CONFIG.sessionGoals, ...(overrides.sessionGoals ?? {}) },
+  filesAndChanges: { ...DEFAULT_COMPACTOR_CONFIG.filesAndChanges, ...(overrides.filesAndChanges ?? {}) },
+  commits: { ...DEFAULT_COMPACTOR_CONFIG.commits, ...(overrides.commits ?? {}) },
+  outstandingContext: { ...DEFAULT_COMPACTOR_CONFIG.outstandingContext, ...(overrides.outstandingContext ?? {}) },
+  userPreferences: { ...DEFAULT_COMPACTOR_CONFIG.userPreferences, ...(overrides.userPreferences ?? {}) },
+  briefTranscript: { ...DEFAULT_COMPACTOR_CONFIG.briefTranscript, ...(overrides.briefTranscript ?? {}) },
+  sessionContinuity: { ...DEFAULT_COMPACTOR_CONFIG.sessionContinuity, ...(overrides.sessionContinuity ?? {}) },
+  fts5Index: { ...DEFAULT_COMPACTOR_CONFIG.fts5Index, ...(overrides.fts5Index ?? {}) },
+  sandboxExecution: { ...DEFAULT_COMPACTOR_CONFIG.sandboxExecution, ...(overrides.sandboxExecution ?? {}) },
+  toolDisplay: { ...DEFAULT_COMPACTOR_CONFIG.toolDisplay, ...(overrides.toolDisplay ?? {}) },
 });
 
 // Pipeline feature defaults per preset:

package/src/executor/executor.ts

@@ -122,11 +122,11 @@ export class PolyglotExecutor {
       }
 
       return result;
-    } catch (err: any) {
+    } catch (err: unknown) {
       try { rmSync(tmpDir, { recursive: true, force: true }); } catch { /* ignore */ }
       return {
         stdout: "",
-        stderr: err?.message ?? String(err),
+        stderr: (err instanceof Error ? err.message : String(err)),
         exitCode: 1,
         timedOut: false,
       };
@@ -228,10 +228,10 @@ export class PolyglotExecutor {
         timeout: timeout / 2,
         stdio: ["ignore", "pipe", "pipe"],
       });
-    } catch (err: any) {
+    } catch (err: unknown) {
       return {
         stdout: "",
-        stderr: err?.stderr?.toString?.() ?? err?.message ?? "Rust compilation failed",
+        stderr: (err instanceof Error && 'stderr' in err ? String((err as Error & { stderr?: Buffer }).stderr) : '') || (err instanceof Error ? err.message : String(err)) || "Rust compilation failed",
         exitCode: 1,
         timedOut: false,
       };

package/src/index.ts

@@ -9,7 +9,6 @@ import { registerCompactionHooks } from "./compaction/hooks.js";
 import { SessionDB, getWorktreeSuffix } from "./session/db.js";
 import { extractEventsFromToolResult } from "./session/extract.js";
 import { injectResumeSnapshot } from "./session/resume-inject.js";
-import { ContentStore } from "./store/index.js";
 import { PolyglotExecutor } from "./executor/executor.js";
 import { registerCommands } from "./commands/index.js";
 import { registerCompactorTools } from "./tools/register.js";
@@ -57,7 +56,6 @@ function isIndexTool(_name: string): boolean {
 
 export default function compactorExtension(pi: ExtensionAPI): void {
   let sessionDB: SessionDB | null = null;
-  let contentStore: ContentStore | null = null;
   let executor: PolyglotExecutor | null = null;
   let config = loadConfig();
   let cachedBlocks: NormalizedBlock[] = [];
@@ -101,28 +99,19 @@ export default function compactorExtension(pi: ExtensionAPI): void {
       sessionDB = null;
     }
 
-    // Initialize ContentStore independently — its failure shouldn't
-    // prevent SessionDB commands from working.
-    if (config.fts5Index.enabled) {
-      try {
-        const cs = new ContentStore();
-        await cs.init();
-        contentStore = cs;
-      } catch {
-        // Silently ignore — ContentStore init failure is handled gracefully.
-        contentStore = null;
-      }
-    }
-
     executor = new PolyglotExecutor();
   };
 
-  registerCompactionHooks(pi);
+  // Register compaction hooks with lazy deps — sessionDB/sessionId may not be
+  // available at registration time, but will be by the time events fire.
+  registerCompactionHooks(pi, {
+    getSessionDB: () => sessionDB,
+    getSessionId: () => currentSessionId,
+  });
 
   // Commands registered inside session_start after init() when deps are ready
   const getCommandDeps = () => ({
     sessionDB,
-    contentStore,
     getSessionId: () => currentSessionId,
     getBlocks: () => cachedBlocks,
     getCounters,
@@ -139,6 +128,17 @@ export default function compactorExtension(pi: ExtensionAPI): void {
 
     debug("session_start", { sessionId: fullSessionId, projectDir });
 
+    // Seed runtime counters from DB so they reflect prior usage
+    if (sessionDB) {
+      try {
+        const allTime = sessionDB.getAllTimeStats();
+        counters.sandboxRuns = allTime.allSandboxRuns;
+        counters.searchQueries = allTime.allSearchQueries;
+      } catch {
+        // Non-fatal: counter seeding from DB failed
+      }
+    }
+
     // Reset runtime stats for new session
     runtimeStats.bytesReturned = {};
     runtimeStats.bytesIndexed = 0;
@@ -154,7 +154,6 @@ export default function compactorExtension(pi: ExtensionAPI): void {
     if (sessionDB) {
       registerCompactorTools(pi, {
         sessionDB,
-        contentStore,
         getSessionId: () => currentSessionId,
         getBlocks: () => cachedBlocks,
         getCounters,
@@ -165,7 +164,7 @@ export default function compactorExtension(pi: ExtensionAPI): void {
     registerCommands(pi, getCommandDeps());
 
     // Register info-screen group
-    const infoRegistry = (globalThis as any).__unipi_info_registry;
+    const infoRegistry = globalThis.__unipi_info_registry;
     if (infoRegistry && sessionDB) {
       const sdb = sessionDB;
       const sid = () => currentSessionId;
@@ -188,7 +187,7 @@ export default function compactorExtension(pi: ExtensionAPI): void {
         dataProvider: async () => {
           try {
             const { getInfoScreenData } = await import("./info-screen.js");
-            const data = await getInfoScreenData(sdb, sid(), runtimeStats);
+            const data = await getInfoScreenData(sdb, sid(), getCounters());
             return {
               tokensSaved: { value: data.tokensSaved.value, detail: data.tokensSaved.detail },
               costSaved: { value: data.costSaved.value, detail: data.costSaved.detail },
@@ -204,7 +203,7 @@ export default function compactorExtension(pi: ExtensionAPI): void {
       });
     }
 
-    emitEvent(pi as any, UNIPI_EVENTS.MODULE_READY, {
+    emitEvent(pi, UNIPI_EVENTS.MODULE_READY, {
       name: MODULES.COMPACTOR,
       version: "0.1.0",
       commands: Object.values(COMPACTOR_COMMANDS),
@@ -213,10 +212,6 @@ export default function compactorExtension(pi: ExtensionAPI): void {
 
     debug("MODULE_READY", { commands: Object.values(COMPACTOR_COMMANDS), tools: Object.values(COMPACTOR_TOOLS) });
 
-    if (config.fts5Index.mode === "auto" && contentStore) {
-      // TODO: index project files
-    }
-
     ctx.ui.notify("🗜️  Compactor ready", "info");
   });
 
@@ -232,7 +227,6 @@ export default function compactorExtension(pi: ExtensionAPI): void {
       const { join } = await import("node:path");
       const strategies: Array<{ key: string; config: CompactorStrategyConfig }> = [
         { key: "commits", config: config.commits },
-        { key: "fts5Index", config: config.fts5Index },
       ];
       for (const { key, config: strat } of strategies) {
         if ((strat as any).autoDetect === "git") {
@@ -283,7 +277,9 @@ export default function compactorExtension(pi: ExtensionAPI): void {
 
   pi.on("session_before_compact", async (event, _ctx) => {
     if (sessionDB) {
-      const sessionId = `${(event as any).sessionId ?? "default"}${getWorktreeSuffix()}`;
+      // Use closure currentSessionId — Pi's session_before_compact event
+      // does not include sessionId at the top level.
+      const sessionId = currentSessionId;
       const events = sessionDB.getEvents(sessionId, { limit: 1000 });
       const stats = sessionDB.getSessionStats(sessionId);
       debug("session_before_compact", { sessionId, eventCount: events.length, compactCount: stats?.compact_count ?? 0 });
@@ -297,44 +293,55 @@ export default function compactorExtension(pi: ExtensionAPI): void {
 
   pi.on("session_compact", async (event, _ctx) => {
     if (sessionDB) {
-      const sessionId = `${(event as any).sessionId ?? "default"}${getWorktreeSuffix()}`;
+      // Use closure currentSessionId — Pi's session_compact event does not
+      // include sessionId at the top level (it's inside compactionEntry).
+      const sessionId = currentSessionId;
       sessionDB.incrementCompactCount(sessionId);
       counters.compactions++;
 
-      // Use actual runtimeStats for byte measurement instead of heuristic
-      const totalBytesReturned = Object.values(runtimeStats.bytesReturned).reduce((s, b) => s + b, 0);
-      const totalBytesProcessed = runtimeStats.bytesIndexed + runtimeStats.bytesSandboxed + totalBytesReturned;
-      // charsBefore = total bytes processed by all tools (proxy for context window usage)
-      // charsKept = bytes that stayed in context (bytesReturned, minus what compaction removed)
-      const tokensBefore = (event as any).tokensBefore ?? 0;
-      if (totalBytesProcessed > 0 && tokensBefore > 0) {
-        // Use actual token count from Pi, estimate chars from it
+      // Pi's session_compact event structure: { compactionEntry, fromExtension }
+      // tokensBefore is inside compactionEntry, not at event root.
+      const compactionEntry = (event as any).compactionEntry;
+      const tokensBefore = compactionEntry?.tokensBefore ?? 0;
+
+      if (tokensBefore > 0) {
+        // Use actual token count from Pi's compactionEntry.
+        // Compaction typically keeps ~10-15% of original context.
         const charsBefore = tokensBefore * 4;
-        // Estimate kept chars: proportional to what remains after compaction
-        const tokensAfter = (event as any).tokensAfter ?? Math.round(tokensBefore * 0.15);
+        const tokensAfter = (event as any).tokensAfter ?? Math.round(tokensBefore * 0.12);
         const charsKept = tokensAfter * 4;
         const messagesSummarized = Math.max(1, Math.round(tokensBefore / 500));
         counters.totalTokensCompacted += tokensBefore - tokensAfter;
         sessionDB.addCompactionStats(sessionId, charsBefore, charsKept, messagesSummarized);
-      } else if (tokensBefore > 0) {
-        // Fallback: only tokensBefore available, use conservative estimate
-        const tokensAfter = (event as any).tokensAfter ?? Math.round(tokensBefore * 0.15);
-        counters.totalTokensCompacted += tokensBefore - tokensAfter;
-        sessionDB.addCompactionStats(sessionId, tokensBefore * 4, tokensAfter * 4, 1);
+      } else {
+        // tokensBefore unavailable — use session event count as a rough heuristic.
+        // This happens when Pi's compaction doesn't report tokensBefore.
+        // We estimate from the number of events stored in this session.
+        try {
+          const eventCount = sessionDB.getEventCount(sessionId);
+          if (eventCount > 0) {
+            // Rough estimate: ~500 tokens per event on average
+            const estTokensBefore = eventCount * 500;
+            const estTokensAfter = Math.round(estTokensBefore * 0.12);
+            const charsBefore = estTokensBefore * 4;
+            const charsKept = estTokensAfter * 4;
+            counters.totalTokensCompacted += estTokensBefore - estTokensAfter;
+            sessionDB.addCompactionStats(sessionId, charsBefore, charsKept, eventCount);
+          }
+        } catch {
+          // Non-fatal: heuristic estimation failed
+        }
       }
-      debug("session_compact", { sessionId, tokensBefore, totalBytesProcessed });
+      debug("session_compact", { sessionId, tokensBefore, hasCompactionEntry: !!compactionEntry });
     }
   });
 
   pi.on("session_shutdown", async (_event, _ctx) => {
     debug("session_shutdown");
-    // WAL checkpoint: TRUNCATE on shutdown to keep DB file size down
-    contentStore?.checkpointWAL("TRUNCATE");
     if (sessionDB) {
       sessionDB.cleanupOldSessions(7);
     }
     executor?.cleanupBackgrounded();
-    contentStore?.close();
     sessionDB?.close();
   });
 
@@ -412,7 +419,8 @@ export default function compactorExtension(pi: ExtensionAPI): void {
 
   pi.on("tool_result", async (event, _ctx) => {
     if (!sessionDB) return;
-    const sessionId = `${(event as any).sessionId ?? "default"}${getWorktreeSuffix()}`;
+    // Use closure currentSessionId — tool_result events use the same session
+    const sessionId = currentSessionId;
     const toolNameRaw = (event as any).toolName ?? "";
     const isError = (event as any).isError ?? false;