@pi-unipi/compactor 0.2.2 → 0.2.3

package/README.md

@@ -1,14 +1,8 @@
 # @pi-unipi/compactor
 
-Context engine for Pi coding agent. Fuses zero-LLM compaction, session continuity, sandbox execution, FTS5 search, and tool display optimization into a single cohesive package.
+Context engine that keeps sessions lean. Compacts conversations, indexes code, searches history, and runs sandboxed code — all without burning LLM tokens on compaction.
 
-## Features
-
-- **Zero-LLM Compaction** — 6-stage pipeline (normalize → filter → build sections → brief → format → merge) achieves 95%+ token reduction with zero API cost
-- **Session Continuity** — XML resume snapshots survive compaction, preserving context across session boundaries
-- **Sandbox Execution** — 11 languages with process isolation, security hardening, and output capping
-- **FTS5 Search** — Full-text search over indexed content with auto-chunking
-- **Tool Display** — Mode-aware rendering for read, grep, find, ls, bash, edit, write tools
+The zero-LLM pipeline compresses context through 6 stages (normalize, filter, build sections, brief, format, merge) to hit 95%+ token reduction at zero API cost. Session continuity preserves context across compaction boundaries with XML resume snapshots.
 
 ## Commands
 
@@ -25,6 +19,12 @@ Context engine for Pi coding agent. Fuses zero-LLM compaction, session continuit
 | `/unipi:compact-preset <name>` | Apply quick preset |
 | `/unipi:compact-help` | Show detailed documentation |
 
+## Special Triggers
+
+Compactor tools are available to the main agent when installed. All workflow skills can use compactor tools for context management.
+
+Compactor registers with the info-screen dashboard, showing compaction count, tokens saved, compression ratio, and indexed documents. The footer subscribes to `COMPACTOR_STATSUPDATED` events to display compaction stats in the status bar.
+
 ## Agent Tools
 
 | Tool | Family | Description |
@@ -34,19 +34,19 @@ Context engine for Pi coding agent. Fuses zero-LLM compaction, session continuit
 | `sandbox` | sandbox | Run code in sandbox (11 languages) |
 | `sandbox_file` | sandbox | Execute file via FILE_CONTENT |
 | `sandbox_batch` | sandbox | Atomic batch of commands + searches |
-| `content_index` | content | Chunk content → FTS5 index |
+| `content_index` | content | Chunk content to FTS5 index |
 | `content_search` | content | Query indexed content |
-| `content_fetch` | content | Fetch URL → index |
+| `content_fetch` | content | Fetch URL and index |
 | `compactor_stats` | compactor | Context savings dashboard |
 | `compactor_doctor` | compactor | Diagnostics checklist |
 | `context_budget` | compactor | Estimate remaining context window |
 
 ## Two-Tier Skills
 
-- **Tier 1** (`compactor`): ~175 tokens, always loaded. Routing + critical rules + Ralph awareness.
+- **Tier 1** (`compactor`): ~175 tokens, always loaded. Routing and critical rules.
 - **Tier 2** (`compactor-detail`): On-demand. Full tool reference, anti-patterns, sandbox languages, FTS5 modes, workflows.
 
-## Configuration
+## Configurables
 
 Config lives at `~/.unipi/config/compactor/config.json`. Per-project overrides at `<project>/.unipi/config/compactor.json`.
 
@@ -78,7 +78,7 @@ Tabbed settings interface (Presets / Strategies / Pipeline):
 - `/` key opens search filter in Strategies tab
 - Preset selection shows 3-line preview
 - Per-project override checkbox (`o` key)
-- Keyboard: `←→` cycle modes, `Space` toggle, `s` save, `Esc` cancel
+- Keyboard: left/right cycle modes, Space toggle, `s` save, Esc cancel
 
 ## Architecture
 
@@ -95,18 +95,6 @@ Tabbed settings interface (Presets / Strategies / Pipeline):
                     └─────────────────────┘
 ```
 
-## Installation
-
-Included in `@pi-unipi/unipi` metapackage. To use standalone:
-
-```json
-{
-  "pi": {
-    "extensions": ["node_modules/@pi-unipi/compactor/src/index.ts"]
-  }
-}
-```
-
 ## License
 
 MIT

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-unipi/compactor",
-  "version": "0.2.2",
+  "version": "0.2.3",
   "description": "Context engine for Pi — zero-LLM compaction, session continuity, sandbox execution, FTS5 search, and tool display optimization",
   "type": "module",
   "license": "MIT",

package/src/index.ts

@@ -117,7 +117,12 @@ export default function compactorExtension(pi: ExtensionAPI): void {
     executor = new PolyglotExecutor();
   };
 
-  registerCompactionHooks(pi);
+  // Register compaction hooks with lazy deps — sessionDB/sessionId may not be
+  // available at registration time, but will be by the time events fire.
+  registerCompactionHooks(pi, {
+    getSessionDB: () => sessionDB,
+    getSessionId: () => currentSessionId,
+  });
 
   // Commands registered inside session_start after init() when deps are ready
   const getCommandDeps = () => ({
@@ -283,7 +288,9 @@ export default function compactorExtension(pi: ExtensionAPI): void {
 
   pi.on("session_before_compact", async (event, _ctx) => {
     if (sessionDB) {
-      const sessionId = `${(event as any).sessionId ?? "default"}${getWorktreeSuffix()}`;
+      // Use closure currentSessionId — Pi's session_before_compact event
+      // does not include sessionId at the top level.
+      const sessionId = currentSessionId;
       const events = sessionDB.getEvents(sessionId, { limit: 1000 });
       const stats = sessionDB.getSessionStats(sessionId);
       debug("session_before_compact", { sessionId, eventCount: events.length, compactCount: stats?.compact_count ?? 0 });
@@ -297,18 +304,23 @@ export default function compactorExtension(pi: ExtensionAPI): void {
 
   pi.on("session_compact", async (event, _ctx) => {
     if (sessionDB) {
-      const sessionId = `${(event as any).sessionId ?? "default"}${getWorktreeSuffix()}`;
+      // Use closure currentSessionId — Pi's session_compact event does not
+      // include sessionId at the top level (it's inside compactionEntry).
+      const sessionId = currentSessionId;
       sessionDB.incrementCompactCount(sessionId);
       counters.compactions++;
 
+      // Pi's session_compact event structure: { compactionEntry, fromExtension }
+      // tokensBefore is inside compactionEntry, not at event root.
+      const compactionEntry = (event as any).compactionEntry;
+      const tokensBefore = compactionEntry?.tokensBefore ?? 0;
+
       // Use actual runtimeStats for byte measurement instead of heuristic
       const totalBytesReturned = Object.values(runtimeStats.bytesReturned).reduce((s, b) => s + b, 0);
       const totalBytesProcessed = runtimeStats.bytesIndexed + runtimeStats.bytesSandboxed + totalBytesReturned;
-      // charsBefore = total bytes processed by all tools (proxy for context window usage)
-      // charsKept = bytes that stayed in context (bytesReturned, minus what compaction removed)
-      const tokensBefore = (event as any).tokensBefore ?? 0;
-      if (totalBytesProcessed > 0 && tokensBefore > 0) {
-        // Use actual token count from Pi, estimate chars from it
+
+      if (tokensBefore > 0) {
+        // Use actual token count from Pi's compactionEntry
         const charsBefore = tokensBefore * 4;
         // Estimate kept chars: proportional to what remains after compaction
         const tokensAfter = (event as any).tokensAfter ?? Math.round(tokensBefore * 0.15);
@@ -316,13 +328,19 @@ export default function compactorExtension(pi: ExtensionAPI): void {
         const messagesSummarized = Math.max(1, Math.round(tokensBefore / 500));
         counters.totalTokensCompacted += tokensBefore - tokensAfter;
         sessionDB.addCompactionStats(sessionId, charsBefore, charsKept, messagesSummarized);
-      } else if (tokensBefore > 0) {
-        // Fallback: only tokensBefore available, use conservative estimate
-        const tokensAfter = (event as any).tokensAfter ?? Math.round(tokensBefore * 0.15);
-        counters.totalTokensCompacted += tokensBefore - tokensAfter;
-        sessionDB.addCompactionStats(sessionId, tokensBefore * 4, tokensAfter * 4, 1);
+      } else {
+        // Fallback: estimate from runtime byte stats when tokensBefore unavailable
+        if (totalBytesProcessed > 0) {
+          const charsBefore = totalBytesProcessed;
+          const charsKept = totalBytesReturned;
+          const messagesSummarized = Math.max(1, Math.round(totalBytesProcessed / 2000));
+          const estTokensBefore = Math.round(totalBytesProcessed / 4);
+          const estTokensAfter = Math.round(totalBytesReturned / 4);
+          counters.totalTokensCompacted += Math.max(0, estTokensBefore - estTokensAfter);
+          sessionDB.addCompactionStats(sessionId, charsBefore, charsKept, messagesSummarized);
+        }
       }
-      debug("session_compact", { sessionId, tokensBefore, totalBytesProcessed });
+      debug("session_compact", { sessionId, tokensBefore, totalBytesProcessed, hasCompactionEntry: !!compactionEntry });
     }
   });
 
@@ -412,7 +430,8 @@ export default function compactorExtension(pi: ExtensionAPI): void {
 
   pi.on("tool_result", async (event, _ctx) => {
     if (!sessionDB) return;
-    const sessionId = `${(event as any).sessionId ?? "default"}${getWorktreeSuffix()}`;
+    // Use closure currentSessionId — tool_result events use the same session
+    const sessionId = currentSessionId;
     const toolNameRaw = (event as any).toolName ?? "";
     const isError = (event as any).isError ?? false;
 

package/src/info-screen.ts

@@ -125,11 +125,11 @@ export async function getInfoScreenData(
         detail: top5Detail,
       },
       compactions: {
-        value: String(report.continuity.compact_count),
+        value: String(Math.max(report.continuity.compact_count, report.continuity.all_time_compact_count)),
         detail: compactStats
           ? `Last: ${compactStats.summarized} msgs summarized, ${compactStats.kept} kept (~${formatTokens(compactStats.keptTokensEst)} tok)`
-          : report.continuity.compact_count > 0
-            ? `${report.continuity.compact_count} compaction(s) this session`
+          : report.continuity.all_time_compact_count > 0
+            ? `${report.continuity.all_time_compact_count} compaction(s) across all sessions`
             : "No compactions yet",
       },
       toolCalls: {

package/src/session/analytics.ts

@@ -66,6 +66,7 @@ export interface FullReport {
   continuity: {
     total_events: number;
     compact_count: number;
+    all_time_compact_count: number;
     resume_ready: boolean;
   };
   /** Persistent project memory — all events across all sessions */
@@ -135,6 +136,11 @@ export class AnalyticsEngine {
     ).get(sid) as { compact_count: number } | undefined;
     const compactCount = meta?.compact_count ?? 0;
 
+    // ── All-time compaction count across all sessions ──
+    const allTimeCompactions = (this.db.prepare(
+      "SELECT COALESCE(SUM(compact_count), 0) as total FROM session_meta",
+    ).get() as { total: number }).total;
+
     const resume = this.db.prepare(
       "SELECT event_count, consumed FROM session_resume WHERE session_id = ? ORDER BY created_at DESC LIMIT 1",
     ).get(sid) as { event_count: number; consumed: number } | undefined;
@@ -165,6 +171,7 @@ export class AnalyticsEngine {
       continuity: {
         total_events: eventTotal,
         compact_count: compactCount,
+        all_time_compact_count: allTimeCompactions,
         resume_ready: resumeReady,
       },
       projectMemory: {
@@ -188,7 +195,7 @@ export function createMinimalDb(): DatabaseAdapter {
   // so AnalyticsEngine queries don't fail.
   const emptyStmt = {
     run: (..._params: unknown[]) => {},
-    get: (..._params: unknown[]) => ({ cnt: 0, sessions: 0, compact_count: 0, session_id: "", event_count: 0, consumed: 1 }),
+    get: (..._params: unknown[]) => ({ cnt: 0, sessions: 0, compact_count: 0, total: 0, session_id: "", event_count: 0, consumed: 1 }),
     all: (..._params: unknown[]) => [] as unknown[],
   };
 

package/src/tools/ctx-stats.ts

@@ -26,10 +26,34 @@ export async function ctxStats(
   const sessionStats = sessionDB.getSessionStats(sessionId);
   const storeStats = await contentStore.getStats();
 
+  // Compute tokensSaved: prefer in-memory counters (current session),
+  // fall back to per-session DB stats, then all-time DB stats.
+  let tokensSaved = counters?.totalTokensCompacted ?? 0;
+  if (tokensSaved === 0 && sessionStats) {
+    const sessionCharsBefore = (sessionStats as any).total_chars_before ?? 0;
+    const sessionCharsKept = (sessionStats as any).total_chars_kept ?? 0;
+    tokensSaved = Math.round((sessionCharsBefore - sessionCharsKept) / 4);
+  }
+  if (tokensSaved === 0) {
+    const allTime = sessionDB.getAllTimeStats();
+    tokensSaved = Math.round((allTime.allCharsBefore - allTime.allCharsKept) / 4);
+  }
+
+  // Compute compactions: prefer in-memory counter (current session),
+  // fall back to per-session DB, then all-time DB.
+  let compactions = counters?.compactions ?? 0;
+  if (compactions === 0) {
+    compactions = sessionStats?.compact_count ?? 0;
+  }
+  if (compactions === 0) {
+    const allTime = sessionDB.getAllTimeStats();
+    compactions = allTime.allCompactions;
+  }
+
   return {
     sessionEvents: sessionStats?.event_count ?? 0,
-    compactions: counters?.compactions ?? sessionStats?.compact_count ?? 0,
-    tokensSaved: counters?.totalTokensCompacted ?? 0,
+    compactions,
+    tokensSaved,
     compressionRatio: "N/A",
     indexedDocs: storeStats.sources,
     indexedChunks: storeStats.chunks,