@pi-unipi/compactor 0.2.2 → 2.0.0

package/README.md

@@ -1,20 +1,14 @@
 # @pi-unipi/compactor
 
-Context engine for Pi coding agent. Fuses zero-LLM compaction, session continuity, sandbox execution, FTS5 search, and tool display optimization into a single cohesive package.
+Context engine that keeps sessions lean. Compacts conversations, indexes code, searches history, and runs sandboxed code — all without burning LLM tokens on compaction.
 
-## Features
-
-- **Zero-LLM Compaction** — 6-stage pipeline (normalize → filter → build sections → brief → format → merge) achieves 95%+ token reduction with zero API cost
-- **Session Continuity** — XML resume snapshots survive compaction, preserving context across session boundaries
-- **Sandbox Execution** — 11 languages with process isolation, security hardening, and output capping
-- **FTS5 Search** — Full-text search over indexed content with auto-chunking
-- **Tool Display** — Mode-aware rendering for read, grep, find, ls, bash, edit, write tools
+The zero-LLM pipeline compresses context through 6 stages (normalize, filter, build sections, brief, format, merge) to hit 95%+ token reduction at zero API cost. Session continuity preserves context across compaction boundaries with XML resume snapshots.
 
 ## Commands
 
 | Command | Description |
 |---------|-------------|
-| `/unipi:compact` | Manual compaction with stats |
+| `/unipi:lossless-compact` | Immediate zero-LLM compaction with structured summary |
 | `/unipi:session-recall` | Search session history (BM25 or regex) |
 | `/unipi:content-index` | Index current project into FTS5 |
 | `/unipi:content-search` | Search indexed content |
@@ -25,6 +19,14 @@ Context engine for Pi coding agent. Fuses zero-LLM compaction, session continuit
 | `/unipi:compact-preset <name>` | Apply quick preset |
 | `/unipi:compact-help` | Show detailed documentation |
 
+> **Note:** `/unipi:compact` still works as a deprecated alias for `/unipi:lossless-compact`.
+
+## Special Triggers
+
+Compactor tools are available to the main agent when installed. All workflow skills can use compactor tools for context management.
+
+Compactor registers with the info-screen dashboard, showing compaction count, tokens saved, compression ratio, and indexed documents. The footer subscribes to `COMPACTOR_STATSUPDATED` events to display compaction stats in the status bar.
+
 ## Agent Tools
 
 | Tool | Family | Description |
@@ -34,19 +36,19 @@ Context engine for Pi coding agent. Fuses zero-LLM compaction, session continuit
 | `sandbox` | sandbox | Run code in sandbox (11 languages) |
 | `sandbox_file` | sandbox | Execute file via FILE_CONTENT |
 | `sandbox_batch` | sandbox | Atomic batch of commands + searches |
-| `content_index` | content | Chunk content → FTS5 index |
+| `content_index` | content | Chunk content to FTS5 index |
 | `content_search` | content | Query indexed content |
-| `content_fetch` | content | Fetch URL → index |
+| `content_fetch` | content | Fetch URL and index |
 | `compactor_stats` | compactor | Context savings dashboard |
 | `compactor_doctor` | compactor | Diagnostics checklist |
 | `context_budget` | compactor | Estimate remaining context window |
 
 ## Two-Tier Skills
 
-- **Tier 1** (`compactor`): ~175 tokens, always loaded. Routing + critical rules + Ralph awareness.
+- **Tier 1** (`compactor`): ~175 tokens, always loaded. Routing and critical rules.
 - **Tier 2** (`compactor-detail`): On-demand. Full tool reference, anti-patterns, sandbox languages, FTS5 modes, workflows.
 
-## Configuration
+## Configurables
 
 Config lives at `~/.unipi/config/compactor/config.json`. Per-project overrides at `<project>/.unipi/config/compactor.json`.
 
@@ -78,7 +80,7 @@ Tabbed settings interface (Presets / Strategies / Pipeline):
 - `/` key opens search filter in Strategies tab
 - Preset selection shows 3-line preview
 - Per-project override checkbox (`o` key)
-- Keyboard: `←→` cycle modes, `Space` toggle, `s` save, `Esc` cancel
+- Keyboard: left/right cycle modes, Space toggle, `s` save, Esc cancel
 
 ## Architecture
 
@@ -95,18 +97,6 @@ Tabbed settings interface (Presets / Strategies / Pipeline):
                     └─────────────────────┘
 ```
 
-## Installation
-
-Included in `@pi-unipi/unipi` metapackage. To use standalone:
-
-```json
-{
-  "pi": {
-    "extensions": ["node_modules/@pi-unipi/compactor/src/index.ts"]
-  }
-}
-```
-
 ## License
 
 MIT

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@pi-unipi/compactor",
-  "version": "0.2.2",
-  "description": "Context engine for Pi — zero-LLM compaction, session continuity, sandbox execution, FTS5 search, and tool display optimization",
+  "version": "2.0.0",
+  "description": "Context engine for Pi — zero-LLM compaction, session continuity, sandbox execution, and tool display optimization",
   "type": "module",
   "license": "MIT",
   "author": "Neuron Mr White",

package/src/commands/index.ts

@@ -2,33 +2,22 @@
  * All /unipi:compact-* commands
  *
  * Commands perform real work by calling tool implementations directly.
- * Dependencies (sessionDB, contentStore, sessionId) are injected at registration time.
- *
- * New command names (v0.2.0):
- *   /unipi:session-recall (was /unipi:compact-recall)
- *   /unipi:content-index (was /unipi:compact-index)
- *   /unipi:content-search (was /unipi:compact-search)
- *   /unipi:content-purge (was /unipi:compact-purge)
- *
- * Old names kept as deprecated aliases for backward compatibility.
+ * Dependencies (sessionDB, sessionId) are injected at registration time.
  */
 
-import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+import type { ExtensionAPI, ExtensionCommandContext } from "@mariozechner/pi-coding-agent";
 import { loadConfig, saveConfig } from "../config/manager.js";
 import { applyPreset, parsePreset } from "../config/presets.js";
+import { COMPACTOR_INSTRUCTION } from "@pi-unipi/core";
 import { getLastCompactionStats } from "../compaction/hooks.js";
-import { compactTool } from "../tools/compact.js";
 import { vccRecall } from "../tools/vcc-recall.js";
 import { ctxStats } from "../tools/ctx-stats.js";
 import { ctxDoctor } from "../tools/ctx-doctor.js";
-import { ctxSearch } from "../tools/ctx-search.js";
-import { ContentStore } from "../store/index.js";
 import type { SessionDB } from "../session/db.js";
 import type { NormalizedBlock, RuntimeCounters } from "../types.js";
 
 export interface CommandDeps {
   sessionDB: SessionDB | null;
-  contentStore: ContentStore | null;
   getSessionId: () => string;
   getBlocks: () => NormalizedBlock[];
   getCounters?: () => RuntimeCounters;
@@ -38,22 +27,70 @@ function deprecationLog(_oldName: string, _newName: string): void {
   // Deprecation logging disabled — was writing to stdout causing TUI rendering issues.
 }
 
+const formatTokens = (n: number): string => {
+  if (n >= 1000) return `${(n / 1000).toFixed(1)}k`;
+  return String(n);
+};
+
 export function registerCommands(pi: ExtensionAPI, deps?: CommandDeps): void {
-  // ── /unipi:compact ──────────────────────────────────
+  // ── /unipi:lossless-compact ──────────────────────────
+  pi.registerCommand("unipi:lossless-compact", {
+    description: "Immediate zero-LLM compaction — structured summary with full recall",
+    handler: async (_args: string, ctx: ExtensionCommandContext) => {
+      ctx.compact({
+        customInstructions: COMPACTOR_INSTRUCTION,
+        onComplete: () => {
+          const stats = getLastCompactionStats();
+          if (stats) {
+            ctx.ui.notify(
+              `Compacted ${stats.totalMessages} messages (~${formatTokens(stats.tokensBefore)} tokens) → ${stats.kept} messages (~${formatTokens(stats.tokensAfterEst)} tokens)`,
+              "info",
+            );
+          } else {
+            ctx.ui.notify("Compaction completed.", "info");
+          }
+        },
+        onError: (err: Error) => {
+          if (err.message === "Compaction cancelled" || err.message === "Already compacted") {
+            ctx.ui.notify("Nothing to compact.", "info");
+          } else {
+            ctx.ui.notify(`Compaction failed: ${err.message}`, "error");
+          }
+        },
+      });
+    },
+  });
+  // Deprecated alias — old name
   pi.registerCommand("unipi:compact", {
-    description: "Trigger manual compaction with stats",
-    handler: async (_args: string, ctx: any) => {
-      const result = compactTool();
-      const stats = getLastCompactionStats();
-      const msg = stats
-        ? `🗜️ Compaction: ${stats.summarized} summarized, ${stats.kept} kept (~${stats.keptTokensEst} tok)\n${result.message}`
-        : `🗜️ ${result.message}`;
-      ctx.ui.notify(msg, "info");
+    description: "(DEPRECATED) Use /unipi:lossless-compact instead",
+    handler: async (_args: string, ctx: ExtensionCommandContext) => {
+      deprecationLog("/unipi:compact", "/unipi:lossless-compact");
+      ctx.compact({
+        customInstructions: COMPACTOR_INSTRUCTION,
+        onComplete: () => {
+          const stats = getLastCompactionStats();
+          if (stats) {
+            ctx.ui.notify(
+              `Compacted ${stats.totalMessages} messages (~${formatTokens(stats.tokensBefore)} tokens) → ${stats.kept} messages (~${formatTokens(stats.tokensAfterEst)} tokens)`,
+              "info",
+            );
+          } else {
+            ctx.ui.notify("Compaction completed.", "info");
+          }
+        },
+        onError: (err: Error) => {
+          if (err.message === "Compaction cancelled" || err.message === "Already compacted") {
+            ctx.ui.notify("Nothing to compact.", "info");
+          } else {
+            ctx.ui.notify(`Compaction failed: ${err.message}`, "error");
+          }
+        },
+      });
     },
   });
 
   // ── /unipi:session-recall (new) ─────────────────────
-  const sessionRecallHandler = async (args: string, ctx: any) => {
+  const sessionRecallHandler = async (args: string, ctx: ExtensionCommandContext) => {
     const query = args.trim();
     if (!query) {
       ctx.ui.notify("Usage: /unipi:session-recall <query>", "warning");
@@ -81,7 +118,7 @@ export function registerCommands(pi: ExtensionAPI, deps?: CommandDeps): void {
   // Deprecated alias
   pi.registerCommand("unipi:compact-recall", {
     description: "(DEPRECATED) Search session history — use /unipi:session-recall instead",
-    handler: async (args: string, ctx: any) => {
+    handler: async (args: string, ctx: ExtensionCommandContext) => {
       deprecationLog("/unipi:compact-recall", "/unipi:session-recall");
       return sessionRecallHandler(args, ctx);
     },
@@ -90,19 +127,18 @@ export function registerCommands(pi: ExtensionAPI, deps?: CommandDeps): void {
   // ── /unipi:compact-stats ─────────────────────────────
   pi.registerCommand("unipi:compact-stats", {
     description: "Show context savings dashboard",
-    handler: async (_args: string, ctx: any) => {
-      if (!deps?.sessionDB || !deps?.contentStore) {
+    handler: async (_args: string, ctx: ExtensionCommandContext) => {
+      if (!deps?.sessionDB) {
         ctx.ui.notify("Compactor services not initialized.", "error");
         return;
       }
       try {
-        const stats = await ctxStats(deps.sessionDB, deps.contentStore, deps.getSessionId(), deps.getCounters?.());
+        const stats = await ctxStats(deps.sessionDB, deps.getSessionId(), deps.getCounters?.());
         const lines = [
           "📊 Compactor Stats",
           `Session events: ${stats.sessionEvents}`,
           `Compactions: ${stats.compactions}`,
           `Tokens saved: ${stats.tokensSaved}`,
-          `Indexed docs: ${stats.indexedDocs} (${stats.indexedChunks} chunks)`,
           `Sandbox runs: ${stats.sandboxRuns}`,
           `Search queries: ${stats.searchQueries}`,
         ];
@@ -116,13 +152,13 @@ export function registerCommands(pi: ExtensionAPI, deps?: CommandDeps): void {
   // ── /unipi:compact-doctor ────────────────────────────
   pi.registerCommand("unipi:compact-doctor", {
     description: "Run diagnostics checklist",
-    handler: async (_args: string, ctx: any) => {
-      if (!deps?.sessionDB || !deps?.contentStore) {
+    handler: async (_args: string, ctx: ExtensionCommandContext) => {
+      if (!deps?.sessionDB) {
         ctx.ui.notify("Compactor services not initialized.", "error");
         return;
       }
       try {
-        const result = await ctxDoctor(deps.sessionDB, deps.contentStore);
+        const result = await ctxDoctor(deps.sessionDB);
         const icon = (s: string) => (s === "pass" ? "✅" : s === "warn" ? "⚠️" : "❌");
         const lines = [
           result.healthy ? "🩺 All checks passed" : "🩺 Issues found",
@@ -139,7 +175,7 @@ export function registerCommands(pi: ExtensionAPI, deps?: CommandDeps): void {
   // ── /unipi:compact-settings ──────────────────────────
   pi.registerCommand("unipi:compact-settings", {
     description: "Open TUI settings overlay",
-    handler: async (_args: string, ctx: any) => {
+    handler: async (_args: string, ctx: ExtensionCommandContext) => {
       try {
         const cwd = (ctx as any).cwd ?? process.cwd();
         const { renderSettingsOverlay } = await import("../tui/settings-overlay.js");
@@ -158,7 +194,7 @@ export function registerCommands(pi: ExtensionAPI, deps?: CommandDeps): void {
   // ── /unipi:compact-preset ────────────────────────────
   pi.registerCommand("unipi:compact-preset", {
     description: "Apply quick preset (precise/balanced/thorough/lean)",
-    handler: async (args: string, ctx: any) => {
+    handler: async (args: string, ctx: ExtensionCommandContext) => {
       const presetName = parsePreset(args.trim());
       if (!presetName) {
         ctx.ui.notify("Unknown preset. Use: precise, balanced, thorough, lean", "error");
@@ -174,150 +210,24 @@ export function registerCommands(pi: ExtensionAPI, deps?: CommandDeps): void {
     },
   });
 
-  // ── /unipi:content-index (new) / /unipi:compact-index (deprecated) ──
-  const contentIndexHandler = async (_args: string, ctx: any) => {
-    if (!deps?.contentStore) {
-      ctx.ui.notify("Content store not initialized. Enable fts5Index in config.", "warning");
-      return;
-    }
-    try {
-      const cwd = (ctx as any).cwd ?? process.cwd();
-      const { readdirSync, readFileSync, statSync } = await import("node:fs");
-      const { join, relative, extname } = await import("node:path");
-
-      const indexable = [".md", ".txt", ".ts", ".js", ".json", ".py", ".sh"];
-      const files: string[] = [];
-
-      const walk = (dir: string, depth = 0) => {
-        if (depth > 3) return;
-        try {
-          for (const entry of readdirSync(dir, { withFileTypes: true })) {
-            if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
-            const full = join(dir, entry.name);
-            if (entry.isDirectory()) {
-              walk(full, depth + 1);
-            } else if (indexable.includes(extname(entry.name))) {
-              files.push(full);
-            }
-          }
-        } catch {
-          // skip unreadable dirs
-        }
-      };
-
-      walk(cwd);
-      let totalChunks = 0;
-      for (const file of files.slice(0, 100)) {
-        try {
-          const content = readFileSync(file, "utf-8");
-          if (content.length < 50) continue;
-          const ext = extname(file);
-          const ct = ext === ".md" ? "markdown" : ext === ".json" ? "json" : "plain";
-          const result = await deps.contentStore!.index(relative(cwd, file), content, {
-            contentType: ct,
-            source: file,
-          });
-          totalChunks += result.totalChunks;
-        } catch {
-          // skip unreadable files
-        }
-      }
-      ctx.ui.notify(`Indexed ${Math.min(files.length, 100)} files (${totalChunks} chunks).`, "info");
-    } catch (err) {
-      ctx.ui.notify(`Index error: ${err}`, "error");
-    }
-  };
-  pi.registerCommand("unipi:content-index", {
-    description: "Index current project files into FTS5",
-    handler: contentIndexHandler,
-  });
-  pi.registerCommand("unipi:compact-index", {
-    description: "(DEPRECATED) Index project files — use /unipi:content-index instead",
-    handler: async (args: string, ctx: any) => {
-      deprecationLog("/unipi:compact-index", "/unipi:content-index");
-      return contentIndexHandler(args, ctx);
-    },
-  });
-
-  // ── /unipi:content-search (new) / /unipi:compact-search (deprecated) ──
-  const contentSearchHandler = async (args: string, ctx: any) => {
-    const query = args.trim();
-    if (!query) {
-      ctx.ui.notify("Usage: /unipi:content-search <query>", "warning");
-      return;
-    }
-    if (!deps?.contentStore) {
-      ctx.ui.notify("Content store not initialized.", "warning");
-      return;
-    }
-    try {
-      const results = await ctxSearch(deps.contentStore!, { query, limit: 10 });
-      if (results.length === 0) {
-        ctx.ui.notify(`No results for "${query}".`, "info");
-        return;
-      }
-      const lines = results.map(
-        (r, i) => `[${i + 1}] ${r.title} (rank: ${r.rank.toFixed(3)})\n${r.content.slice(0, 200)}`,
-      );
-      ctx.ui.notify(`Found ${results.length} results:\n${lines.join("\n\n")}`, "info");
-    } catch (err) {
-      ctx.ui.notify(`Search error: ${err}`, "error");
-    }
-  };
-  pi.registerCommand("unipi:content-search", {
-    description: "Search indexed content",
-    handler: contentSearchHandler,
-  });
-  pi.registerCommand("unipi:compact-search", {
-    description: "(DEPRECATED) Search indexed content — use /unipi:content-search instead",
-    handler: async (args: string, ctx: any) => {
-      deprecationLog("/unipi:compact-search", "/unipi:content-search");
-      return contentSearchHandler(args, ctx);
-    },
-  });
-
-  // ── /unipi:content-purge (new) / /unipi:compact-purge (deprecated) ──
-  const contentPurgeHandler = async (_args: string, ctx: any) => {
-    if (!deps?.contentStore) {
-      ctx.ui.notify("Content store not initialized.", "warning");
-      return;
-    }
-    try {
-      await deps.contentStore!.purge();
-      ctx.ui.notify("All indexed content purged.", "info");
-    } catch (err) {
-      ctx.ui.notify(`Purge error: ${err}`, "error");
-    }
-  };
-  pi.registerCommand("unipi:content-purge", {
-    description: "Wipe all indexed content from FTS5",
-    handler: contentPurgeHandler,
-  });
-  pi.registerCommand("unipi:compact-purge", {
-    description: "(DEPRECATED) Wipe indexed content — use /unipi:content-purge instead",
-    handler: async (args: string, ctx: any) => {
-      deprecationLog("/unipi:compact-purge", "/unipi:content-purge");
-      return contentPurgeHandler(args, ctx);
-    },
-  });
-
   // ── /unipi:compact-help ──────────────────────────────
   pi.registerCommand("unipi:compact-help", {
     description: "Show detailed compactor documentation (tier-2 skill)",
-    handler: async (_args: string, ctx: any) => {
-      // Load tier-2 skill content — delegates to skill loading system
+    handler: async (_args: string, ctx: ExtensionCommandContext) => {
       ctx.ui.notify(
-        "🗜️ Compactor Help — Use your compactor-detail skill for full documentation.\n" +
+        "🗜️ Compactor Help\n" +
         "Quick commands:\n" +
-        "  /unipi:compact — trigger compaction\n" +
+        "  /unipi:lossless-compact — trigger immediate compaction\n" +
         "  /unipi:session-recall <query> — search session history\n" +
-        "  /unipi:content-index — index project files\n" +
-        "  /unipi:content-search <query> — search indexed content\n" +
-        "  /unipi:content-purge — wipe indexed content\n" +
         "  /unipi:compact-stats — view stats\n" +
         "  /unipi:compact-doctor — run diagnostics\n" +
         "  /unipi:compact-settings — TUI settings\n" +
-        "  /unipi:compact-preset <name> — apply preset",
+        "  /unipi:compact-preset <name> — apply preset\n" +
+        "\n" +
+        "Content indexing has moved to @pi-unipi/cocoindex:\n" +
+        "  /unipi:cocoindex-init — initialize pipeline\n" +
+        "  /unipi:cocoindex-update — index project files\n" +
+        "  cocoindex_search — search indexed content",
         "info",
       );
     },

package/src/compaction/content.ts

@@ -7,8 +7,8 @@ export function textOf(content: unknown): string {
   if (typeof content === "string") return content;
   if (Array.isArray(content)) {
     return content
-      .map((c: any) => {
-        if (c?.type === "text") return c.text ?? "";
+      .map((c: Record<string, unknown>) => {
+        if (c?.type === "text") return (c.text as string) ?? "";
         if (c?.type === "toolCall") return `[toolCall:${c.name}]`;
         if (c?.type === "thinking") return "[thinking]";
         if (c?.type === "image") return `[image:${c.mimeType}]`;

package/src/compaction/cut.ts

@@ -7,28 +7,32 @@ export type OwnCutCancelReason =
   | "too_few_live_messages"
   | "no_user_message";
 
+import type { SessionEntry, SessionMessageEntry, CompactionEntry } from "@mariozechner/pi-coding-agent";
+import type { AgentMessage } from "@mariozechner/pi-agent-core";
+
 export type OwnCutResult =
-  | { ok: true; messages: any[]; firstKeptEntryId: string; compactAll: boolean }
+  | { ok: true; messages: AgentMessage[]; firstKeptEntryId: string; compactAll: boolean }
   | { ok: false; reason: OwnCutCancelReason };
 
 interface EntryWithMessage {
-  entry: { id: string; type: string };
-  message: { role: string; content: unknown };
+  entry: SessionEntry;
+  message: AgentMessage;
 }
 
-export function buildOwnCut(branchEntries: any[]): OwnCutResult {
+export function buildOwnCut(branchEntries: SessionEntry[]): OwnCutResult {
   let lastCompactionIdx = -1;
   let lastKeptId: string | undefined;
   for (let i = branchEntries.length - 1; i >= 0; i--) {
     if (branchEntries[i].type === "compaction") {
       lastCompactionIdx = i;
-      lastKeptId = branchEntries[i].firstKeptEntryId;
+      const ce = branchEntries[i] as CompactionEntry;
+      lastKeptId = ce.firstKeptEntryId;
       break;
     }
   }
 
   const hasPriorCompaction = lastCompactionIdx >= 0;
-  const hasValidKeptId = !!lastKeptId && branchEntries.some((e: any) => e.id === lastKeptId);
+  const hasValidKeptId = !!lastKeptId && branchEntries.some((e) => e.id === lastKeptId);
   const orphanRecovery = hasPriorCompaction && !hasValidKeptId;
 
   const liveMessages: EntryWithMessage[] = [];