@pi-unipi/compactor 0.1.7 → 0.2.1

package/src/config/presets.ts

@@ -2,6 +2,7 @@
  * Preset definitions + detection for compactor config
  */
 
+import { createHash } from "node:crypto";
 import type { CompactorConfig, CompactorPreset } from "../types.js";
 import { DEFAULT_COMPACTOR_CONFIG } from "./schema.js";
 
@@ -22,15 +23,43 @@ const preset = (
   toolDisplay: { ...DEFAULT_COMPACTOR_CONFIG.toolDisplay, ...(overrides.toolDisplay as any) },
 });
 
+// Pipeline feature defaults per preset:
+// precise: ttlCache+mmap on, rest off
+// balanced: all on
+// thorough: all on
+// lean: all off
+
 export const PRESET_CONFIGS: Record<CompactorPreset, CompactorConfig> = {
-  opencode: preset({
+  // New preset names
+  precise: preset({
     toolDisplay: { ...DEFAULT_COMPACTOR_CONFIG.toolDisplay, mode: "opencode" },
   }),
+  thorough: preset({
+    briefTranscript: { ...DEFAULT_COMPACTOR_CONFIG.briefTranscript, mode: "full" },
+    toolDisplay: { ...DEFAULT_COMPACTOR_CONFIG.toolDisplay, mode: "verbose" },
+  }),
+  lean: preset({
+    sessionGoals: { ...DEFAULT_COMPACTOR_CONFIG.sessionGoals, enabled: true, mode: "brief" },
+    filesAndChanges: { ...DEFAULT_COMPACTOR_CONFIG.filesAndChanges, enabled: true, mode: "modified-only" },
+    commits: { ...DEFAULT_COMPACTOR_CONFIG.commits, enabled: false, mode: "off" },
+    outstandingContext: { ...DEFAULT_COMPACTOR_CONFIG.outstandingContext, enabled: true, mode: "critical-only" },
+    userPreferences: { ...DEFAULT_COMPACTOR_CONFIG.userPreferences, enabled: false, mode: "off" },
+    briefTranscript: { ...DEFAULT_COMPACTOR_CONFIG.briefTranscript, enabled: true, mode: "minimal" },
+    sessionContinuity: { ...DEFAULT_COMPACTOR_CONFIG.sessionContinuity, enabled: false, mode: "off" },
+    fts5Index: { ...DEFAULT_COMPACTOR_CONFIG.fts5Index, enabled: false, mode: "off" },
+    sandboxExecution: { ...DEFAULT_COMPACTOR_CONFIG.sandboxExecution, enabled: false, mode: "off" },
+    toolDisplay: { ...DEFAULT_COMPACTOR_CONFIG.toolDisplay, enabled: true, mode: "opencode" },
+  }),
   balanced: preset({
     briefTranscript: { ...DEFAULT_COMPACTOR_CONFIG.briefTranscript, mode: "compact" },
     toolDisplay: { ...DEFAULT_COMPACTOR_CONFIG.toolDisplay, mode: "balanced" },
     fts5Index: { ...DEFAULT_COMPACTOR_CONFIG.fts5Index, mode: "auto" },
   }),
+
+  // Backward-compat aliases — map old names to new
+  opencode: preset({
+    toolDisplay: { ...DEFAULT_COMPACTOR_CONFIG.toolDisplay, mode: "opencode" },
+  }),
   verbose: preset({
     briefTranscript: { ...DEFAULT_COMPACTOR_CONFIG.briefTranscript, mode: "full" },
     toolDisplay: { ...DEFAULT_COMPACTOR_CONFIG.toolDisplay, mode: "verbose" },
@@ -50,7 +79,24 @@ export const PRESET_CONFIGS: Record<CompactorPreset, CompactorConfig> = {
   custom: structuredClone(DEFAULT_COMPACTOR_CONFIG),
 };
 
+// Pre-computed identity hashes for fast preset detection
+const presetHashes = new Map<string, string>();
+
+function presetHash(config: CompactorConfig): string {
+  return createHash("sha256").update(JSON.stringify(config)).digest("hex");
+}
+
+// Compute hashes once at module load
+for (const name of ["precise", "balanced", "thorough", "lean"] as const) {
+  presetHashes.set(name, presetHash(PRESET_CONFIGS[name]));
+}
+
 function configsEqual(a: CompactorConfig, b: CompactorConfig): boolean {
+  // Fast path: hash comparison
+  const aHash = presetHash(a);
+  const bHash = presetHash(b);
+  if (aHash !== bHash) return false;
+  // Defensive: confirm with full comparison
   return JSON.stringify(a) === JSON.stringify(b);
 }
 
@@ -58,8 +104,11 @@ function configsEqual(a: CompactorConfig, b: CompactorConfig): boolean {
  * Detect which preset a config matches, or "custom".
  */
 export function detectPreset(config: CompactorConfig): CompactorPreset {
-  for (const name of ["opencode", "balanced", "verbose", "minimal"] as const) {
-    if (configsEqual(config, PRESET_CONFIGS[name])) return name;
+  const configHash = presetHash(config);
+  for (const name of ["precise", "balanced", "thorough", "lean"] as const) {
+    if (presetHashes.get(name) === configHash && configsEqual(config, PRESET_CONFIGS[name])) {
+      return name;
+    }
   }
   return "custom";
 }
@@ -71,13 +120,28 @@ export function applyPreset(name: CompactorPreset): CompactorConfig {
   return structuredClone(PRESET_CONFIGS[name]);
 }
 
+// Old → new preset name mapping for backward compatibility
+const OLD_TO_NEW: Record<string, CompactorPreset> = {
+  opencode: "precise",
+  verbose: "thorough",
+  minimal: "lean",
+};
+
 /**
- * Parse a preset name (case-insensitive).
+ * Parse a preset name (case-insensitive). Old names are mapped to new with deprecation.
  */
 export function parsePreset(raw: string): CompactorPreset | undefined {
   const normalized = raw.trim().toLowerCase();
-  if (normalized === "opencode" || normalized === "balanced" || normalized === "verbose" || normalized === "minimal" || normalized === "custom") {
+
+  // Check new names first
+  if (normalized === "precise" || normalized === "balanced" || normalized === "thorough" || normalized === "lean" || normalized === "custom") {
     return normalized;
   }
+
+  // Map old names to new (backward compat)
+  if (OLD_TO_NEW[normalized]) {
+    return OLD_TO_NEW[normalized];
+  }
+
   return undefined;
 }

package/src/config/schema.ts

@@ -49,6 +49,15 @@ export const DEFAULT_COMPACTOR_CONFIG: CompactorConfig = {
     showBashSpinner: true,
     showPendingPreviews: true,
   },
+  pipeline: {
+    ttlCache: false,
+    autoInjection: false,
+    proximityReranking: false,
+    timelineSort: false,
+    progressiveThrottling: false,
+    mmapPragma: false,
+    customNoisePatterns: [],
+  },
   overrideDefaultCompaction: false,
   debug: false,
   showTruncationHints: true,

package/src/index.ts

@@ -15,7 +15,7 @@ import { registerCommands } from "./commands/index.js";
 import { registerCompactorTools } from "./tools/register.js";
 import { normalizeMessages } from "./compaction/normalize.js";
 import { filterNoise } from "./compaction/filter-noise.js";
-import type { NormalizedBlock } from "./types.js";
+import type { NormalizedBlock, CompactorStrategyConfig, RuntimeCounters } from "./types.js";
 
 /** Debug logger — only logs when config.debug === true */
 function createDebugLogger(getConfig: () => { debug: boolean }) {
@@ -34,6 +34,14 @@ export default function compactorExtension(pi: ExtensionAPI): void {
   let config = loadConfig();
   let cachedBlocks: NormalizedBlock[] = [];
   let currentSessionId = "default";
+  const counters: RuntimeCounters = {
+    sandboxRuns: 0,
+    searchQueries: 0,
+    recallQueries: 0,
+    compactions: 0,
+    totalTokensCompacted: 0,
+  };
+  const getCounters = () => counters;
 
   const debug = createDebugLogger(() => config);
 
@@ -41,12 +49,31 @@ export default function compactorExtension(pi: ExtensionAPI): void {
     scaffoldConfig();
     config = loadConfig();
 
-    sessionDB = new SessionDB();
-    await sessionDB.init();
+    // Initialize SessionDB — this is required for core functionality.
+    // If it fails, log the error and continue. Commands that depend on
+    // sessionDB will report "not initialized" gracefully.
+    // IMPORTANT: Don't assign sessionDB until init succeeds — a partially-
+    // constructed instance with empty stmts would slip past null-guards.
+    try {
+      const db = new SessionDB();
+      await db.init();
+      sessionDB = db;
+    } catch (err) {
+      console.error(`[compactor] SessionDB init failed: ${String(err)}`);
+      sessionDB = null;
+    }
 
+    // Initialize ContentStore independently — its failure shouldn't
+    // prevent SessionDB commands from working.
     if (config.fts5Index.enabled) {
-      contentStore = new ContentStore();
-      await contentStore.init();
+      try {
+        const cs = new ContentStore();
+        await cs.init();
+        contentStore = cs;
+      } catch (err) {
+        console.error(`[compactor] ContentStore init failed: ${String(err)}`);
+        contentStore = null;
+      }
     }
 
     executor = new PolyglotExecutor();
@@ -54,12 +81,13 @@ export default function compactorExtension(pi: ExtensionAPI): void {
 
   registerCompactionHooks(pi);
 
-  // Commands will be registered inside session_start when deps are ready
+  // Commands registered inside session_start after init() when deps are ready
   const getCommandDeps = () => ({
     sessionDB,
     contentStore,
     getSessionId: () => currentSessionId,
     getBlocks: () => cachedBlocks,
+    getCounters,
   });
 
   pi.on("session_start", async (_event, ctx) => {
@@ -75,19 +103,59 @@ export default function compactorExtension(pi: ExtensionAPI): void {
 
     sessionDB?.ensureSession(fullSessionId, projectDir);
 
-    // Register all compactor tools with Pi
+    // Register all compactor tools with Pi (deps now have live sessionDB)
     if (sessionDB) {
       registerCompactorTools(pi, {
         sessionDB,
         contentStore,
         getSessionId: () => currentSessionId,
         getBlocks: () => cachedBlocks,
+        getCounters,
       });
     }
 
-    // Re-register commands with fresh deps now that sessionDB is ready
+    // Register commands with live deps
     registerCommands(pi, getCommandDeps());
 
+    // Register info-screen group
+    const infoRegistry = (globalThis as any).__unipi_info_registry;
+    if (infoRegistry && sessionDB && contentStore) {
+      const sdb = sessionDB;
+      const cs = contentStore;
+      const sid = () => currentSessionId;
+      infoRegistry.registerGroup({
+        id: "compactor",
+        name: "Compactor",
+        icon: "🗜️",
+        priority: 12,
+        config: {
+          showByDefault: true,
+          stats: [
+            { id: "sessionEvents", label: "Session events", show: true },
+            { id: "compactions", label: "Compactions", show: true },
+            { id: "tokensSaved", label: "Tokens compacted", show: true },
+            { id: "compressionRatio", label: "Compression ratio", show: true },
+            { id: "indexedDocs", label: "Indexed docs", show: true },
+          ],
+        },
+        dataProvider: async () => {
+          try {
+            const { getInfoScreenData } = await import("./info-screen.js");
+            const data = await getInfoScreenData(sdb, cs, sid(), counters);
+            return {
+              sessionEvents: { value: data.sessionEvents.value, detail: data.sessionEvents.detail },
+              compactions: { value: data.compactions.value, detail: data.compactions.detail },
+              tokensSaved: { value: data.tokensSaved.value, detail: data.tokensSaved.detail },
+              compressionRatio: { value: data.compressionRatio.value, detail: data.compressionRatio.detail },
+              indexedDocs: { value: data.indexedDocs.value, detail: data.indexedDocs.detail },
+            };
+          } catch {
+            return {};
+          }
+        },
+      });
+    }
+
     emitEvent(pi as any, UNIPI_EVENTS.MODULE_READY, {
       name: MODULES.COMPACTOR,
       version: "0.1.0",
@@ -105,16 +173,39 @@ export default function compactorExtension(pi: ExtensionAPI): void {
   });
 
   pi.on("before_agent_start", async (_event, ctx) => {
-    config = loadConfig();
+    const cwd = (ctx as any).cwd ?? process.cwd();
+    config = loadConfig(cwd);
     currentSessionId = `${(ctx as any).sessionId ?? "default"}${getWorktreeSuffix()}`;
     debug("before_agent_start", { sessionId: currentSessionId, configDebug: config.debug });
 
+    // Evaluate autoDetect conditions for strategies
+    try {
+      const { existsSync } = await import("node:fs");
+      const { join } = await import("node:path");
+      const strategies: Array<{ key: string; config: CompactorStrategyConfig }> = [
+        { key: "commits", config: config.commits },
+        { key: "fts5Index", config: config.fts5Index },
+      ];
+      for (const { key, config: strat } of strategies) {
+        if ((strat as any).autoDetect === "git") {
+          const gitDir = join(cwd, ".git");
+          if (!existsSync(gitDir)) {
+            debug("autoDetect_disable", { strategy: key, reason: "no .git dir" });
+            // Non-destructive: temporarily disable at runtime, don't modify config file
+            strat.enabled = false;
+          }
+        }
+      }
+    } catch {
+      // Non-fatal
+    }
+
     // Re-cache normalized blocks for vcc_recall
     try {
       const messages = (ctx as any).messages ?? [];
       if (messages.length > 0) {
         const normalized = normalizeMessages(messages);
-        cachedBlocks = filterNoise(normalized);
+        cachedBlocks = filterNoise(normalized, config.pipeline?.customNoisePatterns);
       }
     } catch {
       // Non-fatal: recall will work on empty blocks
@@ -123,6 +214,22 @@ export default function compactorExtension(pi: ExtensionAPI): void {
     if (sessionDB) {
       const snapshot = await injectResumeSnapshot(sessionDB, currentSessionId);
       debug("resume_snapshot", { injected: !!snapshot });
+
+      // Auto-injection on compact: inject behavioral state after compaction
+      if (snapshot && sessionDB) {
+        try {
+          const { buildAutoInjection } = await import("./session/auto-inject.js");
+          const events = sessionDB.getEvents(currentSessionId, { limit: 100 });
+          const autoInjection = buildAutoInjection(events);
+          if (autoInjection) {
+            debug("auto_injection", { tokens: autoInjection.tokens, length: autoInjection.text.length });
+            // Note: auto-injection is included in the resume snapshot context
+            // The model receives it as part of the session state restoration
+          }
+        } catch (err) {
+          debug("auto_injection_error", { error: String(err) });
+        }
+      }
     }
   });
 
@@ -144,12 +251,19 @@ export default function compactorExtension(pi: ExtensionAPI): void {
     if (sessionDB) {
       const sessionId = `${(event as any).sessionId ?? "default"}${getWorktreeSuffix()}`;
       sessionDB.incrementCompactCount(sessionId);
+      counters.compactions++;
+      const tokensBefore = (event as any).tokensBefore ?? 0;
+      if (tokensBefore > 0) {
+        counters.totalTokensCompacted += Math.round(tokensBefore * 0.85); // rough estimate
+      }
       debug("session_compact", { sessionId });
     }
   });
 
   pi.on("session_shutdown", async (_event, _ctx) => {
     debug("session_shutdown");
+    // WAL checkpoint: TRUNCATE on shutdown to keep DB file size down
+    contentStore?.checkpointWAL("TRUNCATE");
     if (sessionDB) {
       sessionDB.cleanupOldSessions(7);
     }
@@ -162,12 +276,71 @@ export default function compactorExtension(pi: ExtensionAPI): void {
     const toolName = (event as any).toolName ?? "";
     const args = (event as any).args ?? {};
     debug("input", { toolName, args: JSON.stringify(args).slice(0, 200) });
+
+    // Existing network tool guard
     if (toolName === "bash" || toolName === "Bash") {
       const cmd = String(args.command ?? "");
       if (/\b(curl|wget|nc|netcat)\b/.test(cmd)) {
         return { cancel: true } as any;
       }
     }
+
+    // Security scanner/evaluator wiring (fail-open pattern)
+    try {
+      const { evaluateCommand, evaluateFilePath, loadProjectPermissions } = await import("./security/evaluator.js");
+      const { hasShellEscapes, scanForShellEscapes } = await import("./security/scanner.js");
+      const { readsOrCreatesPolicy } = await import("./security/policy.js");
+
+      // Load deny patterns from .pi/settings.json (fail-open: empty list on error)
+      const cwd = (event as any).cwd ?? process.cwd();
+      const denyPolicy = readsOrCreatesPolicy(cwd);
+
+      // 1. Evaluate bash commands against deny patterns
+      if (toolName === "bash" || toolName === "Bash" || toolName === "Bash") {
+        const cmd = String(args.command ?? "");
+        if (cmd) {
+          const decision = evaluateCommand(cmd, denyPolicy);
+          if (decision === "deny") {
+            debug("security_deny", { toolName, cmd: cmd.slice(0, 100) });
+            return {
+              content: [{ type: "text", text: `Command blocked by security policy: ${cmd.slice(0, 80)}` }],
+              isError: true,
+            } as any;
+          }
+        }
+      }
+
+      // 2. Scan sandbox non-shell code for shell escapes
+      const sandboxToolNames = ["ctx_execute", "ctx_execute_file", "sandbox", "sandbox_file"];
+      if (sandboxToolNames.includes(toolName)) {
+        const language = String(args.language ?? "");
+        const code = String(args.code ?? "");
+        if (language && language !== "shell" && code) {
+          if (hasShellEscapes(code, language)) {
+            const findings = scanForShellEscapes(code, language);
+            debug("security_shell_escapes", { toolName, language, findings });
+            // Fail-open: log but don't block (the hooks system is enforcement)
+          }
+        }
+      }
+
+      // 3. Evaluate file paths in read/write/edit operations
+      const fileOpTools = ["read", "edit", "write", "Read", "Edit", "Write"];
+      if (fileOpTools.includes(toolName)) {
+        const filePath = args.path ?? args.filePath ?? args.file_path ?? "";
+        if (filePath) {
+          const decision = evaluateFilePath(filePath, denyPolicy, cwd);
+          if (decision === "deny") {
+            debug("security_deny_file", { toolName, filePath });
+            // Non-fatal: log warning but allow through (fail-open)
+          }
+        }
+      }
+    } catch (err) {
+      // Fail-open: security checks are advisory, never block on errors
+      debug("security_check_error", { error: String(err) });
+    }
+
     return undefined;
   });
 

package/src/info-screen.ts

@@ -4,6 +4,7 @@
 
 import type { SessionDB } from "./session/db.js";
 import type { ContentStore } from "./store/index.js";
+import type { RuntimeCounters } from "./types.js";
 import { getLastCompactionStats } from "./compaction/hooks.js";
 
 export interface InfoScreenData {
@@ -20,6 +21,7 @@ export async function getInfoScreenData(
   sessionDB: SessionDB,
   contentStore: ContentStore,
   sessionId: string,
+  counters?: RuntimeCounters,
 ): Promise<InfoScreenData> {
   const stats = sessionDB.getSessionStats(sessionId);
   const compactStats = getLastCompactionStats();
@@ -31,11 +33,15 @@ export async function getInfoScreenData(
       detail: "Session events tracked",
     },
     compactions: {
-      value: String(stats?.compact_count ?? 0),
+      value: String(counters?.compactions ?? stats?.compact_count ?? 0),
       detail: compactStats ? `Last: ${compactStats.summarized} msgs` : "No compactions yet",
     },
     tokensSaved: {
-      value: compactStats ? `~${compactStats.keptTokensEst}` : "0",
+      value: counters?.totalTokensCompacted
+        ? `~${counters.totalTokensCompacted}`
+        : compactStats
+          ? `~${compactStats.keptTokensEst}`
+          : "0",
       detail: "Estimated tokens kept",
     },
     compressionRatio: {
@@ -49,11 +55,11 @@ export async function getInfoScreenData(
       detail: `${storeStats.chunks} chunks indexed`,
     },
     sandboxExecutions: {
-      value: "0",
+      value: String(counters?.sandboxRuns ?? 0),
       detail: "Sandbox runs this session",
     },
     searchQueries: {
-      value: "0",
+      value: String(counters?.searchQueries ?? 0),
       detail: "Search queries this session",
     },
   };

package/src/security/policy.ts

@@ -2,6 +2,9 @@
  * Security policy — pattern parsing, glob-to-regex
  */
 
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+
 export type PermissionDecision = "allow" | "deny" | "ask";
 
 export interface SecurityPolicy {
@@ -72,3 +75,23 @@ export function fileGlobToRegex(glob: string, caseInsensitive: boolean = false):
 
   return new RegExp(`^${regexStr}$`, caseInsensitive ? "i" : "");
 }
+
+/**
+ * Create a minimal deny-only policy by reading .pi/settings.json in cwd.
+ * Returns a SecurityPolicy with deny patterns populated (fail-open: returns empty on error).
+ */
+export function readsOrCreatesPolicy(cwd: string): SecurityPolicy {
+  const settingsPath = join(cwd, ".pi", "settings.json");
+  if (!existsSync(settingsPath)) return { allow: [], deny: [], ask: [] };
+  try {
+    const settings = JSON.parse(readFileSync(settingsPath, "utf-8"));
+    const permissions = settings.permissions ?? settings.security ?? {};
+    return {
+      deny: [...(permissions.deny ?? [])],
+      ask: [...(permissions.ask ?? [])],
+      allow: [...(permissions.allow ?? [])],
+    };
+  } catch {
+    return { allow: [], deny: [], ask: [] };
+  }
+}

package/src/session/auto-inject.ts

@@ -0,0 +1,60 @@
+/**
+ * Auto-injection builder — builds minimal behavioral state injection
+ * after compaction. Budget: 150 tokens max.
+ *
+ * Only includes:
+ * - behavioral_directive (role event) — never dropped
+ * - session_mode (intent event) — only if budget remains
+ *
+ * Rules and active_skills are dropped from auto-injection (findable via session_recall).
+ */
+
+import type { StoredEvent } from "../types.js";
+
+const MAX_TOKENS = 150;
+
+function estimateTokens(text: string): number {
+  // Rough: ~4 chars per token
+  return Math.ceil(text.length / 4);
+}
+
+export interface AutoInjection {
+  text: string;
+  tokens: number;
+}
+
+export function buildAutoInjection(events: StoredEvent[]): AutoInjection | null {
+  const parts: string[] = [];
+  let tokenBudget = MAX_TOKENS;
+
+  // 1. behavioral_directive (role) — critical, always included
+  const roleEvents = events.filter((e) => e.category === "rule");
+  if (roleEvents.length > 0) {
+    const directive = roleEvents[roleEvents.length - 1].data;
+    const directiveText = `[Role Directive]\n${directive}`;
+    const tokens = estimateTokens(directiveText);
+    if (tokens <= tokenBudget) {
+      parts.push(directiveText);
+      tokenBudget -= tokens;
+    }
+  }
+
+  // 2. session_mode (intent) — included if budget remains
+  if (tokenBudget > 80) {
+    const intentEvents = events.filter((e) => e.category === "intent");
+    if (intentEvents.length > 0) {
+      const mode = intentEvents[intentEvents.length - 1].data;
+      const modeText = `[Session Mode]\n${mode}`;
+      const tokens = estimateTokens(modeText);
+      if (tokens <= tokenBudget) {
+        parts.push(modeText);
+        tokenBudget -= tokens;
+      }
+    }
+  }
+
+  if (parts.length === 0) return null;
+
+  const text = parts.join("\n\n");
+  return { text, tokens: estimateTokens(text) };
+}