@pi-unipi/compactor 0.1.1

package/src/executor/executor.ts

@@ -0,0 +1,242 @@
+/**
+ * PolyglotExecutor — sandboxed code execution for 11 languages
+ */
+
+import { spawn, execSync, execFileSync } from "node:child_process";
+import { mkdtempSync, writeFileSync, rmSync, existsSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { tmpdir } from "node:os";
+import { detectRuntimes, buildCommand, type RuntimeMap, type Language } from "./runtime.js";
+import type { ExecResult } from "../types.js";
+
+const isWin = process.platform === "win32";
+
+const OS_TMPDIR = (() => {
+  if (isWin) return process.env.TEMP ?? process.env.TMP ?? tmpdir();
+  try {
+    const result = execFileSync(
+      process.platform === "darwin" ? "getconf" : "mktemp",
+      process.platform === "darwin" ? ["DARWIN_USER_TEMP_DIR"] : ["-u", "-d"],
+      { env: { ...process.env, TMPDIR: undefined as unknown as string }, encoding: "utf-8" },
+    ).trim();
+    const dir = process.platform === "darwin" ? result : resolve(result, "..");
+    if (dir && dir !== process.cwd()) return dir;
+  } catch { /* fall through */ }
+  return "/tmp";
+})();
+
+function killTree(proc: ReturnType<typeof spawn>): void {
+  if (isWin && proc.pid) {
+    try {
+      execSync(`taskkill /F /T /PID ${proc.pid}`, { stdio: "pipe" });
+    } catch { /* already dead */ }
+  } else if (proc.pid) {
+    try {
+      process.kill(-proc.pid, "SIGKILL");
+    } catch { /* already dead */ }
+  }
+}
+
+const DANGEROUS_ENV_VARS = [
+  "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN",
+  "AZURE_CLIENT_ID", "AZURE_CLIENT_SECRET", "AZURE_TENANT_ID",
+  "GCP_SERVICE_ACCOUNT_KEY", "GOOGLE_APPLICATION_CREDENTIALS",
+  "GITHUB_TOKEN", "GH_TOKEN", "GITLAB_TOKEN",
+  "DOCKER_AUTH_CONFIG", "KUBECONFIG",
+  "NPM_TOKEN", "NODE_AUTH_TOKEN", "YARN_AUTH_TOKEN",
+  "PYPI_TOKEN", "POETRY_PYPI_TOKEN_PYPI",
+  "SSH_PRIVATE_KEY", "SSH_AUTH_SOCK",
+  "DATABASE_URL", "REDIS_URL", "MONGO_URL",
+  "SECRET_KEY", "JWT_SECRET", "API_KEY", "API_SECRET",
+  "PASSWORD", "PASSPHRASE", "TOKEN", "CREDENTIALS",
+];
+
+function sanitizeEnv(): NodeJS.ProcessEnv {
+  const env = { ...process.env };
+  for (const key of DANGEROUS_ENV_VARS) {
+    delete env[key];
+  }
+  // Also strip anything matching *SECRET*, *PASSWORD*, *TOKEN*, *KEY*
+  for (const key of Object.keys(env)) {
+    const upper = key.toUpperCase();
+    if (upper.includes("SECRET") || upper.includes("PASSWORD") || upper.includes("TOKEN") || upper.includes("PRIVATE_KEY")) {
+      delete env[key];
+    }
+  }
+  return env;
+}
+
+interface ExecuteOptions {
+  language: Language;
+  code: string;
+  timeout?: number;
+  background?: boolean;
+}
+
+interface ExecuteFileOptions extends ExecuteOptions {
+  path: string;
+}
+
+export class PolyglotExecutor {
+  #hardCapBytes: number;
+  #projectRoot: string;
+  #runtimes: RuntimeMap;
+  #backgroundedPids = new Set<number>();
+
+  constructor(opts?: { hardCapBytes?: number; projectRoot?: string; runtimes?: RuntimeMap }) {
+    this.#hardCapBytes = opts?.hardCapBytes ?? 100 * 1024 * 1024;
+    this.#projectRoot = opts?.projectRoot ?? process.cwd();
+    this.#runtimes = opts?.runtimes ?? detectRuntimes();
+  }
+
+  get runtimes(): RuntimeMap {
+    return { ...this.#runtimes };
+  }
+
+  cleanupBackgrounded(): void {
+    for (const pid of this.#backgroundedPids) {
+      try {
+        process.kill(isWin ? pid : -pid, "SIGTERM");
+      } catch { /* already dead */ }
+    }
+    this.#backgroundedPids.clear();
+  }
+
+  async execute(opts: ExecuteOptions): Promise<ExecResult> {
+    const { language, code, timeout = 30_000, background = false } = opts;
+    const tmpDir = mkdtempSync(join(OS_TMPDIR, ".compactor-"));
+
+    try {
+      const filePath = this.#writeScript(tmpDir, code, language);
+      const cmd = buildCommand(this.#runtimes, language, filePath);
+
+      if (cmd[0] === "__rust_compile_run__") {
+        return await this.#compileAndRun(filePath, tmpDir, timeout);
+      }
+
+      const cwd = language === "shell" ? this.#projectRoot : tmpDir;
+      const result = await this.#spawn(cmd, cwd, tmpDir, timeout, background);
+
+      if (!result.backgrounded) {
+        try { rmSync(tmpDir, { recursive: true, force: true }); } catch { /* ignore */ }
+      }
+
+      return result;
+    } catch (err: any) {
+      try { rmSync(tmpDir, { recursive: true, force: true }); } catch { /* ignore */ }
+      return {
+        stdout: "",
+        stderr: err?.message ?? String(err),
+        exitCode: 1,
+        timedOut: false,
+      };
+    }
+  }
+
+  async executeFile(opts: ExecuteFileOptions): Promise<ExecResult> {
+    const { language, path, timeout = 30_000, background = false } = opts;
+    const cmd = buildCommand(this.#runtimes, language, path);
+    const cwd = language === "shell" ? this.#projectRoot : resolve(path, "..");
+    return this.#spawn(cmd, cwd, "", timeout, background);
+  }
+
+  #writeScript(tmpDir: string, code: string, language: Language): string {
+    const extensions: Record<Language, string> = {
+      javascript: "js", typescript: "ts", python: "py", shell: "sh",
+      ruby: "rb", go: "go", rust: "rs", php: "php", perl: "pl", r: "r", elixir: "ex",
+    };
+    const ext = extensions[language];
+    const filePath = join(tmpDir, `script.${ext}`);
+    writeFileSync(filePath, code, "utf-8");
+    return filePath;
+  }
+
+  async #spawn(
+    cmd: string[],
+    cwd: string,
+    tmpDir: string,
+    timeout: number,
+    background: boolean,
+  ): Promise<ExecResult> {
+    return new Promise((resolve) => {
+      const proc = spawn(cmd[0], cmd.slice(1), {
+        cwd,
+        env: sanitizeEnv(),
+        stdio: ["ignore", "pipe", "pipe"],
+        detached: !isWin,
+      });
+
+      let stdout = "";
+      let stderr = "";
+      let killed = false;
+      let stdoutLen = 0;
+      let stderrLen = 0;
+
+      const killIfOversized = () => {
+        if (stdoutLen + stderrLen > this.#hardCapBytes && !killed) {
+          killed = true;
+          killTree(proc);
+        }
+      };
+
+      proc.stdout?.on("data", (chunk: Buffer) => {
+        stdout += chunk.toString("utf-8");
+        stdoutLen += chunk.length;
+        killIfOversized();
+      });
+
+      proc.stderr?.on("data", (chunk: Buffer) => {
+        stderr += chunk.toString("utf-8");
+        stderrLen += chunk.length;
+        killIfOversized();
+      });
+
+      const timer = background ? null : setTimeout(() => {
+        if (!killed) {
+          killed = true;
+          killTree(proc);
+        }
+      }, timeout);
+
+      proc.on("error", (err) => {
+        if (timer) clearTimeout(timer);
+        resolve({ stdout, stderr: err.message, exitCode: 1, timedOut: false });
+      });
+
+      proc.on("close", (code) => {
+        if (timer) clearTimeout(timer);
+        if (background && proc.pid) {
+          this.#backgroundedPids.add(proc.pid);
+        }
+        resolve({
+          stdout: stdout.slice(0, this.#hardCapBytes),
+          stderr: stderr.slice(0, this.#hardCapBytes),
+          exitCode: code ?? 0,
+          timedOut: killed && !background,
+          backgrounded: background && !!proc.pid,
+        });
+      });
+    });
+  }
+
+  async #compileAndRun(filePath: string, tmpDir: string, timeout: number): Promise<ExecResult> {
+    const binPath = join(tmpDir, "script");
+    try {
+      execSync(`rustc "${filePath}" -o "${binPath}"`, {
+        cwd: tmpDir,
+        env: sanitizeEnv(),
+        timeout: timeout / 2,
+        stdio: ["ignore", "pipe", "pipe"],
+      });
+    } catch (err: any) {
+      return {
+        stdout: "",
+        stderr: err?.stderr?.toString?.() ?? err?.message ?? "Rust compilation failed",
+        exitCode: 1,
+        timedOut: false,
+      };
+    }
+
+    return this.#spawn([binPath], tmpDir, tmpDir, timeout / 2, false);
+  }
+}

package/src/executor/runtime.ts

@@ -0,0 +1,125 @@
+/**
+ * Runtime detection for sandbox executor
+ */
+
+export type Language =
+  | "javascript"
+  | "typescript"
+  | "python"
+  | "shell"
+  | "ruby"
+  | "go"
+  | "rust"
+  | "php"
+  | "perl"
+  | "r"
+  | "elixir";
+
+export interface RuntimeMap {
+  javascript: string;
+  typescript: string;
+  python: string;
+  shell: string;
+  ruby?: string;
+  go?: string;
+  rust?: string;
+  php?: string;
+  perl?: string;
+  r?: string;
+  elixir?: string;
+}
+
+export function detectRuntimes(): RuntimeMap {
+  const runtimes: Partial<RuntimeMap> = {};
+
+  // JavaScript / TypeScript — prefer Bun, fall back to Node
+  try {
+    const { execSync } = require("node:child_process");
+    try {
+      execSync("command -v bun", { stdio: "ignore" });
+      runtimes.javascript = "bun";
+      runtimes.typescript = "bun";
+    } catch {
+      runtimes.javascript = "node";
+      runtimes.typescript = "npx tsx";
+    }
+  } catch {
+    runtimes.javascript = "node";
+    runtimes.typescript = "npx tsx";
+  }
+
+  // Shell
+  runtimes.shell = "bash";
+
+  // Python
+  try {
+    const { execSync } = require("node:child_process");
+    try {
+      execSync("command -v python3", { stdio: "ignore" });
+      runtimes.python = "python3";
+    } catch {
+      runtimes.python = "python";
+    }
+  } catch {
+    runtimes.python = "python3";
+  }
+
+  // Optional runtimes
+  const optional = [
+    ["ruby", "ruby"],
+    ["go", "go"],
+    ["rustc", "rust"],
+    ["php", "php"],
+    ["perl", "perl"],
+    ["Rscript", "r"],
+    ["elixir", "elixir"],
+  ] as const;
+
+  for (const [cmd, lang] of optional) {
+    try {
+      const { execSync } = require("node:child_process");
+      execSync(`command -v ${cmd}`, { stdio: "ignore" });
+      (runtimes as any)[lang] = cmd;
+    } catch {
+      // not available
+    }
+  }
+
+  return runtimes as RuntimeMap;
+}
+
+export function buildCommand(
+  runtimes: RuntimeMap,
+  language: Language,
+  filePath: string,
+): string[] {
+  const runtime = runtimes[language];
+  if (!runtime) throw new Error(`No runtime available for ${language}`);
+
+  switch (language) {
+    case "javascript":
+      return runtime === "bun" ? ["bun", "run", filePath] : ["node", filePath];
+    case "typescript":
+      return runtime === "bun" ? ["bun", "run", filePath] : ["npx", "tsx", filePath];
+    case "python":
+      return [runtime, filePath];
+    case "shell":
+      return ["bash", filePath];
+    case "ruby":
+      return [runtime, filePath];
+    case "go":
+      return ["go", "run", filePath];
+    case "rust":
+      return ["__rust_compile_run__", filePath];
+    case "php":
+      return [runtime, filePath];
+    case "perl":
+      return [runtime, filePath];
+    case "r":
+      return [runtime, filePath];
+    case "elixir":
+      return [runtime, filePath];
+    default:
+      throw new Error(`Unsupported language: ${language}`);
+  }
+}

package/src/index.ts

@@ -0,0 +1,211 @@
+/**
+ * @pi-unipi/compactor — Extension entry point
+ */
+
+import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+import { MODULES, UNIPI_EVENTS, COMPACTOR_COMMANDS, COMPACTOR_TOOLS, emitEvent } from "@pi-unipi/core";
+import { scaffoldConfig, loadConfig } from "./config/manager.js";
+import { registerCompactionHooks } from "./compaction/hooks.js";
+import { SessionDB, getWorktreeSuffix } from "./session/db.js";
+import { extractEventsFromToolResult } from "./session/extract.js";
+import { injectResumeSnapshot } from "./session/resume-inject.js";
+import { ContentStore } from "./store/index.js";
+import { PolyglotExecutor } from "./executor/executor.js";
+import { registerCommands } from "./commands/index.js";
+import { registerCompactorTools } from "./tools/register.js";
+import { normalizeMessages } from "./compaction/normalize.js";
+import { filterNoise } from "./compaction/filter-noise.js";
+import type { NormalizedBlock } from "./types.js";
+
+export default function compactorExtension(pi: ExtensionAPI): void {
+  let sessionDB: SessionDB | null = null;
+  let contentStore: ContentStore | null = null;
+  let executor: PolyglotExecutor | null = null;
+  let config = loadConfig();
+  let cachedBlocks: NormalizedBlock[] = [];
+  let currentSessionId = "default";
+
+  const init = async () => {
+    scaffoldConfig();
+    config = loadConfig();
+
+    sessionDB = new SessionDB();
+    await sessionDB.init();
+
+    if (config.fts5Index.enabled) {
+      contentStore = new ContentStore();
+      await contentStore.init();
+    }
+
+    executor = new PolyglotExecutor();
+  };
+
+  registerCompactionHooks(pi);
+
+  // Register commands with deps (they need sessionDB/contentStore)
+  const getCommandDeps = () => ({
+    sessionDB,
+    contentStore,
+    getSessionId: () => currentSessionId,
+    getBlocks: () => cachedBlocks,
+  });
+  registerCommands(pi, getCommandDeps());
+
+  pi.on("session_start", async (_event, ctx) => {
+    await init();
+
+    const sessionId = (ctx as any).sessionId ?? "default";
+    const projectDir = (ctx as any).cwd ?? process.cwd();
+    const suffix = getWorktreeSuffix();
+    const fullSessionId = `${sessionId}${suffix}`;
+    currentSessionId = fullSessionId;
+
+    sessionDB?.ensureSession(fullSessionId, projectDir);
+
+    // Register all compactor tools with Pi
+    if (sessionDB) {
+      registerCompactorTools(pi, {
+        sessionDB,
+        contentStore,
+        getSessionId: () => currentSessionId,
+        getBlocks: () => cachedBlocks,
+      });
+    }
+
+    // Re-register commands with fresh deps now that sessionDB is ready
+    registerCommands(pi, getCommandDeps());
+
+    emitEvent(pi as any, UNIPI_EVENTS.MODULE_READY, {
+      name: MODULES.COMPACTOR,
+      version: "0.1.0",
+      commands: Object.values(COMPACTOR_COMMANDS),
+      tools: Object.values(COMPACTOR_TOOLS),
+    });
+
+    if (config.fts5Index.mode === "auto" && contentStore) {
+      // TODO: index project files
+    }
+
+    ctx.ui.notify("🗜️  Compactor ready", "info");
+  });
+
+  pi.on("before_agent_start", async (_event, ctx) => {
+    config = loadConfig();
+    currentSessionId = `${(ctx as any).sessionId ?? "default"}${getWorktreeSuffix()}`;
+
+    // Re-cache normalized blocks for vcc_recall
+    try {
+      const messages = (ctx as any).messages ?? [];
+      if (messages.length > 0) {
+        const normalized = normalizeMessages(messages);
+        cachedBlocks = filterNoise(normalized);
+      }
+    } catch {
+      // Non-fatal: recall will work on empty blocks
+    }
+
+    if (sessionDB) {
+      const snapshot = await injectResumeSnapshot(sessionDB, currentSessionId);
+      if (snapshot) {
+        // Snapshot injected as context
+      }
+    }
+  });
+
+  pi.on("session_before_compact", async (event, _ctx) => {
+    if (sessionDB) {
+      const sessionId = `${(event as any).sessionId ?? "default"}${getWorktreeSuffix()}`;
+      const events = sessionDB.getEvents(sessionId, { limit: 1000 });
+      const stats = sessionDB.getSessionStats(sessionId);
+      const { buildResumeSnapshot } = await import("./session/snapshot.js");
+      const snapshot = buildResumeSnapshot(events, {
+        compactCount: stats?.compact_count ?? 1,
+      });
+      sessionDB.upsertResume(sessionId, snapshot, events.length);
+    }
+  });
+
+  pi.on("session_compact", async (event, _ctx) => {
+    if (sessionDB) {
+      const sessionId = `${(event as any).sessionId ?? "default"}${getWorktreeSuffix()}`;
+      sessionDB.incrementCompactCount(sessionId);
+    }
+  });
+
+  pi.on("session_shutdown", async (_event, _ctx) => {
+    if (sessionDB) {
+      sessionDB.cleanupOldSessions(7);
+    }
+    executor?.cleanupBackgrounded();
+    contentStore?.close();
+    sessionDB?.close();
+  });
+
+  pi.on("input", async (event, _ctx) => {
+    const toolName = (event as any).toolName ?? "";
+    const args = (event as any).args ?? {};
+    if (toolName === "bash" || toolName === "Bash") {
+      const cmd = String(args.command ?? "");
+      if (/\b(curl|wget|nc|netcat)\b/.test(cmd)) {
+        return { cancel: true } as any;
+      }
+    }
+    return undefined;
+  });
+
+  pi.on("tool_result", async (event, _ctx) => {
+    if (!sessionDB) return;
+    const sessionId = `${(event as any).sessionId ?? "default"}${getWorktreeSuffix()}`;
+
+    // Extract and store session events
+    const toolEvents = extractEventsFromToolResult({
+      toolName: (event as any).toolName ?? "",
+      toolInput: (event as any).input ?? {},
+      toolResponse: (event as any).content ? JSON.stringify((event as any).content).slice(0, 1000) : undefined,
+      isError: (event as any).isError ?? false,
+    });
+
+    for (const ev of toolEvents) {
+      sessionDB.insertEvent(sessionId, ev, "PostToolUse");
+    }
+
+    // Apply display overrides for built-in tools
+    const toolName = (event as any).toolName ?? "";
+    const td = config.toolDisplay;
+    const toolConfig = {
+      readOutputMode: td?.mode as any,
+      searchOutputMode: td?.mode as any,
+      bashOutputMode: td?.mode as any,
+      previewLines: 20,
+      bashCollapsedLines: 5,
+      showTruncationHints: true,
+    };
+    try {
+      const { applyToolDisplayOverride } = await import("./display/tool-overrides.js");
+      const override = applyToolDisplayOverride(toolName, event as any, toolConfig);
+      if (override !== undefined) {
+        return override as any;
+      }
+    } catch {
+      // Non-fatal: display override failed
+    }
+  });
+
+  pi.on("message_update", async (event, _ctx) => {
+    if ((event as any).message?.thinking) {
+      // Handled by display engine
+    }
+  });
+
+  pi.on("message_end", async (_event, _ctx) => {
+    // Thinking label persistence
+  });
+
+  pi.on("context", async (event, _ctx) => {
+    const { sanitizeThinkingArtifacts } = await import("./display/thinking-label.js");
+    const ctx = (event as any).context;
+    if (typeof ctx === "string") {
+      (event as any).context = sanitizeThinkingArtifacts(ctx);
+    }
+  });
+}

package/src/info-screen.ts

@@ -0,0 +1,60 @@
+/**
+ * Info-screen integration for @pi-unipi/compactor
+ */
+
+import type { SessionDB } from "./session/db.js";
+import type { ContentStore } from "./store/index.js";
+import { getLastCompactionStats } from "./compaction/hooks.js";
+
+export interface InfoScreenData {
+  sessionEvents: { value: string; detail: string };
+  compactions: { value: string; detail: string };
+  tokensSaved: { value: string; detail: string };
+  compressionRatio: { value: string; detail: string };
+  indexedDocs: { value: string; detail: string };
+  sandboxExecutions: { value: string; detail: string };
+  searchQueries: { value: string; detail: string };
+}
+
+export async function getInfoScreenData(
+  sessionDB: SessionDB,
+  contentStore: ContentStore,
+  sessionId: string,
+): Promise<InfoScreenData> {
+  const stats = sessionDB.getSessionStats(sessionId);
+  const compactStats = getLastCompactionStats();
+  const storeStats = await contentStore.getStats();
+
+  return {
+    sessionEvents: {
+      value: String(stats?.event_count ?? 0),
+      detail: "Session events tracked",
+    },
+    compactions: {
+      value: String(stats?.compact_count ?? 0),
+      detail: compactStats ? `Last: ${compactStats.summarized} msgs` : "No compactions yet",
+    },
+    tokensSaved: {
+      value: compactStats ? `~${compactStats.keptTokensEst}` : "0",
+      detail: "Estimated tokens kept",
+    },
+    compressionRatio: {
+      value: compactStats && compactStats.summarized > 0
+        ? `${Math.round(compactStats.summarized / Math.max(compactStats.kept, 1))}:1`
+        : "N/A",
+      detail: "Compression ratio",
+    },
+    indexedDocs: {
+      value: String(storeStats.sources),
+      detail: `${storeStats.chunks} chunks indexed`,
+    },
+    sandboxExecutions: {
+      value: "0",
+      detail: "Sandbox runs this session",
+    },
+    searchQueries: {
+      value: "0",
+      detail: "Search queries this session",
+    },
+  };
+}