@pi-unipi/unipi 2.0.3 → 2.0.4

package/packages/compactor/skills/compactor-doctor/SKILL.md

@@ -1,55 +1,53 @@
 ---
 name: compactor-doctor
-description: Diagnostics — validate config, DB, FTS5, runtimes, troubleshoot issues.
+description: Diagnostics — validate config, session DB, runtimes, and troubleshoot compactor issues.
 ---
 
 # Compactor Doctor
 
 Run diagnostics and troubleshoot compactor issues.
 
-## Commands
+## Commands and Tools
 
-- `/unipi:compact-doctor` — run all checks
-- `ctx_doctor` tool — agent-callable diagnostics
+- `/unipi:compact-doctor` — user-facing diagnostics
+- `compactor_doctor` tool — agent-callable diagnostics
+- `ctx_doctor` tool — deprecated alias
 
 ## Checks Performed
 
 | Check | What It Validates |
 |-------|-------------------|
-| **Config file** | `~/.unipi/config/compactor/config.json` exists and is valid |
-| **Session DB** | SQLite connection works, schema correct |
-| **Content Store** | FTS5 index accessible, tables exist |
-| **Runtime: node** | Node.js available for sandbox |
-| **Runtime: python3** | Python 3 available for sandbox |
-| **Runtime: bash** | Bash available for sandbox |
+| **Config file** | `~/.unipi/config/compactor/config.json` exists or defaults can be used |
+| **Session DB** | SQLite connection works and session schema is usable |
+| **Runtime: node** | Node.js available for sandbox/runtime helpers |
+| **Runtime: python3** | Python 3 available for Python sandbox execution |
+| **Runtime: bash** | Bash available for shell sandbox execution |
 
 ## Status Icons
 
 - ✅ **pass** — check succeeded
-- ⚠️ **warn** — non-critical issue (e.g., optional runtime missing)
-- ❌ **fail** — critical issue, feature may not work
+- ⚠️ **warn** — non-critical issue, such as an optional runtime missing
+- ❌ **fail** — critical issue; a feature may not work
 
 ## Common Issues
 
 ### "Config file: Using defaults"
-- Normal on first run
-- Config auto-created on next settings save
-- Fix: `/unipi:compact-settings` → save
+
+- Normal on first run.
+- Config is created by the extension or on settings save.
+- Fix: open `/unipi:compact-settings`, adjust if desired, then save.
 
 ### "Session DB: Connection failed"
-- SQLite not available
-- Check if `better-sqlite3` is installed
-- Fix: `npm install better-sqlite3`
 
-### "Content Store: FTS5 error"
-- SQLite FTS5 extension not available
-- Requires SQLite 3.9+ with FTS5
-- Fix: Update system SQLite
+- SQLite or `better-sqlite3` may be unavailable.
+- Compaction can still degrade gracefully, but stats/recall may be limited.
+- Fix: verify dependencies and run package install/build steps.
 
 ### "Runtime: python3 Not found"
-- Python not installed or not in PATH
-- Only needed for Python sandbox execution
-- Fix: Install Python 3 or ignore if not needed
+
+- Python is not installed or not in `PATH`.
+- Only needed for Python sandbox execution.
+- Fix: install Python 3 or ignore if not needed.
 
 ## Manual Diagnostics
 
@@ -63,12 +61,11 @@ cat ~/.unipi/config/compactor/config.json
 ls -la ~/.unipi/db/compactor/
 ```
 
-### Check SQLite version
+### Check runtimes
 ```bash
-sqlite3 --version
+node --version
+python3 --version
+bash --version
 ```
 
-### Test FTS5
-```bash
-sqlite3 ':memory:' "CREATE VIRTUAL TABLE t USING fts5(x); SELECT 1;"
-```
+Project content indexing diagnostics live in the CocoIndex package, not compactor.

package/packages/compactor/skills/compactor-stats/SKILL.md

@@ -1,49 +1,51 @@
 ---
 name: compactor-stats
-description: Stats display — context savings, session metrics, index health.
+description: Stats display — context savings, session metrics, compactions, sandbox and recall/search counters.
 ---
 
 # Compactor Stats
 
 Display and interpret compactor statistics.
 
-## Commands
+## Commands and Tools
 
-- `/unipi:compact-stats` — quick dashboard
-- `ctx_stats` tool — detailed stats (agent-callable)
+- `/unipi:compact-stats` — user-facing dashboard
+- `compactor_stats` tool — agent-callable stats
+- `ctx_stats` tool — deprecated alias
 
 ## Metrics Explained
 
 | Metric | Description |
 |--------|-------------|
-| **Session events** | Total events tracked in current session |
-| **Compactions** | Number of times context was compacted |
-| **Tokens saved** | Estimated tokens freed by compaction |
-| **Indexed docs** | Number of files/sources in FTS5 index |
-| **Indexed chunks** | Total searchable chunks |
-| **Sandbox runs** | Code executions in sandbox |
-| **Search queries** | FTS5 searches performed |
+| **Session events** | Events tracked for the current session in the compactor session DB |
+| **Compactions** | Number of compaction events recorded |
+| **Tokens saved** | Estimated tokens freed by compaction, using Pi token counts when available and DB fallbacks otherwise |
+| **Compression ratio** | Approximate before/kept ratio from stored compaction stats |
+| **Sandbox runs** | Agent sandbox executions recorded in runtime counters/DB |
+| **Search queries** | Session recall/search queries recorded in runtime counters/DB |
 
 ## Health Indicators
 
-- **Compactions > 0** on long sessions = working correctly
-- **Indexed chunks > 0** = search available
-- **Sandbox runs** = code execution active
+- **Compactions > 0** on long sessions = compaction is happening.
+- **Tokens saved > 0** after compaction = savings were recorded.
+- **Sandbox runs** increasing = sandbox tool usage is tracked.
+- **Search queries** increasing = recall/search activity is tracked.
 
 ## Reading Stats
 
-```
+```text
 📊 Compactor Stats
 Session events: 42
 Compactions: 3
 Tokens saved: 15000
-Indexed docs: 12 (48 chunks)
 Sandbox runs: 7
 Search queries: 15
 ```
 
 This means:
-- 42 tool calls/events tracked
-- 3 compactions saved ~15K tokens
-- 12 files indexed into 48 searchable chunks
-- 7 code executions, 15 searches performed
+
+- 42 events are tracked for the session.
+- 3 compactions saved roughly 15K tokens.
+- 7 sandbox executions and 15 recall/search queries were recorded.
+
+Project content indexing stats are not owned by compactor anymore; use the CocoIndex package for indexed file search/status.

package/packages/compactor/src/commands/index.ts

@@ -229,9 +229,12 @@ export function registerCommands(pi: ExtensionAPI, deps?: CommandDeps): void {
         "  /unipi:session-recall <query> — search session history\n" +
         "  /unipi:compact-stats — view stats\n" +
         "  /unipi:compact-doctor — run diagnostics\n" +
-        "  /unipi:compact-settings — TUI settings\n" +
+        "  /unipi:compact-settings — TUI settings, including optional % auto-compaction\n" +
         "  /unipi:compact-preset <name> — apply preset\n" +
         "\n" +
+        "Percentage trigger:\n" +
+        "  Disabled by default. Enable in /unipi:compact-settings to compact at a context % before Pi's reserve-token limit.\n" +
+        "\n" +
         "Content indexing has moved to @pi-unipi/cocoindex:\n" +
         "  /unipi:cocoindex-init — initialize pipeline\n" +
         "  /unipi:cocoindex-update — index project files\n" +

package/packages/compactor/src/compaction/auto-trigger.ts

@@ -0,0 +1,306 @@
+/**
+ * Pure percentage auto-compaction trigger decisions.
+ *
+ * The runtime wiring owns Pi ctx access; this module only decides whether a
+ * known context-usage sample should trigger UniPi's zero-LLM compaction.
+ */
+
+import type { AutoCompactionConfig } from "../types.js";
+
+export interface AutoCompactionUsage {
+  tokens?: number | null;
+  percent?: number | null;
+  contextWindow?: number | null;
+}
+
+export interface KnownAutoCompactionUsage {
+  tokens: number;
+  percent: number;
+  contextWindow?: number;
+}
+
+export interface AutoCompactionState {
+  /** Previous known context percentage sample. Null before first known usage. */
+  previousPercent: number | null;
+  /** Previous known context token sample. Null before first known usage. */
+  previousTokens: number | null;
+  /** True after UniPi calls ctx.compact() and before Pi reports completion/error. */
+  inFlight: boolean;
+  /** Timestamp of the last UniPi-triggered compaction attempt. */
+  lastTriggerAt: number | null;
+  /** Context tokens reported at the last UniPi-triggered compaction attempt. */
+  lastTriggerTokens: number | null;
+  /** Baseline used to require meaningful token growth for repeat high-usage triggers. */
+  repeatBaselineTokens: number | null;
+  /** After a successful compaction, consume one known usage sample as a fresh baseline. */
+  awaitingPostCompactionSample: boolean;
+}
+
+export type AutoCompactionDecisionReason =
+  | "disabled"
+  | "unknown_usage"
+  | "in_flight"
+  | "below_threshold"
+  | "post_compaction_baseline"
+  | "threshold_reached"
+  | "threshold_crossed"
+  | "cooldown_active"
+  | "repeat_growth_needed"
+  | "repeat_growth_reached";
+
+export interface AutoCompactionDecision {
+  shouldTrigger: boolean;
+  reason: AutoCompactionDecisionReason;
+  state: AutoCompactionState;
+  thresholdPercent: number;
+  usage?: KnownAutoCompactionUsage;
+  cooldownRemainingMs?: number;
+  tokenGrowth?: number;
+  tokensUntilRepeat?: number;
+}
+
+export interface AutoCompactionDecisionInput {
+  config?: Partial<AutoCompactionConfig> | null;
+  usage?: AutoCompactionUsage | null;
+  state: AutoCompactionState;
+  nowMs?: number;
+}
+
+export const AUTO_COMPACTION_DEFAULTS: AutoCompactionConfig = {
+  enabled: false,
+  thresholdPercent: 80,
+  cooldownMs: 60_000,
+  repeatMinGrowthTokens: 4_000,
+  notify: true,
+};
+
+const clamp = (value: unknown, fallback: number, min: number, max: number): number => {
+  if (typeof value !== "number" || !Number.isFinite(value)) return fallback;
+  return Math.min(max, Math.max(min, value));
+};
+
+export function normalizeAutoCompactionConfig(config?: Partial<AutoCompactionConfig> | null): AutoCompactionConfig {
+  return {
+    enabled: config?.enabled ?? AUTO_COMPACTION_DEFAULTS.enabled,
+    thresholdPercent: clamp(
+      config?.thresholdPercent,
+      AUTO_COMPACTION_DEFAULTS.thresholdPercent,
+      1,
+      99,
+    ),
+    cooldownMs: Math.round(clamp(
+      config?.cooldownMs,
+      AUTO_COMPACTION_DEFAULTS.cooldownMs,
+      0,
+      24 * 60 * 60 * 1000,
+    )),
+    repeatMinGrowthTokens: Math.round(clamp(
+      config?.repeatMinGrowthTokens,
+      AUTO_COMPACTION_DEFAULTS.repeatMinGrowthTokens,
+      0,
+      10_000_000,
+    )),
+    notify: config?.notify ?? AUTO_COMPACTION_DEFAULTS.notify,
+  };
+}
+
+export function createAutoCompactionState(): AutoCompactionState {
+  return {
+    previousPercent: null,
+    previousTokens: null,
+    inFlight: false,
+    lastTriggerAt: null,
+    lastTriggerTokens: null,
+    repeatBaselineTokens: null,
+    awaitingPostCompactionSample: false,
+  };
+}
+
+function knownUsage(usage?: AutoCompactionUsage | null): KnownAutoCompactionUsage | null {
+  if (!usage) return null;
+  const { tokens, percent, contextWindow } = usage;
+  if (typeof tokens !== "number" || !Number.isFinite(tokens) || tokens < 0) return null;
+  if (typeof percent !== "number" || !Number.isFinite(percent) || percent < 0) return null;
+  const known: KnownAutoCompactionUsage = { tokens, percent };
+  if (typeof contextWindow === "number" && Number.isFinite(contextWindow) && contextWindow > 0) {
+    known.contextWindow = contextWindow;
+  }
+  return known;
+}
+
+function updatePrevious(state: AutoCompactionState, usage: KnownAutoCompactionUsage): AutoCompactionState {
+  return {
+    ...state,
+    previousPercent: usage.percent,
+    previousTokens: usage.tokens,
+  };
+}
+
+function trigger(
+  reason: Extract<AutoCompactionDecisionReason, "threshold_reached" | "threshold_crossed" | "repeat_growth_reached">,
+  state: AutoCompactionState,
+  usage: KnownAutoCompactionUsage,
+  thresholdPercent: number,
+  nowMs: number,
+  tokenGrowth?: number,
+): AutoCompactionDecision {
+  return {
+    shouldTrigger: true,
+    reason,
+    thresholdPercent,
+    usage,
+    tokenGrowth,
+    state: {
+      ...updatePrevious(state, usage),
+      inFlight: true,
+      lastTriggerAt: nowMs,
+      lastTriggerTokens: usage.tokens,
+      repeatBaselineTokens: usage.tokens,
+      awaitingPostCompactionSample: false,
+    },
+  };
+}
+
+/**
+ * Decide whether the current usage sample should trigger auto-compaction.
+ *
+ * Loop safeguards:
+ * - unknown/null usage never triggers and does not erase the previous baseline;
+ * - in-flight compactions suppress further triggers;
+ * - every repeat trigger after a prior attempt observes cooldown;
+ * - if usage remains above threshold after compaction, the first known sample is
+ *   only a baseline, and a repeat requires sufficient token growth.
+ */
+export function decideAutoCompaction(input: AutoCompactionDecisionInput): AutoCompactionDecision {
+  const config = normalizeAutoCompactionConfig(input.config);
+  const nowMs = input.nowMs ?? Date.now();
+
+  if (!config.enabled) {
+    return {
+      shouldTrigger: false,
+      reason: "disabled",
+      thresholdPercent: config.thresholdPercent,
+      state: input.state,
+    };
+  }
+
+  if (input.state.inFlight) {
+    return {
+      shouldTrigger: false,
+      reason: "in_flight",
+      thresholdPercent: config.thresholdPercent,
+      state: input.state,
+    };
+  }
+
+  const usage = knownUsage(input.usage);
+  if (!usage) {
+    return {
+      shouldTrigger: false,
+      reason: "unknown_usage",
+      thresholdPercent: config.thresholdPercent,
+      state: input.state,
+    };
+  }
+
+  let nextState = updatePrevious(input.state, usage);
+
+  if (input.state.awaitingPostCompactionSample) {
+    nextState = {
+      ...nextState,
+      awaitingPostCompactionSample: false,
+      repeatBaselineTokens: usage.percent >= config.thresholdPercent ? usage.tokens : null,
+    };
+    return {
+      shouldTrigger: false,
+      reason: usage.percent >= config.thresholdPercent ? "post_compaction_baseline" : "below_threshold",
+      thresholdPercent: config.thresholdPercent,
+      usage,
+      state: nextState,
+    };
+  }
+
+  if (usage.percent < config.thresholdPercent) {
+    return {
+      shouldTrigger: false,
+      reason: "below_threshold",
+      thresholdPercent: config.thresholdPercent,
+      usage,
+      state: {
+        ...nextState,
+        repeatBaselineTokens: null,
+      },
+    };
+  }
+
+  if (input.state.lastTriggerAt !== null) {
+    const elapsedMs = Math.max(0, nowMs - input.state.lastTriggerAt);
+    if (elapsedMs < config.cooldownMs) {
+      return {
+        shouldTrigger: false,
+        reason: "cooldown_active",
+        thresholdPercent: config.thresholdPercent,
+        usage,
+        cooldownRemainingMs: config.cooldownMs - elapsedMs,
+        state: nextState,
+      };
+    }
+  }
+
+  const previousPercent = input.state.previousPercent;
+  if (previousPercent === null) {
+    return trigger("threshold_reached", input.state, usage, config.thresholdPercent, nowMs);
+  }
+  if (previousPercent < config.thresholdPercent) {
+    return trigger("threshold_crossed", input.state, usage, config.thresholdPercent, nowMs);
+  }
+
+  // We were already above threshold. Re-trigger only after enough growth.
+  if (input.state.lastTriggerAt === null) {
+    return trigger("threshold_reached", input.state, usage, config.thresholdPercent, nowMs);
+  }
+
+  const baselineTokens = input.state.repeatBaselineTokens
+    ?? input.state.lastTriggerTokens
+    ?? input.state.previousTokens
+    ?? usage.tokens;
+  const tokenGrowth = Math.max(0, usage.tokens - baselineTokens);
+
+  if (tokenGrowth < config.repeatMinGrowthTokens) {
+    return {
+      shouldTrigger: false,
+      reason: "repeat_growth_needed",
+      thresholdPercent: config.thresholdPercent,
+      usage,
+      tokenGrowth,
+      tokensUntilRepeat: config.repeatMinGrowthTokens - tokenGrowth,
+      state: nextState,
+    };
+  }
+
+  return trigger(
+    "repeat_growth_reached",
+    input.state,
+    usage,
+    config.thresholdPercent,
+    nowMs,
+    tokenGrowth,
+  );
+}
+
+export function markAutoCompactionComplete(state: AutoCompactionState): AutoCompactionState {
+  return {
+    ...state,
+    inFlight: false,
+    awaitingPostCompactionSample: true,
+  };
+}
+
+export function markAutoCompactionError(state: AutoCompactionState, nowMs?: number): AutoCompactionState {
+  return {
+    ...state,
+    inFlight: false,
+    awaitingPostCompactionSample: false,
+    lastTriggerAt: state.lastTriggerAt ?? nowMs ?? null,
+  };
+}

package/packages/compactor/src/config/manager.ts

@@ -116,6 +116,7 @@ export function migrateConfig(partial: Partial<CompactorConfig>): CompactorConfi
     sandboxExecution: mergeStrategy("sandboxExecution", defaults.sandboxExecution, partial.sandboxExecution),
     toolDisplay: mergeStrategy("toolDisplay", defaults.toolDisplay, partial.toolDisplay),
     pipeline: mergeStrategy("pipeline", defaults.pipeline, (partial as any).pipeline) as any,
+    autoCompaction: mergeStrategy("autoCompaction", defaults.autoCompaction, partial.autoCompaction),
     overrideDefaultCompaction: partial.overrideDefaultCompaction ?? defaults.overrideDefaultCompaction,
     debug: partial.debug ?? defaults.debug,
     showTruncationHints: partial.showTruncationHints ?? defaults.showTruncationHints,

package/packages/compactor/src/config/presets.ts

@@ -6,6 +6,21 @@ import { createHash } from "node:crypto";
 import type { CompactorConfig, CompactorPreset } from "../types.js";
 import { DEFAULT_COMPACTOR_CONFIG } from "./schema.js";
 
+const pipeline = (overrides: Partial<CompactorConfig["pipeline"]> = {}): CompactorConfig["pipeline"] => ({
+  ...DEFAULT_COMPACTOR_CONFIG.pipeline,
+  ...overrides,
+  customNoisePatterns: [...(overrides.customNoisePatterns ?? DEFAULT_COMPACTOR_CONFIG.pipeline.customNoisePatterns)],
+});
+
+const pipelineAllOn = (): CompactorConfig["pipeline"] => pipeline({
+  ttlCache: true,
+  autoInjection: true,
+  proximityReranking: true,
+  timelineSort: true,
+  progressiveThrottling: true,
+  mmapPragma: true,
+});
+
 const preset = (
   overrides: Partial<CompactorConfig>,
 ): CompactorConfig => ({
@@ -21,6 +36,8 @@ const preset = (
   fts5Index: { ...DEFAULT_COMPACTOR_CONFIG.fts5Index, ...(overrides.fts5Index ?? {}) },
   sandboxExecution: { ...DEFAULT_COMPACTOR_CONFIG.sandboxExecution, ...(overrides.sandboxExecution ?? {}) },
   toolDisplay: { ...DEFAULT_COMPACTOR_CONFIG.toolDisplay, ...(overrides.toolDisplay ?? {}) },
+  pipeline: pipeline(overrides.pipeline),
+  autoCompaction: { ...DEFAULT_COMPACTOR_CONFIG.autoCompaction, ...(overrides.autoCompaction ?? {}) },
 });
 
 // Pipeline feature defaults per preset:
@@ -32,13 +49,17 @@ const preset = (
 export const PRESET_CONFIGS: Record<CompactorPreset, CompactorConfig> = {
   // New preset names
   precise: preset({
+    pipeline: pipeline({ ttlCache: true, mmapPragma: true }),
+    sandboxExecution: { ...DEFAULT_COMPACTOR_CONFIG.sandboxExecution, mode: "safe-only" },
     toolDisplay: { ...DEFAULT_COMPACTOR_CONFIG.toolDisplay, mode: "opencode" },
   }),
   thorough: preset({
+    pipeline: pipelineAllOn(),
     briefTranscript: { ...DEFAULT_COMPACTOR_CONFIG.briefTranscript, mode: "full" },
     toolDisplay: { ...DEFAULT_COMPACTOR_CONFIG.toolDisplay, mode: "verbose" },
   }),
   lean: preset({
+    pipeline: pipeline(),
     sessionGoals: { ...DEFAULT_COMPACTOR_CONFIG.sessionGoals, enabled: true, mode: "brief" },
     filesAndChanges: { ...DEFAULT_COMPACTOR_CONFIG.filesAndChanges, enabled: true, mode: "modified-only" },
     commits: { ...DEFAULT_COMPACTOR_CONFIG.commits, enabled: false, mode: "off" },
@@ -51,6 +72,7 @@ export const PRESET_CONFIGS: Record<CompactorPreset, CompactorConfig> = {
     toolDisplay: { ...DEFAULT_COMPACTOR_CONFIG.toolDisplay, enabled: true, mode: "opencode" },
   }),
   balanced: preset({
+    pipeline: pipelineAllOn(),
     briefTranscript: { ...DEFAULT_COMPACTOR_CONFIG.briefTranscript, mode: "compact" },
     toolDisplay: { ...DEFAULT_COMPACTOR_CONFIG.toolDisplay, mode: "balanced" },
     fts5Index: { ...DEFAULT_COMPACTOR_CONFIG.fts5Index, mode: "auto" },
@@ -58,13 +80,17 @@ export const PRESET_CONFIGS: Record<CompactorPreset, CompactorConfig> = {
 
   // Backward-compat aliases — map old names to new
   opencode: preset({
+    pipeline: pipeline({ ttlCache: true, mmapPragma: true }),
+    sandboxExecution: { ...DEFAULT_COMPACTOR_CONFIG.sandboxExecution, mode: "safe-only" },
     toolDisplay: { ...DEFAULT_COMPACTOR_CONFIG.toolDisplay, mode: "opencode" },
   }),
   verbose: preset({
+    pipeline: pipelineAllOn(),
     briefTranscript: { ...DEFAULT_COMPACTOR_CONFIG.briefTranscript, mode: "full" },
     toolDisplay: { ...DEFAULT_COMPACTOR_CONFIG.toolDisplay, mode: "verbose" },
   }),
   minimal: preset({
+    pipeline: pipeline(),
     sessionGoals: { ...DEFAULT_COMPACTOR_CONFIG.sessionGoals, enabled: true, mode: "brief" },
     filesAndChanges: { ...DEFAULT_COMPACTOR_CONFIG.filesAndChanges, enabled: true, mode: "modified-only" },
     commits: { ...DEFAULT_COMPACTOR_CONFIG.commits, enabled: false, mode: "off" },

package/packages/compactor/src/config/schema.ts

@@ -58,6 +58,13 @@ export const DEFAULT_COMPACTOR_CONFIG: CompactorConfig = {
     mmapPragma: false,
     customNoisePatterns: [],
   },
+  autoCompaction: {
+    enabled: false,
+    thresholdPercent: 80,
+    cooldownMs: 60_000,
+    repeatMinGrowthTokens: 4_000,
+    notify: true,
+  },
   overrideDefaultCompaction: true,
   debug: false,
   showTruncationHints: true,