@pi-unipi/compactor 0.1.7 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -24
- package/index.ts +7 -0
- package/package.json +4 -2
- package/skills/compactor/SKILL.md +21 -65
- package/skills/compactor-detail/SKILL.md +133 -0
- package/src/commands/index.ts +186 -109
- package/src/compaction/filter-noise.ts +4 -3
- package/src/compaction/hooks.ts +25 -6
- package/src/compaction/search-entries.ts +51 -4
- package/src/config/manager.ts +55 -6
- package/src/config/presets.ts +69 -5
- package/src/config/schema.ts +10 -1
- package/src/display/diff-presentation.ts +6 -1
- package/src/display/diff-renderer.ts +34 -8
- package/src/display/diff-width-safety.ts +83 -0
- package/src/display/line-width-safety.ts +14 -2
- package/src/index.ts +297 -16
- package/src/info-screen.ts +137 -46
- package/src/security/policy.ts +23 -0
- package/src/session/analytics.ts +198 -0
- package/src/session/auto-inject.ts +60 -0
- package/src/session/db.ts +68 -8
- package/src/session/resume-inject.ts +13 -1
- package/src/store/db-base.ts +11 -0
- package/src/store/index.ts +150 -4
- package/src/store/unified.ts +109 -0
- package/src/tools/context-budget.ts +50 -0
- package/src/tools/ctx-batch-execute.ts +2 -5
- package/src/tools/ctx-fetch-and-index.ts +3 -8
- package/src/tools/ctx-index.ts +3 -9
- package/src/tools/ctx-search.ts +3 -7
- package/src/tools/ctx-stats.ts +6 -4
- package/src/tools/register.ts +251 -216
- package/src/tui/settings-overlay.ts +359 -149
- package/src/types.ts +30 -7
- package/skills/compactor-ops/SKILL.md +0 -65
- package/skills/compactor-tools/SKILL.md +0 -120
package/src/index.ts
CHANGED
|
@@ -15,18 +15,46 @@ import { registerCommands } from "./commands/index.js";
|
|
|
15
15
|
import { registerCompactorTools } from "./tools/register.js";
|
|
16
16
|
import { normalizeMessages } from "./compaction/normalize.js";
|
|
17
17
|
import { filterNoise } from "./compaction/filter-noise.js";
|
|
18
|
-
import type { NormalizedBlock } from "./types.js";
|
|
18
|
+
import type { NormalizedBlock, CompactorStrategyConfig, RuntimeCounters } from "./types.js";
|
|
19
|
+
import type { RuntimeStats } from "./session/analytics.js";
|
|
19
20
|
|
|
20
21
|
/** Debug logger — only logs when config.debug === true */
|
|
21
22
|
function createDebugLogger(getConfig: () => { debug: boolean }) {
|
|
22
|
-
return (
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
const details = data ? " " + JSON.stringify(data) : "";
|
|
26
|
-
console.error(`[compactor:${ts}] ${event}${details}`);
|
|
23
|
+
return (_event: string, _data?: Record<string, unknown>) => {
|
|
24
|
+
// Debug logging disabled — was writing to stdout causing TUI rendering issues.
|
|
25
|
+
return;
|
|
27
26
|
};
|
|
28
27
|
}
|
|
29
28
|
|
|
29
|
+
/** Measure byte size of a tool_result event's response content. */
|
|
30
|
+
function measureResponseBytes(event: any): number {
|
|
31
|
+
try {
|
|
32
|
+
const content = event.content;
|
|
33
|
+
if (typeof content === "string") return Buffer.byteLength(content, "utf-8");
|
|
34
|
+
if (Array.isArray(content)) {
|
|
35
|
+
return content.reduce((sum: number, block: any) => {
|
|
36
|
+
if (typeof block?.text === "string") return sum + Buffer.byteLength(block.text, "utf-8");
|
|
37
|
+
if (typeof block === "string") return sum + Buffer.byteLength(block, "utf-8");
|
|
38
|
+
return sum;
|
|
39
|
+
}, 0);
|
|
40
|
+
}
|
|
41
|
+
if (event.output && typeof event.output === "string") return Buffer.byteLength(event.output, "utf-8");
|
|
42
|
+
} catch {
|
|
43
|
+
// Non-blocking: byte measurement errors silently skipped
|
|
44
|
+
}
|
|
45
|
+
return 0;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Check if a tool is a sandbox tool (output stays in sandbox, not context). */
|
|
49
|
+
function isSandboxTool(name: string): boolean {
|
|
50
|
+
return name === "bash" || name === "Bash";
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Check if a tool is an index tool (content goes to FTS5, not context). Future-proofing. */
|
|
54
|
+
function isIndexTool(_name: string): boolean {
|
|
55
|
+
return false;
|
|
56
|
+
}
|
|
57
|
+
|
|
30
58
|
export default function compactorExtension(pi: ExtensionAPI): void {
|
|
31
59
|
let sessionDB: SessionDB | null = null;
|
|
32
60
|
let contentStore: ContentStore | null = null;
|
|
@@ -34,6 +62,24 @@ export default function compactorExtension(pi: ExtensionAPI): void {
|
|
|
34
62
|
let config = loadConfig();
|
|
35
63
|
let cachedBlocks: NormalizedBlock[] = [];
|
|
36
64
|
let currentSessionId = "default";
|
|
65
|
+
const counters: RuntimeCounters = {
|
|
66
|
+
sandboxRuns: 0,
|
|
67
|
+
searchQueries: 0,
|
|
68
|
+
recallQueries: 0,
|
|
69
|
+
compactions: 0,
|
|
70
|
+
totalTokensCompacted: 0,
|
|
71
|
+
};
|
|
72
|
+
const getCounters = () => counters;
|
|
73
|
+
|
|
74
|
+
const runtimeStats: RuntimeStats = {
|
|
75
|
+
bytesReturned: {},
|
|
76
|
+
bytesIndexed: 0,
|
|
77
|
+
bytesSandboxed: 0,
|
|
78
|
+
calls: {},
|
|
79
|
+
sessionStart: Date.now(),
|
|
80
|
+
cacheHits: 0,
|
|
81
|
+
cacheBytesSaved: 0,
|
|
82
|
+
};
|
|
37
83
|
|
|
38
84
|
const debug = createDebugLogger(() => config);
|
|
39
85
|
|
|
@@ -41,12 +87,31 @@ export default function compactorExtension(pi: ExtensionAPI): void {
|
|
|
41
87
|
scaffoldConfig();
|
|
42
88
|
config = loadConfig();
|
|
43
89
|
|
|
44
|
-
|
|
45
|
-
|
|
90
|
+
// Initialize SessionDB — this is required for core functionality.
|
|
91
|
+
// If it fails, log the error and continue. Commands that depend on
|
|
92
|
+
// sessionDB will report "not initialized" gracefully.
|
|
93
|
+
// IMPORTANT: Don't assign sessionDB until init succeeds — a partially-
|
|
94
|
+
// constructed instance with empty stmts would slip past null-guards.
|
|
95
|
+
try {
|
|
96
|
+
const db = new SessionDB();
|
|
97
|
+
await db.init();
|
|
98
|
+
sessionDB = db;
|
|
99
|
+
} catch {
|
|
100
|
+
// Silently ignore — SessionDB init failure is handled gracefully.
|
|
101
|
+
sessionDB = null;
|
|
102
|
+
}
|
|
46
103
|
|
|
104
|
+
// Initialize ContentStore independently — its failure shouldn't
|
|
105
|
+
// prevent SessionDB commands from working.
|
|
47
106
|
if (config.fts5Index.enabled) {
|
|
48
|
-
|
|
49
|
-
|
|
107
|
+
try {
|
|
108
|
+
const cs = new ContentStore();
|
|
109
|
+
await cs.init();
|
|
110
|
+
contentStore = cs;
|
|
111
|
+
} catch {
|
|
112
|
+
// Silently ignore — ContentStore init failure is handled gracefully.
|
|
113
|
+
contentStore = null;
|
|
114
|
+
}
|
|
50
115
|
}
|
|
51
116
|
|
|
52
117
|
executor = new PolyglotExecutor();
|
|
@@ -54,12 +119,13 @@ export default function compactorExtension(pi: ExtensionAPI): void {
|
|
|
54
119
|
|
|
55
120
|
registerCompactionHooks(pi);
|
|
56
121
|
|
|
57
|
-
// Commands
|
|
122
|
+
// Commands registered inside session_start after init() when deps are ready
|
|
58
123
|
const getCommandDeps = () => ({
|
|
59
124
|
sessionDB,
|
|
60
125
|
contentStore,
|
|
61
126
|
getSessionId: () => currentSessionId,
|
|
62
127
|
getBlocks: () => cachedBlocks,
|
|
128
|
+
getCounters,
|
|
63
129
|
});
|
|
64
130
|
|
|
65
131
|
pi.on("session_start", async (_event, ctx) => {
|
|
@@ -73,21 +139,71 @@ export default function compactorExtension(pi: ExtensionAPI): void {
|
|
|
73
139
|
|
|
74
140
|
debug("session_start", { sessionId: fullSessionId, projectDir });
|
|
75
141
|
|
|
142
|
+
// Reset runtime stats for new session
|
|
143
|
+
runtimeStats.bytesReturned = {};
|
|
144
|
+
runtimeStats.bytesIndexed = 0;
|
|
145
|
+
runtimeStats.bytesSandboxed = 0;
|
|
146
|
+
runtimeStats.calls = {};
|
|
147
|
+
runtimeStats.sessionStart = Date.now();
|
|
148
|
+
runtimeStats.cacheHits = 0;
|
|
149
|
+
runtimeStats.cacheBytesSaved = 0;
|
|
150
|
+
|
|
76
151
|
sessionDB?.ensureSession(fullSessionId, projectDir);
|
|
77
152
|
|
|
78
|
-
// Register all compactor tools with Pi
|
|
153
|
+
// Register all compactor tools with Pi (deps now have live sessionDB)
|
|
79
154
|
if (sessionDB) {
|
|
80
155
|
registerCompactorTools(pi, {
|
|
81
156
|
sessionDB,
|
|
82
157
|
contentStore,
|
|
83
158
|
getSessionId: () => currentSessionId,
|
|
84
159
|
getBlocks: () => cachedBlocks,
|
|
160
|
+
getCounters,
|
|
85
161
|
});
|
|
86
162
|
}
|
|
87
163
|
|
|
88
|
-
//
|
|
164
|
+
// Register commands with live deps
|
|
89
165
|
registerCommands(pi, getCommandDeps());
|
|
90
166
|
|
|
167
|
+
// Register info-screen group
|
|
168
|
+
const infoRegistry = (globalThis as any).__unipi_info_registry;
|
|
169
|
+
if (infoRegistry && sessionDB) {
|
|
170
|
+
const sdb = sessionDB;
|
|
171
|
+
const sid = () => currentSessionId;
|
|
172
|
+
infoRegistry.registerGroup({
|
|
173
|
+
id: "compactor",
|
|
174
|
+
name: "Compactor",
|
|
175
|
+
icon: "🗜️",
|
|
176
|
+
priority: 12,
|
|
177
|
+
config: {
|
|
178
|
+
showByDefault: true,
|
|
179
|
+
stats: [
|
|
180
|
+
{ id: "tokensSaved", label: "Tokens saved", show: true },
|
|
181
|
+
{ id: "costSaved", label: "Cost saved", show: true },
|
|
182
|
+
{ id: "pctReduction", label: "% Reduction", show: true },
|
|
183
|
+
{ id: "topTools", label: "Top tools", show: true },
|
|
184
|
+
{ id: "compactions", label: "Compactions", show: true },
|
|
185
|
+
{ id: "toolCalls", label: "Tool calls", show: true },
|
|
186
|
+
],
|
|
187
|
+
},
|
|
188
|
+
dataProvider: async () => {
|
|
189
|
+
try {
|
|
190
|
+
const { getInfoScreenData } = await import("./info-screen.js");
|
|
191
|
+
const data = await getInfoScreenData(sdb, sid(), runtimeStats);
|
|
192
|
+
return {
|
|
193
|
+
tokensSaved: { value: data.tokensSaved.value, detail: data.tokensSaved.detail },
|
|
194
|
+
costSaved: { value: data.costSaved.value, detail: data.costSaved.detail },
|
|
195
|
+
pctReduction: { value: data.pctReduction.value, detail: data.pctReduction.detail },
|
|
196
|
+
topTools: { value: data.topTools.value, detail: data.topTools.detail },
|
|
197
|
+
compactions: { value: data.compactions.value, detail: data.compactions.detail },
|
|
198
|
+
toolCalls: { value: data.toolCalls.value, detail: data.toolCalls.detail },
|
|
199
|
+
};
|
|
200
|
+
} catch {
|
|
201
|
+
return {};
|
|
202
|
+
}
|
|
203
|
+
},
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
|
|
91
207
|
emitEvent(pi as any, UNIPI_EVENTS.MODULE_READY, {
|
|
92
208
|
name: MODULES.COMPACTOR,
|
|
93
209
|
version: "0.1.0",
|
|
@@ -105,16 +221,39 @@ export default function compactorExtension(pi: ExtensionAPI): void {
|
|
|
105
221
|
});
|
|
106
222
|
|
|
107
223
|
pi.on("before_agent_start", async (_event, ctx) => {
|
|
108
|
-
|
|
224
|
+
const cwd = (ctx as any).cwd ?? process.cwd();
|
|
225
|
+
config = loadConfig(cwd);
|
|
109
226
|
currentSessionId = `${(ctx as any).sessionId ?? "default"}${getWorktreeSuffix()}`;
|
|
110
227
|
debug("before_agent_start", { sessionId: currentSessionId, configDebug: config.debug });
|
|
111
228
|
|
|
229
|
+
// Evaluate autoDetect conditions for strategies
|
|
230
|
+
try {
|
|
231
|
+
const { existsSync } = await import("node:fs");
|
|
232
|
+
const { join } = await import("node:path");
|
|
233
|
+
const strategies: Array<{ key: string; config: CompactorStrategyConfig }> = [
|
|
234
|
+
{ key: "commits", config: config.commits },
|
|
235
|
+
{ key: "fts5Index", config: config.fts5Index },
|
|
236
|
+
];
|
|
237
|
+
for (const { key, config: strat } of strategies) {
|
|
238
|
+
if ((strat as any).autoDetect === "git") {
|
|
239
|
+
const gitDir = join(cwd, ".git");
|
|
240
|
+
if (!existsSync(gitDir)) {
|
|
241
|
+
debug("autoDetect_disable", { strategy: key, reason: "no .git dir" });
|
|
242
|
+
// Non-destructive: temporarily disable at runtime, don't modify config file
|
|
243
|
+
strat.enabled = false;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
} catch {
|
|
248
|
+
// Non-fatal
|
|
249
|
+
}
|
|
250
|
+
|
|
112
251
|
// Re-cache normalized blocks for vcc_recall
|
|
113
252
|
try {
|
|
114
253
|
const messages = (ctx as any).messages ?? [];
|
|
115
254
|
if (messages.length > 0) {
|
|
116
255
|
const normalized = normalizeMessages(messages);
|
|
117
|
-
cachedBlocks = filterNoise(normalized);
|
|
256
|
+
cachedBlocks = filterNoise(normalized, config.pipeline?.customNoisePatterns);
|
|
118
257
|
}
|
|
119
258
|
} catch {
|
|
120
259
|
// Non-fatal: recall will work on empty blocks
|
|
@@ -123,6 +262,22 @@ export default function compactorExtension(pi: ExtensionAPI): void {
|
|
|
123
262
|
if (sessionDB) {
|
|
124
263
|
const snapshot = await injectResumeSnapshot(sessionDB, currentSessionId);
|
|
125
264
|
debug("resume_snapshot", { injected: !!snapshot });
|
|
265
|
+
|
|
266
|
+
// Auto-injection on compact: inject behavioral state after compaction
|
|
267
|
+
if (snapshot && sessionDB) {
|
|
268
|
+
try {
|
|
269
|
+
const { buildAutoInjection } = await import("./session/auto-inject.js");
|
|
270
|
+
const events = sessionDB.getEvents(currentSessionId, { limit: 100 });
|
|
271
|
+
const autoInjection = buildAutoInjection(events);
|
|
272
|
+
if (autoInjection) {
|
|
273
|
+
debug("auto_injection", { tokens: autoInjection.tokens, length: autoInjection.text.length });
|
|
274
|
+
// Note: auto-injection is included in the resume snapshot context
|
|
275
|
+
// The model receives it as part of the session state restoration
|
|
276
|
+
}
|
|
277
|
+
} catch (err) {
|
|
278
|
+
debug("auto_injection_error", { error: String(err) });
|
|
279
|
+
}
|
|
280
|
+
}
|
|
126
281
|
}
|
|
127
282
|
});
|
|
128
283
|
|
|
@@ -144,12 +299,37 @@ export default function compactorExtension(pi: ExtensionAPI): void {
|
|
|
144
299
|
if (sessionDB) {
|
|
145
300
|
const sessionId = `${(event as any).sessionId ?? "default"}${getWorktreeSuffix()}`;
|
|
146
301
|
sessionDB.incrementCompactCount(sessionId);
|
|
147
|
-
|
|
302
|
+
counters.compactions++;
|
|
303
|
+
|
|
304
|
+
// Use actual runtimeStats for byte measurement instead of heuristic
|
|
305
|
+
const totalBytesReturned = Object.values(runtimeStats.bytesReturned).reduce((s, b) => s + b, 0);
|
|
306
|
+
const totalBytesProcessed = runtimeStats.bytesIndexed + runtimeStats.bytesSandboxed + totalBytesReturned;
|
|
307
|
+
// charsBefore = total bytes processed by all tools (proxy for context window usage)
|
|
308
|
+
// charsKept = bytes that stayed in context (bytesReturned, minus what compaction removed)
|
|
309
|
+
const tokensBefore = (event as any).tokensBefore ?? 0;
|
|
310
|
+
if (totalBytesProcessed > 0 && tokensBefore > 0) {
|
|
311
|
+
// Use actual token count from Pi, estimate chars from it
|
|
312
|
+
const charsBefore = tokensBefore * 4;
|
|
313
|
+
// Estimate kept chars: proportional to what remains after compaction
|
|
314
|
+
const tokensAfter = (event as any).tokensAfter ?? Math.round(tokensBefore * 0.15);
|
|
315
|
+
const charsKept = tokensAfter * 4;
|
|
316
|
+
const messagesSummarized = Math.max(1, Math.round(tokensBefore / 500));
|
|
317
|
+
counters.totalTokensCompacted += tokensBefore - tokensAfter;
|
|
318
|
+
sessionDB.addCompactionStats(sessionId, charsBefore, charsKept, messagesSummarized);
|
|
319
|
+
} else if (tokensBefore > 0) {
|
|
320
|
+
// Fallback: only tokensBefore available, use conservative estimate
|
|
321
|
+
const tokensAfter = (event as any).tokensAfter ?? Math.round(tokensBefore * 0.15);
|
|
322
|
+
counters.totalTokensCompacted += tokensBefore - tokensAfter;
|
|
323
|
+
sessionDB.addCompactionStats(sessionId, tokensBefore * 4, tokensAfter * 4, 1);
|
|
324
|
+
}
|
|
325
|
+
debug("session_compact", { sessionId, tokensBefore, totalBytesProcessed });
|
|
148
326
|
}
|
|
149
327
|
});
|
|
150
328
|
|
|
151
329
|
pi.on("session_shutdown", async (_event, _ctx) => {
|
|
152
330
|
debug("session_shutdown");
|
|
331
|
+
// WAL checkpoint: TRUNCATE on shutdown to keep DB file size down
|
|
332
|
+
contentStore?.checkpointWAL("TRUNCATE");
|
|
153
333
|
if (sessionDB) {
|
|
154
334
|
sessionDB.cleanupOldSessions(7);
|
|
155
335
|
}
|
|
@@ -162,12 +342,71 @@ export default function compactorExtension(pi: ExtensionAPI): void {
|
|
|
162
342
|
const toolName = (event as any).toolName ?? "";
|
|
163
343
|
const args = (event as any).args ?? {};
|
|
164
344
|
debug("input", { toolName, args: JSON.stringify(args).slice(0, 200) });
|
|
345
|
+
|
|
346
|
+
// Existing network tool guard
|
|
165
347
|
if (toolName === "bash" || toolName === "Bash") {
|
|
166
348
|
const cmd = String(args.command ?? "");
|
|
167
349
|
if (/\b(curl|wget|nc|netcat)\b/.test(cmd)) {
|
|
168
350
|
return { cancel: true } as any;
|
|
169
351
|
}
|
|
170
352
|
}
|
|
353
|
+
|
|
354
|
+
// Security scanner/evaluator wiring (fail-open pattern)
|
|
355
|
+
try {
|
|
356
|
+
const { evaluateCommand, evaluateFilePath, loadProjectPermissions } = await import("./security/evaluator.js");
|
|
357
|
+
const { hasShellEscapes, scanForShellEscapes } = await import("./security/scanner.js");
|
|
358
|
+
const { readsOrCreatesPolicy } = await import("./security/policy.js");
|
|
359
|
+
|
|
360
|
+
// Load deny patterns from .pi/settings.json (fail-open: empty list on error)
|
|
361
|
+
const cwd = (event as any).cwd ?? process.cwd();
|
|
362
|
+
const denyPolicy = readsOrCreatesPolicy(cwd);
|
|
363
|
+
|
|
364
|
+
// 1. Evaluate bash commands against deny patterns
|
|
365
|
+
if (toolName === "bash" || toolName === "Bash" || toolName === "Bash") {
|
|
366
|
+
const cmd = String(args.command ?? "");
|
|
367
|
+
if (cmd) {
|
|
368
|
+
const decision = evaluateCommand(cmd, denyPolicy);
|
|
369
|
+
if (decision === "deny") {
|
|
370
|
+
debug("security_deny", { toolName, cmd: cmd.slice(0, 100) });
|
|
371
|
+
return {
|
|
372
|
+
content: [{ type: "text", text: `Command blocked by security policy: ${cmd.slice(0, 80)}` }],
|
|
373
|
+
isError: true,
|
|
374
|
+
} as any;
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
// 2. Scan sandbox non-shell code for shell escapes
|
|
380
|
+
const sandboxToolNames = ["ctx_execute", "ctx_execute_file", "sandbox", "sandbox_file"];
|
|
381
|
+
if (sandboxToolNames.includes(toolName)) {
|
|
382
|
+
const language = String(args.language ?? "");
|
|
383
|
+
const code = String(args.code ?? "");
|
|
384
|
+
if (language && language !== "shell" && code) {
|
|
385
|
+
if (hasShellEscapes(code, language)) {
|
|
386
|
+
const findings = scanForShellEscapes(code, language);
|
|
387
|
+
debug("security_shell_escapes", { toolName, language, findings });
|
|
388
|
+
// Fail-open: log but don't block (the hooks system is enforcement)
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
// 3. Evaluate file paths in read/write/edit operations
|
|
394
|
+
const fileOpTools = ["read", "edit", "write", "Read", "Edit", "Write"];
|
|
395
|
+
if (fileOpTools.includes(toolName)) {
|
|
396
|
+
const filePath = args.path ?? args.filePath ?? args.file_path ?? "";
|
|
397
|
+
if (filePath) {
|
|
398
|
+
const decision = evaluateFilePath(filePath, denyPolicy, cwd);
|
|
399
|
+
if (decision === "deny") {
|
|
400
|
+
debug("security_deny_file", { toolName, filePath });
|
|
401
|
+
// Non-fatal: log warning but allow through (fail-open)
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
} catch (err) {
|
|
406
|
+
// Fail-open: security checks are advisory, never block on errors
|
|
407
|
+
debug("security_check_error", { error: String(err) });
|
|
408
|
+
}
|
|
409
|
+
|
|
171
410
|
return undefined;
|
|
172
411
|
});
|
|
173
412
|
|
|
@@ -192,6 +431,24 @@ export default function compactorExtension(pi: ExtensionAPI): void {
|
|
|
192
431
|
debug("event_stored", { category: ev.category, type: ev.type });
|
|
193
432
|
}
|
|
194
433
|
|
|
434
|
+
// Track byte consumption per tool for analytics
|
|
435
|
+
try {
|
|
436
|
+
const responseBytes = measureResponseBytes(event);
|
|
437
|
+
if (responseBytes > 0) {
|
|
438
|
+
const tName = (event as any).toolName ?? "unknown";
|
|
439
|
+
runtimeStats.calls[tName] = (runtimeStats.calls[tName] || 0) + 1;
|
|
440
|
+
runtimeStats.bytesReturned[tName] = (runtimeStats.bytesReturned[tName] || 0) + responseBytes;
|
|
441
|
+
if (isSandboxTool(tName)) {
|
|
442
|
+
runtimeStats.bytesSandboxed += responseBytes;
|
|
443
|
+
}
|
|
444
|
+
if (isIndexTool(tName)) {
|
|
445
|
+
runtimeStats.bytesIndexed += responseBytes;
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
} catch {
|
|
449
|
+
// Non-blocking: byte tracking errors silently skipped
|
|
450
|
+
}
|
|
451
|
+
|
|
195
452
|
// Apply display overrides for built-in tools
|
|
196
453
|
const toolName = (event as any).toolName ?? "";
|
|
197
454
|
const td = config.toolDisplay;
|
|
@@ -212,6 +469,30 @@ export default function compactorExtension(pi: ExtensionAPI): void {
|
|
|
212
469
|
} catch {
|
|
213
470
|
// Non-fatal: display override failed
|
|
214
471
|
}
|
|
472
|
+
|
|
473
|
+
// Width-safe diff truncation for edit/write tool results.
|
|
474
|
+
// Pi's renderDiff() does not truncate lines to terminal width,
|
|
475
|
+
// causing TUI crashes on narrow terminals. We truncate the
|
|
476
|
+
// diff string in details.diff before it reaches the TUI.
|
|
477
|
+
const diffToolNames = ["edit", "Edit", "write", "Write"];
|
|
478
|
+
if (diffToolNames.includes(toolName)) {
|
|
479
|
+
try {
|
|
480
|
+
const details = (event as any).details as
|
|
481
|
+
{ diff?: string } | undefined;
|
|
482
|
+
if (details?.diff) {
|
|
483
|
+
const { clampDiffToWidth } = await import(
|
|
484
|
+
"./display/diff-width-safety.js"
|
|
485
|
+
);
|
|
486
|
+
const clamped = clampDiffToWidth(details.diff);
|
|
487
|
+
if (clamped !== details.diff) {
|
|
488
|
+
debug("diff_width_clamped", { toolName });
|
|
489
|
+
return { details: { ...details, diff: clamped } } as any;
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
} catch (err) {
|
|
493
|
+
debug("diff_width_clamp_error", { error: String(err) });
|
|
494
|
+
}
|
|
495
|
+
}
|
|
215
496
|
});
|
|
216
497
|
|
|
217
498
|
pi.on("message_update", async (event, _ctx) => {
|
package/src/info-screen.ts
CHANGED
|
@@ -1,60 +1,151 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Info-screen integration for @pi-unipi/compactor
|
|
3
|
+
*
|
|
4
|
+
* Budget-focused stats: tokensSaved, costSaved, pctReduction,
|
|
5
|
+
* topTools, compactions, toolCalls.
|
|
3
6
|
*/
|
|
4
7
|
|
|
5
8
|
import type { SessionDB } from "./session/db.js";
|
|
6
|
-
import type {
|
|
9
|
+
import type { RuntimeStats, FullReport } from "./session/analytics.js";
|
|
10
|
+
import { AnalyticsEngine, createMinimalDb } from "./session/analytics.js";
|
|
7
11
|
import { getLastCompactionStats } from "./compaction/hooks.js";
|
|
12
|
+
import { parseUsageStats } from "@pi-unipi/info-screen/usage-parser.js";
|
|
8
13
|
|
|
9
|
-
export interface
|
|
10
|
-
sessionEvents: { value: string; detail: string };
|
|
11
|
-
compactions: { value: string; detail: string };
|
|
14
|
+
export interface CompactorInfoData {
|
|
12
15
|
tokensSaved: { value: string; detail: string };
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
16
|
+
costSaved: { value: string; detail: string };
|
|
17
|
+
pctReduction: { value: string; detail: string };
|
|
18
|
+
topTools: { value: string; detail: string };
|
|
19
|
+
compactions: { value: string; detail: string };
|
|
20
|
+
toolCalls: { value: string; detail: string };
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** Format token count for display (e.g., "12.4k", "1.2M"). */
|
|
24
|
+
function formatTokens(n: number): string {
|
|
25
|
+
if (n < 1000) return String(n);
|
|
26
|
+
if (n < 10_000) return `${(n / 1000).toFixed(1)}k`;
|
|
27
|
+
if (n < 1_000_000) return `${Math.round(n / 1000)}k`;
|
|
28
|
+
if (n < 10_000_000) return `${(n / 1_000_000).toFixed(1)}M`;
|
|
29
|
+
return `${Math.round(n / 1_000_000)}M`;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Format cost for display (e.g., "$0.34", "<$0.01"). */
|
|
33
|
+
function formatCost(n: number): string {
|
|
34
|
+
if (n === 0) return "$0.00";
|
|
35
|
+
if (n < 0.01) return "<$0.01";
|
|
36
|
+
if (n < 1) return `$${n.toFixed(2)}`;
|
|
37
|
+
return `$${n.toFixed(2)}`;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Estimate cost per token for the most-used model in the current session. */
|
|
41
|
+
function estimateCostPerToken(): number | null {
|
|
42
|
+
try {
|
|
43
|
+
const usage = parseUsageStats();
|
|
44
|
+
// Use today's most-used model if available, otherwise all-time
|
|
45
|
+
const models = usage.byModelToday;
|
|
46
|
+
const todayKeys = Object.keys(models);
|
|
47
|
+
if (todayKeys.length > 0) {
|
|
48
|
+
// Pick the model with most tokens today
|
|
49
|
+
const topModel = todayKeys.reduce((a, b) => models[a].tokens > models[b].tokens ? a : b);
|
|
50
|
+
const entry = models[topModel];
|
|
51
|
+
if (entry.tokens > 0 && entry.cost > 0) {
|
|
52
|
+
return entry.cost / entry.tokens;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
// Fall back to all-time model data
|
|
56
|
+
const allKeys = Object.keys(usage.byModel);
|
|
57
|
+
if (allKeys.length > 0) {
|
|
58
|
+
const topModel = allKeys.reduce((a, b) => usage.byModel[a].tokens > usage.byModel[b].tokens ? a : b);
|
|
59
|
+
const entry = usage.byModel[topModel];
|
|
60
|
+
if (entry.tokens > 0 && entry.cost > 0) {
|
|
61
|
+
return entry.cost / entry.tokens;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return null;
|
|
65
|
+
} catch {
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
17
68
|
}
|
|
18
69
|
|
|
19
70
|
export async function getInfoScreenData(
|
|
20
71
|
sessionDB: SessionDB,
|
|
21
|
-
contentStore: ContentStore,
|
|
22
72
|
sessionId: string,
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
73
|
+
runtimeStats: RuntimeStats,
|
|
74
|
+
): Promise<CompactorInfoData> {
|
|
75
|
+
try {
|
|
76
|
+
const db = sessionDB.getDb();
|
|
77
|
+
const adapter = db ?? createMinimalDb();
|
|
78
|
+
const engine = new AnalyticsEngine(adapter);
|
|
79
|
+
const report = engine.queryAll(runtimeStats);
|
|
80
|
+
const compactStats = getLastCompactionStats();
|
|
81
|
+
|
|
82
|
+
// Tokens saved: bytes kept out of context / 4
|
|
83
|
+
const tokensSaved = Math.round(report.savings.kept_out / 4);
|
|
84
|
+
|
|
85
|
+
// Per-tool breakdown table for tokensSaved detail
|
|
86
|
+
const toolsWithCalls = report.savings.by_tool
|
|
87
|
+
.filter(t => t.calls > 0)
|
|
88
|
+
.sort((a, b) => b.tokens - a.tokens);
|
|
89
|
+
const toolBreakdown = toolsWithCalls.length > 0
|
|
90
|
+
? toolsWithCalls.map(t =>
|
|
91
|
+
` ${t.tool.padEnd(20)} ${String(t.calls).padStart(4)} calls ${formatTokens(t.tokens).padStart(8)} tok`
|
|
92
|
+
).join("\n")
|
|
93
|
+
: "No tool calls yet";
|
|
94
|
+
|
|
95
|
+
// Cost saved: tokensSaved × cost per token
|
|
96
|
+
const costPerToken = estimateCostPerToken();
|
|
97
|
+
const costSaved = costPerToken !== null ? tokensSaved * costPerToken : null;
|
|
98
|
+
|
|
99
|
+
// Top consuming tool
|
|
100
|
+
const topTool = toolsWithCalls[0];
|
|
101
|
+
const top5Tools = toolsWithCalls.slice(0, 5);
|
|
102
|
+
const top5Detail = top5Tools.length > 0
|
|
103
|
+
? top5Tools.map(t =>
|
|
104
|
+
`${t.tool}: ${formatTokens(t.tokens)} (${t.calls} calls)`
|
|
105
|
+
).join("\n")
|
|
106
|
+
: "No tool calls yet";
|
|
107
|
+
|
|
108
|
+
return {
|
|
109
|
+
tokensSaved: {
|
|
110
|
+
value: formatTokens(tokensSaved),
|
|
111
|
+
detail: toolBreakdown,
|
|
112
|
+
},
|
|
113
|
+
costSaved: {
|
|
114
|
+
value: costSaved !== null ? formatCost(costSaved) : "N/A",
|
|
115
|
+
detail: costSaved !== null
|
|
116
|
+
? `~${formatTokens(tokensSaved)} tokens × $${(costPerToken! * 1_000_000).toFixed(2)}/M tokens`
|
|
117
|
+
: "Cost data unavailable for current model",
|
|
118
|
+
},
|
|
119
|
+
pctReduction: {
|
|
120
|
+
value: `${report.savings.pct}%`,
|
|
121
|
+
detail: `${formatTokens(Math.round(report.savings.processed_kb * 1024 / 4))} processed → ${formatTokens(Math.round(report.savings.entered_kb * 1024 / 4))} entered context`,
|
|
122
|
+
},
|
|
123
|
+
topTools: {
|
|
124
|
+
value: topTool ? `${topTool.tool}: ${formatTokens(topTool.tokens)}` : "N/A",
|
|
125
|
+
detail: top5Detail,
|
|
126
|
+
},
|
|
127
|
+
compactions: {
|
|
128
|
+
value: String(report.continuity.compact_count),
|
|
129
|
+
detail: compactStats
|
|
130
|
+
? `Last: ${compactStats.summarized} msgs summarized, ${compactStats.kept} kept (~${formatTokens(compactStats.keptTokensEst)} tok)`
|
|
131
|
+
: report.continuity.compact_count > 0
|
|
132
|
+
? `${report.continuity.compact_count} compaction(s) this session`
|
|
133
|
+
: "No compactions yet",
|
|
134
|
+
},
|
|
135
|
+
toolCalls: {
|
|
136
|
+
value: String(report.savings.total_calls),
|
|
137
|
+
detail: `${report.savings.total_calls} total tool calls across ${toolsWithCalls.length} tool${toolsWithCalls.length !== 1 ? "s" : ""}`,
|
|
138
|
+
},
|
|
139
|
+
};
|
|
140
|
+
} catch {
|
|
141
|
+
// Never throw from dataProvider — return zeroed stats
|
|
142
|
+
return {
|
|
143
|
+
tokensSaved: { value: "0", detail: "No data" },
|
|
144
|
+
costSaved: { value: "N/A", detail: "No data" },
|
|
145
|
+
pctReduction: { value: "0%", detail: "No data" },
|
|
146
|
+
topTools: { value: "N/A", detail: "No data" },
|
|
147
|
+
compactions: { value: "0", detail: "No data" },
|
|
148
|
+
toolCalls: { value: "0", detail: "No data" },
|
|
149
|
+
};
|
|
150
|
+
}
|
|
60
151
|
}
|
package/src/security/policy.ts
CHANGED
|
@@ -2,6 +2,9 @@
|
|
|
2
2
|
* Security policy — pattern parsing, glob-to-regex
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
6
|
+
import { join } from "node:path";
|
|
7
|
+
|
|
5
8
|
export type PermissionDecision = "allow" | "deny" | "ask";
|
|
6
9
|
|
|
7
10
|
export interface SecurityPolicy {
|
|
@@ -72,3 +75,23 @@ export function fileGlobToRegex(glob: string, caseInsensitive: boolean = false):
|
|
|
72
75
|
|
|
73
76
|
return new RegExp(`^${regexStr}$`, caseInsensitive ? "i" : "");
|
|
74
77
|
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Create a minimal deny-only policy by reading .pi/settings.json in cwd.
|
|
81
|
+
* Returns a SecurityPolicy with deny patterns populated (fail-open: returns empty on error).
|
|
82
|
+
*/
|
|
83
|
+
export function readsOrCreatesPolicy(cwd: string): SecurityPolicy {
|
|
84
|
+
const settingsPath = join(cwd, ".pi", "settings.json");
|
|
85
|
+
if (!existsSync(settingsPath)) return { allow: [], deny: [], ask: [] };
|
|
86
|
+
try {
|
|
87
|
+
const settings = JSON.parse(readFileSync(settingsPath, "utf-8"));
|
|
88
|
+
const permissions = settings.permissions ?? settings.security ?? {};
|
|
89
|
+
return {
|
|
90
|
+
deny: [...(permissions.deny ?? [])],
|
|
91
|
+
ask: [...(permissions.ask ?? [])],
|
|
92
|
+
allow: [...(permissions.allow ?? [])],
|
|
93
|
+
};
|
|
94
|
+
} catch {
|
|
95
|
+
return { allow: [], deny: [], ask: [] };
|
|
96
|
+
}
|
|
97
|
+
}
|