@pi-unipi/compactor 2.6.1 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -4
- package/package.json +7 -7
- package/skills/compactor/SKILL.md +1 -1
- package/skills/compactor-detail/SKILL.md +3 -5
- package/skills/compactor-doctor/SKILL.md +1 -1
- package/skills/compactor-stats/SKILL.md +1 -1
- package/src/commands/index.ts +47 -78
- package/src/compaction/brief.ts +161 -90
- package/src/compaction/build-sections.ts +3 -4
- package/src/compaction/compact-args.ts +86 -0
- package/src/compaction/cut.ts +270 -28
- package/src/compaction/drill-down.ts +261 -0
- package/src/compaction/format-recall.ts +96 -0
- package/src/compaction/format.ts +8 -3
- package/src/compaction/hooks.ts +248 -72
- package/src/compaction/merge.ts +34 -4
- package/src/compaction/rank.ts +270 -0
- package/src/compaction/recall-scope.ts +28 -0
- package/src/compaction/search-entries.ts +333 -96
- package/src/compaction/skill-collapse.ts +35 -0
- package/src/compaction/summarize.ts +37 -6
- package/src/compaction/token-estimate.ts +104 -0
- package/src/compaction/touched-files.ts +35 -0
- package/src/config/manager.ts +2 -27
- package/src/config/presets.ts +0 -2
- package/src/config/schema.ts +2 -16
- package/src/executor/executor.ts +6 -15
- package/src/executor/runtime.ts +2 -12
- package/src/index.ts +12 -122
- package/src/info-screen.ts +3 -10
- package/src/security/evaluator.ts +0 -53
- package/src/security/policy.ts +7 -8
- package/src/session/db.ts +0 -6
- package/src/tools/ctx-execute-file.ts +0 -5
- package/src/tools/register.ts +27 -50
- package/src/tools/vcc-recall.ts +86 -48
- package/src/tui/settings-overlay.ts +20 -40
- package/src/types.ts +43 -100
- package/src/display/diff-renderer.ts +0 -281
- package/src/display/line-width-safety.ts +0 -28
- package/src/display/render-utils.ts +0 -52
- package/src/display/thinking-label.ts +0 -18
- package/src/display/tool-overrides.ts +0 -136
- package/src/tools/compact.ts +0 -20
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/** Recall output formatting (pi-vcc parity) */
|
|
2
|
+
|
|
3
|
+
import type { TouchedFile } from "./touched-files.js";
|
|
4
|
+
import type { RecallHit } from "./search-entries.js";
|
|
5
|
+
|
|
6
|
+
// ── Path shortening ───────────────────────────────────────────────────────
|
|
7
|
+
|
|
8
|
+
const CWD = process.cwd();
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Shorten an absolute file path for display:
|
|
12
|
+
* - If within cwd, return `./relative/path`
|
|
13
|
+
* - Otherwise, show last 3 path components with `.../` prefix
|
|
14
|
+
* - Short paths (≤3 components) returned as-is
|
|
15
|
+
*/
|
|
16
|
+
export function shortPath(fullPath: string): string {
|
|
17
|
+
const normalized = fullPath.replace(/\\/g, "/");
|
|
18
|
+
const cwdNormalized = CWD.replace(/\\/g, "/");
|
|
19
|
+
if (normalized.startsWith(cwdNormalized + "/")) {
|
|
20
|
+
return "." + normalized.slice(cwdNormalized.length);
|
|
21
|
+
}
|
|
22
|
+
const parts = normalized.split("/");
|
|
23
|
+
if (parts.length > 3) {
|
|
24
|
+
return ".../" + parts.slice(-3).join("/");
|
|
25
|
+
}
|
|
26
|
+
return normalized;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// ── Touched file output ───────────────────────────────────────────────────
|
|
30
|
+
|
|
31
|
+
export const TOUCHED_PAGE_SIZE = 5;
|
|
32
|
+
|
|
33
|
+
export function formatTouchedOutput(
|
|
34
|
+
touched: TouchedFile[],
|
|
35
|
+
page?: number,
|
|
36
|
+
pageSize?: number,
|
|
37
|
+
): string {
|
|
38
|
+
if (touched.length === 0) {
|
|
39
|
+
return "No file operations found in session history.";
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const ps = pageSize ?? TOUCHED_PAGE_SIZE;
|
|
43
|
+
const totalPages = Math.ceil(touched.length / ps);
|
|
44
|
+
const currentPage = Math.max(1, page ?? 1);
|
|
45
|
+
const start = (currentPage - 1) * ps;
|
|
46
|
+
const pageFiles = touched.slice(start, start + ps);
|
|
47
|
+
|
|
48
|
+
const header =
|
|
49
|
+
totalPages > 1
|
|
50
|
+
? `Page ${currentPage}/${totalPages} (${touched.length} total files)`
|
|
51
|
+
: `${touched.length} files touched`;
|
|
52
|
+
|
|
53
|
+
const lines = pageFiles.map((tf) => {
|
|
54
|
+
const displayPath = shortPath(tf.path);
|
|
55
|
+
const indices = tf.entries
|
|
56
|
+
.map((e) => `#${e.index} (${e.toolName})`)
|
|
57
|
+
.join(", ");
|
|
58
|
+
return ` ${displayPath} ${indices}`;
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
let result = `${header}:\n\n${lines.join("\n")}`;
|
|
62
|
+
|
|
63
|
+
if (currentPage < totalPages) {
|
|
64
|
+
result += `\n\n--- Use page:${currentPage + 1} for more results ---`;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
return result;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export const RECALL_PAGE_SIZE = 5;
|
|
71
|
+
|
|
72
|
+
export const formatRecallOutput = (
|
|
73
|
+
hits: RecallHit[],
|
|
74
|
+
query?: string,
|
|
75
|
+
headerOverride?: string,
|
|
76
|
+
): string => {
|
|
77
|
+
if (hits.length === 0) {
|
|
78
|
+
return query
|
|
79
|
+
? `No matches for "${query}" in session history.`
|
|
80
|
+
: "No entries in session history.";
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const header = headerOverride
|
|
84
|
+
? `${headerOverride} for "${query}":`
|
|
85
|
+
: query
|
|
86
|
+
? `Found ${hits.length} matches for "${query}":`
|
|
87
|
+
: `Session history (${hits.length} entries):`;
|
|
88
|
+
|
|
89
|
+
const lines = hits.map((h) => {
|
|
90
|
+
const fileSuffix = h.files?.length ? ` files:[${h.files.join(", ")}]` : "";
|
|
91
|
+
const body = h.snippet ?? h.text;
|
|
92
|
+
return `#${h.index} [${h.kind}]${fileSuffix} ${body}`;
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
return `${header}\n\n${lines.join("\n\n")}`;
|
|
96
|
+
};
|
package/src/compaction/format.ts
CHANGED
|
@@ -10,7 +10,7 @@ const section = (title: string, items: string[]): string => {
|
|
|
10
10
|
return `[${title}]\n${body}`;
|
|
11
11
|
};
|
|
12
12
|
|
|
13
|
-
const BRIEF_MAX_LINES = 120;
|
|
13
|
+
export const BRIEF_MAX_LINES = 120;
|
|
14
14
|
|
|
15
15
|
export const capBrief = (text: string): string => {
|
|
16
16
|
const lines = text.split("\n");
|
|
@@ -26,7 +26,12 @@ export const RECALL_NOTE =
|
|
|
26
26
|
"Use `vcc_recall` to search for prior work, decisions, and context from before this summary. " +
|
|
27
27
|
"Do not redo work already completed.";
|
|
28
28
|
|
|
29
|
-
export
|
|
29
|
+
export interface FormatSummaryOptions {
|
|
30
|
+
capBriefTranscript?: boolean;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export const formatSummary = (data: SectionData, options: FormatSummaryOptions = {}): string => {
|
|
34
|
+
const capBriefTranscript = options.capBriefTranscript ?? true;
|
|
30
35
|
const headerParts = [
|
|
31
36
|
section("Session Goal", data.sessionGoal),
|
|
32
37
|
section("Files And Changes", data.filesAndChanges),
|
|
@@ -40,7 +45,7 @@ export const formatSummary = (data: SectionData): string => {
|
|
|
40
45
|
parts.push(headerParts.join("\n\n"));
|
|
41
46
|
}
|
|
42
47
|
if (data.briefTranscript) {
|
|
43
|
-
parts.push(capBrief(data.briefTranscript));
|
|
48
|
+
parts.push(capBriefTranscript ? capBrief(data.briefTranscript) : data.briefTranscript);
|
|
44
49
|
}
|
|
45
50
|
|
|
46
51
|
if (parts.length === 0) return "";
|
package/src/compaction/hooks.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Hook integration — session_before_compact + session_compact
|
|
3
|
+
* (parity-aligned with pi-vcc before-compact.ts: token calibration, smart keep,
|
|
4
|
+
* budget-cut rescue, keep:N parsing, invisible auto-continue)
|
|
3
5
|
*/
|
|
4
6
|
|
|
5
7
|
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
@@ -10,46 +12,131 @@ import type {
|
|
|
10
12
|
SessionBeforeCompactEvent,
|
|
11
13
|
SessionCompactEvent,
|
|
12
14
|
} from "@earendil-works/pi-coding-agent";
|
|
13
|
-
import
|
|
14
|
-
import { compile } from "./summarize.js";
|
|
15
|
+
import { compileRanked } from "./summarize.js";
|
|
15
16
|
import { loadConfig } from "../config/manager.js";
|
|
16
|
-
import {
|
|
17
|
+
import {
|
|
18
|
+
buildOwnCut,
|
|
19
|
+
resolveSmartKeepUserTurns,
|
|
20
|
+
applyTailBudget,
|
|
21
|
+
MAX_SMART_TAIL_TOKENS,
|
|
22
|
+
} from "./cut.js";
|
|
23
|
+
import { parseCompactionInstructions } from "./compact-args.js";
|
|
24
|
+
import {
|
|
25
|
+
calibrateCharsPerToken,
|
|
26
|
+
estimateMessageContentChars,
|
|
27
|
+
estimateTokensFromChars,
|
|
28
|
+
} from "./token-estimate.js";
|
|
17
29
|
import type { CompactionStats } from "../types.js";
|
|
18
30
|
import type { SessionDB } from "../session/db.js";
|
|
19
31
|
|
|
20
|
-
import { COMPACTOR_INSTRUCTION } from "@pi-unipi/core";
|
|
32
|
+
import { COMPACTOR_INSTRUCTION, formatTokens } from "@pi-unipi/core";
|
|
21
33
|
|
|
22
34
|
let lastStats: CompactionStats | null = null;
|
|
23
35
|
let lastCompactWasCompactor = false;
|
|
36
|
+
let pendingFollowUpPrompt: string | null = null;
|
|
37
|
+
let pendingAutoContinueTimer: ReturnType<typeof setTimeout> | null = null;
|
|
24
38
|
export const getLastCompactionStats = () => lastStats;
|
|
39
|
+
export const consumePendingFollowUpPrompt = (): string | null => {
|
|
40
|
+
const p = pendingFollowUpPrompt;
|
|
41
|
+
pendingFollowUpPrompt = null;
|
|
42
|
+
return p;
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
// ── Invisible auto-continue (pi-vcc parity) ──────────────────────────────
|
|
46
|
+
// Resume the agent after threshold/overflow compaction without polluting the
|
|
47
|
+
// LLM context: send a custom message marked with a dedicated customType
|
|
48
|
+
// (content:[], display:false, triggerTurn:true, deliverAs:'followUp') so Pi's
|
|
49
|
+
// queue/busy-state stays coherent; the on('context') filter registered below
|
|
50
|
+
// removes that message (by customType ONLY) from the LLM payload — the model
|
|
51
|
+
// simply continues from the compaction summary.
|
|
52
|
+
export const AUTO_CONTINUE_CUSTOM_TYPE = "compactor-auto-continue";
|
|
53
|
+
|
|
54
|
+
const clearPendingAutoContinue = () => {
|
|
55
|
+
if (pendingAutoContinueTimer) {
|
|
56
|
+
clearTimeout(pendingAutoContinueTimer);
|
|
57
|
+
pendingAutoContinueTimer = null;
|
|
58
|
+
}
|
|
59
|
+
};
|
|
25
60
|
|
|
26
|
-
const
|
|
27
|
-
|
|
28
|
-
|
|
61
|
+
export const triggerInvisibleContinue = (pi: ExtensionAPI): void => {
|
|
62
|
+
pi.sendMessage(
|
|
63
|
+
{
|
|
64
|
+
customType: AUTO_CONTINUE_CUSTOM_TYPE,
|
|
65
|
+
content: [],
|
|
66
|
+
display: false,
|
|
67
|
+
details: undefined,
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
triggerTurn: true,
|
|
71
|
+
deliverAs: "followUp",
|
|
72
|
+
},
|
|
73
|
+
);
|
|
29
74
|
};
|
|
30
75
|
|
|
31
|
-
const
|
|
32
|
-
|
|
33
|
-
|
|
76
|
+
const scheduleAutoContinue = (pi: ExtensionAPI) => {
|
|
77
|
+
clearPendingAutoContinue();
|
|
78
|
+
pendingAutoContinueTimer = setTimeout(() => {
|
|
79
|
+
pendingAutoContinueTimer = null;
|
|
80
|
+
try {
|
|
81
|
+
triggerInvisibleContinue(pi);
|
|
82
|
+
} catch {}
|
|
83
|
+
}, 0);
|
|
34
84
|
};
|
|
35
85
|
|
|
36
|
-
const REASON_MESSAGES: Record<
|
|
86
|
+
const REASON_MESSAGES: Record<string, string> = {
|
|
37
87
|
no_live_messages: "compactor: Nothing to compact (no live messages)",
|
|
38
88
|
too_few_live_messages: "compactor: Too few messages to compact",
|
|
39
|
-
no_user_message: "compactor: Cannot compact — no user message found",
|
|
40
89
|
};
|
|
41
90
|
|
|
42
|
-
|
|
91
|
+
const dbg = (debug: boolean, data: Record<string, unknown>) => {
|
|
92
|
+
if (!debug) return;
|
|
93
|
+
try {
|
|
94
|
+
// Lazy import so the debug path never loads node:fs in hot paths.
|
|
95
|
+
import("node:fs").then(({ writeFileSync }) =>
|
|
96
|
+
writeFileSync("/tmp/compactor-debug.json", JSON.stringify(data, null, 2)),
|
|
97
|
+
).catch(() => {});
|
|
98
|
+
} catch {}
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
const previewContent = (content: unknown): string => {
|
|
102
|
+
if (typeof content === "string") return content.slice(0, 300);
|
|
103
|
+
if (Array.isArray(content)) {
|
|
104
|
+
return content
|
|
105
|
+
.map((c: any) => {
|
|
106
|
+
if (c?.type === "text") return c.text ?? "";
|
|
107
|
+
if (c?.type === "toolCall") return `[toolCall:${c.name}]`;
|
|
108
|
+
if (c?.type === "thinking") return `[thinking]`;
|
|
109
|
+
if (c?.type === "image") return `[image:${c.mimeType}]`;
|
|
110
|
+
return `[${c?.type ?? "unknown"}]`;
|
|
111
|
+
})
|
|
112
|
+
.join("\n")
|
|
113
|
+
.slice(0, 300);
|
|
114
|
+
}
|
|
115
|
+
return "";
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
/** Format the post-compaction toast (pi-vcc formatCompactionStats parity). */
|
|
119
|
+
export const formatCompactionStats = (stats: CompactionStats): string => {
|
|
120
|
+
if (stats.budgetCut) {
|
|
121
|
+
const reason = stats.budgetCut === "no_anchor" ? "no user anchor" : "oversized tail";
|
|
122
|
+
return `compactor: kept ~${formatTokens(stats.keptTokensEst)} tok tail (mid-turn cut, ${reason}), summarized ${stats.summarized}.`;
|
|
123
|
+
}
|
|
124
|
+
const notes: string[] = [`summarized ${stats.summarized}`];
|
|
125
|
+
if (stats.smartKeepAdjusted) {
|
|
126
|
+
notes.push("smart-keep");
|
|
127
|
+
}
|
|
128
|
+
return `compactor: kept ${stats.keptUserTurns}/${stats.totalUserTurns} turns, ~${formatTokens(stats.keptTokensEst)} tok (${notes.join(", ")}).`;
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
/** Count chars in a content part array (legacy helper, kept for stats parity). */
|
|
43
132
|
function contentPartsChars(parts: Array<{ text?: string; name?: string; input?: unknown; content?: unknown }>): number {
|
|
44
133
|
return parts.reduce((s: number, p) => {
|
|
45
134
|
if (p.text) return s + p.text.length;
|
|
46
135
|
if (p.name) {
|
|
47
|
-
// ToolCall
|
|
48
136
|
const inputStr = typeof p.input === "string" ? p.input : JSON.stringify(p.input ?? "");
|
|
49
137
|
return s + p.name.length + inputStr.length;
|
|
50
138
|
}
|
|
51
139
|
if (p.content !== undefined) {
|
|
52
|
-
// ToolResult
|
|
53
140
|
const contentStr = typeof p.content === "string" ? p.content : JSON.stringify(p.content ?? "");
|
|
54
141
|
return s + contentStr.length;
|
|
55
142
|
}
|
|
@@ -58,80 +145,135 @@ function contentPartsChars(parts: Array<{ text?: string; name?: string; input?:
|
|
|
58
145
|
}
|
|
59
146
|
|
|
60
147
|
/** Estimate char count for an AgentMessage (unwrapped — has role + content directly) */
|
|
61
|
-
function messageChars(msg:
|
|
62
|
-
const c =
|
|
148
|
+
export function messageChars(msg: { content: unknown }): number {
|
|
149
|
+
const c = msg.content;
|
|
63
150
|
if (typeof c === "string") return c.length;
|
|
64
151
|
if (Array.isArray(c)) return contentPartsChars(c as Array<{ text?: string; name?: string; input?: unknown; content?: unknown }>);
|
|
65
152
|
return 0;
|
|
66
153
|
}
|
|
67
154
|
|
|
68
|
-
/** Estimate char count for a SessionMessageEntry's message */
|
|
69
|
-
function entryMessageChars(entry: SessionMessageEntry): number {
|
|
70
|
-
return messageChars(entry.message);
|
|
71
|
-
}
|
|
72
|
-
|
|
73
155
|
/** Filter entries to only SessionMessageEntry */
|
|
74
156
|
function filterMessageEntries(entries: SessionEntry[]): SessionMessageEntry[] {
|
|
75
157
|
return entries.filter((e): e is SessionMessageEntry => e.type === "message");
|
|
76
158
|
}
|
|
77
159
|
|
|
160
|
+
const readCompactionEventContext = (event: unknown): { reason?: "manual" | "threshold" | "overflow"; willRetry: boolean } => {
|
|
161
|
+
const raw = event as { reason?: unknown; willRetry?: unknown };
|
|
162
|
+
const reason = raw.reason === "manual" || raw.reason === "threshold" || raw.reason === "overflow"
|
|
163
|
+
? raw.reason
|
|
164
|
+
: undefined;
|
|
165
|
+
return { reason, willRetry: raw.willRetry === true };
|
|
166
|
+
};
|
|
167
|
+
|
|
78
168
|
export function registerCompactionHooks(
|
|
79
169
|
pi: ExtensionAPI,
|
|
80
170
|
deps?: { getSessionDB?: () => SessionDB | null; getSessionId?: () => string },
|
|
81
171
|
): void {
|
|
172
|
+
// Filter our invisible-continue marker out of the LLM context payload so the
|
|
173
|
+
// model just continues from the compaction summary (matched by customType ONLY).
|
|
174
|
+
// This replaces the old dead sanitizer branch that read a nonexistent
|
|
175
|
+
// event.context string (prefix-cache audit finding).
|
|
176
|
+
pi.on("context", (event) => {
|
|
177
|
+
const messages = (event as { messages: Array<{ role?: string; customType?: string }> }).messages.filter((message) => {
|
|
178
|
+
if (message.role !== "custom") return true;
|
|
179
|
+
return message.customType !== AUTO_CONTINUE_CUSTOM_TYPE;
|
|
180
|
+
});
|
|
181
|
+
if (messages.length !== (event as { messages: unknown[] }).messages.length) {
|
|
182
|
+
return { messages } as any;
|
|
183
|
+
}
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
pi.on("before_agent_start", () => {
|
|
187
|
+
clearPendingAutoContinue();
|
|
188
|
+
});
|
|
189
|
+
|
|
82
190
|
pi.on("session_before_compact", (event: SessionBeforeCompactEvent, ctx) => {
|
|
83
191
|
const { preparation, branchEntries, customInstructions } = event;
|
|
84
192
|
const config = loadConfig();
|
|
85
|
-
const isCompactor
|
|
86
|
-
|
|
87
|
-
entryCount: branchEntries.length,
|
|
88
|
-
hasPrevSummary: !!preparation?.previousSummary,
|
|
89
|
-
isCompactor,
|
|
90
|
-
});
|
|
193
|
+
const { isCompactor, keepUserTurns, keepUserTurnsExplicit, followUpPrompt } =
|
|
194
|
+
parseCompactionInstructions(customInstructions);
|
|
91
195
|
|
|
196
|
+
// Always handle explicit compactor marker.
|
|
197
|
+
// Otherwise, only handle when user opted in via settings.
|
|
92
198
|
if (!isCompactor && !config.overrideDefaultCompaction) {
|
|
93
|
-
dbg(config.debug, "session_before_compact:skip", { reason: "not_compactor_and_no_override" });
|
|
94
199
|
return;
|
|
95
200
|
}
|
|
96
201
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
202
|
+
pendingFollowUpPrompt = null;
|
|
203
|
+
|
|
204
|
+
// Calibrate chars/token from Pi's real token count vs actual message chars.
|
|
205
|
+
const calibrationCut = buildOwnCut(branchEntries as any[], 0);
|
|
206
|
+
const calibrationMessageChars = calibrationCut.ok
|
|
207
|
+
? calibrationCut.messages.reduce(
|
|
208
|
+
(sum: number, message) => sum + estimateMessageContentChars((message as any).content),
|
|
209
|
+
0,
|
|
210
|
+
)
|
|
211
|
+
: 0;
|
|
212
|
+
const calibrationSummaryChars = typeof preparation.previousSummary === "string"
|
|
213
|
+
? preparation.previousSummary.length
|
|
214
|
+
: 0;
|
|
215
|
+
const tokenEstimate = calibrateCharsPerToken(
|
|
216
|
+
calibrationMessageChars + calibrationSummaryChars,
|
|
217
|
+
preparation.tokensBefore,
|
|
218
|
+
);
|
|
219
|
+
|
|
220
|
+
// Smart keep-tail: boost default keep when the tail is small.
|
|
221
|
+
// Explicit keep:N from the user is always respected (resolver no-ops).
|
|
222
|
+
const smartKeep = resolveSmartKeepUserTurns({
|
|
223
|
+
branchEntries: branchEntries as any[],
|
|
224
|
+
requestedKeepUserTurns: keepUserTurnsExplicit ? keepUserTurns : null,
|
|
225
|
+
explicit: keepUserTurnsExplicit,
|
|
226
|
+
smartKeepTail: config.smartKeepTail,
|
|
227
|
+
charsPerToken: tokenEstimate.charsPerToken,
|
|
101
228
|
});
|
|
229
|
+
let ownCut = buildOwnCut(branchEntries as any[], smartKeep.keepUserTurns);
|
|
230
|
+
// Default path only: rescue autonomous / oversized-tail sessions with a
|
|
231
|
+
// token-budget cut. Explicit keep:N is respected absolutely (no-op here).
|
|
232
|
+
if (ownCut.ok && !keepUserTurnsExplicit) {
|
|
233
|
+
ownCut = applyTailBudget(branchEntries as any[], ownCut, { charsPerToken: tokenEstimate.charsPerToken });
|
|
234
|
+
}
|
|
102
235
|
if (!ownCut.ok) {
|
|
236
|
+
if (!isCompactor && (readCompactionEventContext(event).reason === "overflow")) {
|
|
237
|
+
return; // let pi core retry
|
|
238
|
+
}
|
|
103
239
|
try {
|
|
104
|
-
ctx?.ui?.notify?.(REASON_MESSAGES[
|
|
240
|
+
ctx?.ui?.notify?.(REASON_MESSAGES[ownCut.reason] ?? ownCut.reason, "warning");
|
|
105
241
|
} catch {}
|
|
242
|
+
dbg(config.debug, { cancelled: true, reason: ownCut.reason });
|
|
106
243
|
return { cancel: true };
|
|
107
244
|
}
|
|
108
245
|
|
|
109
|
-
|
|
110
|
-
const
|
|
246
|
+
pendingFollowUpPrompt = followUpPrompt;
|
|
247
|
+
const agentMessages = ownCut.messages;
|
|
248
|
+
const firstKeptEntryId = ownCut.firstKeptEntryId;
|
|
249
|
+
const messages = convertToLlm(agentMessages as any);
|
|
111
250
|
|
|
112
|
-
//
|
|
113
|
-
const keptIdx = branchEntries.findIndex((e: SessionEntry) => e.id === firstKeptEntryId);
|
|
251
|
+
// Count kept messages and estimate tokens
|
|
252
|
+
const keptIdx = (branchEntries as SessionEntry[]).findIndex((e: SessionEntry) => e.id === firstKeptEntryId);
|
|
114
253
|
const keptMessageEntries: SessionMessageEntry[] = keptIdx >= 0
|
|
115
|
-
? filterMessageEntries(branchEntries.slice(keptIdx))
|
|
254
|
+
? filterMessageEntries((branchEntries as SessionEntry[]).slice(keptIdx))
|
|
116
255
|
: [];
|
|
256
|
+
const keptChars = keptMessageEntries.reduce(
|
|
257
|
+
(sum: number, e: SessionMessageEntry) => sum + estimateMessageContentChars((e.message as any)?.content),
|
|
258
|
+
0,
|
|
259
|
+
);
|
|
117
260
|
|
|
118
|
-
// Compute char estimates for proportional token estimation
|
|
119
|
-
const summarizedChars = agentMessages.reduce((sum, msg) => sum + messageChars(msg), 0);
|
|
120
|
-
const keptChars = keptMessageEntries.reduce((sum, e) => sum + entryMessageChars(e), 0);
|
|
121
|
-
const totalChars = summarizedChars + keptChars;
|
|
122
|
-
|
|
123
|
-
// Use Pi's real token count for "before", estimate "after" proportionally
|
|
124
261
|
const tokensBefore = preparation.tokensBefore;
|
|
125
|
-
const tokensAfterEst = totalChars > 0
|
|
126
|
-
? Math.round(tokensBefore * keptChars / totalChars)
|
|
127
|
-
: 0;
|
|
128
|
-
|
|
129
262
|
lastStats = {
|
|
130
263
|
summarized: agentMessages.length,
|
|
131
264
|
kept: keptMessageEntries.length,
|
|
132
265
|
totalMessages: agentMessages.length + keptMessageEntries.length,
|
|
133
266
|
tokensBefore,
|
|
134
|
-
tokensAfterEst,
|
|
267
|
+
tokensAfterEst: estimateTokensFromChars(keptChars, tokenEstimate.charsPerToken),
|
|
268
|
+
keptUserTurns: ownCut.keptUserTurns,
|
|
269
|
+
totalUserTurns: ownCut.totalUserTurns,
|
|
270
|
+
requestedKeepUserTurns: ownCut.requestedKeepUserTurns,
|
|
271
|
+
keepUserTurnsExplicit,
|
|
272
|
+
keepFallbackToCompactAll: ownCut.keepFallbackToCompactAll,
|
|
273
|
+
keptTokensEst: estimateTokensFromChars(keptChars, tokenEstimate.charsPerToken),
|
|
274
|
+
smartKeepAdjusted: smartKeep.smartAdjusted,
|
|
275
|
+
smartFromKeep: smartKeep.fromKeep,
|
|
276
|
+
budgetCut: ownCut.budgetCut,
|
|
135
277
|
};
|
|
136
278
|
|
|
137
279
|
// Persist cumulative compaction stats
|
|
@@ -139,41 +281,64 @@ export function registerCompactionHooks(
|
|
|
139
281
|
if (sessionDB && deps?.getSessionId) {
|
|
140
282
|
try {
|
|
141
283
|
const sessionId = deps.getSessionId();
|
|
284
|
+
const summarizedChars = agentMessages.reduce((sum, msg) => sum + messageChars(msg as any), 0);
|
|
142
285
|
sessionDB.addCompactionStats(sessionId, summarizedChars, keptChars, agentMessages.length);
|
|
143
286
|
} catch {
|
|
144
287
|
// non-fatal
|
|
145
288
|
}
|
|
146
289
|
}
|
|
147
290
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
291
|
+
// Ranked compaction: keep the highest-signal blocks under a token budget
|
|
292
|
+
// instead of the old unranked compile() (fixed 120-line cap). The token
|
|
293
|
+
// budget is converted to a char budget via the session's calibrated
|
|
294
|
+
// charsPerToken so the summary targets ~RANKED_BRIEF_BUDGET_TOKENS tokens
|
|
295
|
+
// regardless of content density. The budget is SIZE-RELATIVE: it scales
|
|
296
|
+
// with transcript length between a floor and a ceiling at
|
|
297
|
+
// RANKED_BRIEF_CHARS_PER_BLOCK per normalized block (pi-vcc parity).
|
|
298
|
+
const RANKED_BRIEF_BUDGET_TOKENS = 1100;
|
|
299
|
+
const RANKED_BRIEF_CEILING_TOKENS = 2000;
|
|
300
|
+
const RANKED_BRIEF_TOKENS_PER_BLOCK = 15;
|
|
301
|
+
const summary = compileRanked({
|
|
302
|
+
messages: messages as any,
|
|
151
303
|
previousSummary: preparation.previousSummary,
|
|
152
|
-
fileOps:
|
|
153
|
-
|
|
154
|
-
|
|
304
|
+
fileOps: preparation.fileOps
|
|
305
|
+
? {
|
|
306
|
+
readFiles: [...preparation.fileOps.read],
|
|
307
|
+
modifiedFiles: [...preparation.fileOps.written, ...preparation.fileOps.edited],
|
|
308
|
+
}
|
|
309
|
+
: undefined,
|
|
310
|
+
ranking: {
|
|
311
|
+
maxBriefChars: Math.round(RANKED_BRIEF_BUDGET_TOKENS * tokenEstimate.charsPerToken),
|
|
312
|
+
maxBriefCharsCeiling: Math.round(RANKED_BRIEF_CEILING_TOKENS * tokenEstimate.charsPerToken),
|
|
313
|
+
briefCharsPerBlock: Math.round(RANKED_BRIEF_TOKENS_PER_BLOCK * tokenEstimate.charsPerToken),
|
|
155
314
|
},
|
|
156
315
|
});
|
|
157
316
|
|
|
158
|
-
dbg(config.debug, "compaction_pipeline", {
|
|
159
|
-
usedOwnCut: true,
|
|
160
|
-
messagesToSummarize: agentMessages.length,
|
|
161
|
-
firstKeptEntryId,
|
|
162
|
-
tokensBefore: preparation.tokensBefore,
|
|
163
|
-
summaryLength: summary.length,
|
|
164
|
-
sections: [...summary.matchAll(/^\[(.+?)\]/gm)].map((m) => m[1]),
|
|
165
|
-
});
|
|
166
|
-
|
|
167
317
|
const details = {
|
|
168
318
|
compactor: "@pi-unipi/compactor",
|
|
169
319
|
version: 1,
|
|
170
320
|
sections: [...summary.matchAll(/^\[(.+?)\]/gm)].map((m) => m[1]),
|
|
171
321
|
sourceMessageCount: agentMessages.length,
|
|
172
322
|
previousSummaryUsed: Boolean(preparation.previousSummary),
|
|
323
|
+
reason: readCompactionEventContext(event).reason,
|
|
324
|
+
budgetCut: ownCut.budgetCut,
|
|
173
325
|
};
|
|
174
326
|
|
|
175
327
|
lastCompactWasCompactor = isCompactor;
|
|
176
328
|
|
|
329
|
+
dbg(config.debug, {
|
|
330
|
+
usedOwnCut: true,
|
|
331
|
+
budgetCut: ownCut.budgetCut,
|
|
332
|
+
messagesToSummarize: agentMessages.length,
|
|
333
|
+
firstKeptEntryId,
|
|
334
|
+
tokensBefore,
|
|
335
|
+
tokenEstimate,
|
|
336
|
+
smartKeep,
|
|
337
|
+
summaryLength: summary.length,
|
|
338
|
+
summaryPreview: summary.slice(0, 500),
|
|
339
|
+
sections: details.sections,
|
|
340
|
+
});
|
|
341
|
+
|
|
177
342
|
return {
|
|
178
343
|
compaction: {
|
|
179
344
|
summary,
|
|
@@ -185,19 +350,30 @@ export function registerCompactionHooks(
|
|
|
185
350
|
});
|
|
186
351
|
|
|
187
352
|
pi.on("session_compact", (event: SessionCompactEvent, ctx) => {
|
|
188
|
-
const config = loadConfig();
|
|
189
|
-
dbg(config.debug, "session_compact", { fromExtension: event.fromExtension, lastCompactWasCompactor });
|
|
190
353
|
if (!event.fromExtension) return;
|
|
191
|
-
if (lastCompactWasCompactor) return;
|
|
354
|
+
if (lastCompactWasCompactor) return; // /unipi:compact handles its own toast
|
|
192
355
|
const stats = lastStats;
|
|
193
356
|
if (!stats) return;
|
|
357
|
+
const { reason, willRetry } = readCompactionEventContext(event);
|
|
358
|
+
if (willRetry) return;
|
|
359
|
+
const followUpPrompt = consumePendingFollowUpPrompt();
|
|
360
|
+
const shouldContinueAfterAutoCompact =
|
|
361
|
+
(reason === "threshold" || reason === "overflow") && loadConfig().continueAfterThresholdCompact;
|
|
194
362
|
setTimeout(() => {
|
|
195
363
|
try {
|
|
196
|
-
ctx?.ui?.notify?.(
|
|
197
|
-
`Compacted ${stats.totalMessages} messages (~${formatTokens(stats.tokensBefore)} tokens) → ${stats.kept} messages (~${formatTokens(stats.tokensAfterEst)} tokens)`,
|
|
198
|
-
"info",
|
|
199
|
-
);
|
|
364
|
+
ctx?.ui?.notify?.(formatCompactionStats(stats), "info");
|
|
200
365
|
} catch {}
|
|
201
366
|
}, 500);
|
|
367
|
+
if (followUpPrompt) {
|
|
368
|
+
setTimeout(() => {
|
|
369
|
+
try {
|
|
370
|
+
void pi.sendUserMessage(followUpPrompt);
|
|
371
|
+
} catch {}
|
|
372
|
+
}, 0);
|
|
373
|
+
} else if (shouldContinueAfterAutoCompact) {
|
|
374
|
+
scheduleAutoContinue(pi);
|
|
375
|
+
}
|
|
202
376
|
});
|
|
203
377
|
}
|
|
378
|
+
|
|
379
|
+
export { MAX_SMART_TAIL_TOKENS };
|
package/src/compaction/merge.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Stage 6: Merge — Merge with previous summary, dedup, rolling window
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
|
-
import { capBrief } from "./format.js";
|
|
5
|
+
import { capBrief, BRIEF_MAX_LINES } from "./format.js";
|
|
6
6
|
|
|
7
7
|
const HEADER_NAMES = ["Session Goal", "Files And Changes", "Commits", "Outstanding Context", "User Preferences"];
|
|
8
8
|
const SEPARATOR = "\n\n---\n\n";
|
|
@@ -88,7 +88,35 @@ const mergeBriefTranscript = (prev: string, fresh: string): string => {
|
|
|
88
88
|
return prev + "\n\n" + fresh;
|
|
89
89
|
};
|
|
90
90
|
|
|
91
|
-
|
|
91
|
+
const briefLineCount = (text: string): number =>
|
|
92
|
+
text ? text.split("\n").length : 0;
|
|
93
|
+
|
|
94
|
+
const capBriefToLineBudget = (text: string, maxLines: number): string => {
|
|
95
|
+
if (!text || maxLines <= 0) return "";
|
|
96
|
+
const lines = text.split("\n");
|
|
97
|
+
if (lines.length <= maxLines) return text;
|
|
98
|
+
const kept = lines.slice(-maxLines);
|
|
99
|
+
const firstHeader = kept.findIndex((l) => /^\[.+\]/.test(l));
|
|
100
|
+
const clean = firstHeader > 0 ? kept.slice(firstHeader) : kept;
|
|
101
|
+
const omitted = lines.length - clean.length;
|
|
102
|
+
return `...(${omitted} earlier lines omitted)\n\n${clean.join("\n")}`;
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
/** Fresh brief gets the line budget; previous brief fills the remainder (pi-vcc parity) */
|
|
106
|
+
export const mergeBriefTranscriptWithFreshBudget = (prev: string, fresh: string): string => {
|
|
107
|
+
if (!prev) return fresh;
|
|
108
|
+
if (!fresh) return capBrief(prev);
|
|
109
|
+
const freshLines = briefLineCount(fresh);
|
|
110
|
+
const remainingPrevLines = Math.max(0, BRIEF_MAX_LINES - freshLines);
|
|
111
|
+
const prevTail = capBriefToLineBudget(prev, remainingPrevLines);
|
|
112
|
+
return prevTail ? `${prevTail}\n\n${fresh}` : fresh;
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
export const mergePrevious = (
|
|
116
|
+
prev: string,
|
|
117
|
+
fresh: string,
|
|
118
|
+
options: { preserveFreshBrief?: boolean } = {},
|
|
119
|
+
): string => {
|
|
92
120
|
const headers = HEADER_NAMES
|
|
93
121
|
.map((header) => {
|
|
94
122
|
const freshSec = sectionOf(fresh, header);
|
|
@@ -99,14 +127,16 @@ export const mergePrevious = (prev: string, fresh: string): string => {
|
|
|
99
127
|
|
|
100
128
|
const prevBrief = briefOf(prev);
|
|
101
129
|
const freshBrief = briefOf(fresh);
|
|
102
|
-
const mergedBrief =
|
|
130
|
+
const mergedBrief = options.preserveFreshBrief
|
|
131
|
+
? mergeBriefTranscriptWithFreshBudget(prevBrief, freshBrief)
|
|
132
|
+
: mergeBriefTranscript(prevBrief, freshBrief);
|
|
103
133
|
|
|
104
134
|
const parts: string[] = [];
|
|
105
135
|
if (headers.length > 0) {
|
|
106
136
|
parts.push(headers.join("\n\n"));
|
|
107
137
|
}
|
|
108
138
|
if (mergedBrief) {
|
|
109
|
-
parts.push(capBrief(mergedBrief));
|
|
139
|
+
parts.push(options.preserveFreshBrief ? mergedBrief : capBrief(mergedBrief));
|
|
110
140
|
}
|
|
111
141
|
|
|
112
142
|
return parts.join(SEPARATOR);
|