@danypops/pi-jittor 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -3
- package/docs/USAGE_PRIOR_ART.md +1 -1
- package/extension/src/index.ts +173 -43
- package/extension/src/{context-breakdown.ts → observability/context-breakdown.ts} +200 -27
- package/extension/src/observability/context-growth.ts +26 -0
- package/extension/src/{context-report.ts → observability/context-report.ts} +35 -3
- package/extension/src/observability/context-view.ts +264 -0
- package/extension/src/{footer.ts → observability/footer.ts} +7 -0
- package/extension/src/observability/provider-context-snapshot.ts +246 -0
- package/extension/src/{capabilities/provider-response-telemetry.ts → observability/provider-response.ts} +1 -1
- package/extension/src/{tui.ts → observability/status.ts} +55 -10
- package/extension/src/{usage.ts → observability/usage.ts} +74 -166
- package/extension/src/{benchmark-tui.ts → optimization/model-selection-panel.ts} +61 -29
- package/extension/src/{capabilities/codex-recovery.ts → optimization/recovery/codex.ts} +2 -2
- package/extension/src/service-client.ts +48 -1
- package/extension/src/settings-tui.ts +57 -32
- package/extension/src/settings.ts +27 -19
- package/package.json +9 -3
- package/extension/src/context-view.ts +0 -140
- /package/extension/src/{capabilities → observability}/context-hub.ts +0 -0
- /package/extension/src/{capabilities → observability}/http-headers.ts +0 -0
- /package/extension/src/{capabilities/local-run-telemetry.ts → observability/model-run.ts} +0 -0
|
@@ -4,6 +4,12 @@ import {
|
|
|
4
4
|
CONTEXT_TREE_MAX_NODES,
|
|
5
5
|
type ContextSegment,
|
|
6
6
|
type ContextSegmentItem,
|
|
7
|
+
countTextWithFallback,
|
|
8
|
+
type RequestTokenReconciliation,
|
|
9
|
+
reconcileRequestTokens,
|
|
10
|
+
StructuralTextTokenCounter,
|
|
11
|
+
type TextTokenCounter,
|
|
12
|
+
type TokenMeasurement,
|
|
7
13
|
} from "@danypops/jittor";
|
|
8
14
|
import type { BuildSystemPromptOptions } from "@earendil-works/pi-coding-agent";
|
|
9
15
|
|
|
@@ -32,29 +38,108 @@ export interface SessionTreeNodeLike {
|
|
|
32
38
|
children: SessionTreeNodeLike[];
|
|
33
39
|
}
|
|
34
40
|
|
|
35
|
-
|
|
36
|
-
|
|
41
|
+
interface MessageContentAnalysis {
|
|
42
|
+
characters: number;
|
|
43
|
+
text: string;
|
|
44
|
+
items: ContextSegmentItem[];
|
|
45
|
+
imageCount: number;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export interface ContextTokenMeasurementOptions {
|
|
49
|
+
provider?: string;
|
|
50
|
+
model?: string;
|
|
51
|
+
counters?: readonly TextTokenCounter[];
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const STRUCTURAL_TEXT_COUNTER = new StructuralTextTokenCounter();
|
|
55
|
+
|
|
56
|
+
function measurementForText(text: string, options: ContextTokenMeasurementOptions = {}): TokenMeasurement {
|
|
57
|
+
return countTextWithFallback(
|
|
58
|
+
{
|
|
59
|
+
text,
|
|
60
|
+
scope: "context-item",
|
|
61
|
+
...(options.provider === undefined ? {} : { provider: options.provider }),
|
|
62
|
+
...(options.model === undefined ? {} : { model: options.model }),
|
|
63
|
+
},
|
|
64
|
+
options.counters ?? [],
|
|
65
|
+
STRUCTURAL_TEXT_COUNTER,
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function measuredItem(label: string, textOrCharacters: string | number, options: ContextTokenMeasurementOptions = {}): ContextSegmentItem {
|
|
70
|
+
const characters = typeof textOrCharacters === "string" ? textOrCharacters.length : textOrCharacters;
|
|
71
|
+
const measurement =
|
|
72
|
+
typeof textOrCharacters === "string"
|
|
73
|
+
? measurementForText(textOrCharacters, options)
|
|
74
|
+
: {
|
|
75
|
+
tokens: toCeilTokens(characters),
|
|
76
|
+
scope: "context-item" as const,
|
|
77
|
+
provenance: "structural-estimate" as const,
|
|
78
|
+
method: `char/${CONTEXT_ESTIMATE_CHARACTERS_PER_TOKEN}`,
|
|
79
|
+
};
|
|
80
|
+
const suffix =
|
|
81
|
+
measurement.provenance === "structural-estimate"
|
|
82
|
+
? `${characters.toLocaleString()} chars (≈ char/4)`
|
|
83
|
+
: `${measurement.tokens.toLocaleString()} tok (${measurement.method}; exact text)`;
|
|
84
|
+
return { label: `${label} · ${suffix}`, estimatedTokens: measurement.tokens, measurement };
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Breaks a message into Pi's public content fields and attaches explicit exact-text or structural provenance without retaining content. */
|
|
88
|
+
function analyzeMessageContent(message: unknown, options: ContextTokenMeasurementOptions = {}): MessageContentAnalysis {
|
|
89
|
+
if (typeof message !== "object" || message === null) return { characters: 0, text: "", items: [], imageCount: 0 };
|
|
37
90
|
const record = message as Record<string, unknown>;
|
|
91
|
+
const effectiveOptions = options;
|
|
38
92
|
if (record.role === "bashExecution") {
|
|
39
93
|
// Pi's own context builder excludes "!!"-prefixed bash output from context; match that.
|
|
40
|
-
if (record.excludeFromContext === true) return 0;
|
|
41
|
-
|
|
94
|
+
if (record.excludeFromContext === true) return { characters: 0, text: "", items: [], imageCount: 0 };
|
|
95
|
+
const command = String(record.command ?? "");
|
|
96
|
+
const output = String(record.output ?? "");
|
|
97
|
+
return {
|
|
98
|
+
characters: command.length + output.length,
|
|
99
|
+
text: command + output,
|
|
100
|
+
items: [measuredItem("command", command, effectiveOptions), measuredItem("output", output, effectiveOptions)].filter(
|
|
101
|
+
(item) => item.estimatedTokens > 0,
|
|
102
|
+
),
|
|
103
|
+
imageCount: 0,
|
|
104
|
+
};
|
|
42
105
|
}
|
|
43
106
|
const content = record.content;
|
|
44
|
-
if (typeof content === "string")
|
|
45
|
-
|
|
107
|
+
if (typeof content === "string")
|
|
108
|
+
return {
|
|
109
|
+
characters: content.length,
|
|
110
|
+
text: content,
|
|
111
|
+
items: [measuredItem("text", content, effectiveOptions)],
|
|
112
|
+
imageCount: 0,
|
|
113
|
+
};
|
|
114
|
+
if (!Array.isArray(content)) return { characters: 0, text: "", items: [], imageCount: 0 };
|
|
46
115
|
let characters = 0;
|
|
47
|
-
|
|
116
|
+
let imageCount = 0;
|
|
117
|
+
let text = "";
|
|
118
|
+
const items: ContextSegmentItem[] = [];
|
|
119
|
+
for (let index = 0; index < content.length; index++) {
|
|
120
|
+
const block = content[index];
|
|
48
121
|
if (typeof block !== "object" || block === null) continue;
|
|
49
122
|
const b = block as Record<string, unknown>;
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
123
|
+
let blockText = "";
|
|
124
|
+
let label = `block ${index + 1}`;
|
|
125
|
+
if (b.type === "text") {
|
|
126
|
+
blockText = String(b.text ?? "");
|
|
127
|
+
label = "text";
|
|
128
|
+
} else if (b.type === "thinking") {
|
|
129
|
+
blockText = String(b.thinking ?? "");
|
|
130
|
+
label = "thinking";
|
|
131
|
+
} else if (b.type === "toolCall") {
|
|
132
|
+
blockText = JSON.stringify(b.arguments ?? {});
|
|
133
|
+
label = `tool call ${String(b.name ?? "(unknown)")} arguments`;
|
|
134
|
+
} else if (b.type === "image") {
|
|
135
|
+
imageCount += 1;
|
|
136
|
+
continue; // image tokens are provider/model-specific and cannot be derived from base64 characters
|
|
137
|
+
} else continue;
|
|
138
|
+
characters += blockText.length;
|
|
139
|
+
text += blockText;
|
|
140
|
+
if (blockText.length > 0) items.push(measuredItem(label, blockText, effectiveOptions));
|
|
56
141
|
}
|
|
57
|
-
return characters;
|
|
142
|
+
return { characters, text, items, imageCount };
|
|
58
143
|
}
|
|
59
144
|
|
|
60
145
|
function messageSnippet(message: unknown, maxLength = 48): string {
|
|
@@ -78,13 +163,55 @@ function messageSnippet(message: unknown, maxLength = 48): string {
|
|
|
78
163
|
return collapsed.length > maxLength ? `${collapsed.slice(0, maxLength - 1)}…` : collapsed;
|
|
79
164
|
}
|
|
80
165
|
|
|
81
|
-
|
|
166
|
+
interface ProviderPromptUsage {
|
|
167
|
+
label: string;
|
|
168
|
+
reconciliation: RequestTokenReconciliation;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
function providerPromptUsage(message: unknown): ProviderPromptUsage | null {
|
|
172
|
+
if (typeof message !== "object" || message === null) return null;
|
|
173
|
+
const messageRecord = message as Record<string, unknown>;
|
|
174
|
+
const usage = messageRecord.usage;
|
|
175
|
+
if (typeof usage !== "object" || usage === null) return null;
|
|
176
|
+
const record = usage as Record<string, unknown>;
|
|
177
|
+
const amount = (value: unknown): number => (typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0);
|
|
178
|
+
const input = amount(record.input);
|
|
179
|
+
const cacheRead = amount(record.cacheRead);
|
|
180
|
+
const cacheWrite = amount(record.cacheWrite);
|
|
181
|
+
const promptTokens = input + cacheRead + cacheWrite;
|
|
182
|
+
if (!Number.isSafeInteger(promptTokens) || promptTokens <= 0) return null;
|
|
183
|
+
const identity =
|
|
184
|
+
typeof messageRecord.provider === "string" && typeof messageRecord.model === "string"
|
|
185
|
+
? { provider: messageRecord.provider, model: messageRecord.model }
|
|
186
|
+
: {};
|
|
187
|
+
const reconciliation = reconcileRequestTokens(
|
|
188
|
+
{
|
|
189
|
+
tokens: promptTokens,
|
|
190
|
+
scope: "request-context",
|
|
191
|
+
provenance: "provider-reported",
|
|
192
|
+
method: "pi-assistant-usage",
|
|
193
|
+
...identity,
|
|
194
|
+
},
|
|
195
|
+
[],
|
|
196
|
+
);
|
|
197
|
+
const cache =
|
|
198
|
+
cacheRead > 0 || cacheWrite > 0
|
|
199
|
+
? `; new ${input.toLocaleString()}, cache read ${cacheRead.toLocaleString()}, write ${cacheWrite.toLocaleString()}`
|
|
200
|
+
: "";
|
|
201
|
+
return {
|
|
202
|
+
label: ` · provider-reported request context ${promptTokens.toLocaleString()} tok${cache} · unattributed residual ${reconciliation.residual.tokens.toLocaleString()} tok`,
|
|
203
|
+
reconciliation,
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function entryLabel(entry: SessionEntryLike, imageCount = 0, providerUsage = ""): string {
|
|
82
208
|
if (entry.type === "compaction") return "compaction summary";
|
|
83
209
|
if (entry.type === "branch_summary") return "branch summary";
|
|
84
210
|
const role = typeof entry.message === "object" && entry.message !== null ? (entry.message as Record<string, unknown>).role : undefined;
|
|
85
211
|
const prefix = typeof role === "string" ? role : entry.type;
|
|
86
212
|
const snippet = messageSnippet(entry.message);
|
|
87
|
-
|
|
213
|
+
const images = imageCount > 0 ? ` · ${imageCount} image${imageCount === 1 ? "" : "s"} (token cost unavailable)` : "";
|
|
214
|
+
return `${snippet ? `${prefix}: ${snippet}` : prefix}${providerUsage}${images}`;
|
|
88
215
|
}
|
|
89
216
|
|
|
90
217
|
export interface MessageHistoryTree {
|
|
@@ -130,6 +257,7 @@ export function buildMessageHistoryTree(
|
|
|
130
257
|
roots: ReadonlyArray<SessionTreeNodeLike>,
|
|
131
258
|
activeEntryIds: ReadonlySet<string>,
|
|
132
259
|
branchEntryIds?: ReadonlySet<string>,
|
|
260
|
+
measurementOptions: ContextTokenMeasurementOptions = {},
|
|
133
261
|
): MessageHistoryTree {
|
|
134
262
|
const visited = new Set<string>();
|
|
135
263
|
let truncated = false;
|
|
@@ -160,23 +288,35 @@ export function buildMessageHistoryTree(
|
|
|
160
288
|
for (let index = order.length - 1; index >= 0; index--) {
|
|
161
289
|
const frame = order[index]!;
|
|
162
290
|
const entry = frame.node.entry;
|
|
163
|
-
const
|
|
291
|
+
const entryMeasurementOptions = measurementOptions;
|
|
292
|
+
const analysis =
|
|
293
|
+
entry.type === "message"
|
|
294
|
+
? analyzeMessageContent(entry.message, measurementOptions)
|
|
295
|
+
: { characters: 0, text: "", items: [], imageCount: 0 };
|
|
296
|
+
const text =
|
|
164
297
|
entry.type === "message"
|
|
165
|
-
?
|
|
298
|
+
? analysis.text
|
|
166
299
|
: entry.type === "compaction" || entry.type === "branch_summary"
|
|
167
|
-
? (entry.summary ?? "")
|
|
168
|
-
:
|
|
169
|
-
const
|
|
300
|
+
? (entry.summary ?? "")
|
|
301
|
+
: "";
|
|
302
|
+
const measurement = measurementForText(text, entryMeasurementOptions);
|
|
303
|
+
const tokens = measurement.tokens;
|
|
170
304
|
const isActive = activeEntryIds.has(entry.id);
|
|
171
305
|
if (isActive) activeTokens += tokens;
|
|
172
306
|
const isOnBranch = branchEntryIds ? branchEntryIds.has(entry.id) : isActive; // no branch set given -- fall back to the old binary active/inactive-branch label
|
|
173
307
|
|
|
174
|
-
const
|
|
175
|
-
|
|
308
|
+
const treeChildren = childItemsByParent.get(index) ?? [];
|
|
309
|
+
const contentChildren = analysis.items;
|
|
310
|
+
const children = [...contentChildren, ...treeChildren];
|
|
311
|
+
if (tokens === 0 && children.length === 0 && analysis.imageCount === 0) continue; // no content, no descendants, and no unknown-cost image -- nothing to show
|
|
176
312
|
|
|
313
|
+
const promptUsage = providerPromptUsage(entry.message);
|
|
314
|
+
const label = entryLabel(entry, analysis.imageCount, promptUsage?.label);
|
|
177
315
|
const item: ContextSegmentItem = {
|
|
178
|
-
label: isActive ?
|
|
316
|
+
label: isActive ? label : isOnBranch ? `${label} (compacted)` : `${label} (inactive branch)`,
|
|
179
317
|
estimatedTokens: tokens,
|
|
318
|
+
measurement,
|
|
319
|
+
...(promptUsage ? { requestTokenReconciliation: promptUsage.reconciliation } : {}),
|
|
180
320
|
...(children.length > 0 ? { children } : {}),
|
|
181
321
|
};
|
|
182
322
|
itemByIndex.set(index, item);
|
|
@@ -228,9 +368,14 @@ export function buildBasePromptItems(options: BuildSystemPromptOptions, totalCha
|
|
|
228
368
|
const toolSnippetEntries = Object.entries(options.toolSnippets ?? {});
|
|
229
369
|
// Mirrors buildSystemPrompt()'s own "- name: snippet\n" line shape closely enough to be a
|
|
230
370
|
// fair estimate without importing Pi-internal formatting code.
|
|
371
|
+
const toolSnippetDetails = toolSnippetEntries.map(([name, snippet]) => measuredItem(name, name.length + snippet.length + 4));
|
|
231
372
|
const toolSnippetsCharacters = toolSnippetEntries.reduce((sum, [name, snippet]) => sum + name.length + snippet.length + 4, 0);
|
|
232
373
|
if (toolSnippetsCharacters > 0) {
|
|
233
|
-
items.push({
|
|
374
|
+
items.push({
|
|
375
|
+
label: `Tool snippets (${toolSnippetEntries.length} tools)`,
|
|
376
|
+
estimatedTokens: toCeilTokens(toolSnippetsCharacters),
|
|
377
|
+
children: toolSnippetDetails,
|
|
378
|
+
});
|
|
234
379
|
}
|
|
235
380
|
|
|
236
381
|
const visibleSkills = (options.skills ?? []).filter((skill) => !skill.disableModelInvocation);
|
|
@@ -239,7 +384,13 @@ export function buildBasePromptItems(options: BuildSystemPromptOptions, totalCha
|
|
|
239
384
|
0,
|
|
240
385
|
);
|
|
241
386
|
if (skillsCharacters > 0) {
|
|
242
|
-
items.push({
|
|
387
|
+
items.push({
|
|
388
|
+
label: `Skills catalog (${visibleSkills.length} skills)`,
|
|
389
|
+
estimatedTokens: toCeilTokens(skillsCharacters),
|
|
390
|
+
children: visibleSkills.map((skill) =>
|
|
391
|
+
measuredItem(skill.name, skill.name.length + skill.description.length + skill.filePath.length + 20),
|
|
392
|
+
),
|
|
393
|
+
});
|
|
243
394
|
}
|
|
244
395
|
|
|
245
396
|
const contextFiles = options.contextFiles ?? [];
|
|
@@ -248,10 +399,32 @@ export function buildBasePromptItems(options: BuildSystemPromptOptions, totalCha
|
|
|
248
399
|
items.push({
|
|
249
400
|
label: `Project context files (${contextFiles.length}, e.g. AGENTS.md)`,
|
|
250
401
|
estimatedTokens: toCeilTokens(contextFilesCharacters),
|
|
402
|
+
children: contextFiles.map((file) => measuredItem(file.path, file.path.length + file.content.length + 40)),
|
|
251
403
|
});
|
|
252
404
|
}
|
|
253
405
|
|
|
254
|
-
const
|
|
406
|
+
const promptGuidelines = options.promptGuidelines ?? [];
|
|
407
|
+
const promptGuidelinesCharacters = promptGuidelines.reduce((sum, guideline) => sum + guideline.length + 2, 0);
|
|
408
|
+
if (promptGuidelinesCharacters > 0) {
|
|
409
|
+
items.push({
|
|
410
|
+
label: `Tool guidelines (${promptGuidelines.length})`,
|
|
411
|
+
estimatedTokens: toCeilTokens(promptGuidelinesCharacters),
|
|
412
|
+
children: promptGuidelines.map((guideline, index) => measuredItem(`guideline ${index + 1}`, guideline.length + 2)),
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
const customPromptCharacters = options.customPrompt?.length ?? 0;
|
|
417
|
+
if (customPromptCharacters > 0) items.push(measuredItem("Custom system prompt", customPromptCharacters));
|
|
418
|
+
const appendedPromptCharacters = options.appendSystemPrompt?.length ?? 0;
|
|
419
|
+
if (appendedPromptCharacters > 0) items.push(measuredItem("Appended system prompt", appendedPromptCharacters));
|
|
420
|
+
|
|
421
|
+
const knownCharacters =
|
|
422
|
+
toolSnippetsCharacters +
|
|
423
|
+
skillsCharacters +
|
|
424
|
+
contextFilesCharacters +
|
|
425
|
+
promptGuidelinesCharacters +
|
|
426
|
+
customPromptCharacters +
|
|
427
|
+
appendedPromptCharacters;
|
|
255
428
|
const remainderCharacters = Math.max(0, totalCharacters - knownCharacters);
|
|
256
429
|
if (remainderCharacters > 0 || items.length === 0) {
|
|
257
430
|
items.push({ label: "Base template, guidelines, and formatting", estimatedTokens: toCeilTokens(remainderCharacters) });
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export interface ContextGrowthPoint {
|
|
2
|
+
turn: number;
|
|
3
|
+
tokens: number;
|
|
4
|
+
}
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Owns the current post-compaction context-growth observation window. Trend fitting and a
|
|
8
|
+
* trailing-size cap deliberately remain outside this task; reset() is the critical invariant
|
|
9
|
+
* that prevents one regression window from spanning a real compaction discontinuity.
|
|
10
|
+
*/
|
|
11
|
+
export class ContextGrowthCapability {
|
|
12
|
+
private points: ContextGrowthPoint[] = [];
|
|
13
|
+
|
|
14
|
+
observe(turn: number, tokens: number): void {
|
|
15
|
+
if (!Number.isInteger(turn) || turn < 0 || !Number.isFinite(tokens) || tokens < 0) return;
|
|
16
|
+
this.points.push({ turn, tokens });
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
observations(): readonly ContextGrowthPoint[] {
|
|
20
|
+
return this.points.map((point) => ({ ...point }));
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
reset(): void {
|
|
24
|
+
this.points = [];
|
|
25
|
+
}
|
|
26
|
+
}
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import type { ContextSegment } from "@danypops/jittor";
|
|
2
|
-
import {
|
|
2
|
+
import type { ContextRow, ContextSegment as MalevichContextSegment } from "malevich-tui-components";
|
|
3
3
|
import type { ContextBreakdown } from "./context-breakdown.ts";
|
|
4
4
|
|
|
5
5
|
/** Bounds how many items render per segment in the plain-text fallback -- a notify-mode report is a scan-at-a-glance summary, not a full dump (the interactive TUI view has no such cap, since it scrolls). */
|
|
6
6
|
const MAX_ITEMS_PER_SEGMENT_LINE = 5;
|
|
7
|
+
const MAX_REPORT_ROWS = 200;
|
|
7
8
|
|
|
8
9
|
function formatTokens(tokens: number): string {
|
|
9
10
|
return tokens >= 1_000 ? `${(tokens / 1_000).toFixed(1)}k` : String(tokens);
|
|
@@ -13,6 +14,34 @@ function percentOf(part: number, whole: number): string {
|
|
|
13
14
|
return whole > 0 ? `${((part / whole) * 100).toFixed(1)}%` : "—";
|
|
14
15
|
}
|
|
15
16
|
|
|
17
|
+
/** Malevich-compatible row projection with an explicit stack, safe for long linear session trees. */
|
|
18
|
+
export function buildContextRowsIterative(segments: readonly MalevichContextSegment[], totalTokens?: number | null): ContextRow[] {
|
|
19
|
+
const rows: ContextRow[] = [];
|
|
20
|
+
const denominator = totalTokens ?? segments.reduce((sum, segment) => sum + segment.estimatedTokens, 0);
|
|
21
|
+
for (const segment of segments) {
|
|
22
|
+
const items = [...(segment.items ?? [])]
|
|
23
|
+
.filter((item) => item.estimatedTokens > 0)
|
|
24
|
+
.sort((a, b) => b.estimatedTokens - a.estimatedTokens);
|
|
25
|
+
if (segment.estimatedTokens <= 0 && items.length === 0 && !segment.unknown) continue;
|
|
26
|
+
rows.push({
|
|
27
|
+
key: segment.key,
|
|
28
|
+
isHeader: true,
|
|
29
|
+
depth: 0,
|
|
30
|
+
text: `${segment.label} — ${segment.estimatedTokens} tok (${percentOf(segment.estimatedTokens, denominator)})`,
|
|
31
|
+
});
|
|
32
|
+
const stack = items.reverse().map((item) => ({ item, depth: 1 }));
|
|
33
|
+
while (stack.length > 0) {
|
|
34
|
+
const { item, depth } = stack.pop()!;
|
|
35
|
+
rows.push({ key: segment.key, isHeader: false, depth, text: `${item.estimatedTokens.toString().padStart(6)} tok ${item.label}` });
|
|
36
|
+
const children = [...(item.children ?? [])]
|
|
37
|
+
.filter((child) => child.estimatedTokens > 0)
|
|
38
|
+
.sort((a, b) => b.estimatedTokens - a.estimatedTokens);
|
|
39
|
+
for (let index = children.length - 1; index >= 0; index--) stack.push({ item: children[index]!, depth: depth + 1 });
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
return rows;
|
|
43
|
+
}
|
|
44
|
+
|
|
16
45
|
/** Malevich's row builder is confidence-unaware (it's a generic segment/item shape); folding the tier into the label is how it survives into the rendered row text, e.g. "Active Rules [exact-cooperative]". */
|
|
17
46
|
function withConfidenceLabel(segment: ContextSegment): MalevichContextSegment {
|
|
18
47
|
const items = [...(segment.items ?? [])]
|
|
@@ -49,12 +78,15 @@ export function buildContextReport(breakdown: ContextBreakdown): string {
|
|
|
49
78
|
lines.push(`Estimates exceed real total by ~${breakdown.overshootTokens} tok -- sizes below are approximate, not exact`);
|
|
50
79
|
|
|
51
80
|
const sorted = [...breakdown.segments].sort((left, right) => right.estimatedTokens - left.estimatedTokens).map(withConfidenceLabel);
|
|
52
|
-
const rows =
|
|
81
|
+
const rows = buildContextRowsIterative(sorted, breakdown.totalTokens ?? undefined);
|
|
53
82
|
if (rows.length === 0) {
|
|
54
83
|
lines.push("", "(no segments observed yet)");
|
|
55
84
|
return lines.join("\n");
|
|
56
85
|
}
|
|
57
86
|
lines.push("");
|
|
58
|
-
|
|
87
|
+
const visibleRows = rows.slice(0, MAX_REPORT_ROWS);
|
|
88
|
+
for (const row of visibleRows) lines.push(row.isHeader ? row.text : `${" ".repeat(row.depth)}${row.text}`);
|
|
89
|
+
if (rows.length > visibleRows.length)
|
|
90
|
+
lines.push(`… ${rows.length - visibleRows.length} more context rows (open /context in TUI mode to search and filter)`);
|
|
59
91
|
return lines.join("\n");
|
|
60
92
|
}
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
import type { ContextDelta, ContextSegment } from "@danypops/jittor";
|
|
2
|
+
import type { ExtensionCommandContext, Theme, ThemeColor } from "@earendil-works/pi-coding-agent";
|
|
3
|
+
import { matchesKey, type TUI, truncateToWidth } from "@earendil-works/pi-tui";
|
|
4
|
+
import {
|
|
5
|
+
type ContextBarTheme,
|
|
6
|
+
type ContextRow,
|
|
7
|
+
type ContextRowsTheme,
|
|
8
|
+
type ContextSegment as MalevichContextSegment,
|
|
9
|
+
renderContextRowLines,
|
|
10
|
+
renderContextUsageBar,
|
|
11
|
+
} from "malevich-tui-components";
|
|
12
|
+
import type { ContextBreakdown } from "./context-breakdown.ts";
|
|
13
|
+
import { buildContextReport, buildContextRowsIterative } from "./context-report.ts";
|
|
14
|
+
|
|
15
|
+
const VISIBLE_ROWS = 22;
|
|
16
|
+
const MIN_TOKEN_FILTERS = [0, 100, 1_000, 10_000] as const;
|
|
17
|
+
|
|
18
|
+
export type ContextRowScope = "all" | "active" | "historical";
|
|
19
|
+
|
|
20
|
+
function rowTokens(row: ContextRow): number {
|
|
21
|
+
const match = row.isHeader ? row.text.match(/—\s+([\d,]+)\s+tok/) : row.text.match(/^\s*([\d,]+)\s+tok/);
|
|
22
|
+
return match ? Number(match[1]!.replaceAll(",", "")) : 0;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** Filters the flattened pre-order tree while retaining every ancestor needed to understand a match. */
|
|
26
|
+
export function filterContextRows(rows: readonly ContextRow[], query: string, scope: ContextRowScope, minimumTokens: number): ContextRow[] {
|
|
27
|
+
const terms = query.toLocaleLowerCase().trim().split(/\s+/).filter(Boolean);
|
|
28
|
+
const included = rows.map(() => false);
|
|
29
|
+
const parentByIndex: Array<number | null> = [];
|
|
30
|
+
const ancestors: number[] = [];
|
|
31
|
+
const historicalByDepth: boolean[] = [];
|
|
32
|
+
for (let index = 0; index < rows.length; index++) {
|
|
33
|
+
const row = rows[index]!;
|
|
34
|
+
while (ancestors.length > row.depth) ancestors.pop();
|
|
35
|
+
parentByIndex[index] = row.depth > 0 ? (ancestors[row.depth - 1] ?? null) : null;
|
|
36
|
+
const lower = row.text.toLocaleLowerCase();
|
|
37
|
+
const inheritedHistorical = row.depth > 0 ? (historicalByDepth[row.depth - 1] ?? false) : false;
|
|
38
|
+
const historical = inheritedHistorical || lower.includes("(inactive branch)") || lower.includes("(compacted)");
|
|
39
|
+
const scopeMatches = scope === "all" || (scope === "historical" ? historical : !historical);
|
|
40
|
+
const queryMatches = terms.every((term) => lower.includes(term));
|
|
41
|
+
included[index] = scopeMatches && queryMatches && rowTokens(row) >= minimumTokens;
|
|
42
|
+
ancestors[row.depth] = index;
|
|
43
|
+
ancestors.length = row.depth + 1;
|
|
44
|
+
historicalByDepth[row.depth] = historical;
|
|
45
|
+
historicalByDepth.length = row.depth + 1;
|
|
46
|
+
}
|
|
47
|
+
// Children follow parents in this pre-order list, so one reverse pass propagates every
|
|
48
|
+
// match to its ancestors in O(rows), even for a 50k-entry linear session tree.
|
|
49
|
+
for (let index = rows.length - 1; index >= 0; index--) {
|
|
50
|
+
if (!included[index]) continue;
|
|
51
|
+
const parent = parentByIndex[index];
|
|
52
|
+
if (parent !== null && parent !== undefined) included[parent] = true;
|
|
53
|
+
}
|
|
54
|
+
return rows.filter((_row, index) => included[index]);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* A dynamically-contributed segment set (any string key from any extension, not a fixed enum)
|
|
59
|
+
* can't use a hardcoded per-key color map the way Papyrus's own ContextViewport did for its
|
|
60
|
+
* fixed seven segments -- this cycles a small categorical palette keyed by a stable hash of the
|
|
61
|
+
* segment key, so the same key always renders the same color within one process without needing
|
|
62
|
+
* every possible contributor's key to be known in advance.
|
|
63
|
+
*/
|
|
64
|
+
const PALETTE: ThemeColor[] = ["accent", "success", "syntaxFunction", "warning", "syntaxKeyword", "syntaxType", "muted"];
|
|
65
|
+
|
|
66
|
+
function paletteColor(key: string): ThemeColor {
|
|
67
|
+
let hash = 0;
|
|
68
|
+
for (let index = 0; index < key.length; index += 1) hash = (hash * 31 + key.charCodeAt(index)) >>> 0;
|
|
69
|
+
return PALETTE[hash % PALETTE.length]!;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function formatTokenCount(tokens: number): string {
|
|
73
|
+
return tokens >= 1_000 ? `${(tokens / 1_000).toFixed(1)}k` : String(tokens);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function percentOf(part: number, whole: number): string {
|
|
77
|
+
return whole > 0 ? `${((part / whole) * 100).toFixed(1)}%` : "—";
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function contextDeltaLines(delta: ContextDelta): string[] {
|
|
81
|
+
const lifecycle = new Map<string, number>();
|
|
82
|
+
for (const change of delta.changes) lifecycle.set(change.lifecycle, (lifecycle.get(change.lifecycle) ?? 0) + 1);
|
|
83
|
+
const lifecycleText = [...lifecycle.entries()].map(([name, count]) => `${name} ${count}`).join(" · ") || "none";
|
|
84
|
+
const growthText = delta.growthBySource
|
|
85
|
+
.filter((growth) => growth.deltaTokens !== 0)
|
|
86
|
+
.map((growth) => `${growth.source} ${growth.deltaTokens > 0 ? "+" : ""}${growth.deltaTokens.toLocaleString()} tok`)
|
|
87
|
+
.join(" · ");
|
|
88
|
+
const changed = delta.firstChangedSegment
|
|
89
|
+
? `first change: ${delta.firstChangedSegment.source} @ request ${delta.firstChangedSegment.requestPosition ?? "historical"}`
|
|
90
|
+
: delta.resetReason
|
|
91
|
+
? `comparison reset: ${delta.resetReason}`
|
|
92
|
+
: "request structure unchanged";
|
|
93
|
+
return [
|
|
94
|
+
`Stable prefix ${delta.stablePrefixTokens.toLocaleString()} tok · ${changed}`,
|
|
95
|
+
`Lifecycle ${lifecycleText} · growth ${growthText || "none"}${delta.truncated ? " · bounded snapshot (truncated)" : ""}`,
|
|
96
|
+
"Stable-prefix correlation is structural evidence, not provider cache proof.",
|
|
97
|
+
];
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Folds each segment's confidence tier into its label so it survives Malevich's confidence-unaware row builder. */
|
|
101
|
+
function withConfidenceLabel(segment: ContextSegment): MalevichContextSegment {
|
|
102
|
+
return {
|
|
103
|
+
key: segment.key,
|
|
104
|
+
label: `${segment.label} [${segment.confidence}]`,
|
|
105
|
+
estimatedTokens: segment.estimatedTokens,
|
|
106
|
+
items: segment.items,
|
|
107
|
+
unknown: segment.unknown,
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
class ContextViewport {
|
|
112
|
+
private offsetY = 0;
|
|
113
|
+
private readonly rows: ContextRow[];
|
|
114
|
+
private readonly segments: readonly MalevichContextSegment[];
|
|
115
|
+
private searchMode = false;
|
|
116
|
+
private query = "";
|
|
117
|
+
private scope: ContextRowScope = "all";
|
|
118
|
+
private minimumFilterIndex = 0;
|
|
119
|
+
|
|
120
|
+
constructor(
|
|
121
|
+
private readonly tui: TUI,
|
|
122
|
+
private readonly theme: Theme,
|
|
123
|
+
private readonly breakdown: ContextBreakdown,
|
|
124
|
+
private readonly delta: ContextDelta | null,
|
|
125
|
+
private readonly close: () => void,
|
|
126
|
+
) {
|
|
127
|
+
// Heaviest-first: Malevich renders segments in the order given, so sorting by weight for a
|
|
128
|
+
// merged multi-producer view is this viewport's own policy, matching context-report.ts.
|
|
129
|
+
this.segments = [...breakdown.segments].sort((left, right) => right.estimatedTokens - left.estimatedTokens).map(withConfidenceLabel);
|
|
130
|
+
this.rows = buildContextRowsIterative(this.segments, breakdown.totalTokens ?? undefined);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
invalidate(): void {}
|
|
134
|
+
|
|
135
|
+
private visibleRows(): ContextRow[] {
|
|
136
|
+
return filterContextRows(this.rows, this.query, this.scope, MIN_TOKEN_FILTERS[this.minimumFilterIndex]!);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
private clampOffset(rows: readonly ContextRow[]): void {
|
|
140
|
+
this.offsetY = Math.min(this.offsetY, Math.max(0, rows.length - VISIBLE_ROWS));
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
render(width: number): string[] {
|
|
144
|
+
const theme = this.theme;
|
|
145
|
+
const contentWidth = Math.max(1, width);
|
|
146
|
+
const border = theme.fg("borderMuted", "─".repeat(contentWidth));
|
|
147
|
+
const lines: string[] = [border, truncateToWidth(theme.fg("accent", theme.bold("Context")), contentWidth, "")];
|
|
148
|
+
|
|
149
|
+
const { totalTokens, effectiveBudget } = this.breakdown;
|
|
150
|
+
if (totalTokens !== null && effectiveBudget !== null) {
|
|
151
|
+
lines.push(
|
|
152
|
+
truncateToWidth(
|
|
153
|
+
`${formatTokenCount(totalTokens)} / ${formatTokenCount(effectiveBudget)} tokens (${percentOf(totalTokens, effectiveBudget)} of usable budget)`,
|
|
154
|
+
contentWidth,
|
|
155
|
+
"",
|
|
156
|
+
),
|
|
157
|
+
);
|
|
158
|
+
} else if (totalTokens !== null) {
|
|
159
|
+
lines.push(truncateToWidth(`${formatTokenCount(totalTokens)} tokens (model context window unknown)`, contentWidth, ""));
|
|
160
|
+
} else {
|
|
161
|
+
lines.push(theme.fg("dim", "No real usage reported yet — sizes below are estimates only"));
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
const colorFor = (key: string) => (s: string) => theme.fg(paletteColor(key), s);
|
|
165
|
+
const barTheme: ContextBarTheme = { colorFor, empty: (s) => theme.fg("dim", s) };
|
|
166
|
+
lines.push(renderContextUsageBar(barTheme, this.segments, contentWidth, effectiveBudget ?? undefined, totalTokens ?? undefined));
|
|
167
|
+
if (this.breakdown.overshootTokens > 0) {
|
|
168
|
+
lines.push(
|
|
169
|
+
truncateToWidth(
|
|
170
|
+
theme.fg(
|
|
171
|
+
"warning",
|
|
172
|
+
`Estimates exceed real total by ~${this.breakdown.overshootTokens} tok — sizes below are approximate, not exact`,
|
|
173
|
+
),
|
|
174
|
+
contentWidth,
|
|
175
|
+
"",
|
|
176
|
+
),
|
|
177
|
+
);
|
|
178
|
+
}
|
|
179
|
+
lines.push(theme.fg("dim", "Exact-text items name the model tokenizer; ≈ uses char/4; provider request totals remain aggregate."));
|
|
180
|
+
if (this.delta) {
|
|
181
|
+
for (const line of contextDeltaLines(this.delta)) lines.push(truncateToWidth(theme.fg("muted", line), contentWidth, ""));
|
|
182
|
+
}
|
|
183
|
+
lines.push("");
|
|
184
|
+
|
|
185
|
+
const rowsTheme: ContextRowsTheme = { colorFor, header: (s) => theme.bold(s) };
|
|
186
|
+
const filteredRows = this.visibleRows();
|
|
187
|
+
this.clampOffset(filteredRows);
|
|
188
|
+
const visible = filteredRows.slice(this.offsetY, this.offsetY + VISIBLE_ROWS);
|
|
189
|
+
lines.push(...renderContextRowLines(visible, contentWidth, rowsTheme));
|
|
190
|
+
if (filteredRows.length === 0) lines.push(theme.fg("dim", " (no matching context items)"));
|
|
191
|
+
else lines.push(theme.fg("muted", ` ${Math.min(this.offsetY + VISIBLE_ROWS, filteredRows.length)}/${filteredRows.length}`));
|
|
192
|
+
|
|
193
|
+
lines.push("");
|
|
194
|
+
const minimum = MIN_TOKEN_FILTERS[this.minimumFilterIndex]!;
|
|
195
|
+
const filterState = `scope: ${this.scope} · min: ${minimum === 0 ? "any" : `${formatTokenCount(minimum)} tok`}`;
|
|
196
|
+
lines.push(
|
|
197
|
+
truncateToWidth(
|
|
198
|
+
this.searchMode
|
|
199
|
+
? theme.fg("accent", `Search: ${this.query}▌ · ${filterState}`)
|
|
200
|
+
: theme.fg("muted", `${this.query ? `search: ${this.query} · ` : ""}${filterState}`),
|
|
201
|
+
contentWidth,
|
|
202
|
+
"",
|
|
203
|
+
),
|
|
204
|
+
);
|
|
205
|
+
lines.push(
|
|
206
|
+
theme.fg(
|
|
207
|
+
"dim",
|
|
208
|
+
this.searchMode
|
|
209
|
+
? "type to search · backspace edit · enter apply · esc clear"
|
|
210
|
+
: "/ search · f scope · m min tokens · g/G top/bottom · ↑↓ scroll · esc close",
|
|
211
|
+
),
|
|
212
|
+
);
|
|
213
|
+
lines.push(border);
|
|
214
|
+
return lines;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
handleInput(data: string): void {
|
|
218
|
+
if (this.searchMode) {
|
|
219
|
+
if (matchesKey(data, "escape")) {
|
|
220
|
+
if (this.query.length > 0) this.query = "";
|
|
221
|
+
else this.searchMode = false;
|
|
222
|
+
} else if (matchesKey(data, "enter")) this.searchMode = false;
|
|
223
|
+
else if (matchesKey(data, "backspace")) this.query = this.query.slice(0, -1);
|
|
224
|
+
else if (/^[\x20-\x7e]+$/.test(data)) this.query += data;
|
|
225
|
+
else return;
|
|
226
|
+
this.offsetY = 0;
|
|
227
|
+
this.tui.requestRender();
|
|
228
|
+
return;
|
|
229
|
+
}
|
|
230
|
+
if (matchesKey(data, "escape") || matchesKey(data, "ctrl+c")) {
|
|
231
|
+
this.close();
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
const rows = this.visibleRows();
|
|
235
|
+
if (data === "/") {
|
|
236
|
+
this.searchMode = true;
|
|
237
|
+
this.offsetY = 0;
|
|
238
|
+
} else if (data === "f") {
|
|
239
|
+
this.scope = this.scope === "all" ? "active" : this.scope === "active" ? "historical" : "all";
|
|
240
|
+
this.offsetY = 0;
|
|
241
|
+
} else if (data === "m") {
|
|
242
|
+
this.minimumFilterIndex = (this.minimumFilterIndex + 1) % MIN_TOKEN_FILTERS.length;
|
|
243
|
+
this.offsetY = 0;
|
|
244
|
+
} else if (data === "g") this.offsetY = 0;
|
|
245
|
+
else if (data === "G") this.offsetY = Math.max(0, rows.length - VISIBLE_ROWS);
|
|
246
|
+
else if (matchesKey(data, "up")) this.offsetY = Math.max(0, this.offsetY - 1);
|
|
247
|
+
else if (matchesKey(data, "down")) this.offsetY = Math.min(Math.max(0, rows.length - VISIBLE_ROWS), this.offsetY + 1);
|
|
248
|
+
else return;
|
|
249
|
+
this.tui.requestRender();
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/** Interactive scrollable Context Hub view in TUI mode; the same plain-text report as /context's non-interactive path otherwise. */
|
|
254
|
+
export async function showContextView(
|
|
255
|
+
ctx: ExtensionCommandContext,
|
|
256
|
+
breakdown: ContextBreakdown,
|
|
257
|
+
delta: ContextDelta | null = null,
|
|
258
|
+
): Promise<void> {
|
|
259
|
+
if (ctx.mode !== "tui") {
|
|
260
|
+
ctx.ui.notify([buildContextReport(breakdown), ...(delta ? ["", ...contextDeltaLines(delta)] : [])].join("\n"), "info");
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
await ctx.ui.custom<void>((tui, theme, _keybindings, done) => new ContextViewport(tui, theme, breakdown, delta, done));
|
|
264
|
+
}
|