@danypops/pi-jittor 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -3
- package/docs/USAGE_PRIOR_ART.md +1 -1
- package/extension/src/index.ts +379 -118
- package/extension/src/{context-breakdown.ts → observability/context-breakdown.ts} +251 -47
- package/extension/src/observability/context-growth.ts +26 -0
- package/extension/src/{capabilities → observability}/context-hub.ts +1 -5
- package/extension/src/observability/context-report.ts +92 -0
- package/extension/src/observability/context-view.ts +264 -0
- package/extension/src/{footer.ts → observability/footer.ts} +63 -26
- package/extension/src/{capabilities/local-run-telemetry.ts → observability/model-run.ts} +14 -10
- package/extension/src/observability/provider-context-snapshot.ts +246 -0
- package/extension/src/{capabilities/provider-response-telemetry.ts → observability/provider-response.ts} +21 -7
- package/extension/src/{tui.ts → observability/status.ts} +202 -72
- package/extension/src/observability/usage.ts +314 -0
- package/extension/src/optimization/model-selection-panel.ts +160 -0
- package/extension/src/{capabilities/codex-recovery.ts → optimization/recovery/codex.ts} +41 -25
- package/extension/src/service-client.ts +49 -2
- package/extension/src/settings-tui.ts +73 -33
- package/extension/src/settings.ts +40 -29
- package/package.json +11 -5
- package/extension/src/benchmark-tui.ts +0 -113
- package/extension/src/context-report.ts +0 -49
- package/extension/src/context-view.ts +0 -108
- package/extension/src/usage.ts +0 -324
- /package/extension/src/{capabilities → observability}/http-headers.ts +0 -0
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
import {
|
|
2
|
+
buildCostGraph,
|
|
3
|
+
buildUsageGraph,
|
|
4
|
+
type CostGraph,
|
|
5
|
+
HUMAN_TEXT_FIELD_MAX_CHARACTERS,
|
|
6
|
+
resolveUsageWindow,
|
|
7
|
+
USAGE_CHART_HEIGHT,
|
|
8
|
+
USAGE_MAX_DISTINCT_SCOPES,
|
|
9
|
+
USAGE_PERIODS,
|
|
10
|
+
USAGE_RENDER_MAX_SERIES,
|
|
11
|
+
USAGE_Y_AXIS_WIDTH,
|
|
12
|
+
type UsageAggregateRow,
|
|
13
|
+
type UsageGraph,
|
|
14
|
+
type UsagePeriod,
|
|
15
|
+
usagePeriod,
|
|
16
|
+
} from "@danypops/jittor";
|
|
17
|
+
import type { ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
|
|
18
|
+
import { matchesKey, truncateToWidth, visibleWidth } from "@earendil-works/pi-tui";
|
|
19
|
+
import { HistoryChart, type HistoryChartTheme, ProgressBar, type TextMeasure } from "malevich-tui-components";
|
|
20
|
+
import type { UsageBudgetControl } from "../settings.ts";
|
|
21
|
+
import type { JittorPanelClient } from "./status.ts";
|
|
22
|
+
|
|
23
|
+
type UsageAction = "period-prev" | "period-next" | "view-next" | "refresh" | "close";
|
|
24
|
+
type UsageColor =
|
|
25
|
+
| "accent"
|
|
26
|
+
| "success"
|
|
27
|
+
| "warning"
|
|
28
|
+
| "error"
|
|
29
|
+
| "thinkingText"
|
|
30
|
+
| "muted"
|
|
31
|
+
| "dim"
|
|
32
|
+
| "borderMuted"
|
|
33
|
+
| "syntaxKeyword"
|
|
34
|
+
| "syntaxFunction"
|
|
35
|
+
| "syntaxVariable"
|
|
36
|
+
| "syntaxString"
|
|
37
|
+
| "syntaxNumber"
|
|
38
|
+
| "syntaxType"
|
|
39
|
+
| "syntaxOperator";
|
|
40
|
+
|
|
41
|
+
export interface UsageTheme {
|
|
42
|
+
fg(color: UsageColor, text: string): string;
|
|
43
|
+
bold(text: string): string;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Categorical palette for per-provider/model series. Deliberately excludes "success"/"warning"/
|
|
48
|
+
* "error": those already carry a fixed status meaning elsewhere in this same panel (the budget
|
|
49
|
+
* threshold line, freshness state), so reusing them for arbitrary model identity would make a
|
|
50
|
+
* model's bar segment look like a warning or a failure to a pre-attentive reader — a real
|
|
51
|
+
* categorical-color-design pitfall (see e.g. ColorBrewer/Okabe-Ito guidance on qualitative
|
|
52
|
+
* palettes: colors for nominal categories should not imply an order, magnitude, or judgment).
|
|
53
|
+
*
|
|
54
|
+
* Instead this reuses the syntax-highlighting color roles, because theme authors already tune
|
|
55
|
+
* those specifically to be simultaneously distinguishable on screen — that is the same design
|
|
56
|
+
* problem as a categorical data palette (many hues coexisting in one view that all need to read
|
|
57
|
+
* as different from each other). The order below interleaves the hue families syntax themes
|
|
58
|
+
* conventionally assign to keyword/function/string/number/type/variable/operator (violet, blue,
|
|
59
|
+
* green, orange, teal, cyan, neutral) so the first colors used are spread around the hue wheel
|
|
60
|
+
* rather than clustered, which is the standard categorical-palette heuristic for maximizing
|
|
61
|
+
* perceptual separation between adjacent categories.
|
|
62
|
+
*
|
|
63
|
+
* Terminal foreground color is a single channel with a hard ceiling on how many hues stay mutually
|
|
64
|
+
* distinguishable (most qualitative-palette guidance caps around 8–12). seriesStyle derives both a
|
|
65
|
+
* hue and optional bold weight from stable identity bits, using a second visual channel to reduce
|
|
66
|
+
* collisions without allowing ranking or render order to change a model's appearance.
|
|
67
|
+
*/
|
|
68
|
+
const SERIES_HUES: UsageColor[] = [
|
|
69
|
+
"accent",
|
|
70
|
+
"syntaxFunction",
|
|
71
|
+
"syntaxString",
|
|
72
|
+
"syntaxNumber",
|
|
73
|
+
"syntaxKeyword",
|
|
74
|
+
"syntaxType",
|
|
75
|
+
"thinkingText",
|
|
76
|
+
"syntaxVariable",
|
|
77
|
+
"syntaxOperator",
|
|
78
|
+
];
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Stable FNV-1a identity hash. Series ordering changes with totals and selected period, so assigning
|
|
82
|
+
* colors by array index makes a model change color whenever another model overtakes it. Deriving
|
|
83
|
+
* both visual channels from its provider/model key keeps that identity fixed across token and cost
|
|
84
|
+
* graphics, period switches, refreshes, and extension restarts.
|
|
85
|
+
*/
|
|
86
|
+
function seriesStyle(identity: string, theme: UsageTheme): (text: string) => string {
|
|
87
|
+
let hash = 0x811c9dc5;
|
|
88
|
+
for (let index = 0; index < identity.length; index += 1) {
|
|
89
|
+
hash ^= identity.charCodeAt(index);
|
|
90
|
+
hash = Math.imul(hash, 0x01000193) >>> 0;
|
|
91
|
+
}
|
|
92
|
+
const hue = SERIES_HUES[hash % SERIES_HUES.length]!;
|
|
93
|
+
const useBold = Math.floor(hash / SERIES_HUES.length) % 2 === 1;
|
|
94
|
+
return (text: string) => (useBold ? theme.bold(theme.fg(hue, text)) : theme.fg(hue, text));
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function compact(value: number): string {
|
|
98
|
+
if (value >= 1_000_000_000) return `${(value / 1_000_000_000).toFixed(value >= 10_000_000_000 ? 0 : 1)}B`;
|
|
99
|
+
if (value >= 1_000_000) return `${(value / 1_000_000).toFixed(value >= 10_000_000 ? 0 : 1)}M`;
|
|
100
|
+
if (value >= 1_000) return `${(value / 1_000).toFixed(value >= 10_000 || value % 1_000 === 0 ? 0 : 1)}k`;
|
|
101
|
+
return String(Math.round(value));
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** Compact USD formatter: full cents below $1k, then the same k/M suffix convention as compact(). */
|
|
105
|
+
function formatUsd(value: number): string {
|
|
106
|
+
const magnitude = Math.abs(value);
|
|
107
|
+
if (magnitude === 0) return "$0";
|
|
108
|
+
if (magnitude < 1_000) return `$${value.toFixed(magnitude < 0.01 ? 4 : 2)}`;
|
|
109
|
+
if (magnitude < 1_000_000) return `$${(value / 1_000).toFixed(1)}k`;
|
|
110
|
+
return `$${(value / 1_000_000).toFixed(1)}M`;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function formatPeriodPoint(value: number, period: UsagePeriod): string {
|
|
114
|
+
const date = new Date(value);
|
|
115
|
+
if (period === "hourly" || period === "daily") return date.toLocaleTimeString(undefined, { hour: "2-digit", minute: "2-digit" });
|
|
116
|
+
return date.toLocaleDateString(undefined, { month: "short", day: "numeric" });
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function displayIdentity(value: string): string {
|
|
120
|
+
return value
|
|
121
|
+
.replace(/[\r\n\t]/g, " ")
|
|
122
|
+
.replace(/ +/g, " ")
|
|
123
|
+
.trim()
|
|
124
|
+
.slice(0, HUMAN_TEXT_FIELD_MAX_CHARACTERS);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function plainTheme(): UsageTheme {
|
|
128
|
+
return { fg: (_color, text) => text, bold: (text) => text };
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const hostTextMeasure: TextMeasure = { visibleWidth, truncateToWidth };
|
|
132
|
+
|
|
133
|
+
function chartTheme(theme: UsageTheme, seriesIdentities: string[]): HistoryChartTheme {
|
|
134
|
+
return {
|
|
135
|
+
title: theme.bold,
|
|
136
|
+
subtitle: (text) => text,
|
|
137
|
+
axis: (text) => theme.fg("borderMuted", text),
|
|
138
|
+
warningLine: (text) => theme.fg("warning", text),
|
|
139
|
+
errorLine: (text) => theme.fg("error", text),
|
|
140
|
+
muted: (text) => theme.fg("muted", text),
|
|
141
|
+
series: (index) => seriesStyle(seriesIdentities[index] ?? `series:${index}`, theme),
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
interface ChartRenderOptions {
|
|
146
|
+
title: string;
|
|
147
|
+
formatValue: (value: number) => string;
|
|
148
|
+
unitSuffix: string;
|
|
149
|
+
subtitle?: string;
|
|
150
|
+
budget?: number;
|
|
151
|
+
noDataText: string;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** Adapts Jittor's domain graph to Malevich's reusable cumulative HistoryChart and budget ProgressBar. */
|
|
155
|
+
function renderChart(
|
|
156
|
+
chart: { period: UsagePeriod; buckets: UsageGraph["buckets"]; series: UsageGraph["series"]; total: number; truncated: boolean },
|
|
157
|
+
width: number,
|
|
158
|
+
theme: UsageTheme,
|
|
159
|
+
options: ChartRenderOptions,
|
|
160
|
+
): string[] {
|
|
161
|
+
const component = new HistoryChart({
|
|
162
|
+
title: options.title,
|
|
163
|
+
buckets: chart.buckets,
|
|
164
|
+
series: chart.series.map((series) => ({
|
|
165
|
+
key: series.key,
|
|
166
|
+
label: `${displayIdentity(series.provider)}/${displayIdentity(series.model)}`,
|
|
167
|
+
})),
|
|
168
|
+
formatValue: options.formatValue,
|
|
169
|
+
unitSuffix: options.unitSuffix,
|
|
170
|
+
...(options.subtitle ? { subtitle: options.subtitle } : {}),
|
|
171
|
+
...(options.budget !== undefined ? { budget: options.budget } : {}),
|
|
172
|
+
noDataText: options.noDataText,
|
|
173
|
+
truncated: chart.truncated,
|
|
174
|
+
formatAxisLabel: (value) => formatPeriodPoint(value, chart.period),
|
|
175
|
+
theme: chartTheme(
|
|
176
|
+
theme,
|
|
177
|
+
chart.series.map((series) => series.key),
|
|
178
|
+
),
|
|
179
|
+
measure: hostTextMeasure,
|
|
180
|
+
height: USAGE_CHART_HEIGHT,
|
|
181
|
+
yAxisWidth: USAGE_Y_AXIS_WIDTH,
|
|
182
|
+
maxSeriesShown: USAGE_RENDER_MAX_SERIES,
|
|
183
|
+
});
|
|
184
|
+
const lines = component.render(width);
|
|
185
|
+
if (options.budget === undefined || !Number.isFinite(options.budget) || options.budget <= 0 || chart.total === 0) return lines;
|
|
186
|
+
const meter = new ProgressBar({
|
|
187
|
+
value: chart.total,
|
|
188
|
+
max: options.budget,
|
|
189
|
+
label: "Budget",
|
|
190
|
+
style: chart.total > options.budget ? (text) => theme.fg("error", text) : (text) => theme.fg("accent", text),
|
|
191
|
+
measure: hostTextMeasure,
|
|
192
|
+
});
|
|
193
|
+
lines.splice(options.subtitle ? 3 : 2, 0, ...meter.render(Math.max(20, width)));
|
|
194
|
+
return lines;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
export function renderUsageGraph(chart: UsageGraph, width: number, theme: UsageTheme, tokenBudget?: number): string[] {
|
|
198
|
+
return renderChart(
|
|
199
|
+
{
|
|
200
|
+
period: chart.period,
|
|
201
|
+
buckets: chart.buckets,
|
|
202
|
+
series: chart.series,
|
|
203
|
+
total: chart.totalTokens,
|
|
204
|
+
truncated: chart.truncated,
|
|
205
|
+
},
|
|
206
|
+
width,
|
|
207
|
+
theme,
|
|
208
|
+
{
|
|
209
|
+
title: `${usagePeriod(chart.period).label} token usage`,
|
|
210
|
+
formatValue: compact,
|
|
211
|
+
unitSuffix: " tokens",
|
|
212
|
+
subtitle: `input ${compact(chart.breakdown.input)} · output ${compact(chart.breakdown.output)} · cache ${compact(chart.breakdown.cacheRead + chart.breakdown.cacheWrite)}`,
|
|
213
|
+
budget: tokenBudget,
|
|
214
|
+
noDataText: "No recorded Pi token usage in this period.",
|
|
215
|
+
},
|
|
216
|
+
);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
export function renderCostGraph(chart: CostGraph, width: number, theme: UsageTheme, costBudget?: number): string[] {
|
|
220
|
+
return renderChart(
|
|
221
|
+
{
|
|
222
|
+
period: chart.period,
|
|
223
|
+
buckets: chart.buckets,
|
|
224
|
+
series: chart.series,
|
|
225
|
+
total: chart.totalUsd,
|
|
226
|
+
truncated: chart.truncated,
|
|
227
|
+
},
|
|
228
|
+
width,
|
|
229
|
+
theme,
|
|
230
|
+
{
|
|
231
|
+
title: `${usagePeriod(chart.period).label} cost`,
|
|
232
|
+
formatValue: formatUsd,
|
|
233
|
+
unitSuffix: "",
|
|
234
|
+
budget: costBudget,
|
|
235
|
+
noDataText: "No recorded Pi cost in this period.",
|
|
236
|
+
},
|
|
237
|
+
);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
export type UsageViewKind = "tokens" | "cost";
|
|
241
|
+
const USAGE_VIEWS: UsageViewKind[] = ["tokens", "cost"];
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* One bounded round trip: the daemon discovers distinct scopes (still capped at
|
|
245
|
+
* USAGE_MAX_DISTINCT_SCOPES -- more scopes than that is still a real, honestly-reported
|
|
246
|
+
* truncation) and SQL-side aggregates every matching observation into (scope, metric, bucket)
|
|
247
|
+
* sums for the exact window this panel renders. Replaces a per-scope fetch of up to
|
|
248
|
+
* USAGE_PER_SCOPE_QUERY_LIMIT raw rows each, which fixed a *different* problem (one heavy scope
|
|
249
|
+
* starving *other* scopes out of a shared row budget) but could still silently truncate a single
|
|
250
|
+
* heavy scope's *own* older history within the same window -- a real incident: a scope logging
|
|
251
|
+
* tens of thousands of rows a week had its "weekly" chart built from a few minutes of its most
|
|
252
|
+
* recent rows alone. Aggregation has no such failure mode: result size scales with (scopes x
|
|
253
|
+
* metrics x buckets), never with raw event count.
|
|
254
|
+
*/
|
|
255
|
+
async function loadPiMetrics(
|
|
256
|
+
client: JittorPanelClient,
|
|
257
|
+
window: ReturnType<typeof resolveUsageWindow>,
|
|
258
|
+
): Promise<{ rows: UsageAggregateRow[]; truncated: boolean }> {
|
|
259
|
+
const result = (await client.call("metrics.usage_series", {
|
|
260
|
+
source: "pi",
|
|
261
|
+
since: window.start,
|
|
262
|
+
until: window.end,
|
|
263
|
+
bucketSizeMs: window.bucketSizeMs,
|
|
264
|
+
bucketCount: window.bucketCount,
|
|
265
|
+
scopeLimit: USAGE_MAX_DISTINCT_SCOPES,
|
|
266
|
+
})) as { rows: UsageAggregateRow[]; truncated: boolean };
|
|
267
|
+
return result;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
export async function showUsagePanel(
|
|
271
|
+
ctx: ExtensionCommandContext,
|
|
272
|
+
client: JittorPanelClient,
|
|
273
|
+
budgets: Pick<UsageBudgetControl, "getUsageTokenBudget">,
|
|
274
|
+
now = Date.now(),
|
|
275
|
+
initialView: UsageViewKind = "tokens",
|
|
276
|
+
): Promise<void> {
|
|
277
|
+
let periodIndex = 0;
|
|
278
|
+
let viewIndex = Math.max(0, USAGE_VIEWS.indexOf(initialView));
|
|
279
|
+
for (;;) {
|
|
280
|
+
const period = USAGE_PERIODS[periodIndex]!.id;
|
|
281
|
+
const window = resolveUsageWindow(period, now);
|
|
282
|
+
// One bounded query serves both views: token and cost metrics share the same "pi" source rows.
|
|
283
|
+
const { rows, truncated } = await loadPiMetrics(client, window);
|
|
284
|
+
const view = USAGE_VIEWS[viewIndex]!;
|
|
285
|
+
const tokenChart = buildUsageGraph(rows, window, { period, truncated });
|
|
286
|
+
const costChart = buildCostGraph(rows, window, { period, truncated });
|
|
287
|
+
const tokenBudget = budgets.getUsageTokenBudget(period);
|
|
288
|
+
const renderActive = (width: number, theme: UsageTheme): string[] =>
|
|
289
|
+
view === "tokens" ? renderUsageGraph(tokenChart, width, theme, tokenBudget) : renderCostGraph(costChart, width, theme);
|
|
290
|
+
if (ctx.mode !== "tui") {
|
|
291
|
+
ctx.ui.notify(renderActive(80, plainTheme()).join("\n"), "info");
|
|
292
|
+
return;
|
|
293
|
+
}
|
|
294
|
+
const action = await ctx.ui.custom<UsageAction>((_tui, theme, _keybindings, done) => ({
|
|
295
|
+
invalidate() {},
|
|
296
|
+
render(width: number): string[] {
|
|
297
|
+
const lines = renderActive(width, theme);
|
|
298
|
+
const controls = theme.fg("dim", "←/→/Tab period · v view · r refresh · Esc close");
|
|
299
|
+
return [...lines, "", truncateToWidth(controls, width, "…")];
|
|
300
|
+
},
|
|
301
|
+
handleInput(data: string): void {
|
|
302
|
+
if (matchesKey(data, "escape") || matchesKey(data, "ctrl+c") || data === "q") done("close");
|
|
303
|
+
else if (matchesKey(data, "left") || matchesKey(data, "shift+tab")) done("period-prev");
|
|
304
|
+
else if (matchesKey(data, "right") || matchesKey(data, "tab")) done("period-next");
|
|
305
|
+
else if (data === "v") done("view-next");
|
|
306
|
+
else if (data === "r") done("refresh");
|
|
307
|
+
},
|
|
308
|
+
}));
|
|
309
|
+
if (!action || action === "close") return;
|
|
310
|
+
if (action === "period-prev") periodIndex = (periodIndex - 1 + USAGE_PERIODS.length) % USAGE_PERIODS.length;
|
|
311
|
+
if (action === "period-next") periodIndex = (periodIndex + 1) % USAGE_PERIODS.length;
|
|
312
|
+
if (action === "view-next") viewIndex = (viewIndex + 1) % USAGE_VIEWS.length;
|
|
313
|
+
}
|
|
314
|
+
}
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import {
|
|
2
|
+
BENCHMARK_TUI_MAX_CANDIDATES,
|
|
3
|
+
BENCHMARK_TUI_MAX_PROVENANCE_PER_CANDIDATE,
|
|
4
|
+
MODEL_RANKING_DEFAULT_CONTEXT_WEIGHT,
|
|
5
|
+
MODEL_RANKING_DEFAULT_COST_WEIGHT,
|
|
6
|
+
MODEL_RANKING_DEFAULT_LATENCY_WEIGHT,
|
|
7
|
+
MODEL_RANKING_DEFAULT_QUALITY_WEIGHT,
|
|
8
|
+
MODEL_RANKING_DEFAULT_RELIABILITY_WEIGHT,
|
|
9
|
+
type ModelCandidate,
|
|
10
|
+
type ModelRankingResult,
|
|
11
|
+
type ModelTaskDomain,
|
|
12
|
+
type ModelTaskType,
|
|
13
|
+
type RankedModel,
|
|
14
|
+
type UtilityComponentName,
|
|
15
|
+
} from "@danypops/jittor";
|
|
16
|
+
import type { ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
|
|
17
|
+
import { matchesKey, truncateToWidth, visibleWidth } from "@earendil-works/pi-tui";
|
|
18
|
+
import { BorderedSelectPanel, Table, type TextMeasure } from "malevich-tui-components";
|
|
19
|
+
import { sessionSecretField } from "../session-identity.ts";
|
|
20
|
+
|
|
21
|
+
export interface BenchmarkPanelClient {
|
|
22
|
+
call(operation: string, input: unknown): Promise<any>;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
interface BenchmarkTheme {
|
|
26
|
+
fg(color: string, text: string): string;
|
|
27
|
+
bold(text: string): string;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
type BenchmarkPanelAction = "refresh" | "close";
|
|
31
|
+
|
|
32
|
+
const COMPONENT_LABELS: Record<UtilityComponentName, string> = { quality: "Q", cost: "$", latency: "L", context: "C", reliability: "R" };
|
|
33
|
+
|
|
34
|
+
function componentText(item: RankedModel): string {
|
|
35
|
+
return item.components
|
|
36
|
+
.map((component) => `${COMPONENT_LABELS[component.name]} ${component.score === null ? "?" : component.score.toFixed(3)}`)
|
|
37
|
+
.join(" · ");
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const hostTextMeasure: TextMeasure = { visibleWidth, truncateToWidth };
|
|
41
|
+
|
|
42
|
+
function benchmarkRows(shown: RankedModel[], currentIdentity: string): Record<string, string>[] {
|
|
43
|
+
return shown.flatMap((item, index) => {
|
|
44
|
+
const current = item.identity.startsWith(`${currentIdentity}:`);
|
|
45
|
+
const localSamples = item.components.find((component) => component.name === "reliability")?.evidenceCount ?? 0;
|
|
46
|
+
const provenance = item.provenance
|
|
47
|
+
.slice(0, BENCHMARK_TUI_MAX_PROVENANCE_PER_CANDIDATE)
|
|
48
|
+
.map((source) => `${source.sourceId}@${source.revision} ${source.freshness}`)
|
|
49
|
+
.join(" · ");
|
|
50
|
+
return [
|
|
51
|
+
{ rank: `${index + 1}.`, detail: `${item.identity}${index === 0 ? " recommended" : ""}${current ? " current" : ""}` },
|
|
52
|
+
{
|
|
53
|
+
rank: "",
|
|
54
|
+
detail: `utility ${item.utility === null ? "?" : item.utility.toFixed(3)} · confidence ${(item.confidence * 100).toFixed(0)}% · ${componentText(item)}`,
|
|
55
|
+
},
|
|
56
|
+
{ rank: "", detail: `local n=${localSamples}${provenance ? ` · ${provenance}` : " · no external provenance"}` },
|
|
57
|
+
];
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function renderBenchmarkView(result: ModelRankingResult, currentIdentity: string, width: number, theme: BenchmarkTheme): string[] {
|
|
62
|
+
const safeWidth = Math.max(1, width);
|
|
63
|
+
const shown = result.ranked.slice(0, BENCHMARK_TUI_MAX_CANDIDATES);
|
|
64
|
+
const currentIndex = result.ranked.findIndex((item) => item.identity.startsWith(`${currentIdentity}:`));
|
|
65
|
+
const recommended = result.ranked[0];
|
|
66
|
+
const reason =
|
|
67
|
+
recommended && currentIndex > 0
|
|
68
|
+
? `Recommendation differs from current: ${recommended.identity} ranks #1; current ranks #${currentIndex + 1}.`
|
|
69
|
+
: recommended && currentIndex === 0
|
|
70
|
+
? "Current model is the top recommendation."
|
|
71
|
+
: "Current model is outside the ranked candidates.";
|
|
72
|
+
const table = new Table({
|
|
73
|
+
columns: [
|
|
74
|
+
{ header: "#", key: "rank", width: 4 },
|
|
75
|
+
{ header: "Candidate / evidence", key: "detail" },
|
|
76
|
+
],
|
|
77
|
+
rows: benchmarkRows(shown, currentIdentity),
|
|
78
|
+
headerStyle: theme.bold,
|
|
79
|
+
measure: hostTextMeasure,
|
|
80
|
+
});
|
|
81
|
+
const content = {
|
|
82
|
+
invalidate: () => table.invalidate(),
|
|
83
|
+
render: (availableWidth: number): string[] => [
|
|
84
|
+
truncateToWidth(
|
|
85
|
+
result.scopeAuthority === "exact-session"
|
|
86
|
+
? "Scope: exact session"
|
|
87
|
+
: "Scope: available models · ADVISORY (exact session scope unavailable)",
|
|
88
|
+
availableWidth,
|
|
89
|
+
"…",
|
|
90
|
+
),
|
|
91
|
+
truncateToWidth(`Domain: ${result.domain} · Type: ${result.type} · evidence ${result.completeness}`, availableWidth, "…"),
|
|
92
|
+
truncateToWidth(reason, availableWidth, "…"),
|
|
93
|
+
...table.render(availableWidth),
|
|
94
|
+
...(result.ranked.length > shown.length
|
|
95
|
+
? [truncateToWidth(`… ${result.ranked.length - shown.length} more candidates omitted`, availableWidth, "…")]
|
|
96
|
+
: []),
|
|
97
|
+
...(result.scopeWarning ? [truncateToWidth(result.scopeWarning, availableWidth, "…")] : []),
|
|
98
|
+
],
|
|
99
|
+
};
|
|
100
|
+
return new BorderedSelectPanel({
|
|
101
|
+
title: "Jittor Benchmark Recommendations",
|
|
102
|
+
list: content,
|
|
103
|
+
helpText: "r refresh · Esc close",
|
|
104
|
+
theme: {
|
|
105
|
+
border: (text) => theme.fg("borderMuted", text),
|
|
106
|
+
title: theme.bold,
|
|
107
|
+
help: (text) => theme.fg("dim", text),
|
|
108
|
+
},
|
|
109
|
+
measure: hostTextMeasure,
|
|
110
|
+
}).render(safeWidth);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export async function showBenchmarkPanel(
|
|
114
|
+
ctx: ExtensionCommandContext,
|
|
115
|
+
client: BenchmarkPanelClient,
|
|
116
|
+
candidates: ModelCandidate[],
|
|
117
|
+
currentIdentity: string,
|
|
118
|
+
domain: ModelTaskDomain,
|
|
119
|
+
type: ModelTaskType,
|
|
120
|
+
): Promise<void> {
|
|
121
|
+
for (;;) {
|
|
122
|
+
const session_id = ctx.sessionManager.getSessionId();
|
|
123
|
+
const result = (await client.call("models.rank", {
|
|
124
|
+
candidates,
|
|
125
|
+
session_id,
|
|
126
|
+
...sessionSecretField(session_id),
|
|
127
|
+
scopeAuthority: "available-models",
|
|
128
|
+
domain,
|
|
129
|
+
type,
|
|
130
|
+
budgetPressure: 0,
|
|
131
|
+
weights: {
|
|
132
|
+
quality: MODEL_RANKING_DEFAULT_QUALITY_WEIGHT,
|
|
133
|
+
cost: MODEL_RANKING_DEFAULT_COST_WEIGHT,
|
|
134
|
+
latency: MODEL_RANKING_DEFAULT_LATENCY_WEIGHT,
|
|
135
|
+
context: MODEL_RANKING_DEFAULT_CONTEXT_WEIGHT,
|
|
136
|
+
reliability: MODEL_RANKING_DEFAULT_RELIABILITY_WEIGHT,
|
|
137
|
+
},
|
|
138
|
+
sourceIds: ["openrouter-models", "lmarena-hf", "artificial-analysis-direct", "openrouter-design-arena"],
|
|
139
|
+
})) as ModelRankingResult;
|
|
140
|
+
if (ctx.mode !== "tui") {
|
|
141
|
+
ctx.ui.notify(
|
|
142
|
+
renderBenchmarkView(result, currentIdentity, 100, { fg: (_color, text) => text, bold: (text) => text }).join("\n"),
|
|
143
|
+
"info",
|
|
144
|
+
);
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
const action = await ctx.ui.custom<BenchmarkPanelAction>((_tui, theme, _keybindings, done) => ({
|
|
148
|
+
invalidate() {},
|
|
149
|
+
render(width: number): string[] {
|
|
150
|
+
return renderBenchmarkView(result, currentIdentity, width, theme);
|
|
151
|
+
},
|
|
152
|
+
handleInput(data: string): void {
|
|
153
|
+
if (matchesKey(data, "escape") || matchesKey(data, "ctrl+c")) done("close");
|
|
154
|
+
else if (data === "r") done("refresh");
|
|
155
|
+
},
|
|
156
|
+
}));
|
|
157
|
+
if (!action || action === "close") return;
|
|
158
|
+
await client.call("benchmark.refresh", { force: true });
|
|
159
|
+
}
|
|
160
|
+
}
|
|
@@ -1,19 +1,19 @@
|
|
|
1
|
-
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
2
1
|
import {
|
|
3
2
|
CODEX_RECOVERY_ATTEMPT_WINDOW_MS,
|
|
4
3
|
CODEX_RECOVERY_BASE_DELAY_MS,
|
|
5
4
|
CODEX_RECOVERY_JITTER_RATIO,
|
|
6
5
|
CODEX_RECOVERY_MAX_ATTEMPTS,
|
|
7
6
|
CODEX_RECOVERY_MAX_DELAY_MS,
|
|
8
|
-
MILLISECONDS_PER_MINUTE,
|
|
9
|
-
MILLISECONDS_PER_SECOND,
|
|
10
|
-
CodexRecoveryPolicy,
|
|
11
|
-
classifyCodexFailure,
|
|
12
7
|
type CodexFailureKind,
|
|
13
8
|
type CodexFailureMetadata,
|
|
9
|
+
CodexRecoveryPolicy,
|
|
10
|
+
classifyCodexFailure,
|
|
11
|
+
MILLISECONDS_PER_MINUTE,
|
|
12
|
+
MILLISECONDS_PER_SECOND,
|
|
14
13
|
} from "@danypops/jittor";
|
|
15
|
-
import type {
|
|
16
|
-
import { headerValue } from "
|
|
14
|
+
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
15
|
+
import { headerValue } from "../../observability/http-headers.ts";
|
|
16
|
+
import type { CodexRecoveryControl } from "../../settings.ts";
|
|
17
17
|
|
|
18
18
|
export interface CodexRecoveryRuntime {
|
|
19
19
|
now(): number;
|
|
@@ -25,8 +25,14 @@ export interface CodexRecoveryRuntime {
|
|
|
25
25
|
export const SYSTEM_RECOVERY_RUNTIME: CodexRecoveryRuntime = {
|
|
26
26
|
now: Date.now,
|
|
27
27
|
random: Math.random,
|
|
28
|
-
setTimeout(callback, delayMs) {
|
|
29
|
-
|
|
28
|
+
setTimeout(callback, delayMs) {
|
|
29
|
+
return setTimeout(() => {
|
|
30
|
+
void callback();
|
|
31
|
+
}, delayMs);
|
|
32
|
+
},
|
|
33
|
+
clearTimeout(handle) {
|
|
34
|
+
clearTimeout(handle as ReturnType<typeof setTimeout>);
|
|
35
|
+
},
|
|
30
36
|
};
|
|
31
37
|
|
|
32
38
|
/**
|
|
@@ -47,13 +53,16 @@ export class CodexRecoveryCapability {
|
|
|
47
53
|
private readonly control: CodexRecoveryControl,
|
|
48
54
|
private readonly runtime: CodexRecoveryRuntime,
|
|
49
55
|
) {
|
|
50
|
-
this.policy = new CodexRecoveryPolicy(
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
56
|
+
this.policy = new CodexRecoveryPolicy(
|
|
57
|
+
{
|
|
58
|
+
baseDelayMs: CODEX_RECOVERY_BASE_DELAY_MS,
|
|
59
|
+
maxDelayMs: CODEX_RECOVERY_MAX_DELAY_MS,
|
|
60
|
+
maxAttempts: CODEX_RECOVERY_MAX_ATTEMPTS,
|
|
61
|
+
attemptWindowMs: CODEX_RECOVERY_ATTEMPT_WINDOW_MS,
|
|
62
|
+
jitterRatio: CODEX_RECOVERY_JITTER_RATIO,
|
|
63
|
+
},
|
|
64
|
+
runtime.random,
|
|
65
|
+
);
|
|
57
66
|
}
|
|
58
67
|
|
|
59
68
|
/** Clears the tracked response at the start of every new turn, before any Codex response for it has arrived. */
|
|
@@ -90,9 +99,13 @@ export class CodexRecoveryCapability {
|
|
|
90
99
|
const attempt = this.cooldown?.attempt ?? (state.pending ? state.attempts + 1 : state.attempts);
|
|
91
100
|
const phase = this.cooldown
|
|
92
101
|
? `cooldown ${Math.ceil(Math.max(0, this.cooldown.until - now) / MILLISECONDS_PER_SECOND)}s`
|
|
93
|
-
: state.pending
|
|
94
|
-
|
|
95
|
-
|
|
102
|
+
: state.pending
|
|
103
|
+
? "pending"
|
|
104
|
+
: state.attempts >= CODEX_RECOVERY_MAX_ATTEMPTS
|
|
105
|
+
? "exhausted"
|
|
106
|
+
: state.attempts > 0
|
|
107
|
+
? "waiting"
|
|
108
|
+
: "idle";
|
|
96
109
|
const failureKind = this.cooldown?.failureKind ?? state.lastFailureKind;
|
|
97
110
|
return [
|
|
98
111
|
`Codex recovery: ${enabled ? "on" : "off"}`,
|
|
@@ -119,12 +132,15 @@ export class CodexRecoveryCapability {
|
|
|
119
132
|
if (!ctx.isIdle() || ctx.hasPendingMessages()) return;
|
|
120
133
|
const attempt = this.policy.recordAttempt(this.runtime.now());
|
|
121
134
|
if (!attempt) return;
|
|
122
|
-
this.pi.sendMessage(
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
135
|
+
this.pi.sendMessage(
|
|
136
|
+
{
|
|
137
|
+
customType: "jittor-codex-recovery",
|
|
138
|
+
content: `Retry the previous Codex request after a transient ${attempt.failureKind} failure. Automatic recovery attempt ${attempt.attempt} of ${CODEX_RECOVERY_MAX_ATTEMPTS}.`,
|
|
139
|
+
display: false,
|
|
140
|
+
details: { attempt: attempt.attempt, failureKind: attempt.failureKind },
|
|
141
|
+
},
|
|
142
|
+
{ triggerTurn: true, deliverAs: "followUp" },
|
|
143
|
+
);
|
|
128
144
|
}, plan.delayMs);
|
|
129
145
|
}
|
|
130
146
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { createRetryingClient, type RetryingClient } from "@danypops/vehicle-client/daemon-client";
|
|
2
1
|
import { connectJittorClient, type JittorClient, type OperationInputs, type OperationName, type OperationOutputs } from "@danypops/jittor";
|
|
2
|
+
import { createRetryingClient, type RetryingClient } from "@danypops/vehicle-client/daemon-client";
|
|
3
3
|
|
|
4
4
|
type JittorConnector = () => Promise<JittorClient>;
|
|
5
5
|
|
|
@@ -10,11 +10,58 @@ function defaultConnector(): Promise<JittorClient> {
|
|
|
10
10
|
let connector: JittorConnector = defaultConnector;
|
|
11
11
|
let retrying: RetryingClient<JittorClient> = createRetryingClient(() => connector(), { label: "Jittor" });
|
|
12
12
|
|
|
13
|
+
/**
|
|
14
|
+
* Exhaustive transport-retry policy. New daemon operations cannot compile until their mutability
|
|
15
|
+
* is classified: only reads may be transparently invoked twice after a connection-shaped error.
|
|
16
|
+
*/
|
|
17
|
+
const OPERATION_RETRY_MODE = {
|
|
18
|
+
"metrics.record": "once",
|
|
19
|
+
"metrics.record_batch": "once",
|
|
20
|
+
"metrics.query": "retry",
|
|
21
|
+
"metrics.distinct_scopes": "retry",
|
|
22
|
+
"metrics.usage_series": "retry",
|
|
23
|
+
"metrics.cost_by_task": "retry",
|
|
24
|
+
"metrics.prune": "once",
|
|
25
|
+
"benchmark.refresh": "once",
|
|
26
|
+
"benchmark.status": "retry",
|
|
27
|
+
"benchmark.query": "retry",
|
|
28
|
+
"catalog.refresh": "once",
|
|
29
|
+
"catalog.status": "retry",
|
|
30
|
+
"catalog.query": "retry",
|
|
31
|
+
"usage.import": "once",
|
|
32
|
+
"usage.import_status": "retry",
|
|
33
|
+
"usage.import_cancel": "once",
|
|
34
|
+
"export.status": "retry",
|
|
35
|
+
"export.flush": "once",
|
|
36
|
+
"session.register": "once",
|
|
37
|
+
"session.release": "once",
|
|
38
|
+
"models.rank": "once",
|
|
39
|
+
"context.assess": "retry",
|
|
40
|
+
"context.delta": "retry",
|
|
41
|
+
"context.snapshot": "once",
|
|
42
|
+
"compaction.estimate": "retry",
|
|
43
|
+
"service.checkpoint": "once",
|
|
44
|
+
"telemetry.poll": "retry",
|
|
45
|
+
"router.status": "retry",
|
|
46
|
+
"router.decide": "retry",
|
|
47
|
+
"router.pause": "once",
|
|
48
|
+
"router.resume": "once",
|
|
49
|
+
"router.override": "once",
|
|
50
|
+
"router.clear_override": "once",
|
|
51
|
+
"router.current_route": "once",
|
|
52
|
+
"router.available_routes": "once",
|
|
53
|
+
} as const satisfies Record<OperationName, "retry" | "once">;
|
|
54
|
+
|
|
55
|
+
export function operationRetryMode(operation: OperationName): "retry" | "once" {
|
|
56
|
+
return OPERATION_RETRY_MODE[operation];
|
|
57
|
+
}
|
|
58
|
+
|
|
13
59
|
export async function callJittor<Name extends OperationName>(
|
|
14
60
|
operation: Name,
|
|
15
61
|
input: OperationInputs[Name],
|
|
16
62
|
): Promise<OperationOutputs[Name]> {
|
|
17
|
-
|
|
63
|
+
const invoke = (client: JittorClient) => client.call(operation, input);
|
|
64
|
+
return operationRetryMode(operation) === "retry" ? retrying.call(invoke) : retrying.callOnce(invoke);
|
|
18
65
|
}
|
|
19
66
|
|
|
20
67
|
export function setJittorClientConnectorForTests(value: JittorConnector): void {
|