@danypops/jittor 0.11.0 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -73
- package/package.json +13 -15
- package/src/cli-commands/service-daemon.ts +1 -1
- package/src/cli-commands/support.ts +1 -1
- package/src/cli.ts +1 -1
- package/src/client.ts +2 -2
- package/src/daemon.ts +8 -8
- package/src/db.ts +2 -2
- package/src/index.ts +137 -0
- package/src/log.ts +3 -3
- package/src/ports/session-identity-store.ts +2 -2
- package/src/service.ts +2 -2
- package/src/session-identity-service.ts +1 -1
- package/src/state.ts +10 -10
- package/src/version.ts +1 -1
- package/docs/USAGE_PRIOR_ART.md +0 -64
- package/extension/src/benchmark-tui.ts +0 -109
- package/extension/src/capabilities/codex-recovery.ts +0 -127
- package/extension/src/capabilities/http-headers.ts +0 -5
- package/extension/src/capabilities/local-run-telemetry.ts +0 -104
- package/extension/src/capabilities/provider-response-telemetry.ts +0 -96
- package/extension/src/footer.ts +0 -323
- package/extension/src/index.ts +0 -648
- package/extension/src/service-client.ts +0 -26
- package/extension/src/session-identity.ts +0 -20
- package/extension/src/settings-tui.ts +0 -153
- package/extension/src/settings.ts +0 -103
- package/extension/src/tui.ts +0 -279
- package/extension/src/usage.ts +0 -320
package/docs/USAGE_PRIOR_ART.md
DELETED
|
@@ -1,64 +0,0 @@
|
|
|
1
|
-
# Token-usage TUI prior art
|
|
2
|
-
|
|
3
|
-
Research performed before implementing Jittor's `/jittor usage` frontend.
|
|
4
|
-
|
|
5
|
-
## Local agent implementations
|
|
6
|
-
|
|
7
|
-
### Claude Code
|
|
8
|
-
|
|
9
|
-
`~/Repositories/claude/src/components/Stats.tsx` contains the strongest terminal chart precedent:
|
|
10
|
-
|
|
11
|
-
- `generateTokenChart()` renders daily model-token series with an eight-row `asciichart` plot.
|
|
12
|
-
- Width adapts to the terminal and is capped near 52 columns.
|
|
13
|
-
- The top three models receive distinct theme colors.
|
|
14
|
-
- The X axis places three or four date labels at even positions.
|
|
15
|
-
- The Y axis abbreviates values as `k` and `M`.
|
|
16
|
-
|
|
17
|
-
`~/Repositories/claude/src/utils/heatmap.ts` adds a GitHub-style activity view using percentile-derived `░▒▓█` intensity. `src/components/design-system/ProgressBar.tsx` uses eighth-cell Unicode blocks (`▏▎▍▌▋▊▉█`) for fractional precision.
|
|
18
|
-
|
|
19
|
-
Useful decisions: adaptive width, short axes, bounded series count, theme colors, Unicode partial cells. The Jittor requirement is a histogram rather than Claude's line chart, so only the layout and scaling ideas are reused.
|
|
20
|
-
|
|
21
|
-
### OpenCode
|
|
22
|
-
|
|
23
|
-
`~/Repositories/opencode/packages/stats/core/src/domain/home.ts` keeps usage projection in the domain layer. It defines explicit ranges (`1D`, `1W`, `2W`, `1M`, `2M`, `3M`, `YTD`, `ALL`), computes date windows, creates deterministic buckets, and formats range-appropriate labels.
|
|
24
|
-
|
|
25
|
-
Useful decision: range/window/bucket projection is separate from rendering and data access.
|
|
26
|
-
|
|
27
|
-
### Codex
|
|
28
|
-
|
|
29
|
-
`~/Repositories/codex` exposes turn-level `last` and `total` token usage plus account rate-limit snapshots. Its TUI focuses on compact totals and quota state; no reusable historical token histogram was found.
|
|
30
|
-
|
|
31
|
-
Useful decision: finalized turn usage is the durable accounting point. Jittor already records Pi assistant usage on `message_end`.
|
|
32
|
-
|
|
33
|
-
### Cline
|
|
34
|
-
|
|
35
|
-
Cline tracks accumulated input/output/cache/cost values and presents context/cost in its status area. Its local CLI sources did not contain a historical terminal histogram comparable to the requested chart.
|
|
36
|
-
|
|
37
|
-
Useful decision: preserve input, output, cache-read, and cache-write categories rather than collapsing accounting at ingestion.
|
|
38
|
-
|
|
39
|
-
## Pi extensions
|
|
40
|
-
|
|
41
|
-
Registry and package-source review covered:
|
|
42
|
-
|
|
43
|
-
- `@pi-vault/pi-usage@0.6.0`: polished framed/tabbed dashboard, Today/This Week/Last Week/All Time tables, live provider quotas, width-safe theme adapters. It has no historical vertical token histogram.
|
|
44
|
-
- `@sreetej510/pi-usage@0.1.20`: `/usage` provider quota reports, cache, retries, statusline, and 20-cell `█░` quota bars.
|
|
45
|
-
- `@narumitw/pi-codex-usage@0.20.0`: Codex 5-hour/weekly quota bars and compact statusline.
|
|
46
|
-
- `@alexanderfortin/pi-token-usage@0.3.0`: session-file aggregation and table/export overlay, but no chart.
|
|
47
|
-
|
|
48
|
-
Useful decisions: native `registerCommand`, `ctx.ui.custom`, width-bounded rendering, theme-derived colors, keyboard refresh/range navigation, and daemon/cache-backed data rather than synchronous file scans in render.
|
|
49
|
-
|
|
50
|
-
## OpenRouter visual reference
|
|
51
|
-
|
|
52
|
-
OpenRouter's authenticated web dashboard is not distributed as reusable terminal source. Its relevant visual grammar is a compact time-bucket histogram with colored model/provider series, readable axes, totals, and a legend. Jittor reproduces that grammar with Unicode blocks rather than copying web implementation details.
|
|
53
|
-
|
|
54
|
-
## Jittor design
|
|
55
|
-
|
|
56
|
-
Jittor combines the best applicable patterns:
|
|
57
|
-
|
|
58
|
-
1. Pure domain projection in `src/domain/usage.ts`.
|
|
59
|
-
2. Explicit `24h`, `7d`, `30d`, and `90d` windows.
|
|
60
|
-
3. Provider/model-preserving series and input/output/cache totals.
|
|
61
|
-
4. Vertically scaled, colored, stacked Unicode bars with fractional top blocks.
|
|
62
|
-
5. Width-safe X/Y axes and provider/model legend.
|
|
63
|
-
6. Native `/jittor usage` panel with Left/Right range switching and refresh.
|
|
64
|
-
7. Data access only through authenticated daemon `metrics.query`; the extension never opens SQLite or reads provider credentials.
|
|
@@ -1,109 +0,0 @@
|
|
|
1
|
-
import type { ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
|
|
2
|
-
import { matchesKey, truncateToWidth } from "@earendil-works/pi-tui";
|
|
3
|
-
import {
|
|
4
|
-
BENCHMARK_TUI_MAX_CANDIDATES,
|
|
5
|
-
BENCHMARK_TUI_MAX_PROVENANCE_PER_CANDIDATE,
|
|
6
|
-
MODEL_RANKING_DEFAULT_CONTEXT_WEIGHT,
|
|
7
|
-
MODEL_RANKING_DEFAULT_COST_WEIGHT,
|
|
8
|
-
MODEL_RANKING_DEFAULT_LATENCY_WEIGHT,
|
|
9
|
-
MODEL_RANKING_DEFAULT_QUALITY_WEIGHT,
|
|
10
|
-
MODEL_RANKING_DEFAULT_RELIABILITY_WEIGHT,
|
|
11
|
-
} from "../../src/constants.ts";
|
|
12
|
-
import type { ModelTaskDomain, ModelTaskType } from "../../src/domain/model-observation.ts";
|
|
13
|
-
import type { ModelCandidate, ModelRankingResult, RankedModel, UtilityComponentName } from "../../src/domain/model-ranking.ts";
|
|
14
|
-
import { sessionSecretField } from "./session-identity.ts";
|
|
15
|
-
|
|
16
|
-
export interface BenchmarkPanelClient {
|
|
17
|
-
call(operation: string, input: unknown): Promise<any>;
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
interface BenchmarkTheme {
|
|
21
|
-
fg(color: string, text: string): string;
|
|
22
|
-
bold(text: string): string;
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
type BenchmarkPanelAction = "refresh" | "close";
|
|
26
|
-
|
|
27
|
-
const COMPONENT_LABELS: Record<UtilityComponentName, string> = { quality: "Q", cost: "$", latency: "L", context: "C", reliability: "R" };
|
|
28
|
-
|
|
29
|
-
function componentText(item: RankedModel): string {
|
|
30
|
-
return item.components.map((component) => `${COMPONENT_LABELS[component.name]} ${component.score === null ? "?" : component.score.toFixed(3)}`).join(" · ");
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
function candidateLines(item: RankedModel, index: number, currentIdentity: string): string[] {
|
|
34
|
-
const current = item.identity.startsWith(`${currentIdentity}:`);
|
|
35
|
-
const localSamples = item.components.find((component) => component.name === "reliability")?.evidenceCount ?? 0;
|
|
36
|
-
const provenance = item.provenance.slice(0, BENCHMARK_TUI_MAX_PROVENANCE_PER_CANDIDATE).map((source) => `${source.sourceId}@${source.revision} ${source.freshness}`).join(" · ");
|
|
37
|
-
return [
|
|
38
|
-
` ${index + 1}. ${item.identity}${index === 0 ? " recommended" : ""}${current ? " current" : ""}`,
|
|
39
|
-
` utility ${item.utility === null ? "?" : item.utility.toFixed(3)} · confidence ${(item.confidence * 100).toFixed(0)}% · ${componentText(item)}`,
|
|
40
|
-
` local n=${localSamples}${provenance ? ` · ${provenance}` : " · no external provenance"}`,
|
|
41
|
-
];
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
export function renderBenchmarkView(result: ModelRankingResult, currentIdentity: string, width: number, theme: BenchmarkTheme): string[] {
|
|
45
|
-
const safeWidth = Math.max(1, width);
|
|
46
|
-
const shown = result.ranked.slice(0, BENCHMARK_TUI_MAX_CANDIDATES);
|
|
47
|
-
const currentIndex = result.ranked.findIndex((item) => item.identity.startsWith(`${currentIdentity}:`));
|
|
48
|
-
const recommended = result.ranked[0];
|
|
49
|
-
const reason = recommended && currentIndex > 0
|
|
50
|
-
? `Recommendation differs from current: ${recommended.identity} ranks #1; current ranks #${currentIndex + 1}.`
|
|
51
|
-
: recommended && currentIndex === 0 ? "Current model is the top recommendation." : "Current model is outside the ranked candidates.";
|
|
52
|
-
const lines = [
|
|
53
|
-
theme.fg("borderMuted", "─".repeat(safeWidth)),
|
|
54
|
-
theme.bold("Jittor Benchmark Recommendations"),
|
|
55
|
-
result.scopeAuthority === "exact-session" ? "Scope: exact session" : "Scope: available models · ADVISORY (exact session scope unavailable)",
|
|
56
|
-
`Domain: ${result.domain} · Type: ${result.type} · evidence ${result.completeness}`,
|
|
57
|
-
reason,
|
|
58
|
-
...shown.flatMap((item, index) => candidateLines(item, index, currentIdentity)),
|
|
59
|
-
...(result.ranked.length > shown.length ? [` … ${result.ranked.length - shown.length} more candidates omitted`] : []),
|
|
60
|
-
...(result.scopeWarning ? [result.scopeWarning] : []),
|
|
61
|
-
theme.fg("dim", "r refresh · Esc close"),
|
|
62
|
-
theme.fg("borderMuted", "─".repeat(safeWidth)),
|
|
63
|
-
];
|
|
64
|
-
return lines.map((line) => truncateToWidth(line, safeWidth, "…"));
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
export async function showBenchmarkPanel(
|
|
68
|
-
ctx: ExtensionCommandContext,
|
|
69
|
-
client: BenchmarkPanelClient,
|
|
70
|
-
candidates: ModelCandidate[],
|
|
71
|
-
currentIdentity: string,
|
|
72
|
-
domain: ModelTaskDomain,
|
|
73
|
-
type: ModelTaskType,
|
|
74
|
-
): Promise<void> {
|
|
75
|
-
for (;;) {
|
|
76
|
-
const session_id = ctx.sessionManager.getSessionId();
|
|
77
|
-
const result = await client.call("models.rank", {
|
|
78
|
-
candidates,
|
|
79
|
-
session_id,
|
|
80
|
-
...sessionSecretField(session_id),
|
|
81
|
-
scopeAuthority: "available-models",
|
|
82
|
-
domain,
|
|
83
|
-
type,
|
|
84
|
-
budgetPressure: 0,
|
|
85
|
-
weights: {
|
|
86
|
-
quality: MODEL_RANKING_DEFAULT_QUALITY_WEIGHT,
|
|
87
|
-
cost: MODEL_RANKING_DEFAULT_COST_WEIGHT,
|
|
88
|
-
latency: MODEL_RANKING_DEFAULT_LATENCY_WEIGHT,
|
|
89
|
-
context: MODEL_RANKING_DEFAULT_CONTEXT_WEIGHT,
|
|
90
|
-
reliability: MODEL_RANKING_DEFAULT_RELIABILITY_WEIGHT,
|
|
91
|
-
},
|
|
92
|
-
sourceIds: ["openrouter-models", "lmarena-hf", "artificial-analysis-direct", "openrouter-design-arena"],
|
|
93
|
-
}) as ModelRankingResult;
|
|
94
|
-
if (ctx.mode !== "tui") {
|
|
95
|
-
ctx.ui.notify(renderBenchmarkView(result, currentIdentity, 100, { fg: (_color, text) => text, bold: (text) => text }).join("\n"), "info");
|
|
96
|
-
return;
|
|
97
|
-
}
|
|
98
|
-
const action = await ctx.ui.custom<BenchmarkPanelAction>((_tui, theme, _keybindings, done) => ({
|
|
99
|
-
invalidate() {},
|
|
100
|
-
render(width: number): string[] { return renderBenchmarkView(result, currentIdentity, width, theme); },
|
|
101
|
-
handleInput(data: string): void {
|
|
102
|
-
if (matchesKey(data, "escape") || matchesKey(data, "ctrl+c")) done("close");
|
|
103
|
-
else if (data === "r") done("refresh");
|
|
104
|
-
},
|
|
105
|
-
}));
|
|
106
|
-
if (!action || action === "close") return;
|
|
107
|
-
await client.call("benchmark.refresh", { force: true });
|
|
108
|
-
}
|
|
109
|
-
}
|
|
@@ -1,127 +0,0 @@
|
|
|
1
|
-
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
2
|
-
import {
|
|
3
|
-
CODEX_RECOVERY_ATTEMPT_WINDOW_MS,
|
|
4
|
-
CODEX_RECOVERY_BASE_DELAY_MS,
|
|
5
|
-
CODEX_RECOVERY_JITTER_RATIO,
|
|
6
|
-
CODEX_RECOVERY_MAX_ATTEMPTS,
|
|
7
|
-
CODEX_RECOVERY_MAX_DELAY_MS,
|
|
8
|
-
MILLISECONDS_PER_MINUTE,
|
|
9
|
-
MILLISECONDS_PER_SECOND,
|
|
10
|
-
} from "../../../src/constants.ts";
|
|
11
|
-
import { CodexRecoveryPolicy, classifyCodexFailure, type CodexFailureKind, type CodexFailureMetadata } from "../../../src/domain/codex-recovery.ts";
|
|
12
|
-
import type { CodexRecoveryControl } from "../settings.ts";
|
|
13
|
-
import { headerValue } from "./http-headers.ts";
|
|
14
|
-
|
|
15
|
-
export interface CodexRecoveryRuntime {
|
|
16
|
-
now(): number;
|
|
17
|
-
random(): number;
|
|
18
|
-
setTimeout(callback: () => void | Promise<void>, delayMs: number): unknown;
|
|
19
|
-
clearTimeout(handle: unknown): void;
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
export const SYSTEM_RECOVERY_RUNTIME: CodexRecoveryRuntime = {
|
|
23
|
-
now: Date.now,
|
|
24
|
-
random: Math.random,
|
|
25
|
-
setTimeout(callback, delayMs) { return setTimeout(() => { void callback(); }, delayMs); },
|
|
26
|
-
clearTimeout(handle) { clearTimeout(handle as ReturnType<typeof setTimeout>); },
|
|
27
|
-
};
|
|
28
|
-
|
|
29
|
-
/**
|
|
30
|
-
* Codex's settled-turn hidden-retry recovery, as its own capability: tracks the most recent
|
|
31
|
-
* Codex response (status/retry-after) across the current turn, classifies a finalized failure,
|
|
32
|
-
* and schedules at most one bounded, jittered follow-up once Pi's own turn has genuinely
|
|
33
|
-
* settled. Fully self-contained -- the only external dependencies are Pi's own message-send API,
|
|
34
|
-
* the persisted on/off control, and a runtime the tests can fake (timers, randomness, clock).
|
|
35
|
-
*/
|
|
36
|
-
export class CodexRecoveryCapability {
|
|
37
|
-
private readonly policy: CodexRecoveryPolicy;
|
|
38
|
-
private lastResponse: CodexFailureMetadata = {};
|
|
39
|
-
private timer: unknown;
|
|
40
|
-
private cooldown: { until: number; attempt: number; failureKind: CodexFailureKind } | undefined;
|
|
41
|
-
|
|
42
|
-
constructor(
|
|
43
|
-
private readonly pi: ExtensionAPI,
|
|
44
|
-
private readonly control: CodexRecoveryControl,
|
|
45
|
-
private readonly runtime: CodexRecoveryRuntime,
|
|
46
|
-
) {
|
|
47
|
-
this.policy = new CodexRecoveryPolicy({
|
|
48
|
-
baseDelayMs: CODEX_RECOVERY_BASE_DELAY_MS,
|
|
49
|
-
maxDelayMs: CODEX_RECOVERY_MAX_DELAY_MS,
|
|
50
|
-
maxAttempts: CODEX_RECOVERY_MAX_ATTEMPTS,
|
|
51
|
-
attemptWindowMs: CODEX_RECOVERY_ATTEMPT_WINDOW_MS,
|
|
52
|
-
jitterRatio: CODEX_RECOVERY_JITTER_RATIO,
|
|
53
|
-
}, runtime.random);
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
/** Clears the tracked response at the start of every new turn, before any Codex response for it has arrived. */
|
|
57
|
-
resetTurn(): void {
|
|
58
|
-
this.lastResponse = {};
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
notifyResponse(status: number, headers: Record<string, string>): void {
|
|
62
|
-
this.lastResponse = { status, ...(headerValue(headers, "retry-after") ? { retryAfter: headerValue(headers, "retry-after") } : {}) };
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
notifyMessageEnd(stopReason: string, errorMessage: string | undefined): void {
|
|
66
|
-
if (stopReason === "error") {
|
|
67
|
-
const failure = classifyCodexFailure(errorMessage, this.lastResponse);
|
|
68
|
-
if (this.control.isCodexRecoveryEnabled() && failure.transient) this.policy.observeFailure(failure, this.runtime.now());
|
|
69
|
-
else this.cancel(true);
|
|
70
|
-
} else if (stopReason !== "aborted") {
|
|
71
|
-
this.cancel(true);
|
|
72
|
-
}
|
|
73
|
-
this.lastResponse = {};
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
cancel(resetPolicy: boolean): void {
|
|
77
|
-
if (this.timer !== undefined) this.runtime.clearTimeout(this.timer);
|
|
78
|
-
this.timer = undefined;
|
|
79
|
-
this.cooldown = undefined;
|
|
80
|
-
if (resetPolicy) this.policy.cancel();
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
statusText(): string {
|
|
84
|
-
const now = this.runtime.now();
|
|
85
|
-
const state = this.policy.state(now);
|
|
86
|
-
const enabled = this.control.isCodexRecoveryEnabled();
|
|
87
|
-
const attempt = this.cooldown?.attempt ?? (state.pending ? state.attempts + 1 : state.attempts);
|
|
88
|
-
const phase = this.cooldown
|
|
89
|
-
? `cooldown ${Math.ceil(Math.max(0, this.cooldown.until - now) / MILLISECONDS_PER_SECOND)}s`
|
|
90
|
-
: state.pending ? "pending"
|
|
91
|
-
: state.attempts >= CODEX_RECOVERY_MAX_ATTEMPTS ? "exhausted"
|
|
92
|
-
: state.attempts > 0 ? "waiting" : "idle";
|
|
93
|
-
const failureKind = this.cooldown?.failureKind ?? state.lastFailureKind;
|
|
94
|
-
return [
|
|
95
|
-
`Codex recovery: ${enabled ? "on" : "off"}`,
|
|
96
|
-
phase,
|
|
97
|
-
`attempt ${attempt}/${CODEX_RECOVERY_MAX_ATTEMPTS}`,
|
|
98
|
-
`window ${CODEX_RECOVERY_ATTEMPT_WINDOW_MS / MILLISECONDS_PER_MINUTE}m`,
|
|
99
|
-
...(failureKind ? [failureKind] : []),
|
|
100
|
-
].join(" · ");
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
scheduleIfIdle(ctx: ExtensionContext): void {
|
|
104
|
-
if (!this.control.isCodexRecoveryEnabled() || this.timer !== undefined || !ctx.isIdle() || ctx.hasPendingMessages()) return;
|
|
105
|
-
const plan = this.policy.plan(this.runtime.now());
|
|
106
|
-
if (plan.action === "exhausted") {
|
|
107
|
-
this.policy.abandonFailure();
|
|
108
|
-
if (ctx.hasUI) ctx.ui.notify(`Jittor Codex recovery stopped: ${plan.reason}.`, "warning");
|
|
109
|
-
return;
|
|
110
|
-
}
|
|
111
|
-
if (plan.action !== "schedule") return;
|
|
112
|
-
this.cooldown = { until: this.runtime.now() + plan.delayMs, attempt: plan.attempt, failureKind: plan.failureKind };
|
|
113
|
-
this.timer = this.runtime.setTimeout(async () => {
|
|
114
|
-
this.timer = undefined;
|
|
115
|
-
this.cooldown = undefined;
|
|
116
|
-
if (!ctx.isIdle() || ctx.hasPendingMessages()) return;
|
|
117
|
-
const attempt = this.policy.recordAttempt(this.runtime.now());
|
|
118
|
-
if (!attempt) return;
|
|
119
|
-
this.pi.sendMessage({
|
|
120
|
-
customType: "jittor-codex-recovery",
|
|
121
|
-
content: `Retry the previous Codex request after a transient ${attempt.failureKind} failure. Automatic recovery attempt ${attempt.attempt} of ${CODEX_RECOVERY_MAX_ATTEMPTS}.`,
|
|
122
|
-
display: false,
|
|
123
|
-
details: { attempt: attempt.attempt, failureKind: attempt.failureKind },
|
|
124
|
-
}, { triggerTurn: true, deliverAs: "followUp" });
|
|
125
|
-
}, plan.delayMs);
|
|
126
|
-
}
|
|
127
|
-
}
|
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
/** Case-insensitive header lookup -- Pi's provider-response event headers are a plain Record, not a Headers instance. */
|
|
2
|
-
export function headerValue(headers: Record<string, string>, name: string): string | undefined {
|
|
3
|
-
const expected = name.toLowerCase();
|
|
4
|
-
return Object.entries(headers).find(([key]) => key.toLowerCase() === expected)?.[1];
|
|
5
|
-
}
|
|
@@ -1,104 +0,0 @@
|
|
|
1
|
-
import { classifyTaskFromTools, modelRunMetrics, type ModelRunObservation } from "../../../src/domain/model-observation.ts";
|
|
2
|
-
import type { MetricObservation } from "../../../src/domain/metric.ts";
|
|
3
|
-
|
|
4
|
-
export interface ActiveLocalModelRun {
|
|
5
|
-
runId: string;
|
|
6
|
-
startedAt: number;
|
|
7
|
-
firstTokenAt: number | null;
|
|
8
|
-
providerResponses: number;
|
|
9
|
-
toolNames: string[];
|
|
10
|
-
toolCalls: number;
|
|
11
|
-
toolFailures: number;
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
/**
|
|
15
|
-
* Content-free local model observations derived only from Pi's public lifecycle: TTFT, wall
|
|
16
|
-
* latency, tool-loop counts and failures, and a bounded tool-name list used only to derive
|
|
17
|
-
* domain/type classification -- never prompts, responses, tool arguments/results, or project
|
|
18
|
-
* paths. Owns the one in-flight run plus the most recently completed one (for `/jittor outcome`).
|
|
19
|
-
*/
|
|
20
|
-
export class LocalRunTelemetry {
|
|
21
|
-
private active: ActiveLocalModelRun | undefined;
|
|
22
|
-
private lastCompleted: ModelRunObservation | undefined;
|
|
23
|
-
private sequence = 0;
|
|
24
|
-
|
|
25
|
-
beginTurn(timestamp: number): void {
|
|
26
|
-
this.active = {
|
|
27
|
-
runId: `local-${timestamp}-${++this.sequence}`,
|
|
28
|
-
startedAt: timestamp,
|
|
29
|
-
firstTokenAt: null,
|
|
30
|
-
providerResponses: 0,
|
|
31
|
-
toolNames: [],
|
|
32
|
-
toolCalls: 0,
|
|
33
|
-
toolFailures: 0,
|
|
34
|
-
};
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
discardTurn(): void {
|
|
38
|
-
this.active = undefined;
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
onMessageUpdate(assistantMessageEventType: string): void {
|
|
42
|
-
if (!this.active || this.active.firstTokenAt !== null) return;
|
|
43
|
-
if (["text_delta", "thinking_delta", "toolcall_delta"].includes(assistantMessageEventType)) this.active.firstTokenAt = Date.now();
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
onToolExecutionEnd(toolName: string, isError: boolean): void {
|
|
47
|
-
if (!this.active) return;
|
|
48
|
-
this.active.toolCalls += 1;
|
|
49
|
-
if (isError) this.active.toolFailures += 1;
|
|
50
|
-
if (this.active.toolNames.length < 100) this.active.toolNames.push(toolName);
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
onProviderResponse(): void {
|
|
54
|
-
if (this.active) this.active.providerResponses += 1;
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
/** Finalizes the active run against a completed assistant turn_end message, returning metrics to record (empty if the message shape doesn't match a completed assistant turn). */
|
|
58
|
-
completeTurn(message: unknown, thinkingLevel: string): MetricObservation[] {
|
|
59
|
-
const active = this.active;
|
|
60
|
-
this.active = undefined;
|
|
61
|
-
if (!active || typeof message !== "object" || message === null || Array.isArray(message)) return [];
|
|
62
|
-
const value = message as Record<string, unknown>;
|
|
63
|
-
if (value["role"] !== "assistant" || typeof value["provider"] !== "string" || typeof value["model"] !== "string") return [];
|
|
64
|
-
const usage = typeof value["usage"] === "object" && value["usage"] !== null ? value["usage"] as Record<string, unknown> : {};
|
|
65
|
-
const amount = (name: string): number => typeof usage[name] === "number" && Number.isFinite(usage[name]) ? usage[name] as number : 0;
|
|
66
|
-
const cost = typeof usage["cost"] === "object" && usage["cost"] !== null && typeof (usage["cost"] as Record<string, unknown>)["total"] === "number"
|
|
67
|
-
? (usage["cost"] as Record<string, number>)["total"] ?? 0 : 0;
|
|
68
|
-
const stopReason = ["stop", "length", "toolUse", "error", "aborted"].includes(String(value["stopReason"]))
|
|
69
|
-
? value["stopReason"] as ModelRunObservation["stopReason"] : "unknown";
|
|
70
|
-
const completedAt = Math.max(Date.now(), active.firstTokenAt ?? active.startedAt, active.startedAt);
|
|
71
|
-
this.lastCompleted = {
|
|
72
|
-
runId: active.runId,
|
|
73
|
-
provider: value["provider"],
|
|
74
|
-
model: value["model"],
|
|
75
|
-
thinking: thinkingLevel,
|
|
76
|
-
...classifyTaskFromTools(active.toolNames),
|
|
77
|
-
startedAt: active.startedAt,
|
|
78
|
-
firstTokenAt: active.firstTokenAt,
|
|
79
|
-
completedAt,
|
|
80
|
-
inputTokens: amount("input"),
|
|
81
|
-
outputTokens: amount("output"),
|
|
82
|
-
cacheReadTokens: amount("cacheRead"),
|
|
83
|
-
cacheWriteTokens: amount("cacheWrite"),
|
|
84
|
-
costUsd: Number.isFinite(cost) && cost >= 0 ? cost : 0,
|
|
85
|
-
providerResponses: Math.max(1, active.providerResponses),
|
|
86
|
-
toolCalls: active.toolCalls,
|
|
87
|
-
toolFailures: active.toolFailures,
|
|
88
|
-
stopReason,
|
|
89
|
-
explicitOutcome: "unknown",
|
|
90
|
-
};
|
|
91
|
-
return modelRunMetrics(this.lastCompleted);
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
/** Builds the single outcome-accepted/outcome-accepted=0 metric for the most recently completed run, or null if none exists yet. */
|
|
95
|
-
explicitOutcomeMetric(explicitOutcome: "accepted" | "rejected"): MetricObservation | null {
|
|
96
|
-
if (!this.lastCompleted) return null;
|
|
97
|
-
return modelRunMetrics({ ...this.lastCompleted, explicitOutcome }).find((metric) => metric.metric === "outcome-accepted") ?? null;
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
reset(): void {
|
|
101
|
-
this.active = undefined;
|
|
102
|
-
this.lastCompleted = undefined;
|
|
103
|
-
}
|
|
104
|
-
}
|
|
@@ -1,96 +0,0 @@
|
|
|
1
|
-
import type { MetricObservation } from "../../../src/domain/metric.ts";
|
|
2
|
-
import { hasAnthropicRateLimitHeaders, parseAnthropicRateLimitHeaders } from "../../../src/providers/anthropic-contracts.ts";
|
|
3
|
-
import { parseCodexRateLimitHeaders } from "../../../src/providers/codex.ts";
|
|
4
|
-
import { classifyGoogleVertexFailure, googleVertexFailureMetrics, type GoogleVertexFailureMetadata } from "../../../src/providers/google-vertex-contracts.ts";
|
|
5
|
-
import { headerValue } from "./http-headers.ts";
|
|
6
|
-
|
|
7
|
-
export interface ProviderTelemetryClient {
|
|
8
|
-
call(operation: string, input: unknown): Promise<any>;
|
|
9
|
-
}
|
|
10
|
-
|
|
11
|
-
async function recordMetrics(client: ProviderTelemetryClient, metrics: MetricObservation[]): Promise<void> {
|
|
12
|
-
if (metrics.length === 0) return;
|
|
13
|
-
await client.call("metrics.record_batch", { observations: metrics });
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
/**
|
|
17
|
-
* Bounded telemetry derived directly from provider HTTP responses and finalized messages, for
|
|
18
|
-
* every provider except Codex (whose response tracking is settled-turn-recovery's own concern --
|
|
19
|
-
* see codex-recovery.ts). Anthropic and anthropic-vertex official rate-limit headers become
|
|
20
|
-
* budget metrics; Google Vertex and anthropic-vertex failures become bounded, content-free
|
|
21
|
-
* failure-count metrics classified from GCP's own `google.rpc.Status` shape. anthropic-vertex is
|
|
22
|
-
* tracked distinctly from both google-vertex (different code path, different quota pool) and
|
|
23
|
-
* direct anthropic (different transport), never conflated with either.
|
|
24
|
-
*/
|
|
25
|
-
export class ProviderResponseTelemetry {
|
|
26
|
-
private lastGoogleVertexResponse: GoogleVertexFailureMetadata = {};
|
|
27
|
-
private lastAnthropicVertexResponse: GoogleVertexFailureMetadata = {};
|
|
28
|
-
|
|
29
|
-
resetTurn(): void {
|
|
30
|
-
this.lastGoogleVertexResponse = {};
|
|
31
|
-
this.lastAnthropicVertexResponse = {};
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
async handleProviderResponse(
|
|
35
|
-
client: ProviderTelemetryClient,
|
|
36
|
-
provider: string | undefined,
|
|
37
|
-
status: number,
|
|
38
|
-
headers: Record<string, string>,
|
|
39
|
-
notifySchemaDrift: (message: string) => void,
|
|
40
|
-
): Promise<void> {
|
|
41
|
-
if (provider === "anthropic") {
|
|
42
|
-
const parsedHeaders = new Headers(headers);
|
|
43
|
-
if (hasAnthropicRateLimitHeaders(parsedHeaders)) {
|
|
44
|
-
try {
|
|
45
|
-
await recordMetrics(client, parseAnthropicRateLimitHeaders(parsedHeaders, Date.now()).metrics);
|
|
46
|
-
} catch {
|
|
47
|
-
notifySchemaDrift("Anthropic telemetry schema drift");
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
if (provider === "anthropic-vertex") {
|
|
52
|
-
// Best-effort only: unverified whether this passthrough ever forwards Anthropic's own
|
|
53
|
-
// rate-limit headers. If it doesn't, hasAnthropicRateLimitHeaders is false and nothing is
|
|
54
|
-
// recorded -- the same honest default as every other unconfirmed signal in this module.
|
|
55
|
-
const parsedHeaders = new Headers(headers);
|
|
56
|
-
if (hasAnthropicRateLimitHeaders(parsedHeaders)) {
|
|
57
|
-
try {
|
|
58
|
-
await recordMetrics(client, parseAnthropicRateLimitHeaders(parsedHeaders, Date.now(), "anthropic-vertex").metrics);
|
|
59
|
-
} catch {
|
|
60
|
-
notifySchemaDrift("Anthropic-on-Vertex telemetry schema drift");
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
// Well-evidenced regardless of headers: GCP's own quota system fronts this transport, so the
|
|
64
|
-
// same failure classification as google-vertex applies below.
|
|
65
|
-
this.lastAnthropicVertexResponse = { status, ...(headerValue(headers, "retry-after") ? { retryAfter: headerValue(headers, "retry-after") } : {}) };
|
|
66
|
-
}
|
|
67
|
-
if (provider === "google-vertex") {
|
|
68
|
-
this.lastGoogleVertexResponse = { status, ...(headerValue(headers, "retry-after") ? { retryAfter: headerValue(headers, "retry-after") } : {}) };
|
|
69
|
-
}
|
|
70
|
-
if (Object.keys(headers).some((name) => name.toLowerCase().startsWith("x-codex-"))) {
|
|
71
|
-
try {
|
|
72
|
-
const updates = parseCodexRateLimitHeaders(new Headers(headers), Date.now());
|
|
73
|
-
await recordMetrics(client, updates.flatMap((update) => update.metrics));
|
|
74
|
-
} catch {
|
|
75
|
-
notifySchemaDrift("Codex telemetry schema drift");
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
async handleMessageEnd(client: ProviderTelemetryClient, provider: string | undefined, stopReason: string | undefined, errorMessage: string | undefined): Promise<void> {
|
|
81
|
-
if (provider === "google-vertex") {
|
|
82
|
-
if (stopReason === "error") {
|
|
83
|
-
const failure = classifyGoogleVertexFailure(errorMessage, this.lastGoogleVertexResponse);
|
|
84
|
-
await recordMetrics(client, googleVertexFailureMetrics(failure, Date.now())).catch(() => undefined);
|
|
85
|
-
}
|
|
86
|
-
this.lastGoogleVertexResponse = {};
|
|
87
|
-
}
|
|
88
|
-
if (provider === "anthropic-vertex") {
|
|
89
|
-
if (stopReason === "error") {
|
|
90
|
-
const failure = classifyGoogleVertexFailure(errorMessage, this.lastAnthropicVertexResponse);
|
|
91
|
-
await recordMetrics(client, googleVertexFailureMetrics(failure, Date.now(), "anthropic-vertex")).catch(() => undefined);
|
|
92
|
-
}
|
|
93
|
-
this.lastAnthropicVertexResponse = {};
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
}
|