@danypops/jittor 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -6
- package/extension/src/benchmark-tui.ts +4 -0
- package/extension/src/capabilities/codex-recovery.ts +127 -0
- package/extension/src/capabilities/http-headers.ts +5 -0
- package/extension/src/capabilities/local-run-telemetry.ts +104 -0
- package/extension/src/capabilities/provider-response-telemetry.ts +96 -0
- package/extension/src/footer.ts +10 -53
- package/extension/src/index.ts +100 -276
- package/extension/src/session-identity.ts +20 -0
- package/extension/src/tui.ts +20 -11
- package/package.json +1 -1
- package/src/adapters/sqlite-metric-store.ts +11 -2
- package/src/adapters/sqlite-session-identity-store.ts +45 -0
- package/src/cli-commands/benchmarks.ts +140 -0
- package/src/cli-commands/compaction.ts +17 -0
- package/src/cli-commands/context.ts +49 -0
- package/src/cli-commands/metrics.ts +296 -0
- package/src/cli-commands/op.ts +40 -0
- package/src/cli-commands/route-args.ts +15 -0
- package/src/cli-commands/router.ts +207 -0
- package/src/cli-commands/service-daemon.ts +72 -0
- package/src/cli-commands/session.ts +42 -0
- package/src/cli-commands/support.ts +33 -0
- package/src/cli.ts +42 -766
- package/src/constants.ts +7 -0
- package/src/daemon.ts +13 -3
- package/src/db.ts +15 -1
- package/src/domain/task-cost.ts +44 -9
- package/src/operations/benchmark-operations.ts +12 -0
- package/src/operations/context-operations.ts +30 -0
- package/src/operations/metrics-operations.ts +77 -0
- package/src/operations/model-ranking-operations.ts +16 -0
- package/src/operations/router-operations.ts +19 -0
- package/src/operations/session-identity-operations.ts +15 -0
- package/src/operations/session-scope.ts +31 -0
- package/src/operations/types.ts +3 -0
- package/src/ports/metric-store.ts +2 -0
- package/src/ports/router-controller.ts +9 -9
- package/src/ports/session-identity-store.ts +5 -0
- package/src/providers/telemetry-sources.ts +2 -1
- package/src/router.ts +124 -67
- package/src/service.ts +60 -118
- package/src/session-identity-service.ts +55 -0
package/extension/src/index.ts
CHANGED
|
@@ -1,39 +1,33 @@
|
|
|
1
1
|
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
2
2
|
import {
|
|
3
|
-
CODEX_RECOVERY_ATTEMPT_WINDOW_MS,
|
|
4
|
-
CODEX_RECOVERY_BASE_DELAY_MS,
|
|
5
|
-
CODEX_RECOVERY_JITTER_RATIO,
|
|
6
|
-
CODEX_RECOVERY_MAX_ATTEMPTS,
|
|
7
|
-
CODEX_RECOVERY_MAX_DELAY_MS,
|
|
8
3
|
FOOTER_COMPACTION_RENDER_INTERVAL_MS,
|
|
9
4
|
MAX_DYNAMIC_ROUTES,
|
|
10
|
-
MILLISECONDS_PER_MINUTE,
|
|
11
|
-
MILLISECONDS_PER_SECOND,
|
|
12
5
|
PAPYRUS_CONTEXT_INJECTION_CHANNEL,
|
|
13
6
|
PAPYRUS_TASK_FOCUS_CHANNEL,
|
|
14
7
|
CONTEXT_EVENT_DEDUP_LIMIT,
|
|
15
8
|
} from "../../src/constants.ts";
|
|
16
|
-
import { CodexRecoveryPolicy, classifyCodexFailure, type CodexFailureKind, type CodexFailureMetadata } from "../../src/domain/codex-recovery.ts";
|
|
17
9
|
import { CompactionTelemetry, papyrusContextMetric, validatePapyrusContextInjection } from "../../src/domain/context-telemetry.ts";
|
|
18
10
|
import { applyTaskFocusEvent, validateTaskFocusEvent } from "../../src/domain/task-focus.ts";
|
|
19
11
|
import type { MetricObservation, StoredMetricObservation } from "../../src/domain/metric.ts";
|
|
20
|
-
import {
|
|
12
|
+
import { TASK_DOMAINS, TASK_TYPES, type ModelTaskDomain, type ModelTaskType } from "../../src/domain/model-observation.ts";
|
|
21
13
|
import type { ModelCandidate } from "../../src/domain/model-ranking.ts";
|
|
22
14
|
import { USAGE_PERIODS, type UsagePeriod } from "../../src/domain/usage.ts";
|
|
23
15
|
import type { PolicyDecision, Route } from "../../src/policy.ts";
|
|
24
16
|
import type { RouterStatus } from "../../src/ports/router-controller.ts";
|
|
25
|
-
import { hasAnthropicRateLimitHeaders, parseAnthropicRateLimitHeaders } from "../../src/providers/anthropic-contracts.ts";
|
|
26
|
-
import { parseCodexRateLimitHeaders } from "../../src/providers/codex.ts";
|
|
27
|
-
import { classifyGoogleVertexFailure, googleVertexFailureMetrics, type GoogleVertexFailureMetadata } from "../../src/providers/google-vertex-contracts.ts";
|
|
28
17
|
import { showBenchmarkPanel } from "./benchmark-tui.ts";
|
|
29
18
|
import { installIntegratedFooter, type CompactionProgress, type IntegratedFooterState } from "./footer.ts";
|
|
30
19
|
import { callJittor } from "./service-client.ts";
|
|
31
20
|
import { persistentEnforcementControl, type CodexRecoveryControl, type EnforcementControl, type UsageBudgetControl } from "./settings.ts";
|
|
32
21
|
import { showSettingsPanel } from "./settings-tui.ts";
|
|
33
|
-
import { buildFooterBudget, formatFooterStatus, showJittorPanel } from "./tui.ts";
|
|
22
|
+
import { buildFooterBudget, formatFooterStatus, providerBudgetMetricQuery, showJittorPanel } from "./tui.ts";
|
|
23
|
+
import { cacheSessionSecret, forgetSessionSecret, sessionSecretField } from "./session-identity.ts";
|
|
34
24
|
import { showUsagePanel } from "./usage.ts";
|
|
25
|
+
import { CodexRecoveryCapability, SYSTEM_RECOVERY_RUNTIME, type CodexRecoveryRuntime } from "./capabilities/codex-recovery.ts";
|
|
26
|
+
import { ProviderResponseTelemetry } from "./capabilities/provider-response-telemetry.ts";
|
|
27
|
+
import { LocalRunTelemetry } from "./capabilities/local-run-telemetry.ts";
|
|
35
28
|
|
|
36
29
|
export { formatFooterStatus } from "./tui.ts";
|
|
30
|
+
export type { CodexRecoveryRuntime } from "./capabilities/codex-recovery.ts";
|
|
37
31
|
|
|
38
32
|
const THINKING_LEVELS = new Set(["off", "minimal", "low", "medium", "high", "xhigh", "max"]);
|
|
39
33
|
const RECOVERY_GUIDANCE = "Run /jittor off to disable blocking, or restart the daemon with: systemctl --user restart jittor.service";
|
|
@@ -46,20 +40,6 @@ const daemonClient: JittorExtensionClient = {
|
|
|
46
40
|
call: (operation, input) => callJittor(operation as Parameters<typeof callJittor>[0], input as never),
|
|
47
41
|
};
|
|
48
42
|
|
|
49
|
-
export interface CodexRecoveryRuntime {
|
|
50
|
-
now(): number;
|
|
51
|
-
random(): number;
|
|
52
|
-
setTimeout(callback: () => void | Promise<void>, delayMs: number): unknown;
|
|
53
|
-
clearTimeout(handle: unknown): void;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
const SYSTEM_RECOVERY_RUNTIME: CodexRecoveryRuntime = {
|
|
57
|
-
now: Date.now,
|
|
58
|
-
random: Math.random,
|
|
59
|
-
setTimeout(callback, delayMs) { return setTimeout(() => { void callback(); }, delayMs); },
|
|
60
|
-
clearTimeout(handle) { clearTimeout(handle as ReturnType<typeof setTimeout>); },
|
|
61
|
-
};
|
|
62
|
-
|
|
63
43
|
function usageBudgetControl(enforcement: EnforcementControl): UsageBudgetControl {
|
|
64
44
|
const candidate = enforcement as EnforcementControl & Partial<UsageBudgetControl>;
|
|
65
45
|
return typeof candidate.getUsageTokenBudget === "function" && typeof candidate.setUsageTokenBudget === "function"
|
|
@@ -81,31 +61,17 @@ function recoveryControl(enforcement: EnforcementControl): CodexRecoveryControl
|
|
|
81
61
|
: { isCodexRecoveryEnabled: () => false, setCodexRecoveryEnabled() {} };
|
|
82
62
|
}
|
|
83
63
|
|
|
84
|
-
function header(headers: Record<string, string>, name: string): string | undefined {
|
|
85
|
-
const expected = name.toLowerCase();
|
|
86
|
-
return Object.entries(headers).find(([key]) => key.toLowerCase() === expected)?.[1];
|
|
87
|
-
}
|
|
88
|
-
|
|
89
64
|
async function recordMetrics(client: JittorExtensionClient, metrics: MetricObservation[]): Promise<void> {
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
startedAt: number;
|
|
96
|
-
firstTokenAt: number | null;
|
|
97
|
-
providerResponses: number;
|
|
98
|
-
toolNames: string[];
|
|
99
|
-
toolCalls: number;
|
|
100
|
-
toolFailures: number;
|
|
65
|
+
if (metrics.length === 0) return;
|
|
66
|
+
// One atomic transaction rather than a per-metric RPC loop: a later observation in the same
|
|
67
|
+
// event failing validation, or the connection dropping mid-loop, must not leave this event's
|
|
68
|
+
// metrics partially persisted.
|
|
69
|
+
await client.call("metrics.record_batch", { observations: metrics });
|
|
101
70
|
}
|
|
102
71
|
|
|
103
|
-
async function refreshFooter(client: JittorExtensionClient, state: IntegratedFooterState): Promise<void> {
|
|
104
|
-
const status = await client.call("router.status", {}) as RouterStatus;
|
|
105
|
-
const
|
|
106
|
-
const query = provider === "openai-codex"
|
|
107
|
-
? { source: "codex-subscription", metric: "used-fraction", limit: 100, order: "desc" }
|
|
108
|
-
: provider === "openrouter" ? { source: "openrouter", limit: 20, order: "desc" } : null;
|
|
72
|
+
async function refreshFooter(client: JittorExtensionClient, state: IntegratedFooterState, sessionId: string): Promise<void> {
|
|
73
|
+
const status = await client.call("router.status", { session_id: sessionId }) as RouterStatus;
|
|
74
|
+
const query = providerBudgetMetricQuery(status);
|
|
109
75
|
const metrics = query ? await client.call("metrics.query", query) as StoredMetricObservation[] : [];
|
|
110
76
|
state.providerBudget = buildFooterBudget(status, metrics);
|
|
111
77
|
state.requestRender?.();
|
|
@@ -168,32 +134,41 @@ export function benchmarkCandidatesFromPi(models: PiRouteModel[], thinking: stri
|
|
|
168
134
|
}
|
|
169
135
|
|
|
170
136
|
export function routesFromPi(models: PiRouteModel[], current: PiRouteModel, thinking: string): Route[] {
|
|
171
|
-
const
|
|
172
|
-
.filter((model) => model.provider
|
|
173
|
-
.filter((model, index, rows) => rows.findIndex((candidate) => candidate.id === model.id) === index);
|
|
174
|
-
if (!
|
|
175
|
-
const routes: Route[] = [{ provider: current.provider, model: current.id, thinking }];
|
|
137
|
+
const catalog = models
|
|
138
|
+
.filter((model) => model.provider.length > 0 && model.id.length > 0)
|
|
139
|
+
.filter((model, index, rows) => rows.findIndex((candidate) => candidate.provider === model.provider && candidate.id === model.id) === index);
|
|
140
|
+
if (!catalog.some((model) => model.provider === current.provider && model.id === current.id)) catalog.push(current);
|
|
176
141
|
const currentLevel = THINKING_DESCENDING.indexOf(thinking as typeof THINKING_DESCENDING[number]);
|
|
177
142
|
const lowerLevels = THINKING_DESCENDING.slice(currentLevel >= 0 ? currentLevel + 1 : 0);
|
|
143
|
+
const routes: Route[] = [];
|
|
144
|
+
const add = (route: Route): void => {
|
|
145
|
+
if (routes.length >= MAX_DYNAMIC_ROUTES || routes.some((candidate) => candidate.provider === route.provider && candidate.model === route.model && candidate.thinking === route.thinking)) return;
|
|
146
|
+
routes.push(route);
|
|
147
|
+
};
|
|
148
|
+
add({ provider: current.provider, model: current.id, thinking });
|
|
178
149
|
for (const level of lowerLevels) {
|
|
179
|
-
if (supportsThinking(current, level))
|
|
150
|
+
if (supportsThinking(current, level)) add({ provider: current.provider, model: current.id, thinking: level });
|
|
180
151
|
}
|
|
181
|
-
const alternatives =
|
|
182
|
-
.filter((model) => model.id !== current.id)
|
|
183
|
-
.sort((left, right) =>
|
|
152
|
+
const alternatives = catalog
|
|
153
|
+
.filter((model) => model.provider !== current.provider || model.id !== current.id)
|
|
154
|
+
.sort((left, right) => {
|
|
155
|
+
const providerPriority = Number(left.provider !== current.provider) - Number(right.provider !== current.provider);
|
|
156
|
+
return providerPriority || modelCost(left) - modelCost(right) || left.provider.localeCompare(right.provider) || left.id.localeCompare(right.id);
|
|
157
|
+
});
|
|
184
158
|
for (const model of alternatives) {
|
|
185
159
|
const level = [thinking, ...lowerLevels].find((candidate) => supportsThinking(model, candidate)) ?? "off";
|
|
186
|
-
|
|
187
|
-
if (routes.length >= MAX_DYNAMIC_ROUTES) break;
|
|
160
|
+
add({ provider: model.provider, model: model.id, thinking: level });
|
|
188
161
|
}
|
|
189
162
|
return routes;
|
|
190
163
|
}
|
|
191
164
|
|
|
192
165
|
async function syncAvailableRoutes(pi: ExtensionAPI, client: JittorExtensionClient, ctx: ExtensionContext): Promise<void> {
|
|
193
|
-
|
|
166
|
+
const session_id = ctx.sessionManager.getSessionId();
|
|
167
|
+
const secret = sessionSecretField(session_id);
|
|
168
|
+
if (!ctx.model) { await client.call("router.available_routes", { routes: [], session_id, ...secret }); return; }
|
|
194
169
|
const models = ctx.modelRegistry.getAvailable() as PiRouteModel[];
|
|
195
170
|
const routes = routesFromPi(models, ctx.model as PiRouteModel, pi.getThinkingLevel());
|
|
196
|
-
await client.call("router.available_routes", { routes });
|
|
171
|
+
await client.call("router.available_routes", { routes, session_id, ...secret });
|
|
197
172
|
}
|
|
198
173
|
|
|
199
174
|
async function syncCurrentRoute(
|
|
@@ -204,7 +179,8 @@ async function syncCurrentRoute(
|
|
|
204
179
|
thinking = pi.getThinkingLevel(),
|
|
205
180
|
): Promise<void> {
|
|
206
181
|
if (!model) return;
|
|
207
|
-
|
|
182
|
+
const session_id = ctx.sessionManager.getSessionId();
|
|
183
|
+
await client.call("router.current_route", { provider: model.provider, model: model.id, thinking, session_id, ...sessionSecretField(session_id) });
|
|
208
184
|
}
|
|
209
185
|
|
|
210
186
|
function halt(ctx: ExtensionContext, reason: string): false {
|
|
@@ -225,13 +201,17 @@ async function applyDecision(
|
|
|
225
201
|
if (!decision.route || await applyRoute(pi, ctx, decision.route)) return true;
|
|
226
202
|
if (allowResync) {
|
|
227
203
|
await syncAvailableRoutes(pi, client, ctx);
|
|
228
|
-
return applyDecision(pi, client, ctx, await client.call("router.decide", {}) as PolicyDecision, false);
|
|
204
|
+
return applyDecision(pi, client, ctx, await client.call("router.decide", { session_id: ctx.sessionManager.getSessionId() }) as PolicyDecision, false);
|
|
229
205
|
}
|
|
230
206
|
return halt(ctx, `Jittor could not apply any authenticated Pi route after ${decision.route.provider}/${decision.route.model} became unavailable`);
|
|
231
207
|
}
|
|
232
208
|
|
|
233
|
-
/**
|
|
234
|
-
|
|
209
|
+
/**
|
|
210
|
+
* taskId, when a Papyrus task is focused, tags the metric for cost-per-task correlation. thinking
|
|
211
|
+
* comes from pi.getThinkingLevel() at message_end time, not from the message itself -- AssistantMessage
|
|
212
|
+
* has no thinking field of its own, and the level can't have changed mid-message.
|
|
213
|
+
*/
|
|
214
|
+
function assistantUsageMetrics(message: unknown, observedAt: number, taskId: string | null = null, thinking: string | null = null): MetricObservation[] {
|
|
235
215
|
if (typeof message !== "object" || message === null || Array.isArray(message)) return [];
|
|
236
216
|
const value = message as Record<string, unknown>;
|
|
237
217
|
if (value["role"] !== "assistant" || typeof value["usage"] !== "object" || value["usage"] === null) return [];
|
|
@@ -239,7 +219,7 @@ function assistantUsageMetrics(message: unknown, observedAt: number, taskId: str
|
|
|
239
219
|
const provider = typeof value["provider"] === "string" ? value["provider"] : "unknown";
|
|
240
220
|
const model = typeof value["model"] === "string" ? value["model"] : "unknown";
|
|
241
221
|
const scope = `${provider}:${model}`;
|
|
242
|
-
const attributes = { provider, model, ...(taskId === null ? {} : { taskId }) };
|
|
222
|
+
const attributes = { provider, model, ...(taskId === null ? {} : { taskId }), ...(thinking === null || thinking.length === 0 ? {} : { thinking }) };
|
|
243
223
|
const metrics: MetricObservation[] = [];
|
|
244
224
|
for (const [field, metric] of [["input", "input-tokens"], ["output", "output-tokens"], ["cacheRead", "cache-read-tokens"], ["cacheWrite", "cache-write-tokens"]] as const) {
|
|
245
225
|
const amount = usage[field];
|
|
@@ -260,9 +240,9 @@ export function registerJittorExtension(
|
|
|
260
240
|
const footerState: IntegratedFooterState = { providerBudget: null };
|
|
261
241
|
const usageBudgets = usageBudgetControl(enforcement);
|
|
262
242
|
let compactionTelemetry = new CompactionTelemetry();
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
243
|
+
const localRunTelemetry = new LocalRunTelemetry();
|
|
244
|
+
const providerResponseTelemetry = new ProviderResponseTelemetry();
|
|
245
|
+
const codexRecoveryCapability = new CodexRecoveryCapability(pi, codexRecovery, recoveryRuntime);
|
|
266
246
|
const contextObservations = new Set<string>();
|
|
267
247
|
const stopPapyrusContext = pi.events?.on?.(PAPYRUS_CONTEXT_INJECTION_CHANNEL, (payload) => {
|
|
268
248
|
try {
|
|
@@ -292,71 +272,9 @@ export function registerJittorExtension(
|
|
|
292
272
|
// Reject malformed or stale cross-extension events without retaining payloads or crashing the extension.
|
|
293
273
|
}
|
|
294
274
|
});
|
|
295
|
-
const
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
maxAttempts: CODEX_RECOVERY_MAX_ATTEMPTS,
|
|
299
|
-
attemptWindowMs: CODEX_RECOVERY_ATTEMPT_WINDOW_MS,
|
|
300
|
-
jitterRatio: CODEX_RECOVERY_JITTER_RATIO,
|
|
301
|
-
}, recoveryRuntime.random);
|
|
302
|
-
let recoveryTimer: unknown;
|
|
303
|
-
let recoveryCooldown: { until: number; attempt: number; failureKind: CodexFailureKind } | undefined;
|
|
304
|
-
let lastCodexResponse: CodexFailureMetadata = {};
|
|
305
|
-
let lastGoogleVertexResponse: GoogleVertexFailureMetadata = {};
|
|
306
|
-
// The third-party "anthropic-vertex" provider (Anthropic Claude via Google Vertex) is tracked
|
|
307
|
-
// separately from "google-vertex" (Pi's own, unrelated native Vertex provider): different code
|
|
308
|
-
// path, different account/quota pool, and its metrics must stay distinguishable -- see
|
|
309
|
-
// google-vertex-contracts.ts and anthropic-contracts.ts.
|
|
310
|
-
let lastAnthropicVertexResponse: GoogleVertexFailureMetadata = {};
|
|
311
|
-
const cancelRecovery = (resetPolicy: boolean): void => {
|
|
312
|
-
if (recoveryTimer !== undefined) recoveryRuntime.clearTimeout(recoveryTimer);
|
|
313
|
-
recoveryTimer = undefined;
|
|
314
|
-
recoveryCooldown = undefined;
|
|
315
|
-
if (resetPolicy) recoveryPolicy.cancel();
|
|
316
|
-
};
|
|
317
|
-
const recoveryStatusText = (): string => {
|
|
318
|
-
const now = recoveryRuntime.now();
|
|
319
|
-
const state = recoveryPolicy.state(now);
|
|
320
|
-
const enabled = codexRecovery.isCodexRecoveryEnabled();
|
|
321
|
-
const attempt = recoveryCooldown?.attempt ?? (state.pending ? state.attempts + 1 : state.attempts);
|
|
322
|
-
const phase = recoveryCooldown
|
|
323
|
-
? `cooldown ${Math.ceil(Math.max(0, recoveryCooldown.until - now) / MILLISECONDS_PER_SECOND)}s`
|
|
324
|
-
: state.pending ? "pending"
|
|
325
|
-
: state.attempts >= CODEX_RECOVERY_MAX_ATTEMPTS ? "exhausted"
|
|
326
|
-
: state.attempts > 0 ? "waiting" : "idle";
|
|
327
|
-
const failureKind = recoveryCooldown?.failureKind ?? state.lastFailureKind;
|
|
328
|
-
return [
|
|
329
|
-
`Codex recovery: ${enabled ? "on" : "off"}`,
|
|
330
|
-
phase,
|
|
331
|
-
`attempt ${attempt}/${CODEX_RECOVERY_MAX_ATTEMPTS}`,
|
|
332
|
-
`window ${CODEX_RECOVERY_ATTEMPT_WINDOW_MS / MILLISECONDS_PER_MINUTE}m`,
|
|
333
|
-
...(failureKind ? [failureKind] : []),
|
|
334
|
-
].join(" · ");
|
|
335
|
-
};
|
|
336
|
-
const scheduleCodexRecovery = (ctx: ExtensionContext): void => {
|
|
337
|
-
if (!codexRecovery.isCodexRecoveryEnabled() || recoveryTimer !== undefined || !ctx.isIdle() || ctx.hasPendingMessages()) return;
|
|
338
|
-
const plan = recoveryPolicy.plan(recoveryRuntime.now());
|
|
339
|
-
if (plan.action === "exhausted") {
|
|
340
|
-
recoveryPolicy.abandonFailure();
|
|
341
|
-
if (ctx.hasUI) ctx.ui.notify(`Jittor Codex recovery stopped: ${plan.reason}.`, "warning");
|
|
342
|
-
return;
|
|
343
|
-
}
|
|
344
|
-
if (plan.action !== "schedule") return;
|
|
345
|
-
recoveryCooldown = { until: recoveryRuntime.now() + plan.delayMs, attempt: plan.attempt, failureKind: plan.failureKind };
|
|
346
|
-
recoveryTimer = recoveryRuntime.setTimeout(async () => {
|
|
347
|
-
recoveryTimer = undefined;
|
|
348
|
-
recoveryCooldown = undefined;
|
|
349
|
-
if (!ctx.isIdle() || ctx.hasPendingMessages()) return;
|
|
350
|
-
const attempt = recoveryPolicy.recordAttempt(recoveryRuntime.now());
|
|
351
|
-
if (!attempt) return;
|
|
352
|
-
pi.sendMessage({
|
|
353
|
-
customType: "jittor-codex-recovery",
|
|
354
|
-
content: `Retry the previous Codex request after a transient ${attempt.failureKind} failure. Automatic recovery attempt ${attempt.attempt} of ${CODEX_RECOVERY_MAX_ATTEMPTS}.`,
|
|
355
|
-
display: false,
|
|
356
|
-
details: { attempt: attempt.attempt, failureKind: attempt.failureKind },
|
|
357
|
-
}, { triggerTurn: true, deliverAs: "followUp" });
|
|
358
|
-
}, plan.delayMs);
|
|
359
|
-
};
|
|
275
|
+
const cancelRecovery = (resetPolicy: boolean): void => codexRecoveryCapability.cancel(resetPolicy);
|
|
276
|
+
const recoveryStatusText = (): string => codexRecoveryCapability.statusText();
|
|
277
|
+
const scheduleCodexRecovery = (ctx: ExtensionContext): void => codexRecoveryCapability.scheduleIfIdle(ctx);
|
|
360
278
|
let compactionTimer: ReturnType<typeof setInterval> | undefined;
|
|
361
279
|
const finishCompactionUi = (): void => {
|
|
362
280
|
if (compactionTimer) clearInterval(compactionTimer);
|
|
@@ -402,11 +320,11 @@ export function registerJittorExtension(
|
|
|
402
320
|
await syncCurrentRoute(pi, client, ctx);
|
|
403
321
|
await syncAvailableRoutes(pi, client, ctx);
|
|
404
322
|
await client.call("telemetry.poll", {});
|
|
405
|
-
const readinessDecision = await client.call("router.decide", {}) as PolicyDecision;
|
|
323
|
+
const readinessDecision = await client.call("router.decide", { session_id: ctx.sessionManager.getSessionId() }) as PolicyDecision;
|
|
406
324
|
if (readinessDecision.action === "halt") throw new Error(readinessDecision.reason);
|
|
407
325
|
enforcement.setEnabled(true);
|
|
408
326
|
showFooter(ctx);
|
|
409
|
-
await refreshFooter(client, footerState);
|
|
327
|
+
await refreshFooter(client, footerState, ctx.sessionManager.getSessionId());
|
|
410
328
|
ctx.ui.notify("Jittor enforcement enabled.", "info");
|
|
411
329
|
} catch (error) {
|
|
412
330
|
enforcement.setEnabled(false);
|
|
@@ -426,7 +344,7 @@ export function registerJittorExtension(
|
|
|
426
344
|
setFooter: async (enabled) => {
|
|
427
345
|
enforcement.setFooterEnabled(enabled);
|
|
428
346
|
showFooter(ctx);
|
|
429
|
-
if (enabled) await refreshFooter(client, footerState).catch(() => undefined);
|
|
347
|
+
if (enabled) await refreshFooter(client, footerState, ctx.sessionManager.getSessionId()).catch(() => undefined);
|
|
430
348
|
},
|
|
431
349
|
setRecovery: (enabled) => {
|
|
432
350
|
if (!enabled) cancelRecovery(true);
|
|
@@ -462,12 +380,12 @@ export function registerJittorExtension(
|
|
|
462
380
|
return;
|
|
463
381
|
}
|
|
464
382
|
if (action === "outcome accepted" || action === "outcome rejected") {
|
|
465
|
-
|
|
383
|
+
const explicitOutcome = action.endsWith("accepted") ? "accepted" as const : "rejected" as const;
|
|
384
|
+
const outcomeMetric = localRunTelemetry.explicitOutcomeMetric(explicitOutcome);
|
|
385
|
+
if (!outcomeMetric) {
|
|
466
386
|
ctx.ui.notify("No completed local model run is available for an explicit outcome.", "warning");
|
|
467
387
|
return;
|
|
468
388
|
}
|
|
469
|
-
const explicitOutcome = action.endsWith("accepted") ? "accepted" as const : "rejected" as const;
|
|
470
|
-
const outcomeMetric = modelRunMetrics({ ...lastCompletedLocalRun, explicitOutcome }).find((metric) => metric.metric === "outcome-accepted")!;
|
|
471
389
|
outcomeMetric.observedAt = Date.now();
|
|
472
390
|
await recordMetrics(client, [outcomeMetric]);
|
|
473
391
|
ctx.ui.notify(`Recorded explicit ${explicitOutcome} outcome for the latest local model run.`, "info");
|
|
@@ -504,7 +422,7 @@ export function registerJittorExtension(
|
|
|
504
422
|
if (action === "footer on" || action === "footer enable") {
|
|
505
423
|
enforcement.setFooterEnabled(true);
|
|
506
424
|
showFooter(ctx);
|
|
507
|
-
await refreshFooter(client, footerState).catch(() => undefined);
|
|
425
|
+
await refreshFooter(client, footerState, ctx.sessionManager.getSessionId()).catch(() => undefined);
|
|
508
426
|
ctx.ui.notify("Jittor informational footer enabled; routing enforcement is unchanged.", "info");
|
|
509
427
|
return;
|
|
510
428
|
}
|
|
@@ -573,19 +491,25 @@ export function registerJittorExtension(
|
|
|
573
491
|
focusedTaskId = null;
|
|
574
492
|
finishCompactionUi();
|
|
575
493
|
compactionTelemetry = new CompactionTelemetry();
|
|
576
|
-
|
|
577
|
-
lastCompletedLocalRun = undefined;
|
|
494
|
+
localRunTelemetry.reset();
|
|
578
495
|
cancelRecovery(true);
|
|
579
|
-
|
|
580
|
-
lastGoogleVertexResponse = {};
|
|
581
|
-
lastAnthropicVertexResponse = {};
|
|
496
|
+
providerResponseTelemetry.resetTurn();
|
|
582
497
|
ctx.ui.setStatus("jittor", undefined);
|
|
583
498
|
showFooter(ctx);
|
|
499
|
+
// Registered before any router-mutating call could plausibly happen, closing most of the
|
|
500
|
+
// first-touch registration window; best-effort -- a registration failure leaves this session
|
|
501
|
+
// unarmored (opt-in armor), never blocked.
|
|
502
|
+
try {
|
|
503
|
+
const { secret } = await client.call("session.register", { session_id: currentSessionId });
|
|
504
|
+
cacheSessionSecret(currentSessionId, secret);
|
|
505
|
+
} catch {
|
|
506
|
+
// Unarmored for this session; every router.* call still works exactly as before.
|
|
507
|
+
}
|
|
584
508
|
try {
|
|
585
509
|
await syncCurrentRoute(pi, client, ctx);
|
|
586
510
|
await syncAvailableRoutes(pi, client, ctx);
|
|
587
511
|
await client.call("telemetry.poll", {});
|
|
588
|
-
await refreshFooter(client, footerState);
|
|
512
|
+
await refreshFooter(client, footerState, ctx.sessionManager.getSessionId());
|
|
589
513
|
} catch {
|
|
590
514
|
footerState.providerBudget = null;
|
|
591
515
|
footerState.requestRender?.();
|
|
@@ -619,7 +543,7 @@ export function registerJittorExtension(
|
|
|
619
543
|
try {
|
|
620
544
|
await syncCurrentRoute(pi, client, ctx);
|
|
621
545
|
await syncAvailableRoutes(pi, client, ctx);
|
|
622
|
-
await refreshFooter(client, footerState);
|
|
546
|
+
await refreshFooter(client, footerState, ctx.sessionManager.getSessionId());
|
|
623
547
|
} catch {
|
|
624
548
|
footerState.providerBudget = null;
|
|
625
549
|
footerState.requestRender?.();
|
|
@@ -630,7 +554,7 @@ export function registerJittorExtension(
|
|
|
630
554
|
if (event.source !== "extension") cancelRecovery(true);
|
|
631
555
|
if (event.source === "extension" || !enforcement.isEnabled()) return { action: "continue" as const };
|
|
632
556
|
try {
|
|
633
|
-
const next = await client.call("router.decide", {}) as PolicyDecision;
|
|
557
|
+
const next = await client.call("router.decide", { session_id: ctx.sessionManager.getSessionId() }) as PolicyDecision;
|
|
634
558
|
if (next.action === "halt") {
|
|
635
559
|
ctx.ui.notify(`Jittor blocked input: ${next.reason}. ${RECOVERY_GUIDANCE}.`, "warning");
|
|
636
560
|
return { action: "handled" as const };
|
|
@@ -644,7 +568,7 @@ export function registerJittorExtension(
|
|
|
644
568
|
|
|
645
569
|
pi.on("model_select", async (event, ctx) => {
|
|
646
570
|
await syncCurrentRoute(pi, client, ctx, event.model).then(() => syncAvailableRoutes(pi, client, ctx)).catch(() => undefined);
|
|
647
|
-
if (enforcement.isFooterEnabled()) await refreshFooter(client, footerState).catch(() => undefined);
|
|
571
|
+
if (enforcement.isFooterEnabled()) await refreshFooter(client, footerState, ctx.sessionManager.getSessionId()).catch(() => undefined);
|
|
648
572
|
});
|
|
649
573
|
|
|
650
574
|
pi.on("thinking_level_select", async (event, ctx) => {
|
|
@@ -654,155 +578,53 @@ export function registerJittorExtension(
|
|
|
654
578
|
pi.on("turn_start", async (event, ctx) => {
|
|
655
579
|
currentSessionId = ctx.sessionManager.getSessionId();
|
|
656
580
|
compactionTelemetry.observeTurn();
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
activeLocalRun = {
|
|
661
|
-
runId: `local-${event.timestamp}-${++localRunSequence}`,
|
|
662
|
-
startedAt: event.timestamp,
|
|
663
|
-
firstTokenAt: null,
|
|
664
|
-
providerResponses: 0,
|
|
665
|
-
toolNames: [],
|
|
666
|
-
toolCalls: 0,
|
|
667
|
-
toolFailures: 0,
|
|
668
|
-
};
|
|
581
|
+
codexRecoveryCapability.resetTurn();
|
|
582
|
+
providerResponseTelemetry.resetTurn();
|
|
583
|
+
localRunTelemetry.beginTurn(event.timestamp);
|
|
669
584
|
if (!enforcement.isEnabled()) return;
|
|
670
585
|
try {
|
|
671
586
|
await syncCurrentRoute(pi, client, ctx);
|
|
672
587
|
await syncAvailableRoutes(pi, client, ctx);
|
|
673
|
-
await applyDecision(pi, client, ctx, await client.call("router.decide", {}) as PolicyDecision);
|
|
674
|
-
await refreshFooter(client, footerState);
|
|
588
|
+
await applyDecision(pi, client, ctx, await client.call("router.decide", { session_id: ctx.sessionManager.getSessionId() }) as PolicyDecision);
|
|
589
|
+
await refreshFooter(client, footerState, ctx.sessionManager.getSessionId());
|
|
675
590
|
} catch {
|
|
676
591
|
halt(ctx, "Jittor could not verify or apply a safe route");
|
|
677
592
|
}
|
|
678
593
|
});
|
|
679
594
|
|
|
680
595
|
pi.on("message_update", async (event) => {
|
|
681
|
-
|
|
682
|
-
if (["text_delta", "thinking_delta", "toolcall_delta"].includes(event.assistantMessageEvent.type)) activeLocalRun.firstTokenAt = Date.now();
|
|
596
|
+
localRunTelemetry.onMessageUpdate(event.assistantMessageEvent.type);
|
|
683
597
|
});
|
|
684
598
|
|
|
685
599
|
pi.on("tool_execution_end", async (event) => {
|
|
686
|
-
|
|
687
|
-
activeLocalRun.toolCalls += 1;
|
|
688
|
-
if (event.isError) activeLocalRun.toolFailures += 1;
|
|
689
|
-
if (activeLocalRun.toolNames.length < 100) activeLocalRun.toolNames.push(event.toolName);
|
|
600
|
+
localRunTelemetry.onToolExecutionEnd(event.toolName, event.isError);
|
|
690
601
|
});
|
|
691
602
|
|
|
692
603
|
pi.on("after_provider_response", async (event, ctx) => {
|
|
693
|
-
|
|
694
|
-
if (ctx.model?.provider === "openai-codex")
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
if (ctx.
|
|
698
|
-
const headers = new Headers(event.headers);
|
|
699
|
-
if (hasAnthropicRateLimitHeaders(headers)) {
|
|
700
|
-
try {
|
|
701
|
-
await recordMetrics(client, parseAnthropicRateLimitHeaders(headers, Date.now()).metrics);
|
|
702
|
-
} catch {
|
|
703
|
-
if (enforcement.isEnabled()) ctx.ui.notify(`Jittor detected Anthropic telemetry schema drift. ${RECOVERY_GUIDANCE}.`, "error");
|
|
704
|
-
}
|
|
705
|
-
}
|
|
706
|
-
}
|
|
707
|
-
if (ctx.model?.provider === "anthropic-vertex") {
|
|
708
|
-
// Best-effort only: unverified whether this passthrough ever forwards Anthropic's own
|
|
709
|
-
// rate-limit headers. If it doesn't, hasAnthropicRateLimitHeaders is false and nothing is
|
|
710
|
-
// recorded -- the same honest default as every other unconfirmed signal in this file.
|
|
711
|
-
const headers = new Headers(event.headers);
|
|
712
|
-
if (hasAnthropicRateLimitHeaders(headers)) {
|
|
713
|
-
try {
|
|
714
|
-
await recordMetrics(client, parseAnthropicRateLimitHeaders(headers, Date.now(), "anthropic-vertex").metrics);
|
|
715
|
-
} catch {
|
|
716
|
-
if (enforcement.isEnabled()) ctx.ui.notify(`Jittor detected Anthropic-on-Vertex telemetry schema drift. ${RECOVERY_GUIDANCE}.`, "error");
|
|
717
|
-
}
|
|
718
|
-
}
|
|
719
|
-
// Well-evidenced regardless of headers: GCP's own quota system fronts this transport, so the
|
|
720
|
-
// same failure classification as google-vertex applies -- see google-vertex-contracts.ts.
|
|
721
|
-
lastAnthropicVertexResponse = { status: event.status, ...(header(event.headers, "retry-after") ? { retryAfter: header(event.headers, "retry-after") } : {}) };
|
|
722
|
-
}
|
|
723
|
-
if (ctx.model?.provider === "google-vertex") {
|
|
724
|
-
lastGoogleVertexResponse = { status: event.status, ...(header(event.headers, "retry-after") ? { retryAfter: header(event.headers, "retry-after") } : {}) };
|
|
725
|
-
}
|
|
726
|
-
if (Object.keys(event.headers).some((name) => name.toLowerCase().startsWith("x-codex-"))) {
|
|
727
|
-
try {
|
|
728
|
-
const updates = parseCodexRateLimitHeaders(new Headers(event.headers), Date.now());
|
|
729
|
-
await recordMetrics(client, updates.flatMap((update) => update.metrics));
|
|
730
|
-
} catch {
|
|
731
|
-
if (enforcement.isEnabled()) ctx.ui.notify(`Jittor detected Codex telemetry schema drift. ${RECOVERY_GUIDANCE}.`, "error");
|
|
732
|
-
}
|
|
733
|
-
}
|
|
734
|
-
if (enforcement.isFooterEnabled()) await refreshFooter(client, footerState).catch(() => undefined);
|
|
604
|
+
localRunTelemetry.onProviderResponse();
|
|
605
|
+
if (ctx.model?.provider === "openai-codex") codexRecoveryCapability.notifyResponse(event.status, event.headers);
|
|
606
|
+
const notifySchemaDrift = (message: string) => { if (enforcement.isEnabled()) ctx.ui.notify(`Jittor detected ${message}. ${RECOVERY_GUIDANCE}.`, "error"); };
|
|
607
|
+
await providerResponseTelemetry.handleProviderResponse(client, ctx.model?.provider, event.status, event.headers, notifySchemaDrift);
|
|
608
|
+
if (enforcement.isFooterEnabled()) await refreshFooter(client, footerState, ctx.sessionManager.getSessionId()).catch(() => undefined);
|
|
735
609
|
});
|
|
736
610
|
|
|
737
611
|
pi.on("turn_end", async (event) => {
|
|
738
|
-
const
|
|
739
|
-
|
|
740
|
-
const message = event.message as unknown;
|
|
741
|
-
if (!active || typeof message !== "object" || message === null || Array.isArray(message)) return;
|
|
742
|
-
const value = message as Record<string, unknown>;
|
|
743
|
-
if (value["role"] !== "assistant" || typeof value["provider"] !== "string" || typeof value["model"] !== "string") return;
|
|
744
|
-
const usage = typeof value["usage"] === "object" && value["usage"] !== null ? value["usage"] as Record<string, unknown> : {};
|
|
745
|
-
const amount = (name: string): number => typeof usage[name] === "number" && Number.isFinite(usage[name]) ? usage[name] as number : 0;
|
|
746
|
-
const cost = typeof usage["cost"] === "object" && usage["cost"] !== null && typeof (usage["cost"] as Record<string, unknown>)["total"] === "number"
|
|
747
|
-
? (usage["cost"] as Record<string, number>)["total"] ?? 0 : 0;
|
|
748
|
-
const stopReason = ["stop", "length", "toolUse", "error", "aborted"].includes(String(value["stopReason"]))
|
|
749
|
-
? value["stopReason"] as ModelRunObservation["stopReason"] : "unknown";
|
|
750
|
-
const completedAt = Math.max(Date.now(), active.firstTokenAt ?? active.startedAt, active.startedAt);
|
|
751
|
-
lastCompletedLocalRun = {
|
|
752
|
-
runId: active.runId,
|
|
753
|
-
provider: value["provider"],
|
|
754
|
-
model: value["model"],
|
|
755
|
-
thinking: pi.getThinkingLevel(),
|
|
756
|
-
...classifyTaskFromTools(active.toolNames),
|
|
757
|
-
startedAt: active.startedAt,
|
|
758
|
-
firstTokenAt: active.firstTokenAt,
|
|
759
|
-
completedAt,
|
|
760
|
-
inputTokens: amount("input"),
|
|
761
|
-
outputTokens: amount("output"),
|
|
762
|
-
cacheReadTokens: amount("cacheRead"),
|
|
763
|
-
cacheWriteTokens: amount("cacheWrite"),
|
|
764
|
-
costUsd: Number.isFinite(cost) && cost >= 0 ? cost : 0,
|
|
765
|
-
providerResponses: Math.max(1, active.providerResponses),
|
|
766
|
-
toolCalls: active.toolCalls,
|
|
767
|
-
toolFailures: active.toolFailures,
|
|
768
|
-
stopReason,
|
|
769
|
-
explicitOutcome: "unknown",
|
|
770
|
-
};
|
|
771
|
-
await recordMetrics(client, modelRunMetrics(lastCompletedLocalRun)).catch(() => undefined);
|
|
612
|
+
const metrics = localRunTelemetry.completeTurn(event.message, pi.getThinkingLevel());
|
|
613
|
+
await recordMetrics(client, metrics).catch(() => undefined);
|
|
772
614
|
});
|
|
773
615
|
|
|
774
|
-
pi.on("message_end", async (event,
|
|
775
|
-
if (event.message.role === "assistant"
|
|
776
|
-
if (event.message.
|
|
777
|
-
|
|
778
|
-
if (codexRecovery.isCodexRecoveryEnabled() && failure.transient) recoveryPolicy.observeFailure(failure, recoveryRuntime.now());
|
|
779
|
-
else cancelRecovery(true);
|
|
780
|
-
} else if (event.message.stopReason !== "aborted") {
|
|
781
|
-
cancelRecovery(true);
|
|
782
|
-
}
|
|
783
|
-
lastCodexResponse = {};
|
|
784
|
-
}
|
|
785
|
-
if (event.message.role === "assistant" && event.message.provider === "google-vertex") {
|
|
786
|
-
if (event.message.stopReason === "error") {
|
|
787
|
-
const failure = classifyGoogleVertexFailure(event.message.errorMessage, lastGoogleVertexResponse);
|
|
788
|
-
await recordMetrics(client, googleVertexFailureMetrics(failure, Date.now())).catch(() => undefined);
|
|
789
|
-
}
|
|
790
|
-
lastGoogleVertexResponse = {};
|
|
791
|
-
}
|
|
792
|
-
if (event.message.role === "assistant" && event.message.provider === "anthropic-vertex") {
|
|
793
|
-
if (event.message.stopReason === "error") {
|
|
794
|
-
const failure = classifyGoogleVertexFailure(event.message.errorMessage, lastAnthropicVertexResponse);
|
|
795
|
-
await recordMetrics(client, googleVertexFailureMetrics(failure, Date.now(), "anthropic-vertex")).catch(() => undefined);
|
|
796
|
-
}
|
|
797
|
-
lastAnthropicVertexResponse = {};
|
|
616
|
+
pi.on("message_end", async (event, ctx) => {
|
|
617
|
+
if (event.message.role === "assistant") {
|
|
618
|
+
if (event.message.provider === "openai-codex") codexRecoveryCapability.notifyMessageEnd(event.message.stopReason, event.message.errorMessage);
|
|
619
|
+
await providerResponseTelemetry.handleMessageEnd(client, event.message.provider, event.message.stopReason, event.message.errorMessage);
|
|
798
620
|
}
|
|
799
|
-
const metrics = assistantUsageMetrics(event.message, Date.now(), focusedTaskId);
|
|
621
|
+
const metrics = assistantUsageMetrics(event.message, Date.now(), focusedTaskId, pi.getThinkingLevel());
|
|
800
622
|
if (metrics.length > 0) {
|
|
801
623
|
const amount = (name: string): number => metrics.filter((metric) => metric.metric === name && typeof metric.value === "number").reduce((sum, metric) => sum + (metric.value ?? 0), 0);
|
|
802
624
|
compactionTelemetry.observeProviderUsage({ input: amount("input-tokens"), output: amount("output-tokens"), cacheRead: amount("cache-read-tokens"), cacheWrite: amount("cache-write-tokens") });
|
|
803
625
|
await recordMetrics(client, metrics).catch(() => undefined);
|
|
804
626
|
}
|
|
805
|
-
if (enforcement.isFooterEnabled()) await refreshFooter(client, footerState).catch(() => undefined);
|
|
627
|
+
if (enforcement.isFooterEnabled()) await refreshFooter(client, footerState, ctx.sessionManager.getSessionId()).catch(() => undefined);
|
|
806
628
|
});
|
|
807
629
|
|
|
808
630
|
pi.on("session_shutdown", async (_event, ctx) => {
|
|
@@ -811,9 +633,11 @@ export function registerJittorExtension(
|
|
|
811
633
|
stopPapyrusContext?.();
|
|
812
634
|
stopPapyrusTaskFocus?.();
|
|
813
635
|
cancelRecovery(true);
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
636
|
+
localRunTelemetry.reset();
|
|
637
|
+
const session_id = ctx.sessionManager.getSessionId();
|
|
638
|
+
const secret = sessionSecretField(session_id);
|
|
639
|
+
if (secret.session_secret) await client.call("session.release", { session_id, ...secret }).catch(() => undefined);
|
|
640
|
+
forgetSessionSecret(session_id);
|
|
817
641
|
ctx.ui.setStatus("jittor", undefined);
|
|
818
642
|
ctx.ui.setFooter(undefined);
|
|
819
643
|
});
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Client-side cache for this extension's own session_secret, registered with the daemon at
|
|
3
|
+
* session_start and released at session_shutdown (see index.ts). Keyed by sessionId rather
|
|
4
|
+
* than a single "current" variable, matching every router call site's own explicit sessionId.
|
|
5
|
+
*/
|
|
6
|
+
const secretsBySessionId = new Map<string, string>();
|
|
7
|
+
|
|
8
|
+
export function cacheSessionSecret(sessionId: string, secret: string): void {
|
|
9
|
+
secretsBySessionId.set(sessionId, secret);
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export function forgetSessionSecret(sessionId: string): void {
|
|
13
|
+
secretsBySessionId.delete(sessionId);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** Spread into any router-mutating request body alongside session_id -- empty object when no secret is cached for this sessionId (unregistered, or registration hasn't completed yet), matching the daemon's opt-in-armor default. */
|
|
17
|
+
export function sessionSecretField(sessionId: string | undefined): { session_secret?: string } {
|
|
18
|
+
const secret = sessionId ? secretsBySessionId.get(sessionId) : undefined;
|
|
19
|
+
return secret ? { session_secret: secret } : {};
|
|
20
|
+
}
|