@danypops/jittor 0.1.2 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -0
- package/extension/src/index.ts +154 -3
- package/extension/src/settings.ts +19 -3
- package/package.json +1 -1
- package/src/constants.ts +7 -1
- package/src/domain/codex-recovery.ts +205 -0
- package/src/service.ts +2 -1
- package/src/version.ts +16 -0
package/README.md
CHANGED
|
@@ -39,6 +39,21 @@ The native Pi extension preflights input and every provider turn, applies model/
|
|
|
39
39
|
|
|
40
40
|
Blocking always has a daemon-independent escape hatch. `/jittor off` immediately enters persisted monitor-only mode and never blocks provider requests. The informational footer is independently controlled with `/jittor footer on` and `/jittor footer off`, so showing status never enables enforcement. `/jittor on` only enables enforcement after telemetry polling and available-route synchronization succeed. Every fail-closed error includes these recovery commands plus the daemon restart command.
|
|
41
41
|
|
|
42
|
+
### Opt-in Codex settled-turn recovery
|
|
43
|
+
|
|
44
|
+
Transient Codex recovery is securely off by default and controlled through the existing Jittor command surface:
|
|
45
|
+
|
|
46
|
+
```text
|
|
47
|
+
/jittor recovery status
|
|
48
|
+
/jittor recovery on
|
|
49
|
+
/jittor recovery off
|
|
50
|
+
/jittor recovery cancel
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
The on/off choice persists privately in `$XDG_CONFIG_HOME/jittor/extension.json` (or `~/.config/jittor/extension.json`). Status reports only enabled state, cooldown, bounded attempt/window counters, and the normalized failure class. `cancel` clears the current cooldown and attempt window without changing the persisted on/off choice.
|
|
54
|
+
|
|
55
|
+
Jittor observes finalized Codex assistant errors through Pi's public message lifecycle, classifies only bounded error metadata, and waits for `agent_settled` before acting. That boundary guarantees Pi's built-in retry, compaction retry, and queued follow-up work has finished. A transient concurrency, rate-limit, overload, or transport failure then schedules one hidden follow-up with Retry-After-aware capped jitter. Recovery is limited to three attempts per ten-minute window, never overlaps pending Pi messages, resets after success, and is canceled by human input or session shutdown. Quota, authentication, invalid-request, unknown, and aborted failures remain terminal. Raw provider payloads are never retained or injected.
|
|
56
|
+
|
|
42
57
|
Run `/jittor usage` for a colored Unicode token histogram with X/Y axes, provider/model series, input/output/cache totals, refresh, and `24h`, `7d`, `30d`, or `90d` ranges. Left/Right changes range and `r` refreshes. Usage is persisted by the daemon from finalized Pi assistant messages.
|
|
43
58
|
|
|
44
59
|
See [`docs/CALIBRATION.md`](docs/CALIBRATION.md) for thresholds and rollback, and [`docs/USAGE_PRIOR_ART.md`](docs/USAGE_PRIOR_ART.md) for the chart design research.
|
package/extension/src/index.ts
CHANGED
|
@@ -1,12 +1,23 @@
|
|
|
1
1
|
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
CODEX_RECOVERY_ATTEMPT_WINDOW_MS,
|
|
4
|
+
CODEX_RECOVERY_BASE_DELAY_MS,
|
|
5
|
+
CODEX_RECOVERY_JITTER_RATIO,
|
|
6
|
+
CODEX_RECOVERY_MAX_ATTEMPTS,
|
|
7
|
+
CODEX_RECOVERY_MAX_DELAY_MS,
|
|
8
|
+
FOOTER_COMPACTION_RENDER_INTERVAL_MS,
|
|
9
|
+
MAX_DYNAMIC_ROUTES,
|
|
10
|
+
MILLISECONDS_PER_MINUTE,
|
|
11
|
+
MILLISECONDS_PER_SECOND,
|
|
12
|
+
} from "../../src/constants.ts";
|
|
13
|
+
import { CodexRecoveryPolicy, classifyCodexFailure, type CodexFailureKind, type CodexFailureMetadata } from "../../src/domain/codex-recovery.ts";
|
|
3
14
|
import type { MetricObservation, StoredMetricObservation } from "../../src/domain/metric.ts";
|
|
4
15
|
import type { PolicyDecision, Route } from "../../src/policy.ts";
|
|
5
16
|
import type { RouterStatus } from "../../src/ports/router-controller.ts";
|
|
6
17
|
import { parseCodexRateLimitHeaders } from "../../src/providers/codex.ts";
|
|
7
18
|
import { installIntegratedFooter, type IntegratedFooterState } from "./footer.ts";
|
|
8
19
|
import { callJittor } from "./service-client.ts";
|
|
9
|
-
import { persistentEnforcementControl, type EnforcementControl } from "./settings.ts";
|
|
20
|
+
import { persistentEnforcementControl, type CodexRecoveryControl, type EnforcementControl } from "./settings.ts";
|
|
10
21
|
import { buildFooterBudget, formatFooterStatus, showJittorPanel } from "./tui.ts";
|
|
11
22
|
import { showUsagePanel } from "./usage.ts";
|
|
12
23
|
|
|
@@ -23,6 +34,36 @@ const daemonClient: JittorExtensionClient = {
|
|
|
23
34
|
call: (operation, input) => callJittor(operation as Parameters<typeof callJittor>[0], input as never),
|
|
24
35
|
};
|
|
25
36
|
|
|
37
|
+
export interface CodexRecoveryRuntime {
|
|
38
|
+
now(): number;
|
|
39
|
+
random(): number;
|
|
40
|
+
setTimeout(callback: () => void | Promise<void>, delayMs: number): unknown;
|
|
41
|
+
clearTimeout(handle: unknown): void;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const SYSTEM_RECOVERY_RUNTIME: CodexRecoveryRuntime = {
|
|
45
|
+
now: Date.now,
|
|
46
|
+
random: Math.random,
|
|
47
|
+
setTimeout(callback, delayMs) { return setTimeout(() => { void callback(); }, delayMs); },
|
|
48
|
+
clearTimeout(handle) { clearTimeout(handle as ReturnType<typeof setTimeout>); },
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
function recoveryControl(enforcement: EnforcementControl): CodexRecoveryControl {
|
|
52
|
+
const candidate = enforcement as EnforcementControl & Partial<CodexRecoveryControl>;
|
|
53
|
+
const set = (candidate as Partial<CodexRecoveryControl>).setCodexRecoveryEnabled;
|
|
54
|
+
return typeof candidate.isCodexRecoveryEnabled === "function" && typeof set === "function"
|
|
55
|
+
? {
|
|
56
|
+
isCodexRecoveryEnabled: () => candidate.isCodexRecoveryEnabled!(),
|
|
57
|
+
setCodexRecoveryEnabled: (enabled) => set.call(candidate, enabled),
|
|
58
|
+
}
|
|
59
|
+
: { isCodexRecoveryEnabled: () => false, setCodexRecoveryEnabled() {} };
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function header(headers: Record<string, string>, name: string): string | undefined {
|
|
63
|
+
const expected = name.toLowerCase();
|
|
64
|
+
return Object.entries(headers).find(([key]) => key.toLowerCase() === expected)?.[1];
|
|
65
|
+
}
|
|
66
|
+
|
|
26
67
|
async function recordMetrics(client: JittorExtensionClient, metrics: MetricObservation[]): Promise<void> {
|
|
27
68
|
for (const metric of metrics) await client.call("metrics.record", metric);
|
|
28
69
|
}
|
|
@@ -169,8 +210,69 @@ export function registerJittorExtension(
|
|
|
169
210
|
pi: ExtensionAPI,
|
|
170
211
|
client: JittorExtensionClient = daemonClient,
|
|
171
212
|
enforcement: EnforcementControl = persistentEnforcementControl(),
|
|
213
|
+
codexRecovery: CodexRecoveryControl = recoveryControl(enforcement),
|
|
214
|
+
recoveryRuntime: CodexRecoveryRuntime = SYSTEM_RECOVERY_RUNTIME,
|
|
172
215
|
): void {
|
|
173
216
|
const footerState: IntegratedFooterState = { providerBudget: null };
|
|
217
|
+
const recoveryPolicy = new CodexRecoveryPolicy({
|
|
218
|
+
baseDelayMs: CODEX_RECOVERY_BASE_DELAY_MS,
|
|
219
|
+
maxDelayMs: CODEX_RECOVERY_MAX_DELAY_MS,
|
|
220
|
+
maxAttempts: CODEX_RECOVERY_MAX_ATTEMPTS,
|
|
221
|
+
attemptWindowMs: CODEX_RECOVERY_ATTEMPT_WINDOW_MS,
|
|
222
|
+
jitterRatio: CODEX_RECOVERY_JITTER_RATIO,
|
|
223
|
+
}, recoveryRuntime.random);
|
|
224
|
+
let recoveryTimer: unknown;
|
|
225
|
+
let recoveryCooldown: { until: number; attempt: number; failureKind: CodexFailureKind } | undefined;
|
|
226
|
+
let lastCodexResponse: CodexFailureMetadata = {};
|
|
227
|
+
const cancelRecovery = (resetPolicy: boolean): void => {
|
|
228
|
+
if (recoveryTimer !== undefined) recoveryRuntime.clearTimeout(recoveryTimer);
|
|
229
|
+
recoveryTimer = undefined;
|
|
230
|
+
recoveryCooldown = undefined;
|
|
231
|
+
if (resetPolicy) recoveryPolicy.cancel();
|
|
232
|
+
};
|
|
233
|
+
const recoveryStatusText = (): string => {
|
|
234
|
+
const now = recoveryRuntime.now();
|
|
235
|
+
const state = recoveryPolicy.state(now);
|
|
236
|
+
const enabled = codexRecovery.isCodexRecoveryEnabled();
|
|
237
|
+
const attempt = recoveryCooldown?.attempt ?? (state.pending ? state.attempts + 1 : state.attempts);
|
|
238
|
+
const phase = recoveryCooldown
|
|
239
|
+
? `cooldown ${Math.ceil(Math.max(0, recoveryCooldown.until - now) / MILLISECONDS_PER_SECOND)}s`
|
|
240
|
+
: state.pending ? "pending"
|
|
241
|
+
: state.attempts >= CODEX_RECOVERY_MAX_ATTEMPTS ? "exhausted"
|
|
242
|
+
: state.attempts > 0 ? "waiting" : "idle";
|
|
243
|
+
const failureKind = recoveryCooldown?.failureKind ?? state.lastFailureKind;
|
|
244
|
+
return [
|
|
245
|
+
`Codex recovery: ${enabled ? "on" : "off"}`,
|
|
246
|
+
phase,
|
|
247
|
+
`attempt ${attempt}/${CODEX_RECOVERY_MAX_ATTEMPTS}`,
|
|
248
|
+
`window ${CODEX_RECOVERY_ATTEMPT_WINDOW_MS / MILLISECONDS_PER_MINUTE}m`,
|
|
249
|
+
...(failureKind ? [failureKind] : []),
|
|
250
|
+
].join(" · ");
|
|
251
|
+
};
|
|
252
|
+
const scheduleCodexRecovery = (ctx: ExtensionContext): void => {
|
|
253
|
+
if (!codexRecovery.isCodexRecoveryEnabled() || recoveryTimer !== undefined || !ctx.isIdle() || ctx.hasPendingMessages()) return;
|
|
254
|
+
const plan = recoveryPolicy.plan(recoveryRuntime.now());
|
|
255
|
+
if (plan.action === "exhausted") {
|
|
256
|
+
recoveryPolicy.abandonFailure();
|
|
257
|
+
if (ctx.hasUI) ctx.ui.notify(`Jittor Codex recovery stopped: ${plan.reason}.`, "warning");
|
|
258
|
+
return;
|
|
259
|
+
}
|
|
260
|
+
if (plan.action !== "schedule") return;
|
|
261
|
+
recoveryCooldown = { until: recoveryRuntime.now() + plan.delayMs, attempt: plan.attempt, failureKind: plan.failureKind };
|
|
262
|
+
recoveryTimer = recoveryRuntime.setTimeout(async () => {
|
|
263
|
+
recoveryTimer = undefined;
|
|
264
|
+
recoveryCooldown = undefined;
|
|
265
|
+
if (!ctx.isIdle() || ctx.hasPendingMessages()) return;
|
|
266
|
+
const attempt = recoveryPolicy.recordAttempt(recoveryRuntime.now());
|
|
267
|
+
if (!attempt) return;
|
|
268
|
+
pi.sendMessage({
|
|
269
|
+
customType: "jittor-codex-recovery",
|
|
270
|
+
content: `Retry the previous Codex request after a transient ${attempt.failureKind} failure. Automatic recovery attempt ${attempt.attempt} of ${CODEX_RECOVERY_MAX_ATTEMPTS}.`,
|
|
271
|
+
display: false,
|
|
272
|
+
details: { attempt: attempt.attempt, failureKind: attempt.failureKind },
|
|
273
|
+
}, { triggerTurn: true, deliverAs: "followUp" });
|
|
274
|
+
}, plan.delayMs);
|
|
275
|
+
};
|
|
174
276
|
let compactionTimer: ReturnType<typeof setInterval> | undefined;
|
|
175
277
|
const finishCompaction = (): void => {
|
|
176
278
|
if (compactionTimer) clearInterval(compactionTimer);
|
|
@@ -223,6 +325,26 @@ export function registerJittorExtension(
|
|
|
223
325
|
description: "Inspect or control Jittor routing, budgets, and usage",
|
|
224
326
|
handler: async (args, ctx) => {
|
|
225
327
|
const action = args.trim().toLowerCase();
|
|
328
|
+
if (action === "recovery" || action === "recovery status") {
|
|
329
|
+
ctx.ui.notify(recoveryStatusText(), "info");
|
|
330
|
+
return;
|
|
331
|
+
}
|
|
332
|
+
if (action === "recovery on" || action === "recovery enable") {
|
|
333
|
+
codexRecovery.setCodexRecoveryEnabled(true);
|
|
334
|
+
ctx.ui.notify("Jittor Codex recovery enabled; bounded retries begin only after transient failures fully settle.", "info");
|
|
335
|
+
return;
|
|
336
|
+
}
|
|
337
|
+
if (action === "recovery off" || action === "recovery disable") {
|
|
338
|
+
cancelRecovery(true);
|
|
339
|
+
codexRecovery.setCodexRecoveryEnabled(false);
|
|
340
|
+
ctx.ui.notify("Jittor Codex recovery disabled and pending recovery cleared.", "info");
|
|
341
|
+
return;
|
|
342
|
+
}
|
|
343
|
+
if (action === "recovery cancel") {
|
|
344
|
+
cancelRecovery(true);
|
|
345
|
+
ctx.ui.notify(`Jittor Codex recovery cooldown and attempt window cleared; recovery remains ${codexRecovery.isCodexRecoveryEnabled() ? "on" : "off"}.`, "info");
|
|
346
|
+
return;
|
|
347
|
+
}
|
|
226
348
|
if (action === "off" || action === "disable") { disable(ctx); return; }
|
|
227
349
|
if (action === "on" || action === "enable") { await enable(ctx); return; }
|
|
228
350
|
if (action === "footer off" || action === "footer disable") {
|
|
@@ -252,6 +374,8 @@ export function registerJittorExtension(
|
|
|
252
374
|
|
|
253
375
|
pi.on("session_start", async (_event, ctx) => {
|
|
254
376
|
finishCompaction();
|
|
377
|
+
cancelRecovery(true);
|
|
378
|
+
lastCodexResponse = {};
|
|
255
379
|
ctx.ui.setStatus("jittor", undefined);
|
|
256
380
|
showFooter(ctx);
|
|
257
381
|
try {
|
|
@@ -273,11 +397,22 @@ export function registerJittorExtension(
|
|
|
273
397
|
finishCompaction();
|
|
274
398
|
});
|
|
275
399
|
|
|
276
|
-
pi.on("agent_settled", () => {
|
|
400
|
+
pi.on("agent_settled", async (_event, ctx) => {
|
|
277
401
|
if (footerState.compaction) finishCompaction();
|
|
402
|
+
scheduleCodexRecovery(ctx);
|
|
403
|
+
if (!enforcement.isFooterEnabled()) return;
|
|
404
|
+
try {
|
|
405
|
+
await syncCurrentRoute(pi, client, ctx);
|
|
406
|
+
await syncAvailableRoutes(pi, client, ctx);
|
|
407
|
+
await refreshFooter(client, footerState);
|
|
408
|
+
} catch {
|
|
409
|
+
footerState.providerBudget = null;
|
|
410
|
+
footerState.requestRender?.();
|
|
411
|
+
}
|
|
278
412
|
});
|
|
279
413
|
|
|
280
414
|
pi.on("input", async (event, ctx) => {
|
|
415
|
+
if (event.source !== "extension") cancelRecovery(true);
|
|
281
416
|
if (event.source === "extension" || !enforcement.isEnabled()) return { action: "continue" as const };
|
|
282
417
|
try {
|
|
283
418
|
const next = await client.call("router.decide", {}) as PolicyDecision;
|
|
@@ -302,6 +437,7 @@ export function registerJittorExtension(
|
|
|
302
437
|
});
|
|
303
438
|
|
|
304
439
|
pi.on("turn_start", async (_event, ctx) => {
|
|
440
|
+
lastCodexResponse = {};
|
|
305
441
|
if (!enforcement.isEnabled()) return;
|
|
306
442
|
try {
|
|
307
443
|
await syncCurrentRoute(pi, client, ctx);
|
|
@@ -314,6 +450,9 @@ export function registerJittorExtension(
|
|
|
314
450
|
});
|
|
315
451
|
|
|
316
452
|
pi.on("after_provider_response", async (event, ctx) => {
|
|
453
|
+
if (ctx.model?.provider === "openai-codex") {
|
|
454
|
+
lastCodexResponse = { status: event.status, ...(header(event.headers, "retry-after") ? { retryAfter: header(event.headers, "retry-after") } : {}) };
|
|
455
|
+
}
|
|
317
456
|
if (!Object.keys(event.headers).some((name) => name.toLowerCase().startsWith("x-codex-"))) return;
|
|
318
457
|
try {
|
|
319
458
|
const updates = parseCodexRateLimitHeaders(new Headers(event.headers), Date.now());
|
|
@@ -325,6 +464,16 @@ export function registerJittorExtension(
|
|
|
325
464
|
});
|
|
326
465
|
|
|
327
466
|
pi.on("message_end", async (event, _ctx) => {
|
|
467
|
+
if (event.message.role === "assistant" && event.message.provider === "openai-codex") {
|
|
468
|
+
if (event.message.stopReason === "error") {
|
|
469
|
+
const failure = classifyCodexFailure(event.message.errorMessage, lastCodexResponse);
|
|
470
|
+
if (codexRecovery.isCodexRecoveryEnabled() && failure.transient) recoveryPolicy.observeFailure(failure, recoveryRuntime.now());
|
|
471
|
+
else cancelRecovery(true);
|
|
472
|
+
} else if (event.message.stopReason !== "aborted") {
|
|
473
|
+
cancelRecovery(true);
|
|
474
|
+
}
|
|
475
|
+
lastCodexResponse = {};
|
|
476
|
+
}
|
|
328
477
|
const metrics = assistantUsageMetrics(event.message, Date.now());
|
|
329
478
|
if (metrics.length > 0) await recordMetrics(client, metrics).catch(() => undefined);
|
|
330
479
|
if (enforcement.isFooterEnabled()) await refreshFooter(client, footerState).catch(() => undefined);
|
|
@@ -332,6 +481,8 @@ export function registerJittorExtension(
|
|
|
332
481
|
|
|
333
482
|
pi.on("session_shutdown", async (_event, ctx) => {
|
|
334
483
|
finishCompaction();
|
|
484
|
+
cancelRecovery(true);
|
|
485
|
+
lastCodexResponse = {};
|
|
335
486
|
ctx.ui.setStatus("jittor", undefined);
|
|
336
487
|
ctx.ui.setFooter(undefined);
|
|
337
488
|
});
|
|
@@ -9,9 +9,19 @@ export interface EnforcementControl {
|
|
|
9
9
|
setFooterEnabled(enabled: boolean): void;
|
|
10
10
|
}
|
|
11
11
|
|
|
12
|
+
export interface CodexRecoveryControl {
|
|
13
|
+
isCodexRecoveryEnabled(): boolean;
|
|
14
|
+
setCodexRecoveryEnabled(enabled: boolean): void;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface PersistentExtensionControl extends EnforcementControl, CodexRecoveryControl {
|
|
18
|
+
setCodexRecoveryEnabled(enabled: boolean): void;
|
|
19
|
+
}
|
|
20
|
+
|
|
12
21
|
interface ExtensionSettings {
|
|
13
22
|
enforcementEnabled: boolean;
|
|
14
23
|
footerEnabled: boolean;
|
|
24
|
+
codexRecoveryEnabled: boolean;
|
|
15
25
|
}
|
|
16
26
|
|
|
17
27
|
function settingsPath(env: Record<string, string | undefined> = process.env): string {
|
|
@@ -22,14 +32,15 @@ function settingsPath(env: Record<string, string | undefined> = process.env): st
|
|
|
22
32
|
function loadSettings(path: string): ExtensionSettings {
|
|
23
33
|
try {
|
|
24
34
|
const value = JSON.parse(readFileSync(path, "utf8")) as unknown;
|
|
25
|
-
if (typeof value !== "object" || value === null || Array.isArray(value)) return { enforcementEnabled: true, footerEnabled: true };
|
|
35
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) return { enforcementEnabled: true, footerEnabled: true, codexRecoveryEnabled: false };
|
|
26
36
|
const record = value as Record<string, unknown>;
|
|
27
37
|
return {
|
|
28
38
|
enforcementEnabled: record["enforcementEnabled"] !== false,
|
|
29
39
|
footerEnabled: record["footerEnabled"] !== false,
|
|
40
|
+
codexRecoveryEnabled: record["codexRecoveryEnabled"] === true,
|
|
30
41
|
};
|
|
31
42
|
} catch {
|
|
32
|
-
return { enforcementEnabled: true, footerEnabled: true };
|
|
43
|
+
return { enforcementEnabled: true, footerEnabled: true, codexRecoveryEnabled: false };
|
|
33
44
|
}
|
|
34
45
|
}
|
|
35
46
|
|
|
@@ -41,7 +52,7 @@ function persistSettings(path: string, settings: ExtensionSettings): void {
|
|
|
41
52
|
renameSync(temporary, path);
|
|
42
53
|
}
|
|
43
54
|
|
|
44
|
-
export function persistentEnforcementControl(env: Record<string, string | undefined> = process.env):
|
|
55
|
+
export function persistentEnforcementControl(env: Record<string, string | undefined> = process.env): PersistentExtensionControl {
|
|
45
56
|
const path = settingsPath(env);
|
|
46
57
|
const settings = loadSettings(path);
|
|
47
58
|
return {
|
|
@@ -55,5 +66,10 @@ export function persistentEnforcementControl(env: Record<string, string | undefi
|
|
|
55
66
|
settings.footerEnabled = value;
|
|
56
67
|
persistSettings(path, settings);
|
|
57
68
|
},
|
|
69
|
+
isCodexRecoveryEnabled: () => settings.codexRecoveryEnabled,
|
|
70
|
+
setCodexRecoveryEnabled(value: boolean): void {
|
|
71
|
+
settings.codexRecoveryEnabled = value;
|
|
72
|
+
persistSettings(path, settings);
|
|
73
|
+
},
|
|
58
74
|
};
|
|
59
75
|
}
|
package/package.json
CHANGED
package/src/constants.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
export const VERSION = "0.1.2";
|
|
2
1
|
export const SQLITE_SCHEMA_VERSION = 1;
|
|
3
2
|
export const SQLITE_BUSY_TIMEOUT_MS = 5_000;
|
|
4
3
|
export const DEFAULT_QUERY_LIMIT = 1_000;
|
|
@@ -30,3 +29,10 @@ export const USAGE_CHART_HEIGHT = 8;
|
|
|
30
29
|
export const USAGE_Y_AXIS_WIDTH = 7;
|
|
31
30
|
export const USAGE_TOKEN_QUERY_LIMIT = 10_000;
|
|
32
31
|
export const MAX_DYNAMIC_ROUTES = 100;
|
|
32
|
+
export const CODEX_ERROR_MESSAGE_LIMIT = 160;
|
|
33
|
+
export const CODEX_RETRY_AFTER_MAX_MS = 5 * MILLISECONDS_PER_MINUTE;
|
|
34
|
+
export const CODEX_RECOVERY_BASE_DELAY_MS = 2 * MILLISECONDS_PER_SECOND;
|
|
35
|
+
export const CODEX_RECOVERY_MAX_DELAY_MS = CODEX_RETRY_AFTER_MAX_MS;
|
|
36
|
+
export const CODEX_RECOVERY_MAX_ATTEMPTS = 3;
|
|
37
|
+
export const CODEX_RECOVERY_ATTEMPT_WINDOW_MS = 10 * MILLISECONDS_PER_MINUTE;
|
|
38
|
+
export const CODEX_RECOVERY_JITTER_RATIO = 0.2;
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
import {
|
|
2
|
+
CODEX_ERROR_MESSAGE_LIMIT,
|
|
3
|
+
CODEX_RETRY_AFTER_MAX_MS,
|
|
4
|
+
MILLISECONDS_PER_SECOND,
|
|
5
|
+
} from "../constants.ts";
|
|
6
|
+
|
|
7
|
+
export type CodexFailureKind =
|
|
8
|
+
| "concurrency"
|
|
9
|
+
| "rate-limit"
|
|
10
|
+
| "overload"
|
|
11
|
+
| "transport"
|
|
12
|
+
| "quota"
|
|
13
|
+
| "authentication"
|
|
14
|
+
| "invalid-request"
|
|
15
|
+
| "unknown";
|
|
16
|
+
|
|
17
|
+
export interface CodexFailure {
|
|
18
|
+
kind: CodexFailureKind;
|
|
19
|
+
transient: boolean;
|
|
20
|
+
code?: string;
|
|
21
|
+
source?: string;
|
|
22
|
+
message?: string;
|
|
23
|
+
retryAfterMs?: number;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface CodexFailureMetadata {
|
|
27
|
+
status?: number;
|
|
28
|
+
retryAfter?: string;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export interface CodexRecoveryOptions {
|
|
32
|
+
baseDelayMs: number;
|
|
33
|
+
maxDelayMs: number;
|
|
34
|
+
maxAttempts: number;
|
|
35
|
+
attemptWindowMs: number;
|
|
36
|
+
jitterRatio: number;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export type CodexRecoveryPlan =
|
|
40
|
+
| { action: "schedule"; attempt: number; delayMs: number; failureKind: CodexFailureKind }
|
|
41
|
+
| { action: "wait"; reason: string }
|
|
42
|
+
| { action: "exhausted"; reason: string };
|
|
43
|
+
|
|
44
|
+
export interface CodexRecoveryAttempt {
|
|
45
|
+
attempt: number;
|
|
46
|
+
failureKind: CodexFailureKind;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export class CodexRecoveryPolicy {
|
|
50
|
+
private pendingFailure: CodexFailure | undefined;
|
|
51
|
+
private attempts = 0;
|
|
52
|
+
private windowStartedAt: number | undefined;
|
|
53
|
+
private lastFailureKind: CodexFailureKind | undefined;
|
|
54
|
+
|
|
55
|
+
constructor(
|
|
56
|
+
private readonly options: CodexRecoveryOptions,
|
|
57
|
+
private readonly random: () => number = Math.random,
|
|
58
|
+
) {
|
|
59
|
+
if (!Number.isFinite(options.baseDelayMs) || options.baseDelayMs < 0) throw new Error("baseDelayMs must be non-negative");
|
|
60
|
+
if (!Number.isFinite(options.maxDelayMs) || options.maxDelayMs < options.baseDelayMs) throw new Error("maxDelayMs must be at least baseDelayMs");
|
|
61
|
+
if (!Number.isInteger(options.maxAttempts) || options.maxAttempts < 1) throw new Error("maxAttempts must be a positive integer");
|
|
62
|
+
if (!Number.isFinite(options.attemptWindowMs) || options.attemptWindowMs <= 0) throw new Error("attemptWindowMs must be positive");
|
|
63
|
+
if (!Number.isFinite(options.jitterRatio) || options.jitterRatio < 0 || options.jitterRatio > 1) throw new Error("jitterRatio must be between 0 and 1");
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
observeFailure(failure: CodexFailure, now: number): void {
|
|
67
|
+
this.normalizeWindow(now);
|
|
68
|
+
this.pendingFailure = failure.transient ? failure : undefined;
|
|
69
|
+
this.lastFailureKind = failure.transient ? failure.kind : undefined;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
observeSuccess(): void {
|
|
73
|
+
this.cancel();
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
cancel(): void {
|
|
77
|
+
this.pendingFailure = undefined;
|
|
78
|
+
this.attempts = 0;
|
|
79
|
+
this.windowStartedAt = undefined;
|
|
80
|
+
this.lastFailureKind = undefined;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
abandonFailure(): void {
|
|
84
|
+
this.pendingFailure = undefined;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
plan(now: number): CodexRecoveryPlan {
|
|
88
|
+
this.normalizeWindow(now);
|
|
89
|
+
if (!this.pendingFailure) return { action: "wait", reason: "no transient Codex failure is pending" };
|
|
90
|
+
if (this.attempts >= this.options.maxAttempts) {
|
|
91
|
+
return { action: "exhausted", reason: `${this.options.maxAttempts} recovery attempts reached within ${this.options.attemptWindowMs}ms` };
|
|
92
|
+
}
|
|
93
|
+
const base = this.pendingFailure.retryAfterMs
|
|
94
|
+
?? this.options.baseDelayMs * (2 ** this.attempts);
|
|
95
|
+
const sample = this.random();
|
|
96
|
+
const unit = Number.isFinite(sample) ? Math.min(1, Math.max(0, sample)) : 0;
|
|
97
|
+
const multiplier = 1 + ((unit * 2) - 1) * this.options.jitterRatio;
|
|
98
|
+
const jittered = Math.max(0, Math.round(base * multiplier));
|
|
99
|
+
const delayMs = this.pendingFailure.retryAfterMs === undefined ? jittered : Math.max(base, jittered);
|
|
100
|
+
return {
|
|
101
|
+
action: "schedule",
|
|
102
|
+
attempt: this.attempts + 1,
|
|
103
|
+
delayMs: Math.min(this.options.maxDelayMs, delayMs),
|
|
104
|
+
failureKind: this.pendingFailure.kind,
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
recordAttempt(now: number): CodexRecoveryAttempt | undefined {
|
|
109
|
+
this.normalizeWindow(now);
|
|
110
|
+
if (!this.pendingFailure || this.attempts >= this.options.maxAttempts) return undefined;
|
|
111
|
+
if (this.windowStartedAt === undefined) this.windowStartedAt = now;
|
|
112
|
+
this.attempts += 1;
|
|
113
|
+
const attempt = { attempt: this.attempts, failureKind: this.pendingFailure.kind };
|
|
114
|
+
this.pendingFailure = undefined;
|
|
115
|
+
return attempt;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
state(now?: number): { attempts: number; pending: boolean; lastFailureKind?: CodexFailureKind; windowStartedAt?: number } {
|
|
119
|
+
if (now !== undefined) this.normalizeWindow(now);
|
|
120
|
+
return {
|
|
121
|
+
attempts: this.attempts,
|
|
122
|
+
pending: this.pendingFailure !== undefined,
|
|
123
|
+
...(this.lastFailureKind ? { lastFailureKind: this.lastFailureKind } : {}),
|
|
124
|
+
...(this.windowStartedAt !== undefined ? { windowStartedAt: this.windowStartedAt } : {}),
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
private normalizeWindow(now: number): void {
|
|
129
|
+
if (this.windowStartedAt !== undefined && now - this.windowStartedAt >= this.options.attemptWindowMs) {
|
|
130
|
+
this.attempts = 0;
|
|
131
|
+
this.windowStartedAt = undefined;
|
|
132
|
+
if (!this.pendingFailure) this.lastFailureKind = undefined;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function asRecord(value: unknown): Record<string, unknown> | undefined {
|
|
138
|
+
return typeof value === "object" && value !== null && !Array.isArray(value)
|
|
139
|
+
? value as Record<string, unknown>
|
|
140
|
+
: undefined;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function firstString(...values: unknown[]): string | undefined {
|
|
144
|
+
for (const value of values) {
|
|
145
|
+
if (typeof value === "string" && value.length > 0) return value;
|
|
146
|
+
}
|
|
147
|
+
return undefined;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function retryAfterMs(value: string | undefined): number | undefined {
|
|
151
|
+
if (!value) return undefined;
|
|
152
|
+
const seconds = Number(value.trim());
|
|
153
|
+
if (!Number.isFinite(seconds) || seconds < 0) return undefined;
|
|
154
|
+
return Math.min(CODEX_RETRY_AFTER_MAX_MS, Math.round(seconds * MILLISECONDS_PER_SECOND));
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function matches(value: string, patterns: readonly string[]): boolean {
|
|
158
|
+
return patterns.some((pattern) => value.includes(pattern));
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
export function classifyCodexFailure(value: unknown, metadata: CodexFailureMetadata = {}): CodexFailure {
|
|
162
|
+
const root = asRecord(value);
|
|
163
|
+
const detail = asRecord(root?.["detail"]);
|
|
164
|
+
const nestedError = asRecord(root?.["error"]);
|
|
165
|
+
const code = firstString(detail?.["code"], detail?.["error_code"], nestedError?.["code"], root?.["code"]);
|
|
166
|
+
const source = firstString(detail?.["source"], nestedError?.["source"], root?.["source"]);
|
|
167
|
+
const rawMessage = firstString(
|
|
168
|
+
detail?.["message"],
|
|
169
|
+
typeof root?.["error"] === "string" ? root["error"] : undefined,
|
|
170
|
+
nestedError?.["message"],
|
|
171
|
+
root?.["message"],
|
|
172
|
+
typeof value === "string" ? value : undefined,
|
|
173
|
+
);
|
|
174
|
+
const message = rawMessage?.slice(0, CODEX_ERROR_MESSAGE_LIMIT);
|
|
175
|
+
const evidence = [code, source, message].filter(Boolean).join(" ").toLowerCase();
|
|
176
|
+
const base = {
|
|
177
|
+
...(code ? { code } : {}),
|
|
178
|
+
...(source ? { source } : {}),
|
|
179
|
+
...(message ? { message } : {}),
|
|
180
|
+
...(retryAfterMs(metadata.retryAfter) !== undefined ? { retryAfterMs: retryAfterMs(metadata.retryAfter) } : {}),
|
|
181
|
+
};
|
|
182
|
+
|
|
183
|
+
if (matches(evidence, ["insufficient_quota", "quota exceeded", "out of credits", "billing"])) {
|
|
184
|
+
return { kind: "quota", transient: false, ...base };
|
|
185
|
+
}
|
|
186
|
+
if (metadata.status === 401 || metadata.status === 403 || matches(evidence, ["invalid_api_key", "authentication", "unauthorized", "permission_denied"])) {
|
|
187
|
+
return { kind: "authentication", transient: false, ...base };
|
|
188
|
+
}
|
|
189
|
+
if (matches(evidence, ["invalid_prompt", "invalid_request", "context_length_exceeded"]) || metadata.status === 400 || metadata.status === 422) {
|
|
190
|
+
return { kind: "invalid-request", transient: false, ...base };
|
|
191
|
+
}
|
|
192
|
+
if (matches(evidence, ["concurrency_limit", "too many concurrent requests", "throttled"])) {
|
|
193
|
+
return { kind: "concurrency", transient: true, ...base };
|
|
194
|
+
}
|
|
195
|
+
if (matches(evidence, ["server_is_overloaded", "slow_down", "service unavailable", "overloaded"]) || (metadata.status !== undefined && metadata.status >= 500 && metadata.status <= 599)) {
|
|
196
|
+
return { kind: "overload", transient: true, ...base };
|
|
197
|
+
}
|
|
198
|
+
if (metadata.status === 429 || matches(evidence, ["rate_limit_exceeded", "rate limit", "too many requests"])) {
|
|
199
|
+
return { kind: "rate-limit", transient: true, ...base };
|
|
200
|
+
}
|
|
201
|
+
if (matches(evidence, ["timeout", "timed out", "network", "connection", "websocket", "fetch failed"])) {
|
|
202
|
+
return { kind: "transport", transient: true, ...base };
|
|
203
|
+
}
|
|
204
|
+
return { kind: "unknown", transient: false, ...base };
|
|
205
|
+
}
|
package/src/service.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { SERVICE_MAX_BODY_BYTES
|
|
1
|
+
import { SERVICE_MAX_BODY_BYTES } from "./constants.ts";
|
|
2
|
+
import { VERSION } from "./version.ts";
|
|
2
3
|
import { validateMetricObservation, type MetricObservation, type MetricQuery, type StoredMetricObservation } from "./domain/metric.ts";
|
|
3
4
|
import type { MetricStore } from "./ports/metric-store.ts";
|
|
4
5
|
import type { RouteOverride, RouterController, RouterStatus, TelemetryPollResult } from "./ports/router-controller.ts";
|
package/src/version.ts
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
|
|
3
|
+
function packageVersion(): string {
|
|
4
|
+
const manifest = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")) as unknown;
|
|
5
|
+
if (typeof manifest !== "object" || manifest === null || Array.isArray(manifest)) {
|
|
6
|
+
throw new Error("Jittor package manifest must be an object");
|
|
7
|
+
}
|
|
8
|
+
const version = (manifest as Record<string, unknown>)["version"];
|
|
9
|
+
if (typeof version !== "string" || !/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/.test(version)) {
|
|
10
|
+
throw new Error("Jittor package manifest has an invalid version");
|
|
11
|
+
}
|
|
12
|
+
return version;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** Runtime package version; package.json is the single release source of truth. */
|
|
16
|
+
export const VERSION = packageVersion();
|