@danypops/jittor 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -0
- package/extension/src/index.ts +105 -3
- package/extension/src/settings.ts +18 -3
- package/package.json +1 -1
- package/src/constants.ts +7 -1
- package/src/domain/codex-recovery.ts +195 -0
- package/src/service.ts +2 -1
- package/src/version.ts +16 -0
package/README.md
CHANGED
|
@@ -39,6 +39,18 @@ The native Pi extension preflights input and every provider turn, applies model/
|
|
|
39
39
|
|
|
40
40
|
Blocking always has a daemon-independent escape hatch. `/jittor off` immediately enters persisted monitor-only mode and never blocks provider requests. The informational footer is independently controlled with `/jittor footer on` and `/jittor footer off`, so showing status never enables enforcement. `/jittor on` only enables enforcement after telemetry polling and available-route synchronization succeed. Every fail-closed error includes these recovery commands plus the daemon restart command.
|
|
41
41
|
|
|
42
|
+
### Opt-in Codex settled-turn recovery
|
|
43
|
+
|
|
44
|
+
Transient Codex recovery is securely off by default. Opt in explicitly through `$XDG_CONFIG_HOME/jittor/extension.json` (or `~/.config/jittor/extension.json`):
|
|
45
|
+
|
|
46
|
+
```json
|
|
47
|
+
{
|
|
48
|
+
"codexRecoveryEnabled": true
|
|
49
|
+
}
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Jittor observes finalized Codex assistant errors through Pi's public message lifecycle, classifies only bounded error metadata, and waits for `agent_settled` before acting. That boundary guarantees Pi's built-in retry, compaction retry, and queued follow-up work has finished. A transient concurrency, rate-limit, overload, or transport failure then schedules one hidden follow-up with Retry-After-aware capped jitter. Recovery is limited to three attempts per ten-minute window, never overlaps pending Pi messages, resets after success, and is canceled by human input or session shutdown. Quota, authentication, invalid-request, unknown, and aborted failures remain terminal. Raw provider payloads are never retained or injected.
|
|
53
|
+
|
|
42
54
|
Run `/jittor usage` for a colored Unicode token histogram with X/Y axes, provider/model series, input/output/cache totals, refresh, and `24h`, `7d`, `30d`, or `90d` ranges. Left/Right changes range and `r` refreshes. Usage is persisted by the daemon from finalized Pi assistant messages.
|
|
43
55
|
|
|
44
56
|
See [`docs/CALIBRATION.md`](docs/CALIBRATION.md) for thresholds and rollback, and [`docs/USAGE_PRIOR_ART.md`](docs/USAGE_PRIOR_ART.md) for the chart design research.
|
package/extension/src/index.ts
CHANGED
|
@@ -1,12 +1,21 @@
|
|
|
1
1
|
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
CODEX_RECOVERY_ATTEMPT_WINDOW_MS,
|
|
4
|
+
CODEX_RECOVERY_BASE_DELAY_MS,
|
|
5
|
+
CODEX_RECOVERY_JITTER_RATIO,
|
|
6
|
+
CODEX_RECOVERY_MAX_ATTEMPTS,
|
|
7
|
+
CODEX_RECOVERY_MAX_DELAY_MS,
|
|
8
|
+
FOOTER_COMPACTION_RENDER_INTERVAL_MS,
|
|
9
|
+
MAX_DYNAMIC_ROUTES,
|
|
10
|
+
} from "../../src/constants.ts";
|
|
11
|
+
import { CodexRecoveryPolicy, classifyCodexFailure, type CodexFailureMetadata } from "../../src/domain/codex-recovery.ts";
|
|
3
12
|
import type { MetricObservation, StoredMetricObservation } from "../../src/domain/metric.ts";
|
|
4
13
|
import type { PolicyDecision, Route } from "../../src/policy.ts";
|
|
5
14
|
import type { RouterStatus } from "../../src/ports/router-controller.ts";
|
|
6
15
|
import { parseCodexRateLimitHeaders } from "../../src/providers/codex.ts";
|
|
7
16
|
import { installIntegratedFooter, type IntegratedFooterState } from "./footer.ts";
|
|
8
17
|
import { callJittor } from "./service-client.ts";
|
|
9
|
-
import { persistentEnforcementControl, type EnforcementControl } from "./settings.ts";
|
|
18
|
+
import { persistentEnforcementControl, type CodexRecoveryControl, type EnforcementControl } from "./settings.ts";
|
|
10
19
|
import { buildFooterBudget, formatFooterStatus, showJittorPanel } from "./tui.ts";
|
|
11
20
|
import { showUsagePanel } from "./usage.ts";
|
|
12
21
|
|
|
@@ -23,6 +32,32 @@ const daemonClient: JittorExtensionClient = {
|
|
|
23
32
|
call: (operation, input) => callJittor(operation as Parameters<typeof callJittor>[0], input as never),
|
|
24
33
|
};
|
|
25
34
|
|
|
35
|
+
export interface CodexRecoveryRuntime {
|
|
36
|
+
now(): number;
|
|
37
|
+
random(): number;
|
|
38
|
+
setTimeout(callback: () => void | Promise<void>, delayMs: number): unknown;
|
|
39
|
+
clearTimeout(handle: unknown): void;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const SYSTEM_RECOVERY_RUNTIME: CodexRecoveryRuntime = {
|
|
43
|
+
now: Date.now,
|
|
44
|
+
random: Math.random,
|
|
45
|
+
setTimeout(callback, delayMs) { return setTimeout(() => { void callback(); }, delayMs); },
|
|
46
|
+
clearTimeout(handle) { clearTimeout(handle as ReturnType<typeof setTimeout>); },
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
function recoveryControl(enforcement: EnforcementControl): CodexRecoveryControl {
|
|
50
|
+
const candidate = enforcement as EnforcementControl & Partial<CodexRecoveryControl>;
|
|
51
|
+
return typeof candidate.isCodexRecoveryEnabled === "function"
|
|
52
|
+
? { isCodexRecoveryEnabled: () => candidate.isCodexRecoveryEnabled!() }
|
|
53
|
+
: { isCodexRecoveryEnabled: () => false };
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function header(headers: Record<string, string>, name: string): string | undefined {
|
|
57
|
+
const expected = name.toLowerCase();
|
|
58
|
+
return Object.entries(headers).find(([key]) => key.toLowerCase() === expected)?.[1];
|
|
59
|
+
}
|
|
60
|
+
|
|
26
61
|
async function recordMetrics(client: JittorExtensionClient, metrics: MetricObservation[]): Promise<void> {
|
|
27
62
|
for (const metric of metrics) await client.call("metrics.record", metric);
|
|
28
63
|
}
|
|
@@ -169,8 +204,46 @@ export function registerJittorExtension(
|
|
|
169
204
|
pi: ExtensionAPI,
|
|
170
205
|
client: JittorExtensionClient = daemonClient,
|
|
171
206
|
enforcement: EnforcementControl = persistentEnforcementControl(),
|
|
207
|
+
codexRecovery: CodexRecoveryControl = recoveryControl(enforcement),
|
|
208
|
+
recoveryRuntime: CodexRecoveryRuntime = SYSTEM_RECOVERY_RUNTIME,
|
|
172
209
|
): void {
|
|
173
210
|
const footerState: IntegratedFooterState = { providerBudget: null };
|
|
211
|
+
const recoveryPolicy = new CodexRecoveryPolicy({
|
|
212
|
+
baseDelayMs: CODEX_RECOVERY_BASE_DELAY_MS,
|
|
213
|
+
maxDelayMs: CODEX_RECOVERY_MAX_DELAY_MS,
|
|
214
|
+
maxAttempts: CODEX_RECOVERY_MAX_ATTEMPTS,
|
|
215
|
+
attemptWindowMs: CODEX_RECOVERY_ATTEMPT_WINDOW_MS,
|
|
216
|
+
jitterRatio: CODEX_RECOVERY_JITTER_RATIO,
|
|
217
|
+
}, recoveryRuntime.random);
|
|
218
|
+
let recoveryTimer: unknown;
|
|
219
|
+
let lastCodexResponse: CodexFailureMetadata = {};
|
|
220
|
+
const cancelRecovery = (resetPolicy: boolean): void => {
|
|
221
|
+
if (recoveryTimer !== undefined) recoveryRuntime.clearTimeout(recoveryTimer);
|
|
222
|
+
recoveryTimer = undefined;
|
|
223
|
+
if (resetPolicy) recoveryPolicy.cancel();
|
|
224
|
+
};
|
|
225
|
+
const scheduleCodexRecovery = (ctx: ExtensionContext): void => {
|
|
226
|
+
if (!codexRecovery.isCodexRecoveryEnabled() || recoveryTimer !== undefined || !ctx.isIdle() || ctx.hasPendingMessages()) return;
|
|
227
|
+
const plan = recoveryPolicy.plan(recoveryRuntime.now());
|
|
228
|
+
if (plan.action === "exhausted") {
|
|
229
|
+
recoveryPolicy.abandonFailure();
|
|
230
|
+
if (ctx.hasUI) ctx.ui.notify(`Jittor Codex recovery stopped: ${plan.reason}.`, "warning");
|
|
231
|
+
return;
|
|
232
|
+
}
|
|
233
|
+
if (plan.action !== "schedule") return;
|
|
234
|
+
recoveryTimer = recoveryRuntime.setTimeout(async () => {
|
|
235
|
+
recoveryTimer = undefined;
|
|
236
|
+
if (!ctx.isIdle() || ctx.hasPendingMessages()) return;
|
|
237
|
+
const attempt = recoveryPolicy.recordAttempt(recoveryRuntime.now());
|
|
238
|
+
if (!attempt) return;
|
|
239
|
+
pi.sendMessage({
|
|
240
|
+
customType: "jittor-codex-recovery",
|
|
241
|
+
content: `Retry the previous Codex request after a transient ${attempt.failureKind} failure. Automatic recovery attempt ${attempt.attempt} of ${CODEX_RECOVERY_MAX_ATTEMPTS}.`,
|
|
242
|
+
display: false,
|
|
243
|
+
details: { attempt: attempt.attempt, failureKind: attempt.failureKind },
|
|
244
|
+
}, { triggerTurn: true, deliverAs: "followUp" });
|
|
245
|
+
}, plan.delayMs);
|
|
246
|
+
};
|
|
174
247
|
let compactionTimer: ReturnType<typeof setInterval> | undefined;
|
|
175
248
|
const finishCompaction = (): void => {
|
|
176
249
|
if (compactionTimer) clearInterval(compactionTimer);
|
|
@@ -252,6 +325,8 @@ export function registerJittorExtension(
|
|
|
252
325
|
|
|
253
326
|
pi.on("session_start", async (_event, ctx) => {
|
|
254
327
|
finishCompaction();
|
|
328
|
+
cancelRecovery(true);
|
|
329
|
+
lastCodexResponse = {};
|
|
255
330
|
ctx.ui.setStatus("jittor", undefined);
|
|
256
331
|
showFooter(ctx);
|
|
257
332
|
try {
|
|
@@ -273,11 +348,22 @@ export function registerJittorExtension(
|
|
|
273
348
|
finishCompaction();
|
|
274
349
|
});
|
|
275
350
|
|
|
276
|
-
pi.on("agent_settled", () => {
|
|
351
|
+
pi.on("agent_settled", async (_event, ctx) => {
|
|
277
352
|
if (footerState.compaction) finishCompaction();
|
|
353
|
+
scheduleCodexRecovery(ctx);
|
|
354
|
+
if (!enforcement.isFooterEnabled()) return;
|
|
355
|
+
try {
|
|
356
|
+
await syncCurrentRoute(pi, client, ctx);
|
|
357
|
+
await syncAvailableRoutes(pi, client, ctx);
|
|
358
|
+
await refreshFooter(client, footerState);
|
|
359
|
+
} catch {
|
|
360
|
+
footerState.providerBudget = null;
|
|
361
|
+
footerState.requestRender?.();
|
|
362
|
+
}
|
|
278
363
|
});
|
|
279
364
|
|
|
280
365
|
pi.on("input", async (event, ctx) => {
|
|
366
|
+
if (event.source !== "extension") cancelRecovery(true);
|
|
281
367
|
if (event.source === "extension" || !enforcement.isEnabled()) return { action: "continue" as const };
|
|
282
368
|
try {
|
|
283
369
|
const next = await client.call("router.decide", {}) as PolicyDecision;
|
|
@@ -302,6 +388,7 @@ export function registerJittorExtension(
|
|
|
302
388
|
});
|
|
303
389
|
|
|
304
390
|
pi.on("turn_start", async (_event, ctx) => {
|
|
391
|
+
lastCodexResponse = {};
|
|
305
392
|
if (!enforcement.isEnabled()) return;
|
|
306
393
|
try {
|
|
307
394
|
await syncCurrentRoute(pi, client, ctx);
|
|
@@ -314,6 +401,9 @@ export function registerJittorExtension(
|
|
|
314
401
|
});
|
|
315
402
|
|
|
316
403
|
pi.on("after_provider_response", async (event, ctx) => {
|
|
404
|
+
if (ctx.model?.provider === "openai-codex") {
|
|
405
|
+
lastCodexResponse = { status: event.status, ...(header(event.headers, "retry-after") ? { retryAfter: header(event.headers, "retry-after") } : {}) };
|
|
406
|
+
}
|
|
317
407
|
if (!Object.keys(event.headers).some((name) => name.toLowerCase().startsWith("x-codex-"))) return;
|
|
318
408
|
try {
|
|
319
409
|
const updates = parseCodexRateLimitHeaders(new Headers(event.headers), Date.now());
|
|
@@ -325,6 +415,16 @@ export function registerJittorExtension(
|
|
|
325
415
|
});
|
|
326
416
|
|
|
327
417
|
pi.on("message_end", async (event, _ctx) => {
|
|
418
|
+
if (event.message.role === "assistant" && event.message.provider === "openai-codex") {
|
|
419
|
+
if (event.message.stopReason === "error") {
|
|
420
|
+
const failure = classifyCodexFailure(event.message.errorMessage, lastCodexResponse);
|
|
421
|
+
if (codexRecovery.isCodexRecoveryEnabled() && failure.transient) recoveryPolicy.observeFailure(failure, recoveryRuntime.now());
|
|
422
|
+
else cancelRecovery(true);
|
|
423
|
+
} else if (event.message.stopReason !== "aborted") {
|
|
424
|
+
cancelRecovery(true);
|
|
425
|
+
}
|
|
426
|
+
lastCodexResponse = {};
|
|
427
|
+
}
|
|
328
428
|
const metrics = assistantUsageMetrics(event.message, Date.now());
|
|
329
429
|
if (metrics.length > 0) await recordMetrics(client, metrics).catch(() => undefined);
|
|
330
430
|
if (enforcement.isFooterEnabled()) await refreshFooter(client, footerState).catch(() => undefined);
|
|
@@ -332,6 +432,8 @@ export function registerJittorExtension(
|
|
|
332
432
|
|
|
333
433
|
pi.on("session_shutdown", async (_event, ctx) => {
|
|
334
434
|
finishCompaction();
|
|
435
|
+
cancelRecovery(true);
|
|
436
|
+
lastCodexResponse = {};
|
|
335
437
|
ctx.ui.setStatus("jittor", undefined);
|
|
336
438
|
ctx.ui.setFooter(undefined);
|
|
337
439
|
});
|
|
@@ -9,9 +9,18 @@ export interface EnforcementControl {
|
|
|
9
9
|
setFooterEnabled(enabled: boolean): void;
|
|
10
10
|
}
|
|
11
11
|
|
|
12
|
+
export interface CodexRecoveryControl {
|
|
13
|
+
isCodexRecoveryEnabled(): boolean;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export interface PersistentExtensionControl extends EnforcementControl, CodexRecoveryControl {
|
|
17
|
+
setCodexRecoveryEnabled(enabled: boolean): void;
|
|
18
|
+
}
|
|
19
|
+
|
|
12
20
|
interface ExtensionSettings {
|
|
13
21
|
enforcementEnabled: boolean;
|
|
14
22
|
footerEnabled: boolean;
|
|
23
|
+
codexRecoveryEnabled: boolean;
|
|
15
24
|
}
|
|
16
25
|
|
|
17
26
|
function settingsPath(env: Record<string, string | undefined> = process.env): string {
|
|
@@ -22,14 +31,15 @@ function settingsPath(env: Record<string, string | undefined> = process.env): st
|
|
|
22
31
|
function loadSettings(path: string): ExtensionSettings {
|
|
23
32
|
try {
|
|
24
33
|
const value = JSON.parse(readFileSync(path, "utf8")) as unknown;
|
|
25
|
-
if (typeof value !== "object" || value === null || Array.isArray(value)) return { enforcementEnabled: true, footerEnabled: true };
|
|
34
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) return { enforcementEnabled: true, footerEnabled: true, codexRecoveryEnabled: false };
|
|
26
35
|
const record = value as Record<string, unknown>;
|
|
27
36
|
return {
|
|
28
37
|
enforcementEnabled: record["enforcementEnabled"] !== false,
|
|
29
38
|
footerEnabled: record["footerEnabled"] !== false,
|
|
39
|
+
codexRecoveryEnabled: record["codexRecoveryEnabled"] === true,
|
|
30
40
|
};
|
|
31
41
|
} catch {
|
|
32
|
-
return { enforcementEnabled: true, footerEnabled: true };
|
|
42
|
+
return { enforcementEnabled: true, footerEnabled: true, codexRecoveryEnabled: false };
|
|
33
43
|
}
|
|
34
44
|
}
|
|
35
45
|
|
|
@@ -41,7 +51,7 @@ function persistSettings(path: string, settings: ExtensionSettings): void {
|
|
|
41
51
|
renameSync(temporary, path);
|
|
42
52
|
}
|
|
43
53
|
|
|
44
|
-
export function persistentEnforcementControl(env: Record<string, string | undefined> = process.env):
|
|
54
|
+
export function persistentEnforcementControl(env: Record<string, string | undefined> = process.env): PersistentExtensionControl {
|
|
45
55
|
const path = settingsPath(env);
|
|
46
56
|
const settings = loadSettings(path);
|
|
47
57
|
return {
|
|
@@ -55,5 +65,10 @@ export function persistentEnforcementControl(env: Record<string, string | undefi
|
|
|
55
65
|
settings.footerEnabled = value;
|
|
56
66
|
persistSettings(path, settings);
|
|
57
67
|
},
|
|
68
|
+
isCodexRecoveryEnabled: () => settings.codexRecoveryEnabled,
|
|
69
|
+
setCodexRecoveryEnabled(value: boolean): void {
|
|
70
|
+
settings.codexRecoveryEnabled = value;
|
|
71
|
+
persistSettings(path, settings);
|
|
72
|
+
},
|
|
58
73
|
};
|
|
59
74
|
}
|
package/package.json
CHANGED
package/src/constants.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
export const VERSION = "0.1.2";
|
|
2
1
|
export const SQLITE_SCHEMA_VERSION = 1;
|
|
3
2
|
export const SQLITE_BUSY_TIMEOUT_MS = 5_000;
|
|
4
3
|
export const DEFAULT_QUERY_LIMIT = 1_000;
|
|
@@ -30,3 +29,10 @@ export const USAGE_CHART_HEIGHT = 8;
|
|
|
30
29
|
export const USAGE_Y_AXIS_WIDTH = 7;
|
|
31
30
|
export const USAGE_TOKEN_QUERY_LIMIT = 10_000;
|
|
32
31
|
export const MAX_DYNAMIC_ROUTES = 100;
|
|
32
|
+
export const CODEX_ERROR_MESSAGE_LIMIT = 160;
|
|
33
|
+
export const CODEX_RETRY_AFTER_MAX_MS = 5 * MILLISECONDS_PER_MINUTE;
|
|
34
|
+
export const CODEX_RECOVERY_BASE_DELAY_MS = 2 * MILLISECONDS_PER_SECOND;
|
|
35
|
+
export const CODEX_RECOVERY_MAX_DELAY_MS = CODEX_RETRY_AFTER_MAX_MS;
|
|
36
|
+
export const CODEX_RECOVERY_MAX_ATTEMPTS = 3;
|
|
37
|
+
export const CODEX_RECOVERY_ATTEMPT_WINDOW_MS = 10 * MILLISECONDS_PER_MINUTE;
|
|
38
|
+
export const CODEX_RECOVERY_JITTER_RATIO = 0.2;
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
import {
|
|
2
|
+
CODEX_ERROR_MESSAGE_LIMIT,
|
|
3
|
+
CODEX_RETRY_AFTER_MAX_MS,
|
|
4
|
+
MILLISECONDS_PER_SECOND,
|
|
5
|
+
} from "../constants.ts";
|
|
6
|
+
|
|
7
|
+
export type CodexFailureKind =
|
|
8
|
+
| "concurrency"
|
|
9
|
+
| "rate-limit"
|
|
10
|
+
| "overload"
|
|
11
|
+
| "transport"
|
|
12
|
+
| "quota"
|
|
13
|
+
| "authentication"
|
|
14
|
+
| "invalid-request"
|
|
15
|
+
| "unknown";
|
|
16
|
+
|
|
17
|
+
export interface CodexFailure {
|
|
18
|
+
kind: CodexFailureKind;
|
|
19
|
+
transient: boolean;
|
|
20
|
+
code?: string;
|
|
21
|
+
source?: string;
|
|
22
|
+
message?: string;
|
|
23
|
+
retryAfterMs?: number;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface CodexFailureMetadata {
|
|
27
|
+
status?: number;
|
|
28
|
+
retryAfter?: string;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export interface CodexRecoveryOptions {
|
|
32
|
+
baseDelayMs: number;
|
|
33
|
+
maxDelayMs: number;
|
|
34
|
+
maxAttempts: number;
|
|
35
|
+
attemptWindowMs: number;
|
|
36
|
+
jitterRatio: number;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export type CodexRecoveryPlan =
|
|
40
|
+
| { action: "schedule"; attempt: number; delayMs: number; failureKind: CodexFailureKind }
|
|
41
|
+
| { action: "wait"; reason: string }
|
|
42
|
+
| { action: "exhausted"; reason: string };
|
|
43
|
+
|
|
44
|
+
export interface CodexRecoveryAttempt {
|
|
45
|
+
attempt: number;
|
|
46
|
+
failureKind: CodexFailureKind;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export class CodexRecoveryPolicy {
|
|
50
|
+
private pendingFailure: CodexFailure | undefined;
|
|
51
|
+
private attempts = 0;
|
|
52
|
+
private windowStartedAt: number | undefined;
|
|
53
|
+
|
|
54
|
+
constructor(
|
|
55
|
+
private readonly options: CodexRecoveryOptions,
|
|
56
|
+
private readonly random: () => number = Math.random,
|
|
57
|
+
) {
|
|
58
|
+
if (!Number.isFinite(options.baseDelayMs) || options.baseDelayMs < 0) throw new Error("baseDelayMs must be non-negative");
|
|
59
|
+
if (!Number.isFinite(options.maxDelayMs) || options.maxDelayMs < options.baseDelayMs) throw new Error("maxDelayMs must be at least baseDelayMs");
|
|
60
|
+
if (!Number.isInteger(options.maxAttempts) || options.maxAttempts < 1) throw new Error("maxAttempts must be a positive integer");
|
|
61
|
+
if (!Number.isFinite(options.attemptWindowMs) || options.attemptWindowMs <= 0) throw new Error("attemptWindowMs must be positive");
|
|
62
|
+
if (!Number.isFinite(options.jitterRatio) || options.jitterRatio < 0 || options.jitterRatio > 1) throw new Error("jitterRatio must be between 0 and 1");
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
observeFailure(failure: CodexFailure, now: number): void {
|
|
66
|
+
this.normalizeWindow(now);
|
|
67
|
+
this.pendingFailure = failure.transient ? failure : undefined;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
observeSuccess(): void {
|
|
71
|
+
this.cancel();
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
cancel(): void {
|
|
75
|
+
this.pendingFailure = undefined;
|
|
76
|
+
this.attempts = 0;
|
|
77
|
+
this.windowStartedAt = undefined;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
abandonFailure(): void {
|
|
81
|
+
this.pendingFailure = undefined;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
plan(now: number): CodexRecoveryPlan {
|
|
85
|
+
this.normalizeWindow(now);
|
|
86
|
+
if (!this.pendingFailure) return { action: "wait", reason: "no transient Codex failure is pending" };
|
|
87
|
+
if (this.attempts >= this.options.maxAttempts) {
|
|
88
|
+
return { action: "exhausted", reason: `${this.options.maxAttempts} recovery attempts reached within ${this.options.attemptWindowMs}ms` };
|
|
89
|
+
}
|
|
90
|
+
const base = this.pendingFailure.retryAfterMs
|
|
91
|
+
?? this.options.baseDelayMs * (2 ** this.attempts);
|
|
92
|
+
const sample = this.random();
|
|
93
|
+
const unit = Number.isFinite(sample) ? Math.min(1, Math.max(0, sample)) : 0;
|
|
94
|
+
const multiplier = 1 + ((unit * 2) - 1) * this.options.jitterRatio;
|
|
95
|
+
const jittered = Math.max(0, Math.round(base * multiplier));
|
|
96
|
+
const delayMs = this.pendingFailure.retryAfterMs === undefined ? jittered : Math.max(base, jittered);
|
|
97
|
+
return {
|
|
98
|
+
action: "schedule",
|
|
99
|
+
attempt: this.attempts + 1,
|
|
100
|
+
delayMs: Math.min(this.options.maxDelayMs, delayMs),
|
|
101
|
+
failureKind: this.pendingFailure.kind,
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
recordAttempt(now: number): CodexRecoveryAttempt | undefined {
|
|
106
|
+
this.normalizeWindow(now);
|
|
107
|
+
if (!this.pendingFailure || this.attempts >= this.options.maxAttempts) return undefined;
|
|
108
|
+
if (this.windowStartedAt === undefined) this.windowStartedAt = now;
|
|
109
|
+
this.attempts += 1;
|
|
110
|
+
const attempt = { attempt: this.attempts, failureKind: this.pendingFailure.kind };
|
|
111
|
+
this.pendingFailure = undefined;
|
|
112
|
+
return attempt;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
state(): { attempts: number; pending: boolean } {
|
|
116
|
+
return { attempts: this.attempts, pending: this.pendingFailure !== undefined };
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
private normalizeWindow(now: number): void {
|
|
120
|
+
if (this.windowStartedAt !== undefined && now - this.windowStartedAt >= this.options.attemptWindowMs) {
|
|
121
|
+
this.attempts = 0;
|
|
122
|
+
this.windowStartedAt = undefined;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function asRecord(value: unknown): Record<string, unknown> | undefined {
|
|
128
|
+
return typeof value === "object" && value !== null && !Array.isArray(value)
|
|
129
|
+
? value as Record<string, unknown>
|
|
130
|
+
: undefined;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function firstString(...values: unknown[]): string | undefined {
|
|
134
|
+
for (const value of values) {
|
|
135
|
+
if (typeof value === "string" && value.length > 0) return value;
|
|
136
|
+
}
|
|
137
|
+
return undefined;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function retryAfterMs(value: string | undefined): number | undefined {
|
|
141
|
+
if (!value) return undefined;
|
|
142
|
+
const seconds = Number(value.trim());
|
|
143
|
+
if (!Number.isFinite(seconds) || seconds < 0) return undefined;
|
|
144
|
+
return Math.min(CODEX_RETRY_AFTER_MAX_MS, Math.round(seconds * MILLISECONDS_PER_SECOND));
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
function matches(value: string, patterns: readonly string[]): boolean {
|
|
148
|
+
return patterns.some((pattern) => value.includes(pattern));
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
export function classifyCodexFailure(value: unknown, metadata: CodexFailureMetadata = {}): CodexFailure {
|
|
152
|
+
const root = asRecord(value);
|
|
153
|
+
const detail = asRecord(root?.["detail"]);
|
|
154
|
+
const nestedError = asRecord(root?.["error"]);
|
|
155
|
+
const code = firstString(detail?.["code"], detail?.["error_code"], nestedError?.["code"], root?.["code"]);
|
|
156
|
+
const source = firstString(detail?.["source"], nestedError?.["source"], root?.["source"]);
|
|
157
|
+
const rawMessage = firstString(
|
|
158
|
+
detail?.["message"],
|
|
159
|
+
typeof root?.["error"] === "string" ? root["error"] : undefined,
|
|
160
|
+
nestedError?.["message"],
|
|
161
|
+
root?.["message"],
|
|
162
|
+
typeof value === "string" ? value : undefined,
|
|
163
|
+
);
|
|
164
|
+
const message = rawMessage?.slice(0, CODEX_ERROR_MESSAGE_LIMIT);
|
|
165
|
+
const evidence = [code, source, message].filter(Boolean).join(" ").toLowerCase();
|
|
166
|
+
const base = {
|
|
167
|
+
...(code ? { code } : {}),
|
|
168
|
+
...(source ? { source } : {}),
|
|
169
|
+
...(message ? { message } : {}),
|
|
170
|
+
...(retryAfterMs(metadata.retryAfter) !== undefined ? { retryAfterMs: retryAfterMs(metadata.retryAfter) } : {}),
|
|
171
|
+
};
|
|
172
|
+
|
|
173
|
+
if (matches(evidence, ["insufficient_quota", "quota exceeded", "out of credits", "billing"])) {
|
|
174
|
+
return { kind: "quota", transient: false, ...base };
|
|
175
|
+
}
|
|
176
|
+
if (metadata.status === 401 || metadata.status === 403 || matches(evidence, ["invalid_api_key", "authentication", "unauthorized", "permission_denied"])) {
|
|
177
|
+
return { kind: "authentication", transient: false, ...base };
|
|
178
|
+
}
|
|
179
|
+
if (matches(evidence, ["invalid_prompt", "invalid_request", "context_length_exceeded"]) || metadata.status === 400 || metadata.status === 422) {
|
|
180
|
+
return { kind: "invalid-request", transient: false, ...base };
|
|
181
|
+
}
|
|
182
|
+
if (matches(evidence, ["concurrency_limit", "too many concurrent requests", "throttled"])) {
|
|
183
|
+
return { kind: "concurrency", transient: true, ...base };
|
|
184
|
+
}
|
|
185
|
+
if (matches(evidence, ["server_is_overloaded", "slow_down", "service unavailable", "overloaded"]) || (metadata.status !== undefined && metadata.status >= 500 && metadata.status <= 599)) {
|
|
186
|
+
return { kind: "overload", transient: true, ...base };
|
|
187
|
+
}
|
|
188
|
+
if (metadata.status === 429 || matches(evidence, ["rate_limit_exceeded", "rate limit", "too many requests"])) {
|
|
189
|
+
return { kind: "rate-limit", transient: true, ...base };
|
|
190
|
+
}
|
|
191
|
+
if (matches(evidence, ["timeout", "timed out", "network", "connection", "websocket", "fetch failed"])) {
|
|
192
|
+
return { kind: "transport", transient: true, ...base };
|
|
193
|
+
}
|
|
194
|
+
return { kind: "unknown", transient: false, ...base };
|
|
195
|
+
}
|
package/src/service.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { SERVICE_MAX_BODY_BYTES
|
|
1
|
+
import { SERVICE_MAX_BODY_BYTES } from "./constants.ts";
|
|
2
|
+
import { VERSION } from "./version.ts";
|
|
2
3
|
import { validateMetricObservation, type MetricObservation, type MetricQuery, type StoredMetricObservation } from "./domain/metric.ts";
|
|
3
4
|
import type { MetricStore } from "./ports/metric-store.ts";
|
|
4
5
|
import type { RouteOverride, RouterController, RouterStatus, TelemetryPollResult } from "./ports/router-controller.ts";
|
package/src/version.ts
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
|
|
3
|
+
function packageVersion(): string {
|
|
4
|
+
const manifest = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")) as unknown;
|
|
5
|
+
if (typeof manifest !== "object" || manifest === null || Array.isArray(manifest)) {
|
|
6
|
+
throw new Error("Jittor package manifest must be an object");
|
|
7
|
+
}
|
|
8
|
+
const version = (manifest as Record<string, unknown>)["version"];
|
|
9
|
+
if (typeof version !== "string" || !/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/.test(version)) {
|
|
10
|
+
throw new Error("Jittor package manifest has an invalid version");
|
|
11
|
+
}
|
|
12
|
+
return version;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** Runtime package version; package.json is the single release source of truth. */
|
|
16
|
+
export const VERSION = packageVersion();
|