oira666_pi-limits-wait 0.5.2 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +149 -97
- package/package.json +8 -5
- package/src/constants.ts +38 -38
- package/src/index.ts +36 -2
- package/src/models.ts +155 -155
- package/src/retry-errors.ts +263 -263
- package/src/retry-summary.ts +139 -0
- package/src/settings.ts +151 -130
- package/src/stream.ts +64 -16
- package/src/telemetry.ts +78 -0
- package/src/ui.ts +234 -179
package/src/settings.ts
CHANGED
|
@@ -1,130 +1,151 @@
|
|
|
1
|
-
import { existsSync, readFileSync } from "fs";
|
|
2
|
-
import { homedir } from "os";
|
|
3
|
-
import { join, resolve } from "path";
|
|
4
|
-
import { getAgentDir, type ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
5
|
-
import type { ModelThinkingLevel } from "@earendil-works/pi-ai";
|
|
6
|
-
import { FALLBACK_MODELS_KEY, SETTINGS_FILE_NAME } from "./constants.js";
|
|
7
|
-
import { formatModel, modelKey } from "./models.js";
|
|
8
|
-
import { state } from "./state.js";
|
|
9
|
-
import type { ConfiguredModel } from "./types.js";
|
|
10
|
-
|
|
11
|
-
function isThinkingLevel(value: unknown): value is ModelThinkingLevel {
|
|
12
|
-
return ["off", "minimal", "low", "medium", "high", "xhigh", "max"].includes(String(value));
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
export function parseConfiguredModels(raw: unknown): ConfiguredModel[] {
|
|
16
|
-
const source = raw && typeof raw === "object" && Array.isArray((raw as { [FALLBACK_MODELS_KEY]?: unknown })[FALLBACK_MODELS_KEY])
|
|
17
|
-
? (raw as { [FALLBACK_MODELS_KEY]: unknown[] })[FALLBACK_MODELS_KEY]
|
|
18
|
-
: [];
|
|
19
|
-
|
|
20
|
-
const models: ConfiguredModel[] = [];
|
|
21
|
-
for (const item of source) {
|
|
22
|
-
if (!item || typeof item !== "object") continue;
|
|
23
|
-
const record = item as Record<string, unknown>;
|
|
24
|
-
const provider = typeof record.provider === "string" ? record.provider.trim() : "";
|
|
25
|
-
const modelname =
|
|
26
|
-
typeof record.modelname === "string" ? record.modelname.trim()
|
|
27
|
-
: typeof record.modelName === "string" ? record.modelName.trim()
|
|
28
|
-
: typeof record.model === "string" ? record.model.trim()
|
|
29
|
-
: "";
|
|
30
|
-
const reasoning = record["reasoning effort"] ?? record.reasoningEffort ?? record.reasoning_effort;
|
|
31
|
-
if (!provider || !modelname) continue;
|
|
32
|
-
models.push({ provider, modelname, reasoningEffort: isThinkingLevel(reasoning) ? reasoning : undefined });
|
|
33
|
-
}
|
|
34
|
-
return models;
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
export function configFileCandidates(
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
return
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
const
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
}
|
|
1
|
+
import { existsSync, readFileSync } from "fs";
|
|
2
|
+
import { homedir } from "os";
|
|
3
|
+
import { join, resolve } from "path";
|
|
4
|
+
import { getAgentDir, type ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
5
|
+
import type { ModelThinkingLevel } from "@earendil-works/pi-ai";
|
|
6
|
+
import { FALLBACK_MODELS_KEY, SETTINGS_FILE_NAME } from "./constants.js";
|
|
7
|
+
import { formatModel, modelKey } from "./models.js";
|
|
8
|
+
import { state } from "./state.js";
|
|
9
|
+
import type { ConfiguredModel } from "./types.js";
|
|
10
|
+
|
|
11
|
+
function isThinkingLevel(value: unknown): value is ModelThinkingLevel {
|
|
12
|
+
return ["off", "minimal", "low", "medium", "high", "xhigh", "max"].includes(String(value));
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export function parseConfiguredModels(raw: unknown): ConfiguredModel[] {
|
|
16
|
+
const source = raw && typeof raw === "object" && Array.isArray((raw as { [FALLBACK_MODELS_KEY]?: unknown })[FALLBACK_MODELS_KEY])
|
|
17
|
+
? (raw as { [FALLBACK_MODELS_KEY]: unknown[] })[FALLBACK_MODELS_KEY]
|
|
18
|
+
: [];
|
|
19
|
+
|
|
20
|
+
const models: ConfiguredModel[] = [];
|
|
21
|
+
for (const item of source) {
|
|
22
|
+
if (!item || typeof item !== "object") continue;
|
|
23
|
+
const record = item as Record<string, unknown>;
|
|
24
|
+
const provider = typeof record.provider === "string" ? record.provider.trim() : "";
|
|
25
|
+
const modelname =
|
|
26
|
+
typeof record.modelname === "string" ? record.modelname.trim()
|
|
27
|
+
: typeof record.modelName === "string" ? record.modelName.trim()
|
|
28
|
+
: typeof record.model === "string" ? record.model.trim()
|
|
29
|
+
: "";
|
|
30
|
+
const reasoning = record["reasoning effort"] ?? record.reasoningEffort ?? record.reasoning_effort;
|
|
31
|
+
if (!provider || !modelname) continue;
|
|
32
|
+
models.push({ provider, modelname, reasoningEffort: isThinkingLevel(reasoning) ? reasoning : undefined });
|
|
33
|
+
}
|
|
34
|
+
return models;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function configFileCandidates(
|
|
38
|
+
cwd: string,
|
|
39
|
+
agentDir = getAgentDir(),
|
|
40
|
+
homeDir = homedir(),
|
|
41
|
+
isProjectTrusted = false,
|
|
42
|
+
): string[] {
|
|
43
|
+
const alwaysLoaded = [
|
|
44
|
+
join(homeDir, ".config", ".pi", SETTINGS_FILE_NAME),
|
|
45
|
+
join(agentDir, SETTINGS_FILE_NAME),
|
|
46
|
+
];
|
|
47
|
+
return isProjectTrusted
|
|
48
|
+
? [...alwaysLoaded, resolve(cwd, `.${SETTINGS_FILE_NAME}`), resolve(cwd, ".pi", SETTINGS_FILE_NAME)]
|
|
49
|
+
: alwaysLoaded;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function mergeConfig(base: Record<string, unknown>, override: Record<string, unknown>): Record<string, unknown> {
|
|
53
|
+
return { ...base, ...override };
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function readFallbackSettings(
|
|
57
|
+
cwd: string,
|
|
58
|
+
agentDir = getAgentDir(),
|
|
59
|
+
homeDir = homedir(),
|
|
60
|
+
isProjectTrusted = false,
|
|
61
|
+
): { config: Record<string, unknown>; loadedPaths: string[]; warnings: string[] } {
|
|
62
|
+
let config: Record<string, unknown> = {};
|
|
63
|
+
const loadedPaths: string[] = [];
|
|
64
|
+
const warnings: string[] = [];
|
|
65
|
+
const seen = new Set<string>();
|
|
66
|
+
|
|
67
|
+
for (const candidate of configFileCandidates(cwd, agentDir, homeDir, isProjectTrusted)) {
|
|
68
|
+
const path = resolve(candidate);
|
|
69
|
+
if (seen.has(path)) continue;
|
|
70
|
+
seen.add(path);
|
|
71
|
+
if (!existsSync(path)) continue;
|
|
72
|
+
|
|
73
|
+
try {
|
|
74
|
+
const parsed = JSON.parse(readFileSync(path, "utf-8")) as unknown;
|
|
75
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
76
|
+
warnings.push(`${path}: expected a JSON object`);
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
config = mergeConfig(config, parsed as Record<string, unknown>);
|
|
80
|
+
loadedPaths.push(path);
|
|
81
|
+
} catch (err) {
|
|
82
|
+
warnings.push(`${path}: ${err instanceof Error ? err.message : String(err)}`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
return { config, loadedPaths, warnings };
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function __readFallbackSettingsForTests(
|
|
90
|
+
cwd: string,
|
|
91
|
+
agentDir: string,
|
|
92
|
+
homeDir: string,
|
|
93
|
+
isProjectTrusted: boolean,
|
|
94
|
+
): { config: Record<string, unknown>; loadedPaths: string[]; warnings: string[] } {
|
|
95
|
+
return readFallbackSettings(cwd, agentDir, homeDir, isProjectTrusted);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export function loadFallbackSettings(ctx: ExtensionContext): void {
|
|
99
|
+
state.fallbackModels = [];
|
|
100
|
+
|
|
101
|
+
try {
|
|
102
|
+
const { config, loadedPaths, warnings } = readFallbackSettings(
|
|
103
|
+
ctx.cwd,
|
|
104
|
+
getAgentDir(),
|
|
105
|
+
homedir(),
|
|
106
|
+
typeof ctx.isProjectTrusted === "function" && ctx.isProjectTrusted(),
|
|
107
|
+
);
|
|
108
|
+
const content = JSON.stringify({ config, loadedPaths, warnings });
|
|
109
|
+
const shouldNotify = state.settingsSignature !== content;
|
|
110
|
+
state.settingsSignature = content;
|
|
111
|
+
|
|
112
|
+
const configured = parseConfiguredModels(config);
|
|
113
|
+
if (configured.length === 0) {
|
|
114
|
+
if (shouldNotify && warnings.length > 0) ctx.ui.notify(`Could not fully load ${SETTINGS_FILE_NAME}: ${warnings.join(", ")}`, "warning");
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const modelWarnings: string[] = [];
|
|
119
|
+
const seen = new Set<string>();
|
|
120
|
+
|
|
121
|
+
for (const entry of configured) {
|
|
122
|
+
const model = ctx.modelRegistry.find(entry.provider, entry.modelname);
|
|
123
|
+
if (!model) {
|
|
124
|
+
modelWarnings.push(`missing ${entry.provider}/${entry.modelname}`);
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
if (!ctx.modelRegistry.hasConfiguredAuth(model)) {
|
|
128
|
+
modelWarnings.push(`no auth ${entry.provider}/${entry.modelname}`);
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
const key = modelKey(model);
|
|
132
|
+
if (seen.has(key)) continue;
|
|
133
|
+
seen.add(key);
|
|
134
|
+
state.fallbackModels.push({ model, reasoningEffort: entry.reasoningEffort });
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
const lines = state.fallbackModels.map((entry, index) =>
|
|
138
|
+
`${index + 1}. ${formatModel(entry.model)}${entry.reasoningEffort ? ` (${entry.reasoningEffort})` : ""}`,
|
|
139
|
+
);
|
|
140
|
+
const source = loadedPaths.length > 0 ? ` from ${loadedPaths.join(", ")}` : "";
|
|
141
|
+
const message = lines.length > 0
|
|
142
|
+
? `Loaded ${SETTINGS_FILE_NAME}.${FALLBACK_MODELS_KEY}${source}:\n${lines.join("\n")}`
|
|
143
|
+
: `Loaded ${SETTINGS_FILE_NAME}.${FALLBACK_MODELS_KEY}${source}, but no usable fallback models were found.`;
|
|
144
|
+
const allWarnings = [...warnings, ...modelWarnings];
|
|
145
|
+
if (shouldNotify) {
|
|
146
|
+
ctx.ui.notify(allWarnings.length > 0 ? `${message}\nSkipped: ${allWarnings.join(", ")}` : message, allWarnings.length > 0 ? "warning" : "info");
|
|
147
|
+
}
|
|
148
|
+
} catch (err) {
|
|
149
|
+
ctx.ui.notify(`Could not load ${SETTINGS_FILE_NAME}.${FALLBACK_MODELS_KEY}: ${err instanceof Error ? err.message : String(err)}`, "warning");
|
|
150
|
+
}
|
|
151
|
+
}
|
package/src/stream.ts
CHANGED
|
@@ -28,9 +28,10 @@ import {
|
|
|
28
28
|
} from "./models.js";
|
|
29
29
|
import { withAttemptResponseObserver } from "./response-observer.js";
|
|
30
30
|
import { errorMessageWithCauses, getRetryableError, isAbortMessage, parseRetryDelayMs, retryAfterHeaderMs } from "./retry-errors.js";
|
|
31
|
+
import type { RetryPeriodTracker } from "./retry-summary.js";
|
|
31
32
|
import { loadFallbackSettings } from "./settings.js";
|
|
32
33
|
import { state } from "./state.js";
|
|
33
|
-
import type { FallbackModel, RetryableError, RuntimeStreamSimpleFn } from "./types.js";
|
|
34
|
+
import type { FallbackModel, RetryReason, RetryableError, RuntimeStreamSimpleFn } from "./types.js";
|
|
34
35
|
import { clearModelStatus, freezingEnabled, reasonLabel, waitForRetry } from "./ui.js";
|
|
35
36
|
|
|
36
37
|
export function __setNonRetryableTuningForTests(maxAttempts: number, retryDelayMs: number): void {
|
|
@@ -168,6 +169,7 @@ export function streamWithLimitsRetry(
|
|
|
168
169
|
context: Context,
|
|
169
170
|
options?: ModelsSimpleStreamOptions,
|
|
170
171
|
ownerSignal?: AbortSignal,
|
|
172
|
+
retryPeriod?: RetryPeriodTracker,
|
|
171
173
|
): AssistantMessageEventStream {
|
|
172
174
|
const output = createAssistantMessageEventStream();
|
|
173
175
|
const signal = options?.signal && ownerSignal
|
|
@@ -189,12 +191,22 @@ export function streamWithLimitsRetry(
|
|
|
189
191
|
const nonRetryableAttempts = new Map<string, number>();
|
|
190
192
|
const triedModels = new Set<string>();
|
|
191
193
|
|
|
194
|
+
const managedWait = async (reason: RetryReason, waitMs: number, error: string) =>
|
|
195
|
+
waitForRetry(reason, waitMs, signal, {
|
|
196
|
+
periodId: retryPeriod?.periodId,
|
|
197
|
+
model: attempt.model,
|
|
198
|
+
error,
|
|
199
|
+
events: retryPeriod?.events,
|
|
200
|
+
onEnd: (elapsedMs) => retryPeriod?.recordWait(elapsedMs),
|
|
201
|
+
});
|
|
202
|
+
|
|
192
203
|
const flush = (buffer: AssistantMessageEvent[]) => {
|
|
193
204
|
for (const event of buffer) output.push(event);
|
|
194
205
|
committed = true;
|
|
195
206
|
};
|
|
196
207
|
|
|
197
208
|
const pushAbort = (message: string) => {
|
|
209
|
+
retryPeriod?.finalize();
|
|
198
210
|
if (!committed) {
|
|
199
211
|
output.push({ type: "start", partial: freshMessage(attempt.model) });
|
|
200
212
|
committed = true;
|
|
@@ -281,8 +293,10 @@ export function streamWithLimitsRetry(
|
|
|
281
293
|
await priorOnResponse?.(response, responseModel);
|
|
282
294
|
if (signal?.aborted) return;
|
|
283
295
|
// Throw only after the canonical callback, so after_provider_response
|
|
284
|
-
// still sees
|
|
285
|
-
|
|
296
|
+
// still sees every HTTP failure. All statuses remain retry eligible:
|
|
297
|
+
// classified failures use their normal policy and 400/403 deliberately
|
|
298
|
+
// enter the configured unknown-error retry loop.
|
|
299
|
+
if (response.status >= 400) throw new Error(observedHttpError);
|
|
286
300
|
};
|
|
287
301
|
|
|
288
302
|
try {
|
|
@@ -347,6 +361,7 @@ export function streamWithLimitsRetry(
|
|
|
347
361
|
output.push({ type: "start", partial: freshMessage(attempt.model) });
|
|
348
362
|
committed = true;
|
|
349
363
|
}
|
|
364
|
+
if (event.type === "done") retryPeriod?.finalize();
|
|
350
365
|
output.push(event);
|
|
351
366
|
if (event.type === "done") {
|
|
352
367
|
finished = true;
|
|
@@ -356,6 +371,8 @@ export function streamWithLimitsRetry(
|
|
|
356
371
|
continue;
|
|
357
372
|
}
|
|
358
373
|
|
|
374
|
+
if (event.type === "error") retryPeriod?.recordReason(event.error.errorMessage ?? "Provider stream error");
|
|
375
|
+
if (event.type === "done" || event.type === "error") retryPeriod?.finalize();
|
|
359
376
|
output.push(event);
|
|
360
377
|
if (event.type === "done" || event.type === "error") {
|
|
361
378
|
finished = true;
|
|
@@ -394,6 +411,8 @@ export function streamWithLimitsRetry(
|
|
|
394
411
|
const failureEvent = nonRetryableError?.event;
|
|
395
412
|
const failureMessage = nonRetryableError?.message ?? "Provider stream ended without a terminal event.";
|
|
396
413
|
const emitFailure = () => {
|
|
414
|
+
retryPeriod?.recordReason(failureMessage);
|
|
415
|
+
retryPeriod?.finalize();
|
|
397
416
|
if (!committed) {
|
|
398
417
|
if (buffer.length > 0) flush(buffer);
|
|
399
418
|
else {
|
|
@@ -417,10 +436,13 @@ export function streamWithLimitsRetry(
|
|
|
417
436
|
nonRetryableAttempts.set(key, attempts);
|
|
418
437
|
if (attempts < state.nonRetryableMaxAttempts) {
|
|
419
438
|
notifyRetryingAfterError(attempt.model, state.nonRetryableRetryDelayMs, failureMessage);
|
|
420
|
-
|
|
439
|
+
retryPeriod?.beginRetry(failureMessage);
|
|
440
|
+
const wait = await managedWait("retry", state.nonRetryableRetryDelayMs, failureMessage);
|
|
441
|
+
if (wait === "aborted" || signal?.aborted) {
|
|
421
442
|
pushAbort("Request aborted while retrying after error.");
|
|
422
443
|
return;
|
|
423
444
|
}
|
|
445
|
+
retryPeriod?.completeRetry();
|
|
424
446
|
continue;
|
|
425
447
|
}
|
|
426
448
|
|
|
@@ -430,16 +452,19 @@ export function streamWithLimitsRetry(
|
|
|
430
452
|
rememberNonRetryableFailure(attempt.model, failureMessage);
|
|
431
453
|
const next = nextAvailableCandidate(attempt.model);
|
|
432
454
|
if (next) {
|
|
455
|
+
retryPeriod?.recordImmediateRetry(failureMessage);
|
|
433
456
|
attempt = next;
|
|
434
457
|
continue;
|
|
435
458
|
}
|
|
436
459
|
const deadline = earliestCandidateDeadline(attempt.model);
|
|
437
460
|
if (deadline) {
|
|
438
|
-
|
|
439
|
-
|
|
461
|
+
retryPeriod?.beginRetry(failureMessage);
|
|
462
|
+
const wait = await managedWait("model-frozen", Math.max(0, deadline - Date.now()), failureMessage);
|
|
463
|
+
if (wait === "aborted" || signal?.aborted) {
|
|
440
464
|
pushAbort("Request aborted while all fallback models were frozen.");
|
|
441
465
|
return;
|
|
442
466
|
}
|
|
467
|
+
retryPeriod?.completeRetry();
|
|
443
468
|
attempt = nextAvailableCandidate(attempt.model)
|
|
444
469
|
?? configuredAttempt(getPrimaryModel(attempt.model));
|
|
445
470
|
continue;
|
|
@@ -447,6 +472,7 @@ export function streamWithLimitsRetry(
|
|
|
447
472
|
} else {
|
|
448
473
|
const next = candidateOrder(attempt.model).find((entry) => !triedModels.has(modelKey(entry.model)));
|
|
449
474
|
if (next) {
|
|
475
|
+
retryPeriod?.recordImmediateRetry(failureMessage);
|
|
450
476
|
attempt = next;
|
|
451
477
|
continue;
|
|
452
478
|
}
|
|
@@ -460,37 +486,49 @@ export function streamWithLimitsRetry(
|
|
|
460
486
|
if (allowFallback) {
|
|
461
487
|
if (retryable.reason === "network") {
|
|
462
488
|
notifyRetryableError(attempt.model, retryable, retryableErrorMessage);
|
|
463
|
-
|
|
489
|
+
retryPeriod?.beginRetry(retryableErrorMessage);
|
|
490
|
+
const wait = await managedWait(retryable.reason, retryable.waitMs, retryableErrorMessage);
|
|
491
|
+
if (wait === "aborted" || signal?.aborted) {
|
|
464
492
|
pushAbort(`Request aborted during ${reasonLabel(retryable.reason).toLowerCase()} wait.`);
|
|
465
493
|
return;
|
|
466
494
|
}
|
|
495
|
+
retryPeriod?.completeRetry();
|
|
467
496
|
continue;
|
|
468
497
|
}
|
|
469
498
|
|
|
470
499
|
rememberRateLimit(attempt.model, retryable, retryableErrorMessage);
|
|
471
500
|
const next = nextAvailableCandidate(attempt.model);
|
|
472
501
|
if (next) {
|
|
502
|
+
retryPeriod?.recordImmediateRetry(retryableErrorMessage);
|
|
473
503
|
attempt = next;
|
|
474
504
|
continue;
|
|
475
505
|
}
|
|
476
506
|
const deadline = earliestCandidateDeadline(attempt.model);
|
|
477
507
|
const waitMs = deadline ? Math.max(0, deadline - Date.now()) : retryable.waitMs;
|
|
478
|
-
|
|
508
|
+
retryPeriod?.beginRetry(retryableErrorMessage);
|
|
509
|
+
const wait = await managedWait(retryable.reason, waitMs, retryableErrorMessage);
|
|
510
|
+
if (wait === "aborted" || signal?.aborted) {
|
|
479
511
|
pushAbort(`Request aborted during ${reasonLabel(retryable.reason).toLowerCase()} wait.`);
|
|
480
512
|
return;
|
|
481
513
|
}
|
|
514
|
+
retryPeriod?.completeRetry();
|
|
482
515
|
attempt = nextAvailableCandidate(attempt.model)
|
|
483
516
|
?? configuredAttempt(getPrimaryModel(attempt.model));
|
|
484
517
|
continue;
|
|
485
518
|
}
|
|
486
519
|
|
|
487
520
|
notifyRetryableError(attempt.model, retryable, retryableErrorMessage);
|
|
488
|
-
|
|
521
|
+
retryPeriod?.beginRetry(retryableErrorMessage);
|
|
522
|
+
const wait = await managedWait(retryable.reason, retryable.waitMs, retryableErrorMessage);
|
|
523
|
+
if (wait === "aborted" || signal?.aborted) {
|
|
489
524
|
pushAbort(`Request aborted during ${reasonLabel(retryable.reason).toLowerCase()} wait.`);
|
|
490
525
|
return;
|
|
491
526
|
}
|
|
527
|
+
retryPeriod?.completeRetry();
|
|
492
528
|
}
|
|
493
529
|
})().catch((error) => {
|
|
530
|
+
retryPeriod?.recordReason(errorMessageWithCauses(error));
|
|
531
|
+
retryPeriod?.finalize();
|
|
494
532
|
const failure = freshMessage(model);
|
|
495
533
|
failure.stopReason = signal?.aborted ? "aborted" : "error";
|
|
496
534
|
failure.errorMessage = errorMessageWithCauses(error);
|
|
@@ -526,23 +564,30 @@ interface InterceptionSlot {
|
|
|
526
564
|
type InterceptablePrototype = ModelRuntime & { [INTERCEPTION_SYMBOL]?: InterceptionSlot };
|
|
527
565
|
|
|
528
566
|
/** Install one stable, reload-safe process-wide trampoline. */
|
|
529
|
-
export function installModelRuntimeInterception(
|
|
567
|
+
export function installModelRuntimeInterception(
|
|
568
|
+
createRetryPeriod?: () => RetryPeriodTracker,
|
|
569
|
+
): () => boolean {
|
|
530
570
|
const prototype = ModelRuntime.prototype as InterceptablePrototype;
|
|
531
571
|
const owner = {};
|
|
532
572
|
const controller = new AbortController();
|
|
533
573
|
let slot = prototype[INTERCEPTION_SYMBOL];
|
|
534
574
|
|
|
535
|
-
const handler: InterceptionHandler = (runtime, model, context, options, delegate) =>
|
|
536
|
-
|
|
575
|
+
const handler: InterceptionHandler = (runtime, model, context, options, delegate) => {
|
|
576
|
+
let retryPeriod: RetryPeriodTracker | undefined;
|
|
577
|
+
try { retryPeriod = createRetryPeriod?.(); } catch { /* summary tracking is observational */ }
|
|
578
|
+
return streamWithLimitsRetry(runtime, delegate, model, context, options, controller.signal, retryPeriod);
|
|
579
|
+
};
|
|
537
580
|
|
|
538
581
|
if (slot && prototype.streamSimple === slot.wrapper) {
|
|
539
|
-
slot.abortOwner
|
|
582
|
+
const previousAbort = slot.abortOwner;
|
|
540
583
|
slot.owner = owner;
|
|
541
584
|
slot.abortOwner = () => controller.abort();
|
|
542
585
|
slot.handler = handler;
|
|
586
|
+
// Publish the new owner before abort telemetry can re-enter interception.
|
|
587
|
+
previousAbort?.();
|
|
543
588
|
} else {
|
|
589
|
+
const previousAbort = slot?.abortOwner;
|
|
544
590
|
if (slot) {
|
|
545
|
-
slot.abortOwner?.();
|
|
546
591
|
slot.owner = undefined;
|
|
547
592
|
slot.abortOwner = undefined;
|
|
548
593
|
slot.handler = undefined;
|
|
@@ -565,14 +610,17 @@ export function installModelRuntimeInterception(): () => boolean {
|
|
|
565
610
|
value: slot,
|
|
566
611
|
});
|
|
567
612
|
prototype.streamSimple = slot.wrapper;
|
|
613
|
+
previousAbort?.();
|
|
568
614
|
}
|
|
569
615
|
|
|
570
616
|
return () => {
|
|
571
617
|
if (slot?.owner !== owner) return false;
|
|
572
|
-
slot.abortOwner
|
|
618
|
+
const abortOwner = slot.abortOwner;
|
|
573
619
|
slot.owner = undefined;
|
|
574
620
|
slot.abortOwner = undefined;
|
|
575
621
|
slot.handler = undefined;
|
|
576
|
-
|
|
622
|
+
abortOwner?.();
|
|
623
|
+
// A re-entrant install during abort owns the shared slot now.
|
|
624
|
+
return slot.owner === undefined;
|
|
577
625
|
};
|
|
578
626
|
}
|
package/src/telemetry.ts
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import type { EventBus } from "@earendil-works/pi-coding-agent";
|
|
2
|
+
import type { Api, Model } from "@earendil-works/pi-ai";
|
|
3
|
+
import type { RetryReason } from "./types.js";
|
|
4
|
+
|
|
5
|
+
/** Public, versioned pi.events channel for limits-wait wait telemetry. */
|
|
6
|
+
export const LIMITS_WAIT_EVENT_CHANNEL = "oira666.pi-limits-wait.wait.v1";
|
|
7
|
+
|
|
8
|
+
export type LimitsWaitModel = {
|
|
9
|
+
provider: string;
|
|
10
|
+
id: string;
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
export type LimitsWaitWaitOutcome = "waited" | "skipped" | "aborted";
|
|
14
|
+
|
|
15
|
+
export type LimitsWaitWaitStartPayload = {
|
|
16
|
+
version: 1;
|
|
17
|
+
phase: "start";
|
|
18
|
+
waitId: string;
|
|
19
|
+
periodId?: string;
|
|
20
|
+
reason: RetryReason;
|
|
21
|
+
plannedDurationMs: number;
|
|
22
|
+
plannedDeadline: number;
|
|
23
|
+
startedAt: number;
|
|
24
|
+
model?: LimitsWaitModel;
|
|
25
|
+
error?: string;
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
export type LimitsWaitWaitEndPayload = {
|
|
29
|
+
version: 1;
|
|
30
|
+
phase: "end";
|
|
31
|
+
waitId: string;
|
|
32
|
+
periodId?: string;
|
|
33
|
+
reason: RetryReason;
|
|
34
|
+
plannedDurationMs: number;
|
|
35
|
+
plannedDeadline: number;
|
|
36
|
+
startedAt: number;
|
|
37
|
+
endedAt: number;
|
|
38
|
+
actualElapsedMs: number;
|
|
39
|
+
outcome: LimitsWaitWaitOutcome;
|
|
40
|
+
model?: LimitsWaitModel;
|
|
41
|
+
error?: string;
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
export type LimitsWaitWaitEventPayload = LimitsWaitWaitStartPayload | LimitsWaitWaitEndPayload;
|
|
45
|
+
|
|
46
|
+
// Concise public aliases; the longer names remain descriptive and compatible.
|
|
47
|
+
export type LimitsWaitStartEventPayload = LimitsWaitWaitStartPayload;
|
|
48
|
+
export type LimitsWaitEndEventPayload = LimitsWaitWaitEndPayload;
|
|
49
|
+
export type LimitsWaitEventPayload = LimitsWaitWaitEventPayload;
|
|
50
|
+
export type LimitsWaitOutcome = LimitsWaitWaitOutcome;
|
|
51
|
+
|
|
52
|
+
export type WaitTelemetryOptions = {
|
|
53
|
+
periodId?: string;
|
|
54
|
+
model?: Pick<Model<Api>, "provider" | "id">;
|
|
55
|
+
error?: string;
|
|
56
|
+
events?: EventBus;
|
|
57
|
+
onEnd?: (actualElapsedMs: number, outcome: LimitsWaitWaitOutcome) => void;
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
let nextWaitId = 0;
|
|
61
|
+
|
|
62
|
+
export function createWaitId(): string {
|
|
63
|
+
nextWaitId = (nextWaitId + 1) % Number.MAX_SAFE_INTEGER;
|
|
64
|
+
return `wait-${Date.now().toString(36)}-${nextWaitId.toString(36)}`;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export function waitModel(model: WaitTelemetryOptions["model"]): LimitsWaitModel | undefined {
|
|
68
|
+
return model ? { provider: model.provider, id: model.id } : undefined;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Event listeners are observational and must never alter retry control flow. */
|
|
72
|
+
export function emitWaitTelemetry(events: EventBus | undefined, payload: LimitsWaitWaitEventPayload): void {
|
|
73
|
+
try {
|
|
74
|
+
events?.emit(LIMITS_WAIT_EVENT_CHANNEL, payload);
|
|
75
|
+
} catch {
|
|
76
|
+
// Custom/test event buses may throw even though Pi's standard bus isolates listeners.
|
|
77
|
+
}
|
|
78
|
+
}
|