@north-light/crouter 0.3.273 → 0.3.275
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.d.ts +11 -0
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +35 -31
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +35 -27
- package/dist/clients/attach/viewer.js +601 -601
- package/dist/core/__tests__/fault-classifier.test.js +47 -0
- package/dist/core/canvas/browse/render.js +82 -67
- package/dist/core/canvas/status-glyph.d.ts +4 -3
- package/dist/core/canvas/status-glyph.js +17 -9
- package/dist/core/fault-classifier.d.ts +0 -15
- package/dist/core/fault-classifier.js +25 -90
- package/dist/core/provider-error-recognition.d.ts +27 -0
- package/dist/core/provider-error-recognition.js +74 -0
- package/dist/core/runtime/surface-bg.d.ts +6 -4
- package/dist/core/runtime/surface-bg.js +39 -22
- package/dist/core/tui/draw.d.ts +2 -0
- package/dist/core/tui/draw.js +15 -0
- package/package.json +1 -1
- package/runtime.lock.json +2 -2
|
@@ -6,6 +6,17 @@ export interface ProviderRotationOptions {
|
|
|
6
6
|
nodeCwd?: string;
|
|
7
7
|
profileId?: string;
|
|
8
8
|
}
|
|
9
|
+
type ProviderErrorClassification = {
|
|
10
|
+
kind: "rate_limit" | "transient" | "fatal";
|
|
11
|
+
status?: number;
|
|
12
|
+
reason: string;
|
|
13
|
+
hadRetryAfterHeader: boolean;
|
|
14
|
+
};
|
|
15
|
+
/** Test seam for the policy mapping over the shared recognition signal. */
|
|
16
|
+
export declare function __classifyProviderErrorForTest(error: unknown, observed?: {
|
|
17
|
+
status?: number;
|
|
18
|
+
retryAfterMs?: number;
|
|
19
|
+
}): ProviderErrorClassification;
|
|
9
20
|
declare function defaultSleep(ms: number): Promise<void>;
|
|
10
21
|
/** Test-only seam: override the sleep implementation used for in-place/wait-out retries. Pass undefined to restore the default. */
|
|
11
22
|
export declare function __setSleepForTest(fn: typeof defaultSleep | undefined): void;
|
|
@@ -6,6 +6,7 @@ import { readMergedLaunchConfig } from '../../../core/config.js';
|
|
|
6
6
|
import { expandModelCandidates, modelRequestFromConfig, probeRouteAvailability } from '../../../core/model-routes.js';
|
|
7
7
|
import { envModelExact } from '../../../shared/env.js';
|
|
8
8
|
import { ManagedProviderCoolingError } from '../../../core/runtime/managed-provider-cooling.js';
|
|
9
|
+
import { recognizeProviderError } from '../../../core/provider-error-recognition.js';
|
|
9
10
|
// A response without Retry-After proves only that THIS attempt was throttled; it
|
|
10
11
|
// does not authorize crouter to invent a five-minute provider-wide outage. Keep
|
|
11
12
|
// a short turn-local probe backoff to avoid an immediate retry loop, then let a
|
|
@@ -35,39 +36,42 @@ function configuredValue(state, key, envName) {
|
|
|
35
36
|
function isExactModel(state) {
|
|
36
37
|
return state.options === undefined && envModelExact();
|
|
37
38
|
}
|
|
38
|
-
|
|
39
|
-
const
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
39
|
+
function classifyProviderSignal(signal) {
|
|
40
|
+
const hadRetryAfterHeader = signal.retryAfterMs !== undefined;
|
|
41
|
+
if (signal.kind === 'rate-limit' && signal.matchedOn === 'status') {
|
|
42
|
+
return { kind: 'rate_limit', status: signal.status, reason: 'http 429', hadRetryAfterHeader };
|
|
43
|
+
}
|
|
44
|
+
if (signal.kind === 'transient-http') {
|
|
45
|
+
if (signal.status === 503 && hadRetryAfterHeader) {
|
|
46
|
+
return { kind: 'rate_limit', status: signal.status, reason: 'http 503 with retry-after', hadRetryAfterHeader };
|
|
47
|
+
}
|
|
48
|
+
return { kind: 'transient', status: signal.status, reason: `http ${signal.status}`, hadRetryAfterHeader };
|
|
49
|
+
}
|
|
50
|
+
const textMatches = signal.textMatches ?? (signal.matchedOn === 'text' && signal.match ? [{ kind: signal.kind, match: signal.match }] : []);
|
|
51
|
+
const transientMatch = textMatches.find(({ kind }) => kind === 'connection' || kind === 'socket' || kind === 'service-unavailable' || kind === 'overloaded');
|
|
52
|
+
if (transientMatch) {
|
|
53
|
+
return { kind: 'transient', status: signal.status, reason: transientMatch.match.toLowerCase(), hadRetryAfterHeader };
|
|
54
|
+
}
|
|
55
|
+
const rateLimitMatch = textMatches.find(({ kind }) => kind === 'rate-limit');
|
|
56
|
+
if (rateLimitMatch) {
|
|
57
|
+
return { kind: 'rate_limit', status: signal.status, reason: rateLimitMatch.match.toLowerCase(), hadRetryAfterHeader };
|
|
58
|
+
}
|
|
59
|
+
return { kind: 'fatal', status: signal.status, reason: 'unclassified', hadRetryAfterHeader };
|
|
60
|
+
}
|
|
46
61
|
function classifyProviderError(error, observed) {
|
|
47
62
|
const value = error;
|
|
48
|
-
const
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
.
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
}
|
|
61
|
-
const transientMatch = message.match(TRANSIENT_MESSAGE_PATTERN);
|
|
62
|
-
if (transientMatch)
|
|
63
|
-
return { kind: "transient", status, reason: transientMatch[0].toLowerCase(), hadRetryAfterHeader };
|
|
64
|
-
const rateLimitMatch = message.match(RATE_LIMIT_MESSAGE_PATTERN);
|
|
65
|
-
if (rateLimitMatch)
|
|
66
|
-
return { kind: "rate_limit", status, reason: rateLimitMatch[0].toLowerCase(), hadRetryAfterHeader };
|
|
67
|
-
const usageLimitMatch = message.match(USAGE_LIMIT_MESSAGE_PATTERN);
|
|
68
|
-
if (usageLimitMatch)
|
|
69
|
-
return { kind: "rate_limit", status, reason: usageLimitMatch[0].toLowerCase(), hadRetryAfterHeader };
|
|
70
|
-
return { kind: "fatal", status, reason: "unclassified", hadRetryAfterHeader };
|
|
63
|
+
const message = [value?.errorMessage, value?.message, error instanceof Error ? error.message : '']
|
|
64
|
+
.filter((part) => typeof part === 'string' && part.length > 0)
|
|
65
|
+
.join(' ');
|
|
66
|
+
return classifyProviderSignal(recognizeProviderError({
|
|
67
|
+
status: observed.status ?? value?.status,
|
|
68
|
+
retryAfterMs: observed.retryAfterMs,
|
|
69
|
+
message,
|
|
70
|
+
}));
|
|
71
|
+
}
|
|
72
|
+
/** Test seam for the policy mapping over the shared recognition signal. */
|
|
73
|
+
export function __classifyProviderErrorForTest(error, observed = {}) {
|
|
74
|
+
return classifyProviderError(error, observed);
|
|
71
75
|
}
|
|
72
76
|
function providerErrorMessage(error) {
|
|
73
77
|
const value = error;
|
|
@@ -41,6 +41,7 @@ import { readMergedLaunchConfig } from '../../../core/config.js';
|
|
|
41
41
|
import { expandModelCandidates, modelRequestFromConfig, probeRouteAvailability } from '../../../core/model-routes.js';
|
|
42
42
|
import { envModelExact } from '../../../shared/env.js';
|
|
43
43
|
import { ManagedProviderCoolingError } from '../../../core/runtime/managed-provider-cooling.js';
|
|
44
|
+
import { recognizeProviderError, type ProviderErrorSignal } from '../../../core/provider-error-recognition.js';
|
|
44
45
|
|
|
45
46
|
// A response without Retry-After proves only that THIS attempt was throttled; it
|
|
46
47
|
// does not authorize crouter to invent a five-minute provider-wide outage. Keep
|
|
@@ -123,37 +124,44 @@ type ProviderErrorClassification = {
|
|
|
123
124
|
hadRetryAfterHeader: boolean;
|
|
124
125
|
};
|
|
125
126
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
127
|
+
function classifyProviderSignal(signal: ProviderErrorSignal): ProviderErrorClassification {
|
|
128
|
+
const hadRetryAfterHeader = signal.retryAfterMs !== undefined;
|
|
129
|
+
if (signal.kind === 'rate-limit' && signal.matchedOn === 'status') {
|
|
130
|
+
return { kind: 'rate_limit', status: signal.status, reason: 'http 429', hadRetryAfterHeader };
|
|
131
|
+
}
|
|
132
|
+
if (signal.kind === 'transient-http') {
|
|
133
|
+
if (signal.status === 503 && hadRetryAfterHeader) {
|
|
134
|
+
return { kind: 'rate_limit', status: signal.status, reason: 'http 503 with retry-after', hadRetryAfterHeader };
|
|
135
|
+
}
|
|
136
|
+
return { kind: 'transient', status: signal.status, reason: `http ${signal.status}`, hadRetryAfterHeader };
|
|
137
|
+
}
|
|
138
|
+
const textMatches = signal.textMatches ?? (signal.matchedOn === 'text' && signal.match ? [{ kind: signal.kind, match: signal.match }] : []);
|
|
139
|
+
const transientMatch = textMatches.find(({ kind }) => kind === 'connection' || kind === 'socket' || kind === 'service-unavailable' || kind === 'overloaded');
|
|
140
|
+
if (transientMatch) {
|
|
141
|
+
return { kind: 'transient', status: signal.status, reason: transientMatch.match.toLowerCase(), hadRetryAfterHeader };
|
|
142
|
+
}
|
|
143
|
+
const rateLimitMatch = textMatches.find(({ kind }) => kind === 'rate-limit');
|
|
144
|
+
if (rateLimitMatch) {
|
|
145
|
+
return { kind: 'rate_limit', status: signal.status, reason: rateLimitMatch.match.toLowerCase(), hadRetryAfterHeader };
|
|
146
|
+
}
|
|
147
|
+
return { kind: 'fatal', status: signal.status, reason: 'unclassified', hadRetryAfterHeader };
|
|
148
|
+
}
|
|
135
149
|
|
|
136
150
|
function classifyProviderError(error: unknown, observed: { status?: number; retryAfterMs?: number }): ProviderErrorClassification {
|
|
137
151
|
const value = error as { status?: unknown; errorMessage?: unknown; message?: unknown } | undefined;
|
|
138
|
-
const
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
.
|
|
143
|
-
|
|
152
|
+
const message = [value?.errorMessage, value?.message, error instanceof Error ? error.message : '']
|
|
153
|
+
.filter((part): part is string => typeof part === 'string' && part.length > 0)
|
|
154
|
+
.join(' ');
|
|
155
|
+
return classifyProviderSignal(recognizeProviderError({
|
|
156
|
+
status: observed.status ?? value?.status,
|
|
157
|
+
retryAfterMs: observed.retryAfterMs,
|
|
158
|
+
message,
|
|
159
|
+
}));
|
|
160
|
+
}
|
|
144
161
|
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
return { kind: "transient", status, reason: `http ${status}`, hadRetryAfterHeader };
|
|
149
|
-
}
|
|
150
|
-
const transientMatch = message.match(TRANSIENT_MESSAGE_PATTERN);
|
|
151
|
-
if (transientMatch) return { kind: "transient", status, reason: transientMatch[0].toLowerCase(), hadRetryAfterHeader };
|
|
152
|
-
const rateLimitMatch = message.match(RATE_LIMIT_MESSAGE_PATTERN);
|
|
153
|
-
if (rateLimitMatch) return { kind: "rate_limit", status, reason: rateLimitMatch[0].toLowerCase(), hadRetryAfterHeader };
|
|
154
|
-
const usageLimitMatch = message.match(USAGE_LIMIT_MESSAGE_PATTERN);
|
|
155
|
-
if (usageLimitMatch) return { kind: "rate_limit", status, reason: usageLimitMatch[0].toLowerCase(), hadRetryAfterHeader };
|
|
156
|
-
return { kind: "fatal", status, reason: "unclassified", hadRetryAfterHeader };
|
|
162
|
+
/** Test seam for the policy mapping over the shared recognition signal. */
|
|
163
|
+
export function __classifyProviderErrorForTest(error: unknown, observed: { status?: number; retryAfterMs?: number } = {}): ProviderErrorClassification {
|
|
164
|
+
return classifyProviderError(error, observed);
|
|
157
165
|
}
|
|
158
166
|
|
|
159
167
|
function providerErrorMessage(error: unknown): string | undefined {
|