@centerforagenticai/pi-multi-account 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -3
- package/package.json +5 -5
- package/packages/pi-anthropic-oauth/package.json +2 -2
- package/packages/pi-anthropic-oauth/src/stream.ts +18 -8
- package/packages/pi-anthropic-oauth/src/transport-activity.ts +59 -0
- package/packages/pi-antigravity/package.json +2 -2
- package/packages/pi-antigravity/src/models/discovery.ts +2 -1
- package/packages/pi-antigravity/src/models/grouping.ts +12 -10
- package/packages/pi-antigravity/src/models/models.ts +11 -4
- package/src/account-group-failure.ts +149 -0
- package/src/account-group-members.ts +139 -0
- package/src/anthropic-adaptive-stream.ts +18 -9
- package/src/anthropic-alias-stream.ts +13 -68
- package/src/codex-adapter.ts +5 -51
- package/src/commands.ts +3 -3
- package/src/config.ts +46 -2
- package/src/diagnostic-store.ts +17 -15
- package/src/diagnostics.ts +97 -35
- package/src/host-final-stop-message.ts +18 -128
- package/src/index.ts +436 -88
- package/src/logical-dispatch.ts +12 -13
- package/src/logical-provider.ts +1208 -407
- package/src/models-declaration.ts +15 -8
- package/src/public-assistant-projection.ts +163 -0
- package/src/recovery-engine.ts +311 -59
- package/src/routing.ts +4 -8
- package/src/runtime-state.ts +7 -0
- package/src/upstream-anthropic.ts +12 -4
- package/src/upstream-antigravity.ts +2 -38
|
@@ -35,6 +35,7 @@ import {
|
|
|
35
35
|
type Api,
|
|
36
36
|
type AssistantMessage,
|
|
37
37
|
type AssistantMessageEventStream,
|
|
38
|
+
type JsonObject,
|
|
38
39
|
calculateCost,
|
|
39
40
|
type Context,
|
|
40
41
|
createAssistantMessageEventStream,
|
|
@@ -53,7 +54,7 @@ type IndexedBlock =
|
|
|
53
54
|
type: "toolCall";
|
|
54
55
|
id: string;
|
|
55
56
|
name: string;
|
|
56
|
-
arguments:
|
|
57
|
+
arguments: JsonObject;
|
|
57
58
|
partialJson: string;
|
|
58
59
|
} & { index: number });
|
|
59
60
|
type UpstreamHelpers = {
|
|
@@ -67,15 +68,20 @@ type UpstreamHelpers = {
|
|
|
67
68
|
convertPiToolsToAnthropic: (tools: NonNullable<Context["tools"]>, isOAuth: boolean) => unknown;
|
|
68
69
|
fromClaudeCodeToolName: (name: string, tools?: NonNullable<Context["tools"]>) => string;
|
|
69
70
|
buildAnthropicSystemPrompt: (systemPrompt: string | undefined, isOAuth: boolean) => unknown;
|
|
71
|
+
transportActivityListener: (options: unknown) => (() => void) | undefined;
|
|
72
|
+
createTransportActivityFetch: (onActivity: () => void) => typeof fetch;
|
|
70
73
|
};
|
|
71
74
|
|
|
72
75
|
const upstreamAuthPath = "../packages/pi-anthropic-oauth/src/auth.ts";
|
|
73
76
|
const upstreamConvertPath = "../packages/pi-anthropic-oauth/src/convert.ts";
|
|
74
77
|
const upstreamPromptPath = "../packages/pi-anthropic-oauth/src/prompt.ts";
|
|
78
|
+
const upstreamTransportActivityPath =
|
|
79
|
+
"../packages/pi-anthropic-oauth/src/transport-activity.ts";
|
|
75
80
|
const upstreamHelpers = {
|
|
76
81
|
...(await import(upstreamAuthPath)),
|
|
77
82
|
...(await import(upstreamConvertPath)),
|
|
78
83
|
...(await import(upstreamPromptPath)),
|
|
84
|
+
...(await import(upstreamTransportActivityPath)),
|
|
79
85
|
} as unknown as UpstreamHelpers;
|
|
80
86
|
const {
|
|
81
87
|
isClaudeOAuthAccessToken,
|
|
@@ -84,6 +90,8 @@ const {
|
|
|
84
90
|
convertPiToolsToAnthropic,
|
|
85
91
|
fromClaudeCodeToolName,
|
|
86
92
|
buildAnthropicSystemPrompt,
|
|
93
|
+
transportActivityListener,
|
|
94
|
+
createTransportActivityFetch,
|
|
87
95
|
} = upstreamHelpers;
|
|
88
96
|
|
|
89
97
|
const REQUIRED_BETAS = [
|
|
@@ -230,12 +238,19 @@ export function streamAnthropicAdaptive(
|
|
|
230
238
|
|
|
231
239
|
if (isOAuth) defaultHeaders.authorization = `Bearer ${apiKey}`;
|
|
232
240
|
|
|
241
|
+
// A caller that supplies onTransportActivity hears about every response
|
|
242
|
+
// body chunk, including the pings the SDK drops; otherwise the SDK keeps
|
|
243
|
+
// its default fetch.
|
|
244
|
+
const onTransportActivity = transportActivityListener(options);
|
|
233
245
|
const client = new Anthropic({
|
|
234
246
|
baseURL: model.baseUrl,
|
|
235
247
|
apiKey: isOAuth ? null : apiKey,
|
|
236
248
|
authToken: isOAuth ? apiKey : null,
|
|
237
249
|
defaultHeaders,
|
|
238
250
|
dangerouslyAllowBrowser: true,
|
|
251
|
+
...(onTransportActivity === undefined
|
|
252
|
+
? {}
|
|
253
|
+
: { fetch: createTransportActivityFetch(onTransportActivity) }),
|
|
239
254
|
});
|
|
240
255
|
|
|
241
256
|
const maxTokens =
|
|
@@ -468,10 +483,7 @@ export function streamAnthropicAdaptive(
|
|
|
468
483
|
) {
|
|
469
484
|
block.partialJson += event.delta.partial_json;
|
|
470
485
|
try {
|
|
471
|
-
block.arguments = JSON.parse(block.partialJson) as
|
|
472
|
-
string,
|
|
473
|
-
unknown
|
|
474
|
-
>;
|
|
486
|
+
block.arguments = JSON.parse(block.partialJson) as JsonObject;
|
|
475
487
|
} catch {}
|
|
476
488
|
stream.push({
|
|
477
489
|
type: "toolcall_delta",
|
|
@@ -507,10 +519,7 @@ export function streamAnthropicAdaptive(
|
|
|
507
519
|
});
|
|
508
520
|
} else if (block.type === "toolCall") {
|
|
509
521
|
try {
|
|
510
|
-
block.arguments = JSON.parse(block.partialJson) as
|
|
511
|
-
string,
|
|
512
|
-
unknown
|
|
513
|
-
>;
|
|
522
|
+
block.arguments = JSON.parse(block.partialJson) as JsonObject;
|
|
514
523
|
} catch {}
|
|
515
524
|
delete (block as { partialJson?: string }).partialJson;
|
|
516
525
|
stream.push({
|
|
@@ -1,23 +1,26 @@
|
|
|
1
1
|
import {
|
|
2
2
|
createAssistantMessageEventStream,
|
|
3
3
|
type Api,
|
|
4
|
-
type AssistantMessage,
|
|
5
|
-
type AssistantMessageEvent,
|
|
6
4
|
type AssistantMessageEventStream,
|
|
7
|
-
type Context,
|
|
8
5
|
type Model,
|
|
6
|
+
type Context,
|
|
7
|
+
type TranscriptContext,
|
|
9
8
|
type ProviderResponse,
|
|
10
9
|
type SimpleStreamOptions,
|
|
11
10
|
} from "@earendil-works/pi-ai";
|
|
12
|
-
import {
|
|
13
|
-
|
|
14
|
-
sanitizeHeaderValue,
|
|
15
|
-
} from "./diagnostics.js";
|
|
16
|
-
import { hostFinalStopMessage } from "./host-final-stop-message.js";
|
|
11
|
+
import { projectResponseHeaders } from "./diagnostics.js";
|
|
12
|
+
import { projectAliasAssistantEvent } from "./public-assistant-projection.js";
|
|
17
13
|
|
|
18
14
|
export const ANTHROPIC_ALIAS_API = "hypha-anthropic-oauth" as const;
|
|
19
15
|
|
|
20
16
|
export type AnthropicUpstreamStream = (
|
|
17
|
+
model: Model<Api>,
|
|
18
|
+
context: Context | TranscriptContext,
|
|
19
|
+
options?: SimpleStreamOptions,
|
|
20
|
+
) => AssistantMessageEventStream;
|
|
21
|
+
|
|
22
|
+
/** Legacy Pi 0.84 context-shaped stream consumed by the exact-provenance adaptive adapter. */
|
|
23
|
+
export type AnthropicLegacyStream = (
|
|
21
24
|
model: Model<Api>,
|
|
22
25
|
context: Context,
|
|
23
26
|
options?: SimpleStreamOptions,
|
|
@@ -33,68 +36,10 @@ export function sanitizeAnthropicProviderResponse(
|
|
|
33
36
|
): ProviderResponse {
|
|
34
37
|
return {
|
|
35
38
|
status: response.status,
|
|
36
|
-
headers:
|
|
37
|
-
Object.entries(response.headers).map(([name, value]) => [
|
|
38
|
-
sanitizeDiagnosticText(name),
|
|
39
|
-
sanitizeHeaderValue(name, value),
|
|
40
|
-
]),
|
|
41
|
-
),
|
|
42
|
-
};
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
function withAliasAttribution(
|
|
46
|
-
message: AssistantMessage,
|
|
47
|
-
aliasModel: Model<Api>,
|
|
48
|
-
): AssistantMessage {
|
|
49
|
-
return {
|
|
50
|
-
...message,
|
|
51
|
-
api: aliasModel.api,
|
|
52
|
-
provider: aliasModel.provider,
|
|
53
|
-
model: aliasModel.id,
|
|
39
|
+
headers: projectResponseHeaders(response.headers),
|
|
54
40
|
};
|
|
55
41
|
}
|
|
56
42
|
|
|
57
|
-
/**
|
|
58
|
-
* Bounds an upstream error and, for a structured refusal or unknown stop,
|
|
59
|
-
* publishes the shared host-final message. A direct alias turn reaches the
|
|
60
|
-
* host's retry and compaction predicates without the unified provider, so
|
|
61
|
-
* provider-authored stop wording must not make the host resend the request.
|
|
62
|
-
*/
|
|
63
|
-
function sanitizeUpstreamError(message: AssistantMessage): AssistantMessage {
|
|
64
|
-
if (message.errorMessage === undefined) return message;
|
|
65
|
-
const sanitized: AssistantMessage = {
|
|
66
|
-
...message,
|
|
67
|
-
errorMessage: sanitizeDiagnosticText(message.errorMessage),
|
|
68
|
-
};
|
|
69
|
-
return { ...sanitized, ...hostFinalStopMessage(sanitized) };
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
function withAliasEvent(
|
|
73
|
-
event: AssistantMessageEvent,
|
|
74
|
-
aliasModel: Model<Api>,
|
|
75
|
-
): AssistantMessageEvent {
|
|
76
|
-
switch (event.type) {
|
|
77
|
-
case "done":
|
|
78
|
-
return {
|
|
79
|
-
...event,
|
|
80
|
-
message: withAliasAttribution(event.message, aliasModel),
|
|
81
|
-
};
|
|
82
|
-
case "error":
|
|
83
|
-
return {
|
|
84
|
-
...event,
|
|
85
|
-
error: withAliasAttribution(
|
|
86
|
-
sanitizeUpstreamError(event.error),
|
|
87
|
-
aliasModel,
|
|
88
|
-
),
|
|
89
|
-
};
|
|
90
|
-
default:
|
|
91
|
-
return {
|
|
92
|
-
...event,
|
|
93
|
-
partial: withAliasAttribution(event.partial, aliasModel),
|
|
94
|
-
};
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
|
|
98
43
|
function reattributeStream(
|
|
99
44
|
upstream: AssistantMessageEventStream,
|
|
100
45
|
aliasModel: Model<Api>,
|
|
@@ -103,7 +48,7 @@ function reattributeStream(
|
|
|
103
48
|
|
|
104
49
|
void (async () => {
|
|
105
50
|
for await (const event of upstream) {
|
|
106
|
-
attributed.push(
|
|
51
|
+
attributed.push(projectAliasAssistantEvent(event, aliasModel));
|
|
107
52
|
}
|
|
108
53
|
})();
|
|
109
54
|
|
package/src/codex-adapter.ts
CHANGED
|
@@ -28,9 +28,11 @@ import {
|
|
|
28
28
|
import { cloneProviderModelCatalog } from "./catalog-rebinding.js";
|
|
29
29
|
import {
|
|
30
30
|
sanitizeDiagnosticText,
|
|
31
|
-
|
|
31
|
+
projectResponseHeaders,
|
|
32
32
|
} from "./diagnostics.js";
|
|
33
33
|
|
|
34
|
+
import { projectAliasAssistantEvent } from "./public-assistant-projection.js";
|
|
35
|
+
|
|
34
36
|
const CODEX_BASE_API = "openai-codex-responses" as const;
|
|
35
37
|
/**
|
|
36
38
|
* Keep alias models on Pi's host-known API id. The provider-scoped
|
|
@@ -141,56 +143,8 @@ type CodexUpstreamStream = (
|
|
|
141
143
|
options?: SimpleStreamOptions,
|
|
142
144
|
) => AssistantMessageEventStream;
|
|
143
145
|
|
|
144
|
-
function withAliasAttribution(
|
|
145
|
-
message: AssistantMessage,
|
|
146
|
-
aliasModel: Model<Api>,
|
|
147
|
-
): AssistantMessage {
|
|
148
|
-
return {
|
|
149
|
-
...message,
|
|
150
|
-
api: aliasModel.api,
|
|
151
|
-
provider: aliasModel.provider,
|
|
152
|
-
model: aliasModel.id,
|
|
153
|
-
};
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
function sanitizeUpstreamError(message: AssistantMessage): AssistantMessage {
|
|
157
|
-
if (message.errorMessage === undefined) return message;
|
|
158
|
-
return {
|
|
159
|
-
...message,
|
|
160
|
-
errorMessage: sanitizeDiagnosticText(message.errorMessage),
|
|
161
|
-
};
|
|
162
|
-
}
|
|
163
|
-
|
|
164
146
|
function sanitizeProviderResponse(response: ProviderResponse): ProviderResponse {
|
|
165
|
-
return {
|
|
166
|
-
status: response.status,
|
|
167
|
-
headers: Object.fromEntries(
|
|
168
|
-
Object.entries(response.headers).map(([name, value]) => [
|
|
169
|
-
sanitizeDiagnosticText(name),
|
|
170
|
-
sanitizeHeaderValue(name, value),
|
|
171
|
-
]),
|
|
172
|
-
),
|
|
173
|
-
};
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
function withAliasEvent(
|
|
177
|
-
event: AssistantMessageEvent,
|
|
178
|
-
aliasModel: Model<Api>,
|
|
179
|
-
): AssistantMessageEvent {
|
|
180
|
-
switch (event.type) {
|
|
181
|
-
case "done":
|
|
182
|
-
return { ...event, message: withAliasAttribution(event.message, aliasModel) };
|
|
183
|
-
case "error":
|
|
184
|
-
return {
|
|
185
|
-
...event,
|
|
186
|
-
error: withAliasAttribution(
|
|
187
|
-
sanitizeUpstreamError(event.error),
|
|
188
|
-
aliasModel,
|
|
189
|
-
),
|
|
190
|
-
};
|
|
191
|
-
default:
|
|
192
|
-
return { ...event, partial: withAliasAttribution(event.partial, aliasModel) };
|
|
193
|
-
}
|
|
147
|
+
return { status: response.status, headers: projectResponseHeaders(response.headers) };
|
|
194
148
|
}
|
|
195
149
|
|
|
196
150
|
/**
|
|
@@ -254,7 +208,7 @@ function reattributeStream(
|
|
|
254
208
|
for await (const event of await upstream) {
|
|
255
209
|
if (sawTerminal) continue;
|
|
256
210
|
if (isTerminalEvent(event)) sawTerminal = true;
|
|
257
|
-
attributed.push(
|
|
211
|
+
attributed.push(projectAliasAssistantEvent(event, aliasModel));
|
|
258
212
|
}
|
|
259
213
|
} catch (error) {
|
|
260
214
|
if (!sawTerminal) {
|
package/src/commands.ts
CHANGED
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
import { basename, dirname, join } from "node:path";
|
|
15
15
|
import { randomUUID } from "node:crypto";
|
|
16
16
|
import type { Api, Model } from "@earendil-works/pi-ai";
|
|
17
|
+
import type { AccountGroupMemberAvailability } from "./account-group-members.js";
|
|
17
18
|
import { acquireMachineLease } from "./machine-lease.js";
|
|
18
19
|
import {
|
|
19
20
|
DECLARATION_BASE_URL,
|
|
@@ -290,9 +291,8 @@ export function declarationNoticeMessage(notice: DeclarationNotice): string {
|
|
|
290
291
|
: `LOGICAL ROUTING OFF: The managed model declaration is unreadable. Run ${notice.remedy}.`;
|
|
291
292
|
}
|
|
292
293
|
|
|
293
|
-
export interface AccountGroupCommandMemberStatus {
|
|
294
|
-
|
|
295
|
-
readonly eligible: boolean;
|
|
294
|
+
export interface AccountGroupCommandMemberStatus extends Pick<AccountGroupMemberAvailability, "providerId" | "eligible"> {
|
|
295
|
+
/** Membership availability or an additional managed-routing/metered block. */
|
|
296
296
|
readonly reason: string;
|
|
297
297
|
}
|
|
298
298
|
|
package/src/config.ts
CHANGED
|
@@ -30,6 +30,7 @@ import {
|
|
|
30
30
|
import { randomUUID } from "node:crypto";
|
|
31
31
|
import { basename, dirname, isAbsolute, join } from "node:path";
|
|
32
32
|
import { sanitizeDiagnosticText } from "./diagnostics.js";
|
|
33
|
+
import { isAccountGroupMemberReference } from "./account-group-members.js";
|
|
33
34
|
import { PROJECT_KEY_PATTERN } from "./project-identity.js";
|
|
34
35
|
import {
|
|
35
36
|
AccountRateHistoryError,
|
|
@@ -177,6 +178,9 @@ export interface ModelFallbackEgressAuthorization {
|
|
|
177
178
|
/** Exact source unified model id -> ordered exact fallback model ids. */
|
|
178
179
|
export type ModelFallbackMap = Readonly<Record<string, readonly string[]>>;
|
|
179
180
|
|
|
181
|
+
/** Upper bound for `recoveryStallTimeoutMs`: one stalled attempt never waits longer. */
|
|
182
|
+
export const MAX_RECOVERY_STALL_TIMEOUT_MS = 30 * 60_000;
|
|
183
|
+
|
|
180
184
|
export interface MultiAccountConfig {
|
|
181
185
|
readonly accountLimit: number;
|
|
182
186
|
readonly sameFamilyFailover: boolean;
|
|
@@ -188,6 +192,14 @@ export interface MultiAccountConfig {
|
|
|
188
192
|
readonly recoveryIdleTimeoutMs: number;
|
|
189
193
|
/** Total elapsed time allowed for one complete recovery invocation. */
|
|
190
194
|
readonly recoveryAbsoluteTimeoutMs: number;
|
|
195
|
+
/**
|
|
196
|
+
* Longest wait for the next event of one unified physical attempt, opening
|
|
197
|
+
* included. A stall before any content ends that attempt as a pre-start
|
|
198
|
+
* transient failure that may recover once on another account; a stall after
|
|
199
|
+
* content ends the call with no retry. Must be less than
|
|
200
|
+
* `recoveryIdleTimeoutMs`, so the stall fires before the invocation idles out.
|
|
201
|
+
*/
|
|
202
|
+
readonly recoveryStallTimeoutMs: number;
|
|
191
203
|
/**
|
|
192
204
|
* Operator-chosen display labels keyed by canonical provider id, so managed
|
|
193
205
|
* accounts are distinguishable in Pi's login list and the status view.
|
|
@@ -289,6 +301,7 @@ export const DEFAULT_CONFIG: MultiAccountConfig = {
|
|
|
289
301
|
cooldownMaxMs: 300_000,
|
|
290
302
|
recoveryIdleTimeoutMs: 5 * 60_000,
|
|
291
303
|
recoveryAbsoluteTimeoutMs: 30 * 60_000,
|
|
304
|
+
recoveryStallTimeoutMs: 3 * 60_000,
|
|
292
305
|
accountLabels: {},
|
|
293
306
|
projectLabels: {},
|
|
294
307
|
accountGroups: {},
|
|
@@ -315,6 +328,7 @@ const CONFIG_KEYS = new Set<keyof MultiAccountConfig>([
|
|
|
315
328
|
"cooldownMaxMs",
|
|
316
329
|
"recoveryIdleTimeoutMs",
|
|
317
330
|
"recoveryAbsoluteTimeoutMs",
|
|
331
|
+
"recoveryStallTimeoutMs",
|
|
318
332
|
"accountLabels",
|
|
319
333
|
"projectLabels",
|
|
320
334
|
"accountGroups",
|
|
@@ -961,10 +975,10 @@ function parseAccountGroups(
|
|
|
961
975
|
const providerIds = members.map((member, index) => {
|
|
962
976
|
if (
|
|
963
977
|
typeof member !== "string" ||
|
|
964
|
-
!
|
|
978
|
+
!isAccountGroupMemberReference(member, accountLimit, MANAGED_FAMILIES)
|
|
965
979
|
) {
|
|
966
980
|
throw new ConfigValidationError(
|
|
967
|
-
`accountGroups.${groupId}[${index}] must be a canonical managed
|
|
981
|
+
`accountGroups.${groupId}[${index}] must be a safe provider reference with canonical managed slots within accountLimit.`,
|
|
968
982
|
);
|
|
969
983
|
}
|
|
970
984
|
return member;
|
|
@@ -1327,6 +1341,35 @@ export function parseConfig(value: unknown): MultiAccountConfig {
|
|
|
1327
1341
|
);
|
|
1328
1342
|
}
|
|
1329
1343
|
}
|
|
1344
|
+
// An omitted stall limit defaults below the effective idle limit, so a
|
|
1345
|
+
// config that sets only a short `recoveryIdleTimeoutMs` stays valid and its
|
|
1346
|
+
// stall can still fire first.
|
|
1347
|
+
const recoveryStallTimeoutMs =
|
|
1348
|
+
value["recoveryStallTimeoutMs"] ??
|
|
1349
|
+
Math.min(
|
|
1350
|
+
DEFAULT_CONFIG.recoveryStallTimeoutMs,
|
|
1351
|
+
(recoveryIdleTimeoutMs as number) - 1,
|
|
1352
|
+
);
|
|
1353
|
+
if (
|
|
1354
|
+
typeof recoveryStallTimeoutMs !== "number" ||
|
|
1355
|
+
!Number.isFinite(recoveryStallTimeoutMs) ||
|
|
1356
|
+
recoveryStallTimeoutMs < 1_000 ||
|
|
1357
|
+
recoveryStallTimeoutMs > MAX_RECOVERY_STALL_TIMEOUT_MS
|
|
1358
|
+
) {
|
|
1359
|
+
throw new ConfigValidationError(
|
|
1360
|
+
value["recoveryStallTimeoutMs"] === undefined
|
|
1361
|
+
? "recoveryIdleTimeoutMs must be more than 1000 ms, so the 1000 ms minimum recoveryStallTimeoutMs can end first."
|
|
1362
|
+
: `recoveryStallTimeoutMs must be a finite number from 1000 through ${MAX_RECOVERY_STALL_TIMEOUT_MS} ms.`,
|
|
1363
|
+
);
|
|
1364
|
+
}
|
|
1365
|
+
// The engine's idle timer covers the whole invocation. A stall limit that is
|
|
1366
|
+
// not shorter would let it abort the call before a stalled attempt could
|
|
1367
|
+
// move to another account.
|
|
1368
|
+
if (recoveryStallTimeoutMs >= (recoveryIdleTimeoutMs as number)) {
|
|
1369
|
+
throw new ConfigValidationError(
|
|
1370
|
+
"recoveryStallTimeoutMs must be less than recoveryIdleTimeoutMs.",
|
|
1371
|
+
);
|
|
1372
|
+
}
|
|
1330
1373
|
if (
|
|
1331
1374
|
typeof preemptiveExpiryWindowMs !== "number" ||
|
|
1332
1375
|
!Number.isFinite(preemptiveExpiryWindowMs) ||
|
|
@@ -1346,6 +1389,7 @@ export function parseConfig(value: unknown): MultiAccountConfig {
|
|
|
1346
1389
|
cooldownMaxMs,
|
|
1347
1390
|
recoveryIdleTimeoutMs: recoveryIdleTimeoutMs as number,
|
|
1348
1391
|
recoveryAbsoluteTimeoutMs: recoveryAbsoluteTimeoutMs as number,
|
|
1392
|
+
recoveryStallTimeoutMs,
|
|
1349
1393
|
accountLabels,
|
|
1350
1394
|
projectLabels,
|
|
1351
1395
|
accountGroups,
|
package/src/diagnostic-store.ts
CHANGED
|
@@ -12,7 +12,11 @@ import { randomUUID } from "node:crypto";
|
|
|
12
12
|
import { dirname, join } from "node:path";
|
|
13
13
|
import {
|
|
14
14
|
sanitizeDiagnosticText,
|
|
15
|
+
projectResponseHeaders,
|
|
16
|
+
isHeaderDiagnostic,
|
|
17
|
+
HEADER_DIAGNOSTIC_MESSAGE,
|
|
15
18
|
sanitizeForJson,
|
|
19
|
+
retainDiagnosticEvents,
|
|
16
20
|
type DiagnosticEvent,
|
|
17
21
|
type DiagnosticLevel,
|
|
18
22
|
type DiagnosticPersistence,
|
|
@@ -84,13 +88,16 @@ function projectEvent(value: unknown): DiagnosticEvent | undefined {
|
|
|
84
88
|
if (typeof candidate.level !== "string" || !LEVELS.has(candidate.level as DiagnosticLevel)) {
|
|
85
89
|
return undefined;
|
|
86
90
|
}
|
|
87
|
-
const fields =
|
|
91
|
+
const fields = isHeaderDiagnostic(candidate.category, candidate.message)
|
|
92
|
+
? projectResponseHeaders(candidate.fields)
|
|
93
|
+
: projectFields(candidate.fields);
|
|
88
94
|
if (fields === undefined) return undefined;
|
|
89
95
|
return {
|
|
90
96
|
timestampMs: candidate.timestampMs,
|
|
91
97
|
level: candidate.level as DiagnosticLevel,
|
|
92
98
|
category: sanitizeDiagnosticText(candidate.category),
|
|
93
|
-
message:
|
|
99
|
+
message: isHeaderDiagnostic(candidate.category, candidate.message)
|
|
100
|
+
? HEADER_DIAGNOSTIC_MESSAGE : sanitizeDiagnosticText(candidate.message),
|
|
94
101
|
fields,
|
|
95
102
|
};
|
|
96
103
|
}
|
|
@@ -110,20 +117,15 @@ function parseCompleteEvents(raw: string): readonly DiagnosticEvent[] {
|
|
|
110
117
|
return events;
|
|
111
118
|
}
|
|
112
119
|
|
|
113
|
-
function
|
|
120
|
+
function priorityLinesWithin(
|
|
114
121
|
events: readonly DiagnosticEvent[],
|
|
115
122
|
budgetBytes: number,
|
|
116
123
|
): string {
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
if (bytes > budgetBytes - used) break;
|
|
123
|
-
retained.push(line);
|
|
124
|
-
used += bytes;
|
|
125
|
-
}
|
|
126
|
-
return retained.reverse().join("");
|
|
124
|
+
return retainDiagnosticEvents(
|
|
125
|
+
events,
|
|
126
|
+
budgetBytes,
|
|
127
|
+
(event) => Buffer.byteLength(`${JSON.stringify(event)}\n`, "utf8"),
|
|
128
|
+
).map((event) => `${JSON.stringify(event)}\n`).join("");
|
|
127
129
|
}
|
|
128
130
|
|
|
129
131
|
/** Machine-global, credential-free, bounded diagnostic event history. */
|
|
@@ -164,7 +166,7 @@ export class DiagnosticStore implements DiagnosticPersistence {
|
|
|
164
166
|
: 100;
|
|
165
167
|
if (safeLimit === 0) return [];
|
|
166
168
|
try {
|
|
167
|
-
return parseCompleteEvents(readFileSync(this.#path, "utf8"))
|
|
169
|
+
return retainDiagnosticEvents(parseCompleteEvents(readFileSync(this.#path, "utf8")), safeLimit);
|
|
168
170
|
} catch {
|
|
169
171
|
return [];
|
|
170
172
|
}
|
|
@@ -251,7 +253,7 @@ export class DiagnosticStore implements DiagnosticPersistence {
|
|
|
251
253
|
Math.floor(this.#maxBytes / 2),
|
|
252
254
|
this.#maxBytes - tailBytes,
|
|
253
255
|
);
|
|
254
|
-
const compacted =
|
|
256
|
+
const compacted = priorityLinesWithin(events, targetBytes);
|
|
255
257
|
const encoded = compacted + tail;
|
|
256
258
|
if (Buffer.byteLength(encoded, "utf8") > this.#maxBytes) return false;
|
|
257
259
|
temporaryPath = `${this.#path}.${process.pid}.${randomUUID()}.tmp`;
|
package/src/diagnostics.ts
CHANGED
|
@@ -2,17 +2,38 @@ const MAX_DIAGNOSTIC_LENGTH = 512;
|
|
|
2
2
|
const REDACTED = "[REDACTED]";
|
|
3
3
|
const SENSITIVE_NAME_FRAGMENT = /(?:authorization|authentication|auth|token|credential|secret|cookie|apikey)/;
|
|
4
4
|
const PRIVATE_REASONING_KEY = /^(?:thinking|reasoning)/;
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
"
|
|
5
|
+
// Closed response observations: unknown names (including identifiers) never cross
|
|
6
|
+
// the callback or retained-diagnostic boundary, even when their value looks safe.
|
|
7
|
+
const COUNT_HEADERS = new Set([
|
|
8
|
+
"x-ratelimit-remaining", "x-ratelimit-remaining-requests", "x-ratelimit-requests-remaining",
|
|
9
|
+
"x-ratelimit-remaining-tokens", "x-ratelimit-tokens-remaining",
|
|
10
|
+
"anthropic-ratelimit-requests-remaining", "anthropic-ratelimit-tokens-remaining",
|
|
11
|
+
"x-ratelimit-limit-requests", "x-ratelimit-limit-tokens",
|
|
12
|
+
"anthropic-ratelimit-requests-limit", "anthropic-ratelimit-tokens-limit",
|
|
13
|
+
"retry-after-ms",
|
|
9
14
|
]);
|
|
10
|
-
const
|
|
11
|
-
|
|
15
|
+
const DURATION_RESET_HEADERS = new Set(["x-ratelimit-reset-requests", "x-ratelimit-reset-tokens"]);
|
|
16
|
+
const ABSOLUTE_RESET_HEADERS = new Set([
|
|
17
|
+
"x-ratelimit-reset", "anthropic-ratelimit-requests-reset", "anthropic-ratelimit-tokens-reset",
|
|
18
|
+
"x-codex-primary-reset-at", "x-codex-secondary-reset-at",
|
|
12
19
|
]);
|
|
13
|
-
const
|
|
14
|
-
"
|
|
20
|
+
const SECONDS_HEADERS = new Set([
|
|
21
|
+
"x-codex-primary-reset-after-seconds", "x-codex-secondary-reset-after-seconds",
|
|
15
22
|
]);
|
|
23
|
+
const PERCENT_HEADERS = new Set(["x-codex-primary-used-percent", "x-codex-secondary-used-percent"]);
|
|
24
|
+
const UTILIZATION_HEADERS = new Set([
|
|
25
|
+
"anthropic-ratelimit-unified-5h-utilization", "anthropic-ratelimit-unified-7d-utilization",
|
|
26
|
+
"anthropic-ratelimit-unified-7d-sonnet-utilization", "anthropic-ratelimit-unified-7d-opus-utilization",
|
|
27
|
+
]);
|
|
28
|
+
const STATUS_HEADERS = new Set(["anthropic-ratelimit-unified-status"]);
|
|
29
|
+
const RESPONSE_HEADER_NAMES = new Set([
|
|
30
|
+
...COUNT_HEADERS, ...DURATION_RESET_HEADERS, ...ABSOLUTE_RESET_HEADERS,
|
|
31
|
+
...SECONDS_HEADERS, ...PERCENT_HEADERS, ...UTILIZATION_HEADERS, ...STATUS_HEADERS, "retry-after",
|
|
32
|
+
]);
|
|
33
|
+
export const HEADER_DIAGNOSTIC_MESSAGE = "Provider response headers observed.";
|
|
34
|
+
export function isHeaderDiagnostic(category: unknown, message: unknown): boolean {
|
|
35
|
+
return category === "provider.response" || message === HEADER_DIAGNOSTIC_MESSAGE;
|
|
36
|
+
}
|
|
16
37
|
|
|
17
38
|
export type DiagnosticLevel = "info" | "warning" | "error";
|
|
18
39
|
|
|
@@ -128,22 +149,22 @@ function validAbsoluteReset(value: string): boolean {
|
|
|
128
149
|
return Number.isFinite(epochMs) && epochMs >= 0 && new Date(epochMs).toUTCString() === value;
|
|
129
150
|
}
|
|
130
151
|
|
|
131
|
-
function
|
|
152
|
+
function validatedResponseHeaderValue(name: string, value: unknown): string | undefined {
|
|
132
153
|
const normalized = name.toLowerCase();
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
const trimmed = value.trim();
|
|
139
|
-
if (value !== trimmed) return undefined;
|
|
140
|
-
if (TOKEN_QUOTA_REMAINING_HEADERS.has(normalized)) {
|
|
141
|
-
return validTokenCount(trimmed) ? trimmed : undefined;
|
|
154
|
+
if (!RESPONSE_HEADER_NAMES.has(normalized) || typeof value !== "string" || value !== value.trim()) return undefined;
|
|
155
|
+
if (COUNT_HEADERS.has(normalized)) return validTokenCount(value) ? value : undefined;
|
|
156
|
+
if (DURATION_RESET_HEADERS.has(normalized)) {
|
|
157
|
+
return validDuration(value) || validAbsoluteReset(value) ? value : undefined;
|
|
142
158
|
}
|
|
143
|
-
if (
|
|
144
|
-
|
|
159
|
+
if (ABSOLUTE_RESET_HEADERS.has(normalized)) return validAbsoluteReset(value) ? value : undefined;
|
|
160
|
+
if (STATUS_HEADERS.has(normalized)) {
|
|
161
|
+
return ["allowed", "allowed_warning", "rejected"].includes(value) ? value : undefined;
|
|
145
162
|
}
|
|
146
|
-
|
|
163
|
+
if (normalized === "retry-after" && validAbsoluteReset(value)) return value;
|
|
164
|
+
if (value.length > 32 || !/^\d+(?:\.\d+)?$/.test(value)) return undefined;
|
|
165
|
+
const number = Number(value);
|
|
166
|
+
const maximum = PERCENT_HEADERS.has(normalized) ? 100 : UTILIZATION_HEADERS.has(normalized) ? 1 : Number.MAX_SAFE_INTEGER / 1000;
|
|
167
|
+
return Number.isFinite(number) && number >= 0 && number <= maximum ? value : undefined;
|
|
147
168
|
}
|
|
148
169
|
|
|
149
170
|
/** Sanitizes and bounds one upstream-derived diagnostic value before storage. */
|
|
@@ -151,11 +172,20 @@ export function sanitizeDiagnosticText(value: unknown): string {
|
|
|
151
172
|
return bound(redactDiagnosticText(value));
|
|
152
173
|
}
|
|
153
174
|
|
|
154
|
-
/**
|
|
175
|
+
/** Compatibility scalar API; unsupported names and malformed observations are redacted. */
|
|
155
176
|
export function sanitizeHeaderValue(name: string, value: unknown): string {
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
177
|
+
return validatedResponseHeaderValue(name, value) ?? REDACTED;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/** Copy only named, validated quota/retry facts. Never retain caller keys or prose. */
|
|
181
|
+
export function projectResponseHeaders(headers: unknown): Record<string, string> {
|
|
182
|
+
const projected: Record<string, string> = {};
|
|
183
|
+
if (typeof headers !== "object" || headers === null || Array.isArray(headers)) return projected;
|
|
184
|
+
for (const [name, value] of Object.entries(headers)) {
|
|
185
|
+
const validated = validatedResponseHeaderValue(name, value);
|
|
186
|
+
if (validated !== undefined) projected[name.toLowerCase()] = validated;
|
|
187
|
+
}
|
|
188
|
+
return projected;
|
|
159
189
|
}
|
|
160
190
|
|
|
161
191
|
function sanitizeFields(value: unknown, depth = 0): unknown {
|
|
@@ -202,6 +232,35 @@ export function sanitizedJson(value: unknown): string {
|
|
|
202
232
|
return JSON.stringify(sanitizeForJson(value));
|
|
203
233
|
}
|
|
204
234
|
|
|
235
|
+
/**
|
|
236
|
+
* Reserve the budget for newest failures/routing facts before routine observations.
|
|
237
|
+
* Both classes share one hard bound; a priority flood evicts older priority facts.
|
|
238
|
+
* Return append order, not priority order or wall-clock order. Oversized records
|
|
239
|
+
* are skipped so one record cannot prevent smaller, useful facts from fitting.
|
|
240
|
+
*/
|
|
241
|
+
export function retainDiagnosticEvents(
|
|
242
|
+
events: readonly DiagnosticEvent[],
|
|
243
|
+
budget: number,
|
|
244
|
+
sizeOf: (event: DiagnosticEvent) => number = () => 1,
|
|
245
|
+
): readonly DiagnosticEvent[] {
|
|
246
|
+
const retained = new Set<number>();
|
|
247
|
+
const sizes = events.map(sizeOf);
|
|
248
|
+
let used = 0;
|
|
249
|
+
for (const priority of [true, false]) {
|
|
250
|
+
for (let index = events.length - 1; index >= 0; index -= 1) {
|
|
251
|
+
const event = events[index]!;
|
|
252
|
+
const isPriority = event.level !== "info" || event.category === "routing" ||
|
|
253
|
+
event.category.startsWith("routing.") || event.category === "logical.provider";
|
|
254
|
+
if (isPriority !== priority) continue;
|
|
255
|
+
const size = sizes[index]!;
|
|
256
|
+
if (size > budget - used) continue;
|
|
257
|
+
retained.add(index);
|
|
258
|
+
used += size;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
return events.filter((_event, index) => retained.has(index));
|
|
262
|
+
}
|
|
263
|
+
|
|
205
264
|
export class DiagnosticLog {
|
|
206
265
|
readonly #events: DiagnosticEvent[] = [];
|
|
207
266
|
readonly #maxEvents: number;
|
|
@@ -224,7 +283,9 @@ export class DiagnosticLog {
|
|
|
224
283
|
for (const event of persisted.slice(-this.#maxEvents)) {
|
|
225
284
|
this.#events.push(Object.freeze({
|
|
226
285
|
...event,
|
|
227
|
-
|
|
286
|
+
message: isHeaderDiagnostic(event.category, event.message) ? HEADER_DIAGNOSTIC_MESSAGE : event.message,
|
|
287
|
+
fields: Object.freeze(isHeaderDiagnostic(event.category, event.message)
|
|
288
|
+
? projectResponseHeaders(event.fields) : { ...event.fields }),
|
|
228
289
|
}));
|
|
229
290
|
}
|
|
230
291
|
} catch {
|
|
@@ -239,18 +300,22 @@ export class DiagnosticLog {
|
|
|
239
300
|
fields: Readonly<Record<string, unknown>> = {},
|
|
240
301
|
): DiagnosticEvent {
|
|
241
302
|
const safeFields: Record<string, string> = {};
|
|
242
|
-
|
|
303
|
+
const projectedFields = isHeaderDiagnostic(category, message) ? projectResponseHeaders(fields) : fields;
|
|
304
|
+
for (const [key, value] of Object.entries(projectedFields)) {
|
|
243
305
|
safeFields[sanitizeDiagnosticText(key)] = sanitizeDiagnosticText(value);
|
|
244
306
|
}
|
|
245
307
|
const event = Object.freeze({
|
|
246
308
|
timestampMs: this.#now(),
|
|
247
309
|
level,
|
|
248
310
|
category: sanitizeDiagnosticText(category),
|
|
249
|
-
message: sanitizeDiagnosticText(message),
|
|
311
|
+
message: isHeaderDiagnostic(category, message) ? HEADER_DIAGNOSTIC_MESSAGE : sanitizeDiagnosticText(message),
|
|
250
312
|
fields: Object.freeze(safeFields),
|
|
251
313
|
});
|
|
252
314
|
this.#events.push(event);
|
|
253
|
-
if (this.#events.length > this.#maxEvents)
|
|
315
|
+
if (this.#events.length > this.#maxEvents) {
|
|
316
|
+
const retained = retainDiagnosticEvents(this.#events, this.#maxEvents);
|
|
317
|
+
this.#events.splice(0, this.#events.length, ...retained);
|
|
318
|
+
}
|
|
254
319
|
try {
|
|
255
320
|
this.#persistence?.append(event);
|
|
256
321
|
} catch {
|
|
@@ -273,16 +338,13 @@ export class DiagnosticLog {
|
|
|
273
338
|
}
|
|
274
339
|
|
|
275
340
|
recordHeaders(category: string, headers: Readonly<Record<string, unknown>>): DiagnosticEvent {
|
|
276
|
-
|
|
277
|
-
for (const [name, value] of Object.entries(headers)) {
|
|
278
|
-
safeHeaders[sanitizeDiagnosticText(name)] = sanitizeHeaderValue(name, value);
|
|
279
|
-
}
|
|
280
|
-
return this.record("info", category, "Provider response headers observed.", safeHeaders);
|
|
341
|
+
return this.record("info", category, HEADER_DIAGNOSTIC_MESSAGE, projectResponseHeaders(headers));
|
|
281
342
|
}
|
|
282
343
|
|
|
283
344
|
recent(limit = 20): readonly DiagnosticEvent[] {
|
|
284
345
|
const safeLimit = Number.isSafeInteger(limit) ? Math.max(0, Math.min(limit, 100)) : 20;
|
|
285
|
-
return this.#events
|
|
346
|
+
return retainDiagnosticEvents(this.#events, safeLimit)
|
|
347
|
+
.map((event) => ({ ...event, fields: { ...event.fields } }));
|
|
286
348
|
}
|
|
287
349
|
|
|
288
350
|
formatRecent(limit = 20): string {
|