@centerforagenticai/pi-multi-account 0.1.2 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/packages/pi-anthropic-oauth/src/stream.ts +8 -0
- package/src/anthropic-adaptive-stream.ts +81 -15
- package/src/anthropic-alias-stream.ts +9 -1
- package/src/codex-adapter.ts +109 -11
- package/src/config.ts +151 -0
- package/src/error-classification.ts +13 -1
- package/src/host-final-stop-message.ts +136 -0
- package/src/index.ts +41 -4
- package/src/logical-provider.ts +157 -2
- package/src/model-fallback-policy.ts +384 -0
- package/src/recovery-engine.ts +354 -68
- package/src/recovery-plan.ts +7 -1
- package/src/recovery-send-evidence.ts +29 -0
- package/src/refusal-advice.ts +139 -0
- package/src/routing.ts +17 -1
- package/src/shared-usage.ts +100 -176
- package/src/usage-fetch.ts +52 -32
- package/src/usage.ts +29 -26
package/src/recovery-plan.ts
CHANGED
|
@@ -30,6 +30,9 @@ export type RecoveryCandidateTier =
|
|
|
30
30
|
| "owning-vendor-api"
|
|
31
31
|
| "cross-family-subscription";
|
|
32
32
|
|
|
33
|
+
/** The one recovery dimension a candidate may change for this logical call. */
|
|
34
|
+
export type RecoveryActionKind = "account" | "model";
|
|
35
|
+
|
|
33
36
|
/** One finite provider/model pair. Logical and physical model identity stay distinct. */
|
|
34
37
|
export interface RecoveryCandidate {
|
|
35
38
|
readonly providerId: string;
|
|
@@ -39,6 +42,7 @@ export interface RecoveryCandidate {
|
|
|
39
42
|
readonly selectedModelId: string;
|
|
40
43
|
readonly tier: RecoveryCandidateTier;
|
|
41
44
|
readonly substitution: "exact" | "configured";
|
|
45
|
+
readonly recoveryAction: RecoveryActionKind;
|
|
42
46
|
readonly capability: RecoveryModelCapability;
|
|
43
47
|
}
|
|
44
48
|
|
|
@@ -91,7 +95,7 @@ function immutableCapability(model: RecoveryModelCapability): RecoveryModelCapab
|
|
|
91
95
|
});
|
|
92
96
|
}
|
|
93
97
|
|
|
94
|
-
/** Build the immutable, deterministic candidate
|
|
98
|
+
/** Build the immutable, deterministic candidate order for one logical call. */
|
|
95
99
|
export function buildRecoveryCandidatePlan(
|
|
96
100
|
request: RecoveryCandidatePlanRequest,
|
|
97
101
|
): readonly RecoveryCandidate[] {
|
|
@@ -146,6 +150,8 @@ export function buildRecoveryCandidatePlan(
|
|
|
146
150
|
selectedModelId: request.selectedModelId,
|
|
147
151
|
tier,
|
|
148
152
|
substitution,
|
|
153
|
+
recoveryAction:
|
|
154
|
+
model.modelId === request.selectedModelId ? "account" : "model",
|
|
149
155
|
capability: immutableCapability(model),
|
|
150
156
|
}),
|
|
151
157
|
);
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { AssistantMessage } from "@earendil-works/pi-ai";
|
|
2
|
+
|
|
3
|
+
export type CodexRecoverySendEvidence = "pre-execution-rejected" | "uncertain";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Classify whether a pinned Codex WebSocket terminal proves that one request was
|
|
7
|
+
* rejected before execution without an internal reconnect or SSE fallback.
|
|
8
|
+
*
|
|
9
|
+
* Pinned pi-ai 0.84.4 sends `response.create` before it observes stream events
|
|
10
|
+
* (`dist/api/openai-codex-responses.js:1182`). Its WebSocket loop can reconnect
|
|
11
|
+
* for two special errors or fall back to SSE (`:218-245`). The
|
|
12
|
+
* `provider_transport_failure` diagnostic is appended only on that transport
|
|
13
|
+
* branch (`:229-237`). Structured `CodexApiError` fields are instead normalized
|
|
14
|
+
* to terminal `errorMessage` (`:483-539`, `:344-346`), which retains neither the
|
|
15
|
+
* code/payload nor proof that no prior reconnect occurred.
|
|
16
|
+
*
|
|
17
|
+
* The per-session `getOpenAICodexWebSocketDebugStats` counters (`:632-654`)
|
|
18
|
+
* are process-global, shared by concurrent requests, and absent from the
|
|
19
|
+
* terminal, so they cannot attribute sends to one invocation either.
|
|
20
|
+
*
|
|
21
|
+
* Consequently no terminal `AssistantMessage` exposed by the pinned package is
|
|
22
|
+
* trustworthy proof of the complete condition. Do not infer safety from prose,
|
|
23
|
+
* status-like text, or the absence of a transport diagnostic.
|
|
24
|
+
*/
|
|
25
|
+
export function classifyCodexRecoverySendEvidence(
|
|
26
|
+
_terminal: AssistantMessage,
|
|
27
|
+
): CodexRecoverySendEvidence {
|
|
28
|
+
return "uncertain";
|
|
29
|
+
}
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Operator-only advice for a structured provider refusal.
|
|
3
|
+
*
|
|
4
|
+
* A refusal is an outcome, not an account failure: routing keeps the account
|
|
5
|
+
* and nothing is resent (see `src/routing.ts`, status `retained`). This module
|
|
6
|
+
* only tells the operator what happened and which choices remain. It never
|
|
7
|
+
* resends, rephrases, starts a turn, changes the model, rotates an account, or
|
|
8
|
+
* writes state.
|
|
9
|
+
*
|
|
10
|
+
* Only the structured `code` field selects advice. The adapter sets it from the
|
|
11
|
+
* provider's own stop reason (`src/anthropic-adaptive-stream.ts`). Assistant
|
|
12
|
+
* prose, the error message, the raw stop reason, and content are never read,
|
|
13
|
+
* so a refusal is never guessed and no provider text can reach the advice.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
/** Identical advice inside this window is shown once. */
|
|
17
|
+
export const REFUSAL_ADVICE_REPEAT_WINDOW_MS = 60_000;
|
|
18
|
+
|
|
19
|
+
/** Provider and model IDs are named only when they are short, plain tokens. */
|
|
20
|
+
const SAFE_ROUTE_ID = /^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,127}$/;
|
|
21
|
+
|
|
22
|
+
const CHOICES =
|
|
23
|
+
"Multi-account kept the account and did not resend, reroute, or retry it. " +
|
|
24
|
+
"You can revise the request, or pick another model explicitly with /model.";
|
|
25
|
+
|
|
26
|
+
export type RefusalAdviceRoute = Readonly<{
|
|
27
|
+
providerId?: unknown;
|
|
28
|
+
modelId?: unknown;
|
|
29
|
+
}>;
|
|
30
|
+
|
|
31
|
+
/** Reads one own data property; accessors and proxies count as absent. */
|
|
32
|
+
function ownDataValue(value: object, key: string): unknown {
|
|
33
|
+
const descriptor = Object.getOwnPropertyDescriptor(value, key);
|
|
34
|
+
return descriptor !== undefined && "value" in descriptor
|
|
35
|
+
? descriptor.value
|
|
36
|
+
: undefined;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* The allowlisted projection: whether `message` is an assistant error terminal
|
|
41
|
+
* carrying the exact structured `refusal` code. Nothing else is read.
|
|
42
|
+
*/
|
|
43
|
+
export function isStructuredRefusal(message: unknown): boolean {
|
|
44
|
+
try {
|
|
45
|
+
if (typeof message !== "object" || message === null) return false;
|
|
46
|
+
return (
|
|
47
|
+
ownDataValue(message, "role") === "assistant" &&
|
|
48
|
+
ownDataValue(message, "stopReason") === "error" &&
|
|
49
|
+
ownDataValue(message, "code") === "refusal"
|
|
50
|
+
);
|
|
51
|
+
} catch {
|
|
52
|
+
return false;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function safeRouteId(value: unknown): string | undefined {
|
|
57
|
+
return typeof value === "string" && SAFE_ROUTE_ID.test(value)
|
|
58
|
+
? value
|
|
59
|
+
: undefined;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function safeRouteLabel(route: RefusalAdviceRoute): string | undefined {
|
|
63
|
+
try {
|
|
64
|
+
const providerId = safeRouteId(route.providerId);
|
|
65
|
+
const modelId = safeRouteId(route.modelId);
|
|
66
|
+
return providerId === undefined || modelId === undefined
|
|
67
|
+
? undefined
|
|
68
|
+
: `${providerId}/${modelId}`;
|
|
69
|
+
} catch {
|
|
70
|
+
return undefined;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Bounded advice text for a structured refusal, or `undefined` when the message
|
|
76
|
+
* carries no structured refusal code. The text names only the route identity
|
|
77
|
+
* the caller resolved, and drops it when it is not a short plain token.
|
|
78
|
+
*/
|
|
79
|
+
export function refusalAdvice(
|
|
80
|
+
message: unknown,
|
|
81
|
+
route: RefusalAdviceRoute,
|
|
82
|
+
): string | undefined {
|
|
83
|
+
if (!isStructuredRefusal(message)) return undefined;
|
|
84
|
+
const label = safeRouteLabel(route);
|
|
85
|
+
const subject =
|
|
86
|
+
label === undefined ? "The model refused this request." : `${label} refused this request.`;
|
|
87
|
+
return `${subject} ${CHOICES}`;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
type NotifyContext = {
|
|
91
|
+
readonly hasUI?: unknown;
|
|
92
|
+
readonly ui?: { notify?: unknown };
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
export type RefusalAdvisorOptions = Readonly<{
|
|
96
|
+
/** False for delegate-owned sessions that share the foreground UI. */
|
|
97
|
+
foreground: boolean;
|
|
98
|
+
now?: () => number;
|
|
99
|
+
}>;
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Shows refusal advice in the foreground UI at most once per terminal and once
|
|
103
|
+
* per route inside {@link REFUSAL_ADVICE_REPEAT_WINDOW_MS}. Fail-soft: a UI
|
|
104
|
+
* failure never reaches the caller. Returns whether advice was shown.
|
|
105
|
+
*/
|
|
106
|
+
export function createRefusalAdvisor(options: RefusalAdvisorOptions) {
|
|
107
|
+
const now = options.now ?? Date.now;
|
|
108
|
+
const advised = new WeakSet<object>();
|
|
109
|
+
let lastText: string | undefined;
|
|
110
|
+
let lastAtMs = Number.NEGATIVE_INFINITY;
|
|
111
|
+
return {
|
|
112
|
+
advise(
|
|
113
|
+
message: unknown,
|
|
114
|
+
context: NotifyContext | undefined,
|
|
115
|
+
route: RefusalAdviceRoute,
|
|
116
|
+
): boolean {
|
|
117
|
+
try {
|
|
118
|
+
if (!options.foreground) return false;
|
|
119
|
+
const text = refusalAdvice(message, route);
|
|
120
|
+
if (text === undefined) return false;
|
|
121
|
+
if (advised.has(message as object)) return false;
|
|
122
|
+
advised.add(message as object);
|
|
123
|
+
if (context?.hasUI !== true) return false;
|
|
124
|
+
const notify = context.ui?.notify;
|
|
125
|
+
if (typeof notify !== "function") return false;
|
|
126
|
+
const atMs = now();
|
|
127
|
+
if (text === lastText && atMs - lastAtMs < REFUSAL_ADVICE_REPEAT_WINDOW_MS) {
|
|
128
|
+
return false;
|
|
129
|
+
}
|
|
130
|
+
lastText = text;
|
|
131
|
+
lastAtMs = atMs;
|
|
132
|
+
notify.call(context.ui, text, "warning");
|
|
133
|
+
return true;
|
|
134
|
+
} catch {
|
|
135
|
+
return false;
|
|
136
|
+
}
|
|
137
|
+
},
|
|
138
|
+
};
|
|
139
|
+
}
|
package/src/routing.ts
CHANGED
|
@@ -98,7 +98,18 @@ export interface PausedRoute {
|
|
|
98
98
|
readonly retryAfterMs: number | null;
|
|
99
99
|
}
|
|
100
100
|
|
|
101
|
-
|
|
101
|
+
/**
|
|
102
|
+
* A structured refusal or unknown provider stop. The failed account is kept
|
|
103
|
+
* as-is: no cooldown, no invalidation, no alternative, and no continuation.
|
|
104
|
+
* The stop is deterministic for the request context, so another account would
|
|
105
|
+
* only spend another request on the same answer.
|
|
106
|
+
*/
|
|
107
|
+
export interface RetainedRoute {
|
|
108
|
+
readonly status: "retained";
|
|
109
|
+
readonly classification: FailureClassification;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export type ReactiveRouteDecision = SelectedRoute | PausedRoute | RetainedRoute;
|
|
102
113
|
|
|
103
114
|
export interface HealthSelectionDecision {
|
|
104
115
|
readonly providerId: string;
|
|
@@ -1092,6 +1103,11 @@ export function routeAfterFailure(options: {
|
|
|
1092
1103
|
throw new TypeError("failedAccount must be a canonical managed provider.");
|
|
1093
1104
|
}
|
|
1094
1105
|
const classification = classifyFailure(failure);
|
|
1106
|
+
// Before any state observation: a retained stop writes nothing and routes
|
|
1107
|
+
// nowhere, so settlement cannot turn it into a switch or a continuation.
|
|
1108
|
+
if (classification.accountAction === "retain-account") {
|
|
1109
|
+
return { status: "retained", classification };
|
|
1110
|
+
}
|
|
1095
1111
|
if (
|
|
1096
1112
|
classification.category === "config" &&
|
|
1097
1113
|
failure.code === "model_not_found" &&
|
package/src/shared-usage.ts
CHANGED
|
@@ -65,25 +65,22 @@ export const EXHAUSTION_HOLD_MS = 60 * 60_000;
|
|
|
65
65
|
* policy.
|
|
66
66
|
*/
|
|
67
67
|
const MAX_REFRESH_DEBOUNCE_MS = 60 * 60_000;
|
|
68
|
+
/** Longest persisted usage-attempt delay produced by the capped retry ladder. */
|
|
69
|
+
export const MAX_USAGE_ATTEMPT_DELAY_MS = 15 * 60_000;
|
|
68
70
|
|
|
69
71
|
export const SHARED_USAGE_MAX_BYTES = 512 * 1024;
|
|
70
72
|
const MAX_RECORD_BYTES = 4_096;
|
|
71
73
|
const MAX_OBSERVER_ID_LENGTH = 256;
|
|
72
74
|
/**
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
* identifiers.
|
|
81
|
-
*
|
|
82
|
-
* There is no companion value-length bound: unknown string VALUES are refused
|
|
83
|
-
* outright rather than length-capped, because a bounded short string is
|
|
84
|
-
* exactly the shape of a leaked bearer token.
|
|
75
|
+
* Exactly the grammar `defaultObserverId()` can emit: a hostname mapped into
|
|
76
|
+
* `[A-Za-z0-9._-]`, then at most one `:` followed only by decimal pid digits,
|
|
77
|
+
* sliced to MAX_OBSERVER_ID_LENGTH. The rule is derived from the producer
|
|
78
|
+
* rather than from a guess about what a credential looks like
|
|
79
|
+
* (internal issue #108), and it deliberately does not require the
|
|
80
|
+
* colon or the pid: on a host whose name fills the slice, the delimiter and pid
|
|
81
|
+
* are cut off, and the store must not refuse or delete those records.
|
|
85
82
|
*/
|
|
86
|
-
const
|
|
83
|
+
const OBSERVER_ID_PATTERN = /^(?=.{1,256}$)[A-Za-z0-9._-]*(?::[0-9]*)?$/;
|
|
87
84
|
|
|
88
85
|
export type UsageObservationSource = "rate-limit-header" | "usage-endpoint";
|
|
89
86
|
|
|
@@ -123,6 +120,25 @@ const USAGE_FAILURE_DETAILS = new Set<UsageFailureDetail>([
|
|
|
123
120
|
"quota-summary-error",
|
|
124
121
|
]);
|
|
125
122
|
|
|
123
|
+
/**
|
|
124
|
+
* The closed set `failureReason()` in usage-fetch.ts can produce. A persisted
|
|
125
|
+
* attempt may carry only one of these, so the field cannot hold caller text.
|
|
126
|
+
*/
|
|
127
|
+
export type SharedUsageFailureReason =
|
|
128
|
+
| "rate-limit"
|
|
129
|
+
| "server-error"
|
|
130
|
+
| "credential-unavailable"
|
|
131
|
+
| "malformed-response"
|
|
132
|
+
| "network-error";
|
|
133
|
+
|
|
134
|
+
const USAGE_FAILURE_REASONS = new Set<SharedUsageFailureReason>([
|
|
135
|
+
"rate-limit",
|
|
136
|
+
"server-error",
|
|
137
|
+
"credential-unavailable",
|
|
138
|
+
"malformed-response",
|
|
139
|
+
"network-error",
|
|
140
|
+
]);
|
|
141
|
+
|
|
126
142
|
/** Machine-global fetch-attempt state; deliberately ignored by usage aggregation. */
|
|
127
143
|
export interface SharedUsageAttemptRecord {
|
|
128
144
|
readonly recordType: "usage-attempt";
|
|
@@ -133,9 +149,10 @@ export interface SharedUsageAttemptRecord {
|
|
|
133
149
|
readonly observedAtMs: number;
|
|
134
150
|
readonly observerId: string;
|
|
135
151
|
readonly failureCount: number;
|
|
152
|
+
/** Bounded relative to `observedAtMs` by the capped retry ladder. */
|
|
136
153
|
readonly nextAttemptAtMs: number;
|
|
137
154
|
readonly disabled: boolean;
|
|
138
|
-
readonly failureReason?:
|
|
155
|
+
readonly failureReason?: SharedUsageFailureReason;
|
|
139
156
|
/** Fixed, sanitized classification detail; never an upstream error body. */
|
|
140
157
|
readonly failureDetail?: UsageFailureDetail;
|
|
141
158
|
/**
|
|
@@ -238,141 +255,9 @@ function validTimestamp(value: unknown): value is number {
|
|
|
238
255
|
return typeof value === "number" && Number.isFinite(value) && value >= 0;
|
|
239
256
|
}
|
|
240
257
|
|
|
241
|
-
/**
|
|
242
|
-
* Whether a record this build cannot interpret may survive compaction.
|
|
243
|
-
*
|
|
244
|
-
* Compaction rebuilds the file from recognised records, so anything not carried
|
|
245
|
-
* here is deleted. That is how an older build destroys a record type added
|
|
246
|
-
* after it. Carrying unknown records forward keeps a newer build's state alive
|
|
247
|
-
* across a mixed-version fleet.
|
|
248
|
-
*
|
|
249
|
-
* The filter exists because "unrecognised" also covers records this build
|
|
250
|
-
* rejected as malformed, including the credential-bearing ones that
|
|
251
|
-
* `append` refuses and compaction currently scrubs. Carrying those would turn a
|
|
252
|
-
* durability fix into a privacy regression, so a carried record must still look
|
|
253
|
-
* like a usage record for a known account: a `recordType` string this build
|
|
254
|
-
* does not know, the same account identity fields every record carries, and no
|
|
255
|
-
* field outside that shape. A future record type satisfies this; a leaked
|
|
256
|
-
* authorization header does not.
|
|
257
|
-
*/
|
|
258
|
-
function carryableUnknownRecord(value: unknown): boolean {
|
|
259
|
-
if (typeof value !== "object" || value === null || Array.isArray(value))
|
|
260
|
-
return false;
|
|
261
|
-
const record = value as Record<string, unknown>;
|
|
262
|
-
// `recordType` must look like a record type, not merely be non-empty.
|
|
263
|
-
//
|
|
264
|
-
// Round 2 proved "non-empty string" is not a constraint: `Bearer <token>`
|
|
265
|
-
// is a non-empty string, so a credential placed here survived compaction.
|
|
266
|
-
// A real record type is a lower-kebab identifier, and nothing that fails
|
|
267
|
-
// this shape is a record type this build should carry blind.
|
|
268
|
-
if (
|
|
269
|
-
typeof record.recordType !== "string" ||
|
|
270
|
-
!CARRYABLE_RECORD_TYPE.test(record.recordType)
|
|
271
|
-
)
|
|
272
|
-
return false;
|
|
273
|
-
if (
|
|
274
|
-
typeof record.providerId !== "string" ||
|
|
275
|
-
typeof record.family !== "string" ||
|
|
276
|
-
!isAllowedFamily(record.family) ||
|
|
277
|
-
!isCanonicalManagedProviderId(record.providerId, record.family) ||
|
|
278
|
-
!validTimestamp(record.observedAtMs) ||
|
|
279
|
-
!validObserverId(record.observerId) ||
|
|
280
|
-
// Stricter than `validObserverId` on purpose: that only bounds length,
|
|
281
|
-
// and a bearer token is a bounded string. See CARRYABLE_OBSERVER_ID.
|
|
282
|
-
!CARRYABLE_OBSERVER_ID.test(record.observerId)
|
|
283
|
-
)
|
|
284
|
-
return false;
|
|
285
|
-
// Beyond the identity fields above, a carried record may hold only
|
|
286
|
-
// NON-STRING values.
|
|
287
|
-
//
|
|
288
|
-
// The first version of this filter bounded value shape -- primitives only,
|
|
289
|
-
// length-capped -- and review proved it unsound: `{ authorization: "Bearer
|
|
290
|
-
// SHORT" }` is a bounded primitive and survived compaction, which is exactly
|
|
291
|
-
// the privacy regression the carry-through was not allowed to create.
|
|
292
|
-
//
|
|
293
|
-
// Refusing unknown strings outright is the only defensible rule here. A
|
|
294
|
-
// denylist of sensitive-looking field names would be a guess about what a
|
|
295
|
-
// future record type calls its fields, and every credential this project
|
|
296
|
-
// handles is a string. Timestamps, counts, fractions and flags -- what a
|
|
297
|
-
// forward-compatible usage record actually needs -- are unaffected. A future
|
|
298
|
-
// type that genuinely needs a string field must teach this build about
|
|
299
|
-
// itself rather than rely on being carried blind.
|
|
300
|
-
for (const [key, entry] of Object.entries(record)) {
|
|
301
|
-
// Field NAMES are constrained by shape, not only length. Round 3 found
|
|
302
|
-
// a length bound alone lets a key called `Bearer SECRET` through: the
|
|
303
|
-
// credential rides in the key rather than the value. A field name in a
|
|
304
|
-
// JSON record written by this project is a lower-camel identifier.
|
|
305
|
-
if (!CARRYABLE_FIELD_NAME.test(key)) return false;
|
|
306
|
-
if (CARRYABLE_IDENTITY_FIELDS.has(key)) continue;
|
|
307
|
-
if (!carryableUnknownValue(entry)) return false;
|
|
308
|
-
}
|
|
309
|
-
return true;
|
|
310
|
-
}
|
|
311
|
-
|
|
312
|
-
/**
|
|
313
|
-
* Record types compaction may carry forward: the project's own namespace.
|
|
314
|
-
*
|
|
315
|
-
* Two weaker rules were tried and both refuted. "Non-empty" fell to
|
|
316
|
-
* `Bearer <token>` in round 2. A lower-kebab shape fell in round 3 to
|
|
317
|
-
* `sk-ant-api03-deadbeef`, which IS lower-kebab -- an API key and a record
|
|
318
|
-
* type are not distinguishable by shape, so no amount of character-class
|
|
319
|
-
* tightening can separate them.
|
|
320
|
-
*
|
|
321
|
-
* A namespace can. Every record type this file writes is `usage-`-prefixed
|
|
322
|
-
* (`usage-attempt`, `usage-exhaustion-hold`), so a future type from a newer
|
|
323
|
-
* build will be too. That is a property of the writer rather than a guess
|
|
324
|
-
* about what a credential looks like, which is why it holds where the shape
|
|
325
|
-
* checks did not.
|
|
326
|
-
*/
|
|
327
|
-
const CARRYABLE_RECORD_TYPE = /^usage-[a-z][a-z0-9-]{0,56}$/;
|
|
328
|
-
|
|
329
|
-
/**
|
|
330
|
-
* Shape a carried record's `observerId` must have.
|
|
331
|
-
*
|
|
332
|
-
* `validObserverId` only bounds length, and round 2 proved that insufficient:
|
|
333
|
-
* a bearer token is a bounded string. This restricts the CHARACTER SET instead,
|
|
334
|
-
* which is what makes the field unusable for smuggling while still accepting
|
|
335
|
-
* everything the producer can emit.
|
|
336
|
-
*
|
|
337
|
-
* The colon is deliberately NOT required. `defaultObserverId` builds
|
|
338
|
-
* `${hostname()}:${process.pid}` and then truncates to MAX_OBSERVER_ID_LENGTH,
|
|
339
|
-
* so on a host with a very long name the pid -- and the colon with it -- is cut
|
|
340
|
-
* off entirely. Round 3 found an earlier version of this expression required
|
|
341
|
-
* the colon, which would have made compaction DELETE legitimate records on such
|
|
342
|
-
* a machine: the precise data loss this carry-through exists to prevent, caused
|
|
343
|
-
* by the fix for it. A verified probe produced a 256-character id with no colon
|
|
344
|
-
* at all.
|
|
345
|
-
*
|
|
346
|
-
* The length bound matches MAX_OBSERVER_ID_LENGTH rather than guessing a
|
|
347
|
-
* narrower one, so the accepted domain covers every value the producer can
|
|
348
|
-
* actually return.
|
|
349
|
-
*/
|
|
350
|
-
const CARRYABLE_OBSERVER_ID = /^[A-Za-z0-9._:-]{1,256}$/;
|
|
351
|
-
|
|
352
|
-
/**
|
|
353
|
-
* Identity fields every record carries. They are the only strings a carried
|
|
354
|
-
* unknown record may contain, and each is format-checked above rather than
|
|
355
|
-
* merely bounded.
|
|
356
|
-
*/
|
|
357
|
-
const CARRYABLE_IDENTITY_FIELDS = new Set([
|
|
358
|
-
"recordType",
|
|
359
|
-
"providerId",
|
|
360
|
-
"family",
|
|
361
|
-
"observerId",
|
|
362
|
-
]);
|
|
363
|
-
|
|
364
|
-
function carryableUnknownValue(value: unknown): boolean {
|
|
365
|
-
return (
|
|
366
|
-
value === null || typeof value === "boolean" || typeof value === "number"
|
|
367
|
-
);
|
|
368
|
-
}
|
|
369
258
|
|
|
370
259
|
function validObserverId(value: unknown): value is string {
|
|
371
|
-
return (
|
|
372
|
-
typeof value === "string" &&
|
|
373
|
-
value.length > 0 &&
|
|
374
|
-
value.length <= MAX_OBSERVER_ID_LENGTH
|
|
375
|
-
);
|
|
260
|
+
return typeof value === "string" && OBSERVER_ID_PATTERN.test(value);
|
|
376
261
|
}
|
|
377
262
|
|
|
378
263
|
const TOKEN_FIELDS = [
|
|
@@ -474,10 +359,15 @@ function validAttemptRecord(value: unknown): value is SharedUsageAttemptRecord {
|
|
|
474
359
|
validObserverId(record.observerId) &&
|
|
475
360
|
nonNegativeInteger(record.failureCount) &&
|
|
476
361
|
validTimestamp(record.nextAttemptAtMs) &&
|
|
362
|
+
record.nextAttemptAtMs >= record.observedAtMs &&
|
|
363
|
+
record.nextAttemptAtMs - record.observedAtMs <=
|
|
364
|
+
MAX_USAGE_ATTEMPT_DELAY_MS &&
|
|
477
365
|
typeof record.disabled === "boolean" &&
|
|
478
366
|
(record.failureReason === undefined ||
|
|
479
367
|
(typeof record.failureReason === "string" &&
|
|
480
|
-
|
|
368
|
+
USAGE_FAILURE_REASONS.has(
|
|
369
|
+
record.failureReason as SharedUsageFailureReason,
|
|
370
|
+
))) &&
|
|
481
371
|
(record.failureDetail === undefined ||
|
|
482
372
|
(typeof record.failureDetail === "string" &&
|
|
483
373
|
USAGE_FAILURE_DETAILS.has(record.failureDetail as UsageFailureDetail))) &&
|
|
@@ -551,9 +441,19 @@ function defaultStorePath(): string {
|
|
|
551
441
|
return join(agentDir, "pi-multi-account", "usage.ndjson");
|
|
552
442
|
}
|
|
553
443
|
|
|
554
|
-
/**
|
|
555
|
-
|
|
556
|
-
|
|
444
|
+
/**
|
|
445
|
+
* Identity is bounded metadata only; it contains no credential-derived value.
|
|
446
|
+
*
|
|
447
|
+
* Hostname characters outside the observer-id alphabet are mapped to `-`, so
|
|
448
|
+
* every value this producer returns is one `validObserverId` accepts. Without
|
|
449
|
+
* that, an unusual hostname would make the default store constructor throw.
|
|
450
|
+
*/
|
|
451
|
+
export function defaultObserverId(
|
|
452
|
+
host: string = hostname(),
|
|
453
|
+
pid: number = process.pid,
|
|
454
|
+
): string {
|
|
455
|
+
const safeHost = host.replace(/[^A-Za-z0-9._-]/g, "-");
|
|
456
|
+
return `${safeHost}:${pid}`.slice(0, MAX_OBSERVER_ID_LENGTH);
|
|
557
457
|
}
|
|
558
458
|
|
|
559
459
|
function projectRecord(record: SharedUsageRecord): SharedUsageRecord {
|
|
@@ -829,24 +729,26 @@ function compactUsageFile(options: {
|
|
|
829
729
|
const latestRateLimits = new Map<string, SharedUsageRecord>();
|
|
830
730
|
const latestAttempts = new Map<string, SharedUsageAttemptRecord>();
|
|
831
731
|
const latestHolds = new Map<string, SharedUsageExhaustionHoldRecord>();
|
|
832
|
-
//
|
|
833
|
-
//
|
|
834
|
-
//
|
|
835
|
-
// process running an older build silently deletes every record type added
|
|
836
|
-
// after it -- including the exhaustion holds that keep a spent account out
|
|
837
|
-
// of rotation. The account then looks healthy
|
|
838
|
-
// to the next process and gets routed to again.
|
|
732
|
+
// Compaction is a CLOSED REGISTRY: only the three record types this
|
|
733
|
+
// build validates survive, and each survives as the JSON of its own
|
|
734
|
+
// validated projection, never as the source bytes.
|
|
839
735
|
//
|
|
840
|
-
//
|
|
841
|
-
//
|
|
842
|
-
//
|
|
736
|
+
// internal MR !83 carried unrecognised records verbatim so an
|
|
737
|
+
// older build would not delete a newer build's record types. Four rounds
|
|
738
|
+
// of shape rules (non-empty, lower-kebab, `usage-` namespace, lower-camel
|
|
739
|
+
// field names) tried to tell a future record from credential text and
|
|
740
|
+
// all were refuted (internal issue #110): `sk-ant-api03-deadbeef`
|
|
741
|
+
// is indistinguishable by shape from an identifier, a `usage-` prefix is
|
|
742
|
+
// free to any writer, and a retained source line keeps every field name
|
|
743
|
+
// and raw numeric text too. A record this build cannot interpret cannot
|
|
744
|
+
// be proved credential-free, so it is dropped.
|
|
843
745
|
//
|
|
844
|
-
//
|
|
845
|
-
//
|
|
846
|
-
//
|
|
847
|
-
//
|
|
848
|
-
//
|
|
849
|
-
|
|
746
|
+
// Mixed-version safety is kept for every type this build knows, the
|
|
747
|
+
// exhaustion hold included: each is re-emitted from its validated fields,
|
|
748
|
+
// so a peer on this build never deletes a live hold. The accepted cost
|
|
749
|
+
// is that a record type added after this build is dropped when this
|
|
750
|
+
// build compacts; a new type must stay advisory until every build in the
|
|
751
|
+
// fleet recognises it.
|
|
850
752
|
|
|
851
753
|
for (const line of completeUsageLines(options.path, completePrefixBytes)) {
|
|
852
754
|
let parsed: unknown;
|
|
@@ -893,10 +795,7 @@ function compactUsageFile(options: {
|
|
|
893
795
|
if (!previous || hold.observedAtMs >= previous.observedAtMs) {
|
|
894
796
|
latestHolds.set(hold.providerId, hold);
|
|
895
797
|
}
|
|
896
|
-
} else if (carryableUnknownRecord(parsed)) {
|
|
897
|
-
unrecognisedLines.push(line);
|
|
898
798
|
}
|
|
899
|
-
|
|
900
799
|
}
|
|
901
800
|
const compactedRecords = new Set([
|
|
902
801
|
...latest.values(),
|
|
@@ -905,12 +804,9 @@ function compactUsageFile(options: {
|
|
|
905
804
|
...latestAttempts.values(),
|
|
906
805
|
...latestHolds.values(),
|
|
907
806
|
]);
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
...unrecognisedLines.map((line) => `${line}\n`),
|
|
912
|
-
...[...compactedRecords].map((record) => `${JSON.stringify(record)}\n`),
|
|
913
|
-
].join("");
|
|
807
|
+
const compacted = [...compactedRecords]
|
|
808
|
+
.map((record) => `${JSON.stringify(record)}\n`)
|
|
809
|
+
.join("");
|
|
914
810
|
const compactedBytes = Buffer.byteLength(compacted, "utf8");
|
|
915
811
|
if (compactedBytes > options.maxBytes) return false;
|
|
916
812
|
|
|
@@ -1009,6 +905,15 @@ export class SharedUsageStore {
|
|
|
1009
905
|
|
|
1010
906
|
append(record: SharedUsageLogRecord): boolean {
|
|
1011
907
|
try {
|
|
908
|
+
// The store is the ONLY producer of a persisted observer id
|
|
909
|
+
// (internal issue #133). The grammar check alone cannot keep a
|
|
910
|
+
// credential out: `sk-ant-api03-...` is a valid hostname-shaped id, and
|
|
911
|
+
// no character rule separates a hostname from a key. So a caller may
|
|
912
|
+
// only restate this store's own id, which the constructor validated;
|
|
913
|
+
// any other value, a peer's included, is refused rather than written.
|
|
914
|
+
// Judged on the caller's own value, before projection slices it.
|
|
915
|
+
if ((record as { observerId?: unknown }).observerId !== this.#observerId)
|
|
916
|
+
return false;
|
|
1012
917
|
const projected = validExhaustionHoldRecord(record)
|
|
1013
918
|
? projectExhaustionHold(record)
|
|
1014
919
|
: validAttemptRecord(record)
|
|
@@ -1134,16 +1039,22 @@ export class SharedUsageStore {
|
|
|
1134
1039
|
* records, so a hold that has lapsed simply stops being reported. Callers get
|
|
1135
1040
|
* a time to compare, not a boolean, because routing has to combine it with an
|
|
1136
1041
|
* authoritative recovery time and take whichever is later.
|
|
1042
|
+
*
|
|
1043
|
+
* `clearedAtMs` excludes holds whose own validated `failedAtMs` is at or
|
|
1044
|
+
* before it, so a caller's operator clear is judged against the failure
|
|
1045
|
+
* the record states rather than one reconstructed from its deadline.
|
|
1137
1046
|
*/
|
|
1138
1047
|
activeExhaustionHoldUntilMs(
|
|
1139
1048
|
providerId: string,
|
|
1140
1049
|
family: AllowedFamily,
|
|
1141
1050
|
nowMs: number,
|
|
1051
|
+
clearedAtMs?: number,
|
|
1142
1052
|
): number | undefined {
|
|
1143
1053
|
let latest: number | undefined;
|
|
1144
1054
|
for (const hold of this.readExhaustionHolds()) {
|
|
1145
1055
|
if (hold.providerId !== providerId || hold.family !== family) continue;
|
|
1146
1056
|
if (hold.holdUntilMs <= nowMs) continue;
|
|
1057
|
+
if (clearedAtMs !== undefined && hold.failedAtMs <= clearedAtMs) continue;
|
|
1147
1058
|
// A relative bound alone is not enough. `holdUntilMs - failedAtMs` can
|
|
1148
1059
|
// be a legitimate 60 minutes while `failedAtMs` itself sits in the year
|
|
1149
1060
|
// 3138, which would exclude the account for centuries. No honest hold
|
|
@@ -1156,15 +1067,28 @@ export class SharedUsageStore {
|
|
|
1156
1067
|
return latest;
|
|
1157
1068
|
}
|
|
1158
1069
|
|
|
1070
|
+
/**
|
|
1071
|
+
* The newest attempt for an account that was observed at or before `nowMs`.
|
|
1072
|
+
*
|
|
1073
|
+
* An attempt observed in the future is ignored here, at the single source,
|
|
1074
|
+
* rather than by each reader (internal issue #133). Such a record always
|
|
1075
|
+
* sorts as the newest, so a reader that merely declined to be suppressed by
|
|
1076
|
+
* it still took its `failureCount` as the prior rung: every real failure
|
|
1077
|
+
* rewrote the same count, the next read returned the future record again,
|
|
1078
|
+
* and the backoff ladder froze. `nowMs` is required so each reader judges
|
|
1079
|
+
* against its own (possibly injected) clock.
|
|
1080
|
+
*/
|
|
1159
1081
|
latestAttempt(
|
|
1160
1082
|
providerId: string,
|
|
1161
1083
|
family: AllowedFamily,
|
|
1084
|
+
nowMs: number,
|
|
1162
1085
|
): SharedUsageAttemptRecord | undefined {
|
|
1163
1086
|
let latest: SharedUsageAttemptRecord | undefined;
|
|
1164
1087
|
for (const record of this.readAttempts()) {
|
|
1165
1088
|
if (
|
|
1166
1089
|
record.providerId === providerId &&
|
|
1167
1090
|
record.family === family &&
|
|
1091
|
+
record.observedAtMs <= nowMs &&
|
|
1168
1092
|
(latest === undefined || record.observedAtMs >= latest.observedAtMs)
|
|
1169
1093
|
) {
|
|
1170
1094
|
latest = record;
|