@centerforagenticai/pi-multi-account 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/NOTICE +29 -0
- package/README.md +999 -0
- package/config/models/pi-multi-account.v1.json +32 -0
- package/config/subscription-plans.v1.json +122 -0
- package/package.json +76 -0
- package/packages/pi-anthropic-oauth/LICENSE +21 -0
- package/packages/pi-anthropic-oauth/package.json +54 -0
- package/packages/pi-anthropic-oauth/src/auth.ts +396 -0
- package/packages/pi-anthropic-oauth/src/context.ts +116 -0
- package/packages/pi-anthropic-oauth/src/convert.ts +303 -0
- package/packages/pi-anthropic-oauth/src/index.ts +37 -0
- package/packages/pi-anthropic-oauth/src/prompt.ts +137 -0
- package/packages/pi-anthropic-oauth/src/stream.ts +476 -0
- package/packages/pi-antigravity/LICENSE +21 -0
- package/packages/pi-antigravity/package.json +77 -0
- package/packages/pi-antigravity/src/auth/index.ts +14 -0
- package/packages/pi-antigravity/src/auth/oauth.ts +442 -0
- package/packages/pi-antigravity/src/client/client.ts +561 -0
- package/packages/pi-antigravity/src/client/index.ts +1 -0
- package/packages/pi-antigravity/src/context.ts +110 -0
- package/packages/pi-antigravity/src/diagnostics/diagnostics.ts +96 -0
- package/packages/pi-antigravity/src/diagnostics/index.ts +1 -0
- package/packages/pi-antigravity/src/image/image.ts +336 -0
- package/packages/pi-antigravity/src/image/index.ts +1 -0
- package/packages/pi-antigravity/src/index.ts +280 -0
- package/packages/pi-antigravity/src/models/discovery.ts +154 -0
- package/packages/pi-antigravity/src/models/grouping.ts +424 -0
- package/packages/pi-antigravity/src/models/index.ts +3 -0
- package/packages/pi-antigravity/src/models/models.ts +500 -0
- package/packages/pi-antigravity/src/stream/index.ts +1 -0
- package/packages/pi-antigravity/src/stream/stream.ts +1478 -0
- package/packages/pi-antigravity/src/types/enums.ts +42 -0
- package/packages/pi-antigravity/src/types/index.ts +2 -0
- package/packages/pi-antigravity/src/types/types.ts +292 -0
- package/packages/pi-antigravity/src/usage/index.ts +1 -0
- package/packages/pi-antigravity/src/usage/usage.ts +416 -0
- package/packages/pi-antigravity/src/utils/http.ts +91 -0
- package/packages/pi-antigravity/src/utils/index.ts +3 -0
- package/packages/pi-antigravity/src/utils/security.ts +73 -0
- package/packages/pi-antigravity/src/utils/util.ts +132 -0
- package/scripts/multi-account.mjs +44 -0
- package/src/account-labels.ts +223 -0
- package/src/account-plan-assignment.ts +340 -0
- package/src/account-rate-history.ts +372 -0
- package/src/anthropic-adaptive-stream.ts +531 -0
- package/src/anthropic-alias-stream.ts +140 -0
- package/src/anthropic-context-compat.ts +80 -0
- package/src/api-pricing.ts +579 -0
- package/src/bounded-file-lines.ts +97 -0
- package/src/catalog-rebinding.ts +177 -0
- package/src/catalog-registration-probe.ts +111 -0
- package/src/codex-adapter.ts +345 -0
- package/src/codex-model-defaults.ts +785 -0
- package/src/command-completions.ts +404 -0
- package/src/commands.ts +2000 -0
- package/src/compaction.ts +14 -0
- package/src/config.ts +1317 -0
- package/src/continuation.ts +569 -0
- package/src/cooldowns.ts +110 -0
- package/src/cost-digest-store.ts +332 -0
- package/src/cost-digest.ts +1044 -0
- package/src/cost-history.ts +251 -0
- package/src/cost-period-closer.ts +160 -0
- package/src/cost-report-json.ts +318 -0
- package/src/cost-report-reader.ts +368 -0
- package/src/cost-report-render.ts +207 -0
- package/src/cost-report.ts +1104 -0
- package/src/coverage-attestation.ts +397 -0
- package/src/credential-lifecycle.ts +169 -0
- package/src/credential-refresh.ts +248 -0
- package/src/declaration-notice-marker.ts +238 -0
- package/src/diagnostic-store.ts +276 -0
- package/src/diagnostics.ts +309 -0
- package/src/discovery.ts +471 -0
- package/src/duration.ts +13 -0
- package/src/error-classification.ts +256 -0
- package/src/fuzzy.ts +15 -0
- package/src/group-policy.ts +81 -0
- package/src/history-store.ts +897 -0
- package/src/index.ts +5572 -0
- package/src/lifecycle.ts +378 -0
- package/src/logical-dispatch.ts +279 -0
- package/src/logical-model-selector.ts +254 -0
- package/src/logical-model-switcher.ts +430 -0
- package/src/logical-provider-attribution.ts +544 -0
- package/src/logical-provider.ts +1237 -0
- package/src/logical-route-indicator.ts +215 -0
- package/src/machine-lease.ts +445 -0
- package/src/model-support.ts +66 -0
- package/src/models-declaration.ts +1091 -0
- package/src/openai-adapter.ts +117 -0
- package/src/openrouter-budget.ts +304 -0
- package/src/openrouter-fallback.ts +146 -0
- package/src/period-boundaries.ts +376 -0
- package/src/pi-anthropic-oauth.d.ts +6 -0
- package/src/preflight.ts +253 -0
- package/src/pricing-cache.ts +235 -0
- package/src/project-identity.ts +100 -0
- package/src/provider-registration.ts +942 -0
- package/src/rate-formula.ts +163 -0
- package/src/recovery-engine.ts +853 -0
- package/src/recovery-output.ts +837 -0
- package/src/recovery-plan.ts +239 -0
- package/src/report-range.ts +203 -0
- package/src/route-resolver.ts +789 -0
- package/src/routing-config-transaction.ts +232 -0
- package/src/routing.ts +1163 -0
- package/src/runtime-state.ts +630 -0
- package/src/session-account-groups.ts +284 -0
- package/src/session-restore.ts +287 -0
- package/src/shared-usage.ts +1392 -0
- package/src/standalone-cli.ts +720 -0
- package/src/status-view.ts +578 -0
- package/src/subscription-plan-catalog.ts +346 -0
- package/src/tier-model-resolver.ts +46 -0
- package/src/upstream-anthropic.ts +315 -0
- package/src/upstream-antigravity.ts +327 -0
- package/src/usage-fetch.ts +1634 -0
- package/src/usage.ts +1026 -0
- package/src/vendor.ts +87 -0
- package/src/warmer.ts +231 -0
- package/src/watchdog.ts +219 -0
- package/src/window-history.ts +270 -0
|
@@ -0,0 +1,569 @@
|
|
|
1
|
+
import type { Api, Model } from "@earendil-works/pi-ai";
|
|
2
|
+
import { DiagnosticLog } from "./diagnostics.js";
|
|
3
|
+
import type {
|
|
4
|
+
ContinuationDestinationFamily,
|
|
5
|
+
ContinuationRoutingReason,
|
|
6
|
+
RuntimeReference,
|
|
7
|
+
} from "./runtime-state.js";
|
|
8
|
+
import { RuntimeState } from "./runtime-state.js";
|
|
9
|
+
|
|
10
|
+
export const FIXED_CONTINUATION_MESSAGE =
|
|
11
|
+
"Continue after the confirmed account switch. Do not repeat the failed request.";
|
|
12
|
+
|
|
13
|
+
export interface ContinuationDestination {
|
|
14
|
+
readonly providerId: string;
|
|
15
|
+
readonly family: ContinuationDestinationFamily;
|
|
16
|
+
readonly model: Model<Api>;
|
|
17
|
+
readonly routingReason: ContinuationRoutingReason;
|
|
18
|
+
/** Skip physical setModel and pin this exact account for one unified dispatch. */
|
|
19
|
+
readonly logicalRoutePin?: Readonly<{ readonly requestedModelId: string }>;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface ContinuationRecord {
|
|
23
|
+
readonly failedTurnRef: RuntimeReference;
|
|
24
|
+
readonly continuationTurnRef: RuntimeReference;
|
|
25
|
+
readonly destinationProviderId: string;
|
|
26
|
+
readonly destinationFamily: ContinuationDestinationFamily;
|
|
27
|
+
readonly routingReason: ContinuationRoutingReason;
|
|
28
|
+
readonly selectedAtMs: number;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export interface ContinuationScheduler {
|
|
32
|
+
set(delayMs: number, callback: () => void): unknown;
|
|
33
|
+
clear(handle: unknown): void;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface ContinuationDependencies {
|
|
37
|
+
readonly state: RuntimeState;
|
|
38
|
+
readonly diagnostics: DiagnosticLog;
|
|
39
|
+
readonly setModel: (model: Model<Api>) => Promise<boolean>;
|
|
40
|
+
readonly dispatchFollowUp: (message: string, signal: AbortSignal) => void | Promise<void>;
|
|
41
|
+
readonly scheduler?: ContinuationScheduler;
|
|
42
|
+
readonly now?: () => number;
|
|
43
|
+
readonly onDispatched?: (record: ContinuationRecord) => void | Promise<void>;
|
|
44
|
+
readonly onAutomaticRoutingSelected?: (
|
|
45
|
+
destination: ContinuationDestination,
|
|
46
|
+
) => void | Promise<void>;
|
|
47
|
+
readonly onAutomaticRoutingCleared?: () => void | Promise<void>;
|
|
48
|
+
/**
|
|
49
|
+
* How often a parked turn re-checks whether an account has recovered.
|
|
50
|
+
* Defaults to 30s. Injected so tests drive it with a fake clock.
|
|
51
|
+
*/
|
|
52
|
+
readonly parkPollMs?: number;
|
|
53
|
+
/**
|
|
54
|
+
* Resolves whether a parked turn may resume yet. Returning a destination
|
|
55
|
+
* resumes on it; returning undefined keeps the turn parked.
|
|
56
|
+
*
|
|
57
|
+
* This is the RESUME PREDICATE, injected rather than computed inside the
|
|
58
|
+
* controller so a test can make the clock and the usage view DISAGREE. A park
|
|
59
|
+
* that resumes merely because a timer fired is the defect; it must resume
|
|
60
|
+
* because an account is genuinely available. Callers are expected to consult
|
|
61
|
+
* live usage here, not a recorded estimate.
|
|
62
|
+
*/
|
|
63
|
+
readonly resolveParkedDestination?: (
|
|
64
|
+
nowMs: number,
|
|
65
|
+
) =>
|
|
66
|
+
| ContinuationDestination
|
|
67
|
+
| undefined
|
|
68
|
+
| Promise<ContinuationDestination | undefined>;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* How long a turn may stay parked before it is abandoned.
|
|
73
|
+
*
|
|
74
|
+
* A park MUST be bounded. Without this the poll re-arms forever: an account that
|
|
75
|
+
* never recovers leaves a timer cycling for the life of the process, and under
|
|
76
|
+
* fake timers it is a literal infinite loop (vitest aborts at 10000 timers).
|
|
77
|
+
* Found by an existing test failing after the park landed, which is exactly the
|
|
78
|
+
* kind of unbounded-retry bug this spec exists to remove -- a park that never
|
|
79
|
+
* gives up is its own silent dead end.
|
|
80
|
+
*/
|
|
81
|
+
export const PARK_MAX_DURATION_MS = 30 * 60 * 1000;
|
|
82
|
+
|
|
83
|
+
/** A turn held because no account was available, awaiting genuine recovery. */
|
|
84
|
+
interface ParkedTurn {
|
|
85
|
+
readonly failedTurnRef: RuntimeReference;
|
|
86
|
+
readonly abortController: AbortController;
|
|
87
|
+
readonly parkedAtMs: number;
|
|
88
|
+
timer?: unknown;
|
|
89
|
+
resuming: boolean;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Why a selection was rejected. The two cases warrant opposite responses, so
|
|
94
|
+
* they must never be collapsed into one status.
|
|
95
|
+
*
|
|
96
|
+
* - `unconfigured` — the host's model-selection call returned false because the
|
|
97
|
+
* destination is absent from its `configuredProviders` snapshot
|
|
98
|
+
* (agent-session.ts:2403-2407 -> model-runtime.ts:458-460). The destination
|
|
99
|
+
* cannot serve this snapshot, so exclude it and try another candidate.
|
|
100
|
+
* - `auth-error` — that same call THREW, because `checkAuth` missed
|
|
101
|
+
* (agent-session.ts:1586-1589). The check is async and network-capable, so
|
|
102
|
+
* the failure is transient and MUST NOT exclude the destination: the same
|
|
103
|
+
* account may authenticate on the next attempt.
|
|
104
|
+
*
|
|
105
|
+
* The host call is named indirectly above because test/lifecycle.test.ts scans
|
|
106
|
+
* these modules for that identifier as a substring to prove selection stays
|
|
107
|
+
* confined to the continuation and operator-command paths.
|
|
108
|
+
*/
|
|
109
|
+
export type SelectionRejection = "unconfigured" | "auth-error";
|
|
110
|
+
|
|
111
|
+
export type ContinuationStartResult =
|
|
112
|
+
| { readonly status: "scheduled"; readonly record: ContinuationRecord }
|
|
113
|
+
| {
|
|
114
|
+
readonly status: "selection-failed";
|
|
115
|
+
/** Which rejection path fired. Callers branch on this to decide whether to exclude. */
|
|
116
|
+
readonly rejection: SelectionRejection;
|
|
117
|
+
/**
|
|
118
|
+
* True only when the destination should be excluded from further
|
|
119
|
+
* candidate selection FOR THE CURRENT SNAPSHOT. Never session-scoped:
|
|
120
|
+
* `configuredProviders` mutates mid-session (model-runtime.ts:341-363
|
|
121
|
+
* adds on a successful checkAuth and deletes on a failure), so a
|
|
122
|
+
* session-long exclusion would permanently shrink the usable fleet after
|
|
123
|
+
* a single transient blip.
|
|
124
|
+
*/
|
|
125
|
+
readonly excludeDestination: boolean;
|
|
126
|
+
}
|
|
127
|
+
| { readonly status: "duplicate" | "cancelled" };
|
|
128
|
+
|
|
129
|
+
interface Reservation {
|
|
130
|
+
readonly failedTurnRef: RuntimeReference;
|
|
131
|
+
readonly abortController: AbortController;
|
|
132
|
+
active: boolean;
|
|
133
|
+
dispatchStarted: boolean;
|
|
134
|
+
timer?: unknown;
|
|
135
|
+
record?: ContinuationRecord;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
const defaultScheduler: ContinuationScheduler = {
|
|
139
|
+
set(delayMs, callback) {
|
|
140
|
+
return setTimeout(callback, delayMs);
|
|
141
|
+
},
|
|
142
|
+
clear(handle) {
|
|
143
|
+
clearTimeout(handle as ReturnType<typeof setTimeout>);
|
|
144
|
+
},
|
|
145
|
+
};
|
|
146
|
+
|
|
147
|
+
/** Coordinates reactive, post-failure continuation only; it registers no proactive hook. */
|
|
148
|
+
export class ContinuationController {
|
|
149
|
+
readonly #state: RuntimeState;
|
|
150
|
+
readonly #diagnostics: DiagnosticLog;
|
|
151
|
+
readonly #setModel: (model: Model<Api>) => Promise<boolean>;
|
|
152
|
+
readonly #dispatchFollowUp: (message: string, signal: AbortSignal) => void | Promise<void>;
|
|
153
|
+
readonly #scheduler: ContinuationScheduler;
|
|
154
|
+
readonly #now: () => number;
|
|
155
|
+
readonly #onDispatched: ((record: ContinuationRecord) => void | Promise<void>) | undefined;
|
|
156
|
+
readonly #onAutomaticRoutingSelected:
|
|
157
|
+
| ((destination: ContinuationDestination) => void | Promise<void>)
|
|
158
|
+
| undefined;
|
|
159
|
+
readonly #onAutomaticRoutingCleared: (() => void | Promise<void>) | undefined;
|
|
160
|
+
readonly #parkPollMs: number;
|
|
161
|
+
readonly #resolveParkedDestination:
|
|
162
|
+
| ((
|
|
163
|
+
nowMs: number,
|
|
164
|
+
) =>
|
|
165
|
+
| ContinuationDestination
|
|
166
|
+
| undefined
|
|
167
|
+
| Promise<ContinuationDestination | undefined>)
|
|
168
|
+
| undefined;
|
|
169
|
+
/**
|
|
170
|
+
* Parked turns awaiting an account, keyed by the failed turn each resumes.
|
|
171
|
+
*
|
|
172
|
+
* PER-INSTANCE, NEVER PROCESS-GLOBAL. `delegate-session-scope.ts:60-96` makes
|
|
173
|
+
* some foreground handlers no-op under a delegate bind, so a process-global
|
|
174
|
+
* park would leak a parked turn across sessions and forks and resume it in the
|
|
175
|
+
* wrong one. This Map is also deliberately per-process only: no cross-process
|
|
176
|
+
* reservation is added or implied here.
|
|
177
|
+
*/
|
|
178
|
+
readonly #parked = new Map<RuntimeReference, ParkedTurn>();
|
|
179
|
+
readonly #reservations = new Map<RuntimeReference, Reservation>();
|
|
180
|
+
readonly #records: ContinuationRecord[] = [];
|
|
181
|
+
|
|
182
|
+
constructor(dependencies: ContinuationDependencies) {
|
|
183
|
+
this.#state = dependencies.state;
|
|
184
|
+
this.#diagnostics = dependencies.diagnostics;
|
|
185
|
+
this.#setModel = dependencies.setModel;
|
|
186
|
+
this.#dispatchFollowUp = dependencies.dispatchFollowUp;
|
|
187
|
+
this.#scheduler = dependencies.scheduler ?? defaultScheduler;
|
|
188
|
+
this.#now = dependencies.now ?? Date.now;
|
|
189
|
+
this.#onDispatched = dependencies.onDispatched;
|
|
190
|
+
this.#onAutomaticRoutingSelected = dependencies.onAutomaticRoutingSelected;
|
|
191
|
+
this.#onAutomaticRoutingCleared = dependencies.onAutomaticRoutingCleared;
|
|
192
|
+
this.#parkPollMs = dependencies.parkPollMs ?? 30_000;
|
|
193
|
+
this.#resolveParkedDestination = dependencies.resolveParkedDestination;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
async start(
|
|
197
|
+
failedTurnRef: RuntimeReference,
|
|
198
|
+
destination: ContinuationDestination,
|
|
199
|
+
): Promise<ContinuationStartResult> {
|
|
200
|
+
if (this.#reservations.has(failedTurnRef)) return { status: "duplicate" };
|
|
201
|
+
|
|
202
|
+
// Reservation and state guard are synchronous and precede every await.
|
|
203
|
+
const reservation: Reservation = {
|
|
204
|
+
failedTurnRef,
|
|
205
|
+
abortController: new AbortController(),
|
|
206
|
+
active: true,
|
|
207
|
+
dispatchStarted: false,
|
|
208
|
+
};
|
|
209
|
+
this.#reservations.set(failedTurnRef, reservation);
|
|
210
|
+
try {
|
|
211
|
+
if (destination.model.provider !== destination.providerId) {
|
|
212
|
+
throw new TypeError("continuation destination model/provider mismatch.");
|
|
213
|
+
}
|
|
214
|
+
if (
|
|
215
|
+
destination.logicalRoutePin !== undefined &&
|
|
216
|
+
(destination.family === "openrouter" ||
|
|
217
|
+
destination.model.id !== destination.logicalRoutePin.requestedModelId)
|
|
218
|
+
) {
|
|
219
|
+
throw new TypeError("logical route pin must name the exact managed destination model.");
|
|
220
|
+
}
|
|
221
|
+
const established = this.#state.establishContinuationGuard({
|
|
222
|
+
failedTurnRef,
|
|
223
|
+
destinationProviderId: destination.providerId,
|
|
224
|
+
destinationFamily: destination.family,
|
|
225
|
+
routingReason: destination.routingReason,
|
|
226
|
+
guardSetAtMs: this.#now(),
|
|
227
|
+
});
|
|
228
|
+
if (!established) {
|
|
229
|
+
this.#reservations.delete(failedTurnRef);
|
|
230
|
+
return { status: "duplicate" };
|
|
231
|
+
}
|
|
232
|
+
} catch (error) {
|
|
233
|
+
this.#reservations.delete(failedTurnRef);
|
|
234
|
+
this.#diagnostics.recordError("continuation.reserve", error);
|
|
235
|
+
// Guard/reservation failure says nothing about the destination's auth, so
|
|
236
|
+
// it must not exclude it.
|
|
237
|
+
return { status: "selection-failed", rejection: "auth-error", excludeDestination: false };
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
let selected = false;
|
|
241
|
+
// Distinguishes the two rejection paths. A throw means checkAuth missed and
|
|
242
|
+
// is transient; a false return means the provider is absent from the
|
|
243
|
+
// snapshot. Collapsing them loses the difference that decides exclusion.
|
|
244
|
+
let rejection: SelectionRejection = "unconfigured";
|
|
245
|
+
try {
|
|
246
|
+
if (destination.logicalRoutePin === undefined) {
|
|
247
|
+
// A physical continuation supersedes any older logical one-shot pin. Clear
|
|
248
|
+
// it BEFORE setModel so the stale pin can never be consumed by a later
|
|
249
|
+
// unified dispatch regardless of whether setModel succeeds, returns false,
|
|
250
|
+
// or throws -- leaving at most one continuation intent.
|
|
251
|
+
this.#state.clearLogicalRoutePin();
|
|
252
|
+
selected = (await this.#setModel(destination.model)) === true;
|
|
253
|
+
} else {
|
|
254
|
+
if (destination.family === "openrouter") {
|
|
255
|
+
throw new TypeError("logical route pin cannot target OpenRouter.");
|
|
256
|
+
}
|
|
257
|
+
this.#state.setLogicalRoutePin({
|
|
258
|
+
destinationProviderId: destination.providerId,
|
|
259
|
+
destinationFamily: destination.family,
|
|
260
|
+
requestedModelId: destination.logicalRoutePin.requestedModelId,
|
|
261
|
+
});
|
|
262
|
+
selected = true;
|
|
263
|
+
}
|
|
264
|
+
} catch (error) {
|
|
265
|
+
rejection = "auth-error";
|
|
266
|
+
if (reservation.active) this.#diagnostics.recordError("continuation.set-model", error);
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
if (!reservation.active) return { status: "cancelled" };
|
|
270
|
+
if (!selected) {
|
|
271
|
+
this.#releaseFailedSelection(reservation);
|
|
272
|
+
this.#diagnostics.record(
|
|
273
|
+
"info",
|
|
274
|
+
"continuation.selection-failed",
|
|
275
|
+
`Selection rejected for ${destination.providerId} (${rejection}); ` +
|
|
276
|
+
(rejection === "unconfigured"
|
|
277
|
+
? "excluding it for this snapshot."
|
|
278
|
+
: "retaining it as a candidate because checkAuth failures are transient."),
|
|
279
|
+
);
|
|
280
|
+
return {
|
|
281
|
+
status: "selection-failed",
|
|
282
|
+
rejection,
|
|
283
|
+
// Only a snapshot-membership rejection excludes. An auth error is
|
|
284
|
+
// transient and must leave the destination selectable.
|
|
285
|
+
excludeDestination: rejection === "unconfigured",
|
|
286
|
+
};
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
await this.#reportAutomaticRoutingSelected(destination);
|
|
290
|
+
if (!reservation.active) return { status: "cancelled" };
|
|
291
|
+
|
|
292
|
+
const record: ContinuationRecord = Object.freeze({
|
|
293
|
+
failedTurnRef,
|
|
294
|
+
continuationTurnRef: this.#state.mintReference(),
|
|
295
|
+
destinationProviderId: destination.providerId,
|
|
296
|
+
destinationFamily: destination.family,
|
|
297
|
+
routingReason: destination.routingReason,
|
|
298
|
+
selectedAtMs: this.#now(),
|
|
299
|
+
});
|
|
300
|
+
reservation.record = record;
|
|
301
|
+
this.#records.push(record);
|
|
302
|
+
try {
|
|
303
|
+
reservation.timer = this.#scheduler.set(0, () => {
|
|
304
|
+
reservation.timer = undefined;
|
|
305
|
+
void this.#dispatchReserved(reservation, record);
|
|
306
|
+
});
|
|
307
|
+
} catch (error) {
|
|
308
|
+
// Selection already succeeded. Fall back to direct dispatch so a broken
|
|
309
|
+
// scheduler cannot turn a confirmed switch into zero continuations.
|
|
310
|
+
this.#diagnostics.recordError("continuation.schedule", error);
|
|
311
|
+
void this.#dispatchReserved(reservation, record);
|
|
312
|
+
}
|
|
313
|
+
return { status: "scheduled", record };
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
async #dispatchReserved(reservation: Reservation, record: ContinuationRecord): Promise<void> {
|
|
317
|
+
if (!reservation.active || reservation.dispatchStarted) return;
|
|
318
|
+
reservation.dispatchStarted = true;
|
|
319
|
+
try {
|
|
320
|
+
await this.#dispatchFollowUp(FIXED_CONTINUATION_MESSAGE, reservation.abortController.signal);
|
|
321
|
+
if (reservation.active) await this.#onDispatched?.(record);
|
|
322
|
+
} catch (error) {
|
|
323
|
+
// Keep the guard/reservation after dispatch failure: automatic retry is forbidden.
|
|
324
|
+
// Cancellation suppresses late callback diagnostics after teardown.
|
|
325
|
+
if (reservation.active) this.#diagnostics.recordError("continuation.dispatch", error);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
/**
|
|
330
|
+
* Hold a turn that could not be routed, and re-check on an interval until an
|
|
331
|
+
* account genuinely recovers.
|
|
332
|
+
*
|
|
333
|
+
* REQ-FAILOVER-PARK. This replaces a no-op: the `paused` branch previously
|
|
334
|
+
* recorded a diagnostic and returned, so a turn that hit a fully-exhausted
|
|
335
|
+
* fleet was abandoned with nothing ever re-checking.
|
|
336
|
+
*
|
|
337
|
+
* Returns false when this turn is already parked, so a repeated settlement
|
|
338
|
+
* cannot stack timers on one turn.
|
|
339
|
+
*/
|
|
340
|
+
park(failedTurnRef: RuntimeReference): boolean {
|
|
341
|
+
if (this.#parked.has(failedTurnRef)) return false;
|
|
342
|
+
if (this.#resolveParkedDestination === undefined) return false;
|
|
343
|
+
const parked: ParkedTurn = {
|
|
344
|
+
failedTurnRef,
|
|
345
|
+
abortController: new AbortController(),
|
|
346
|
+
parkedAtMs: this.#now(),
|
|
347
|
+
resuming: false,
|
|
348
|
+
};
|
|
349
|
+
this.#parked.set(failedTurnRef, parked);
|
|
350
|
+
this.#armParkPoll(parked);
|
|
351
|
+
return true;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/** True when this turn is currently parked. Lets callers avoid double-parking. */
|
|
355
|
+
isParked(failedTurnRef: RuntimeReference): boolean {
|
|
356
|
+
return this.#parked.has(failedTurnRef);
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
/** Number of turns currently parked, for the operator surface. */
|
|
360
|
+
parkedCount(): number {
|
|
361
|
+
return this.#parked.size;
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
/**
|
|
365
|
+
* Discard a parked turn. REQ-FAILOVER-CANCEL: the operator must be able to
|
|
366
|
+
* escape a park, and the announcement that a turn was parked states how.
|
|
367
|
+
*/
|
|
368
|
+
cancelParked(failedTurnRef: RuntimeReference): boolean {
|
|
369
|
+
const parked = this.#parked.get(failedTurnRef);
|
|
370
|
+
if (!parked) return false;
|
|
371
|
+
parked.abortController.abort();
|
|
372
|
+
if (parked.timer !== undefined) {
|
|
373
|
+
try {
|
|
374
|
+
this.#scheduler.clear(parked.timer);
|
|
375
|
+
} catch (error) {
|
|
376
|
+
this.#diagnostics.recordError("continuation.park-cancel-timer", error);
|
|
377
|
+
}
|
|
378
|
+
parked.timer = undefined;
|
|
379
|
+
}
|
|
380
|
+
this.#parked.delete(failedTurnRef);
|
|
381
|
+
this.#state.clearContinuationGuard(failedTurnRef);
|
|
382
|
+
this.#reportAutomaticRoutingCleared();
|
|
383
|
+
return true;
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
/** Discard every parked turn, used on stop and shutdown. */
|
|
387
|
+
cancelAllParked(): number {
|
|
388
|
+
let cancelled = 0;
|
|
389
|
+
for (const ref of [...this.#parked.keys()]) {
|
|
390
|
+
if (this.cancelParked(ref)) cancelled += 1;
|
|
391
|
+
}
|
|
392
|
+
return cancelled;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
#armParkPoll(parked: ParkedTurn): void {
|
|
396
|
+
// Bounded, deliberately. An account that never recovers must not leave a
|
|
397
|
+
// timer cycling for the life of the process.
|
|
398
|
+
if (this.#now() - parked.parkedAtMs >= PARK_MAX_DURATION_MS) {
|
|
399
|
+
this.#parked.delete(parked.failedTurnRef);
|
|
400
|
+
this.#state.clearContinuationGuard(parked.failedTurnRef);
|
|
401
|
+
this.#reportAutomaticRoutingCleared();
|
|
402
|
+
this.#diagnostics.record(
|
|
403
|
+
"warning",
|
|
404
|
+
"continuation.park-expired",
|
|
405
|
+
"A parked turn was abandoned after no account recovered within the park window.",
|
|
406
|
+
);
|
|
407
|
+
return;
|
|
408
|
+
}
|
|
409
|
+
try {
|
|
410
|
+
parked.timer = this.#scheduler.set(this.#parkPollMs, () => {
|
|
411
|
+
parked.timer = undefined;
|
|
412
|
+
void this.#attemptParkedResume(parked);
|
|
413
|
+
});
|
|
414
|
+
} catch (error) {
|
|
415
|
+
// A broken scheduler must not silently strand the turn forever.
|
|
416
|
+
this.#diagnostics.recordError("continuation.park-schedule", error);
|
|
417
|
+
this.#parked.delete(parked.failedTurnRef);
|
|
418
|
+
this.#reportAutomaticRoutingCleared();
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
async #attemptParkedResume(parked: ParkedTurn): Promise<void> {
|
|
423
|
+
if (parked.resuming) return;
|
|
424
|
+
if (!this.#parked.has(parked.failedTurnRef)) return;
|
|
425
|
+
if (parked.abortController.signal.aborted) return;
|
|
426
|
+
const resolve = this.#resolveParkedDestination;
|
|
427
|
+
if (resolve === undefined) return;
|
|
428
|
+
|
|
429
|
+
let destination: ContinuationDestination | undefined;
|
|
430
|
+
try {
|
|
431
|
+
destination = await resolve(this.#now());
|
|
432
|
+
} catch (error) {
|
|
433
|
+
this.#diagnostics.recordError("continuation.park-resolve", error);
|
|
434
|
+
destination = undefined;
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
// Cancellation may have landed during the await.
|
|
438
|
+
if (parked.abortController.signal.aborted) return;
|
|
439
|
+
if (!this.#parked.has(parked.failedTurnRef)) return;
|
|
440
|
+
|
|
441
|
+
if (destination === undefined) {
|
|
442
|
+
// Still nothing available. Re-arm rather than giving up: the whole point
|
|
443
|
+
// of a park is that it outlives one failed check.
|
|
444
|
+
this.#armParkPoll(parked);
|
|
445
|
+
return;
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
parked.resuming = true;
|
|
449
|
+
this.#parked.delete(parked.failedTurnRef);
|
|
450
|
+
// Release the park's hold, then resume under a FRESH reference.
|
|
451
|
+
//
|
|
452
|
+
// The original ref cannot be reused: `clearContinuationGuard` retires it, and
|
|
453
|
+
// RuntimeState then rejects it with "failedTurnRef must be minted by this
|
|
454
|
+
// RuntimeState", which surfaced as a silent `selection-failed` -- the resume
|
|
455
|
+
// appeared to run and simply never switched. Minting a new reference is also
|
|
456
|
+
// the honest model: a park that outlived its window resumes as a NEW turn
|
|
457
|
+
// rather than pretending to continue the original request.
|
|
458
|
+
this.#state.clearContinuationGuard(parked.failedTurnRef);
|
|
459
|
+
const resumeRef = this.#state.mintReference();
|
|
460
|
+
try {
|
|
461
|
+
const result = await this.start(resumeRef, destination);
|
|
462
|
+
if (result.status !== "scheduled") {
|
|
463
|
+
this.#reportAutomaticRoutingCleared();
|
|
464
|
+
this.#diagnostics.record(
|
|
465
|
+
"warning",
|
|
466
|
+
"continuation.park-resume-failed",
|
|
467
|
+
`A parked turn could not resume on ${destination.providerId} (${result.status}).`,
|
|
468
|
+
);
|
|
469
|
+
}
|
|
470
|
+
} catch (error) {
|
|
471
|
+
this.#reportAutomaticRoutingCleared();
|
|
472
|
+
this.#diagnostics.recordError("continuation.park-resume", error);
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
#releaseFailedSelection(reservation: Reservation): void {
|
|
477
|
+
reservation.active = false;
|
|
478
|
+
reservation.abortController.abort();
|
|
479
|
+
this.#reservations.delete(reservation.failedTurnRef);
|
|
480
|
+
this.#state.clearContinuationGuard(reservation.failedTurnRef);
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
cancelFailedTurn(failedTurnRef: RuntimeReference): boolean {
|
|
484
|
+
const reservation = this.#reservations.get(failedTurnRef);
|
|
485
|
+
if (!reservation) return false;
|
|
486
|
+
reservation.active = false;
|
|
487
|
+
reservation.abortController.abort();
|
|
488
|
+
if (reservation.timer !== undefined) {
|
|
489
|
+
try {
|
|
490
|
+
this.#scheduler.clear(reservation.timer);
|
|
491
|
+
} catch (error) {
|
|
492
|
+
this.#diagnostics.recordError("continuation.cancel-timer", error);
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
this.#reservations.delete(failedTurnRef);
|
|
496
|
+
try {
|
|
497
|
+
this.#state.clearContinuationGuard(failedTurnRef);
|
|
498
|
+
if (reservation.record) this.#state.clearWatchdog(reservation.record.continuationTurnRef);
|
|
499
|
+
} catch (error) {
|
|
500
|
+
this.#diagnostics.recordError("continuation.cancel", error);
|
|
501
|
+
}
|
|
502
|
+
this.#reportAutomaticRoutingCleared();
|
|
503
|
+
return true;
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
cancelAll(): void {
|
|
507
|
+
// REQ-FAILOVER-CANCEL. Parked turns are cancelled here rather than through a
|
|
508
|
+
// new subcommand, so the existing `/multi-account stop` -- which the park
|
|
509
|
+
// announcement tells the operator to run -- is the single escape hatch.
|
|
510
|
+
this.cancelAllParked();
|
|
511
|
+
for (const reservation of this.#reservations.values()) {
|
|
512
|
+
reservation.active = false;
|
|
513
|
+
reservation.abortController.abort();
|
|
514
|
+
if (reservation.timer !== undefined) {
|
|
515
|
+
try {
|
|
516
|
+
this.#scheduler.clear(reservation.timer);
|
|
517
|
+
} catch (error) {
|
|
518
|
+
this.#diagnostics.recordError("continuation.cancel-timer", error);
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
this.#reservations.clear();
|
|
523
|
+
this.#state.clearPendingActivity();
|
|
524
|
+
this.#reportAutomaticRoutingCleared();
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
async #reportAutomaticRoutingSelected(
|
|
528
|
+
destination: ContinuationDestination,
|
|
529
|
+
): Promise<void> {
|
|
530
|
+
try {
|
|
531
|
+
await this.#onAutomaticRoutingSelected?.(destination);
|
|
532
|
+
} catch {
|
|
533
|
+
this.#diagnostics.record(
|
|
534
|
+
"warning",
|
|
535
|
+
"continuation.route-callback",
|
|
536
|
+
"Automatic route selection bookkeeping failed safely.",
|
|
537
|
+
);
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
#reportAutomaticRoutingCleared(): void {
|
|
542
|
+
try {
|
|
543
|
+
const pending = this.#onAutomaticRoutingCleared?.();
|
|
544
|
+
if (pending !== undefined) {
|
|
545
|
+
void Promise.resolve(pending).catch(() => {
|
|
546
|
+
this.#diagnostics.record(
|
|
547
|
+
"warning",
|
|
548
|
+
"continuation.route-callback",
|
|
549
|
+
"Automatic route cleanup bookkeeping failed safely.",
|
|
550
|
+
);
|
|
551
|
+
});
|
|
552
|
+
}
|
|
553
|
+
} catch {
|
|
554
|
+
this.#diagnostics.record(
|
|
555
|
+
"warning",
|
|
556
|
+
"continuation.route-callback",
|
|
557
|
+
"Automatic route cleanup bookkeeping failed safely.",
|
|
558
|
+
);
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
records(): readonly ContinuationRecord[] {
|
|
563
|
+
return this.#records.map((record) => ({ ...record }));
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
pendingCount(): number {
|
|
567
|
+
return this.#reservations.size;
|
|
568
|
+
}
|
|
569
|
+
}
|
package/src/cooldowns.ts
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import type { ManagedFamily } from "./config.js";
|
|
2
|
+
import type { FailureClassification } from "./error-classification.js";
|
|
3
|
+
import type {
|
|
4
|
+
CooldownReason,
|
|
5
|
+
CooldownRecord,
|
|
6
|
+
NumericServerHint,
|
|
7
|
+
} from "./runtime-state.js";
|
|
8
|
+
|
|
9
|
+
export const ABSOLUTE_COOLDOWN_MAX_MS = 3_600_000;
|
|
10
|
+
|
|
11
|
+
export const BASE_COOLDOWN_MS: Readonly<Record<CooldownReason, number>> = {
|
|
12
|
+
quota: 60_000,
|
|
13
|
+
"rate-limit": 60_000,
|
|
14
|
+
"auth-transient": 30_000,
|
|
15
|
+
permission: 120_000,
|
|
16
|
+
transport: 30_000,
|
|
17
|
+
unknown: 30_000,
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
function finiteNonNegative(value: number | undefined): number | undefined {
|
|
21
|
+
return value !== undefined && Number.isFinite(value) && value >= 0
|
|
22
|
+
? value
|
|
23
|
+
: undefined;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Returns a bounded numeric-only copy suitable for process-local state. */
|
|
27
|
+
export function boundServerHint(
|
|
28
|
+
hint: NumericServerHint | undefined,
|
|
29
|
+
nowMs: number,
|
|
30
|
+
): NumericServerHint | undefined {
|
|
31
|
+
if (hint === undefined) return undefined;
|
|
32
|
+
const retryAfter = finiteNonNegative(hint.retryAfterSeconds);
|
|
33
|
+
const resetAt = finiteNonNegative(hint.resetAtMs);
|
|
34
|
+
if (retryAfter === undefined && resetAt === undefined) return undefined;
|
|
35
|
+
return {
|
|
36
|
+
...(retryAfter === undefined
|
|
37
|
+
? {}
|
|
38
|
+
: { retryAfterSeconds: Math.min(Math.floor(retryAfter), 3_600) }),
|
|
39
|
+
...(resetAt === undefined
|
|
40
|
+
? {}
|
|
41
|
+
: { resetAtMs: Math.min(resetAt, nowMs + ABSOLUTE_COOLDOWN_MAX_MS) }),
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Computes the fixed-base MVP cooldown. Retry-After may shorten the base;
|
|
47
|
+
* reset-at may lengthen it. Every path is capped by both config and one hour.
|
|
48
|
+
*/
|
|
49
|
+
export function calculateCooldownDurationMs(options: {
|
|
50
|
+
reason: CooldownReason;
|
|
51
|
+
nowMs: number;
|
|
52
|
+
configuredMaxMs: number;
|
|
53
|
+
serverHint?: NumericServerHint;
|
|
54
|
+
}): number {
|
|
55
|
+
const { reason, nowMs, configuredMaxMs } = options;
|
|
56
|
+
if (!Number.isFinite(nowMs) || nowMs < 0) {
|
|
57
|
+
throw new TypeError("nowMs must be finite and non-negative.");
|
|
58
|
+
}
|
|
59
|
+
if (!Number.isFinite(configuredMaxMs) || configuredMaxMs < 0) {
|
|
60
|
+
throw new TypeError("configuredMaxMs must be finite and non-negative.");
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const maximum = Math.min(configuredMaxMs, ABSOLUTE_COOLDOWN_MAX_MS);
|
|
64
|
+
const hint = boundServerHint(options.serverHint, nowMs);
|
|
65
|
+
let duration = BASE_COOLDOWN_MS[reason];
|
|
66
|
+
if (hint?.retryAfterSeconds !== undefined) {
|
|
67
|
+
duration = Math.min(duration, hint.retryAfterSeconds * 1_000);
|
|
68
|
+
}
|
|
69
|
+
if (hint?.resetAtMs !== undefined) {
|
|
70
|
+
duration = Math.max(duration, hint.resetAtMs - nowMs);
|
|
71
|
+
}
|
|
72
|
+
return Math.max(0, Math.min(Math.ceil(duration), maximum));
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export function createCooldownRecord(options: {
|
|
76
|
+
providerId: string;
|
|
77
|
+
family: ManagedFamily;
|
|
78
|
+
classification: FailureClassification;
|
|
79
|
+
nowMs: number;
|
|
80
|
+
configuredMaxMs: number;
|
|
81
|
+
}): CooldownRecord | undefined {
|
|
82
|
+
const { classification } = options;
|
|
83
|
+
if (
|
|
84
|
+
classification.accountAction !== "cooldown-and-route" ||
|
|
85
|
+
classification.cooldownReason === undefined
|
|
86
|
+
) {
|
|
87
|
+
return undefined;
|
|
88
|
+
}
|
|
89
|
+
const serverHint = boundServerHint(classification.serverHint, options.nowMs);
|
|
90
|
+
const durationMs = calculateCooldownDurationMs({
|
|
91
|
+
reason: classification.cooldownReason,
|
|
92
|
+
nowMs: options.nowMs,
|
|
93
|
+
configuredMaxMs: options.configuredMaxMs,
|
|
94
|
+
...(serverHint === undefined ? {} : { serverHint }),
|
|
95
|
+
});
|
|
96
|
+
return {
|
|
97
|
+
providerId: options.providerId,
|
|
98
|
+
family: options.family,
|
|
99
|
+
reason: classification.cooldownReason,
|
|
100
|
+
untilMs: options.nowMs + durationMs,
|
|
101
|
+
...(serverHint === undefined ? {} : { serverHint }),
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export function remainingCooldownMs(
|
|
106
|
+
record: CooldownRecord,
|
|
107
|
+
nowMs: number,
|
|
108
|
+
): number {
|
|
109
|
+
return Math.max(0, record.untilMs - nowMs);
|
|
110
|
+
}
|