@yolk-sdk/pool 0.0.0-stage → 0.1.0-canary.116

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/pool.mjs ADDED
@@ -0,0 +1,290 @@
1
+ import { PoolAuthCooldown, PoolConfigError, PoolPin, PoolQuotaCooldown, PoolUnknownAccountError } from "./model.mjs";
2
+ import { isCooldownActive, poolReadingFromSnapshot, poolRoute, rankPool, staleQuotaCooldowns } from "./rank.mjs";
3
+ import { PoolStoredReading } from "./store.mjs";
4
+ import { Clock, Effect, Match, Predicate, Ref, Stream } from "effect";
5
+ import { LLMError } from "@yolk-sdk/agent/loop";
6
+ import { ProviderErrorInfo } from "@yolk-sdk/agent/protocol";
7
+ //#region src/pool.ts
8
+ /** Default `maxAgeMs`: readings older than 30 minutes no longer rank accounts. */
9
+ const defaultPoolMaxAgeMs = 30 * 6e4;
10
+ /**
11
+ * A `rate_limit` whose `Retry-After` reaches this is a quota lockout, not a transient limit
12
+ * (CLIProxyAPI `expiring-first` `QuotaCooldownFloor`).
13
+ */
14
+ const poolQuotaRetryAfterFloorMs = 30 * 6e4;
15
+ /** The longest cooldown the pool records; a later or non-finite hint is not trusted. */
16
+ const poolMaxCooldownMs = 366 * 24 * 60 * 6e4;
17
+ /** How long an account whose credentials were rejected sits out. */
18
+ const poolAuthCooldownMs = 30 * 6e4;
19
+ const configError = (reason, message) => Effect.fail(new PoolConfigError({
20
+ reason,
21
+ message
22
+ }));
23
+ /**
24
+ * The cooldown an account-scoped failure puts its account in, or undefined when the failure is not
25
+ * the account's (the loop's retry handles those): a `rate_limit` with a quota reset or a
26
+ * `Retry-After` of at least `poolQuotaRetryAfterFloorMs` (quota), or `auth`.
27
+ */
28
+ const poolFailureCooldown = (error, accountId, nowMs) => {
29
+ const info = error.provider;
30
+ if (info === void 0) return;
31
+ if (info.kind === "auth") return PoolAuthCooldown.make({
32
+ accountId,
33
+ startedAtMs: nowMs,
34
+ untilMs: nowMs + poolAuthCooldownMs
35
+ });
36
+ if (info.kind !== "rate_limit") return;
37
+ const trusted = (untilMs) => Number.isFinite(untilMs) && untilMs > nowMs && untilMs - nowMs <= 316224e5;
38
+ if (info.quotaResetsAtMs !== void 0 && trusted(info.quotaResetsAtMs)) return PoolQuotaCooldown.make({
39
+ accountId,
40
+ startedAtMs: nowMs,
41
+ untilMs: info.quotaResetsAtMs
42
+ });
43
+ if (info.retryAfterMs !== void 0 && info.retryAfterMs >= 18e5 && trusted(nowMs + info.retryAfterMs)) return PoolQuotaCooldown.make({
44
+ accountId,
45
+ startedAtMs: nowMs,
46
+ untilMs: nowMs + info.retryAfterMs
47
+ });
48
+ };
49
+ const iso = (ms) => {
50
+ const date = new Date(ms);
51
+ return Number.isFinite(date.getTime()) ? date.toISOString() : String(ms);
52
+ };
53
+ const minUntil = (cooldowns) => cooldowns.reduce((earliest, cooldown) => earliest === void 0 || cooldown.untilMs < earliest ? cooldown.untilMs : earliest, void 0);
54
+ const emptySnapshot = {
55
+ pin: void 0,
56
+ cooldowns: [],
57
+ readings: []
58
+ };
59
+ /**
60
+ * Pools several subscription accounts of one provider behind one `LLMProvider` (ADR 0011).
61
+ *
62
+ * Each request goes to the account whose longest usage window resets soonest and still has quota
63
+ * (expiring-first), or, without fresh readings, stays on the route's pinned account. An
64
+ * account-scoped failure before the first event (an exhausted quota, rejected credentials) cools the
65
+ * account down and the same request moves to the next account; every other failure, and any failure
66
+ * after an event, reaches the caller unchanged. With every account out it fails `rate_limit`
67
+ * (non-retryable) carrying the earliest quota reset, or `auth` when only auth cooldowns remain.
68
+ *
69
+ * Store failures never fail a request: reads fall back to stateless fill-first and writes are
70
+ * logged.
71
+ */
72
+ const makePool = (options) => Effect.gen(function* () {
73
+ const poolId = options.id.trim();
74
+ if (poolId === "") return yield* configError("invalid_pool_id", "Pool id must be a non-empty string");
75
+ if (options.accounts.length === 0) return yield* configError("no_accounts", `Pool ${poolId} needs at least one account`);
76
+ const maxAgeMs = options.maxAgeMs ?? 18e5;
77
+ const reservePercent = options.reservePercent ?? 0;
78
+ if (!Number.isFinite(maxAgeMs) || maxAgeMs <= 0) return yield* configError("invalid_max_age", "maxAgeMs must be a positive number");
79
+ if (!Number.isFinite(reservePercent) || reservePercent < 0 || reservePercent >= 100) return yield* configError("invalid_reserve_percent", "reservePercent must be at least 0 and below 100");
80
+ const accounts = [];
81
+ for (const account of options.accounts) {
82
+ const id = account.id.trim();
83
+ const priority = account.priority ?? 0;
84
+ if (id === "" || id !== account.id) return yield* configError("invalid_account_id", "Account ids must be non-empty without surrounding whitespace");
85
+ if (accounts.some((existing) => existing.id === id)) return yield* configError("duplicate_account_id", `Duplicate account id: ${id}`);
86
+ if (!Number.isFinite(priority)) return yield* configError("invalid_priority", `Account ${id} has a non-finite priority`);
87
+ accounts.push({
88
+ id,
89
+ priority,
90
+ provider: account.provider
91
+ });
92
+ }
93
+ const accountIds = accounts.map((account) => account.id);
94
+ const accountsById = new Map(accounts.map((account) => [account.id, account]));
95
+ const store = options.store;
96
+ const loadSnapshot = (route) => store.load({
97
+ poolId,
98
+ route,
99
+ accountIds
100
+ }).pipe(Effect.catch((error) => Effect.logWarning("pool: store load failed; routing without pool state").pipe(Effect.annotateLogs({ operation: error.operation }), Effect.as(emptySnapshot))));
101
+ const logStoreFailure = (operation) => () => Effect.logWarning(`pool: store ${operation} failed`);
102
+ const readingsOf = (snapshot) => {
103
+ const readings = /* @__PURE__ */ new Map();
104
+ for (const stored of snapshot.readings) {
105
+ const reading = poolReadingFromSnapshot(stored.snapshot);
106
+ if (reading !== void 0) readings.set(stored.accountId, reading);
107
+ }
108
+ return readings;
109
+ };
110
+ const moveReason = (input) => Match.value(input.decision).pipe(Match.tag("Quota", (decision) => decision.resetsAtMs === void 0 ? "quota ranking, window not started" : `quota ranking, window resets ${iso(decision.resetsAtMs)}`), Match.orElse(() => {
111
+ const cooldown = input.cooldowns.find((current) => current.accountId === input.from);
112
+ if (cooldown !== void 0) return Match.value(cooldown).pipe(Match.tag("PoolQuotaCooldown", (current) => `quota cooldown until ${iso(current.untilMs)}`), Match.tag("PoolAuthCooldown", (current) => `auth cooldown until ${iso(current.untilMs)}`), Match.exhaustive);
113
+ if (input.excluded.has(input.from)) return "failed this request";
114
+ return accountsById.has(input.from) ? "a fresh reading shows it used up" : "not configured";
115
+ }));
116
+ const updatePin = (input) => {
117
+ const current = input.snapshot.pin?.accountId;
118
+ const next = input.decision.accountId;
119
+ if (current === next) return Effect.void;
120
+ const pin = PoolPin.make({
121
+ poolId,
122
+ route: input.route,
123
+ accountId: next,
124
+ pinnedAtMs: input.nowMs
125
+ });
126
+ const log = current === void 0 ? Effect.logDebug("pool: route pinned").pipe(Effect.annotateLogs({ to: next })) : Effect.logInfo(`pool: pin moved: ${moveReason({
127
+ decision: input.decision,
128
+ from: current,
129
+ cooldowns: input.cooldowns,
130
+ excluded: input.excluded
131
+ })}`).pipe(Effect.annotateLogs({
132
+ from: current,
133
+ to: next
134
+ }));
135
+ return store.movePin({
136
+ expected: current,
137
+ pin
138
+ }).pipe(Effect.flatMap((written) => written ? log : Effect.void), Effect.catch(logStoreFailure("movePin")));
139
+ };
140
+ const exhaustedError = (input) => {
141
+ const isQuota = (cooldown) => Predicate.isTagged(cooldown, "PoolQuotaCooldown");
142
+ const quotaResetsAtMs = minUntil(input.active.filter(isQuota));
143
+ const until = minUntil(input.active);
144
+ const provider = input.lastError?.provider?.provider ?? poolId;
145
+ const message = until === void 0 ? `Every account of pool ${poolId} is unavailable` : `Every account of pool ${poolId} is unavailable until ${iso(until)}`;
146
+ if (!input.all.some(isQuota)) return new LLMError({
147
+ cause: "provider_error",
148
+ message,
149
+ retryable: false,
150
+ provider: ProviderErrorInfo.make({
151
+ provider,
152
+ kind: "auth"
153
+ })
154
+ });
155
+ return new LLMError({
156
+ cause: "rate_limit",
157
+ message,
158
+ retryable: false,
159
+ provider: quotaResetsAtMs === void 0 ? ProviderErrorInfo.make({
160
+ provider,
161
+ kind: "rate_limit"
162
+ }) : ProviderErrorInfo.make({
163
+ provider,
164
+ kind: "rate_limit",
165
+ quotaResetsAtMs
166
+ })
167
+ });
168
+ };
169
+ const attempt = (input) => Stream.unwrap(Ref.make(false).pipe(Effect.map((emitted) => input.account.provider.stream(input.request).pipe(Stream.tap(() => Ref.set(emitted, true)), Stream.catchTags({ LLMError: (error) => Stream.unwrap(Effect.gen(function* () {
170
+ const nowMs = yield* Clock.currentTimeMillis;
171
+ const cooldown = poolFailureCooldown(error, input.account.id, nowMs);
172
+ if ((yield* Ref.get(emitted)) || cooldown === void 0) return Stream.fail(error);
173
+ yield* store.putCooldown(cooldown).pipe(Effect.catch(logStoreFailure("putCooldown")));
174
+ yield* Effect.logInfo(`pool: failover after ${Predicate.isTagged(cooldown, "PoolQuotaCooldown") ? "quota exhaustion" : "auth failure"}, cooldown until ${iso(cooldown.untilMs)}`).pipe(Effect.annotateLogs({ account: input.account.id }));
175
+ return pick({
176
+ request: input.request,
177
+ route: input.route,
178
+ excluded: new Set([...input.excluded, input.account.id]),
179
+ recorded: [...input.recorded, cooldown],
180
+ lastError: error
181
+ });
182
+ })) })))));
183
+ const pick = (input) => Stream.unwrap(Effect.gen(function* () {
184
+ const nowMs = yield* Clock.currentTimeMillis;
185
+ const snapshot = yield* loadSnapshot(input.route);
186
+ const readings = readingsOf(snapshot);
187
+ const stale = staleQuotaCooldowns({
188
+ cooldowns: snapshot.cooldowns,
189
+ readings,
190
+ nowMs,
191
+ maxAgeMs
192
+ });
193
+ yield* Effect.forEach(stale, (cooldown) => store.clearCooldown({
194
+ accountId: cooldown.accountId,
195
+ startedAtMs: cooldown.startedAtMs
196
+ }).pipe(Effect.flatMap((cleared) => cleared ? Effect.logInfo(`pool: cleared cooldown (until ${iso(cooldown.untilMs)}): a newer reading shows quota left`).pipe(Effect.annotateLogs({ account: cooldown.accountId })) : Effect.void), Effect.catch(logStoreFailure("clearCooldown"))), { discard: true });
197
+ const cooldowns = [...snapshot.cooldowns.filter((cooldown) => isCooldownActive(cooldown, nowMs) && !stale.includes(cooldown)), ...input.recorded.filter((cooldown) => isCooldownActive(cooldown, nowMs))];
198
+ const decision = rankPool({
199
+ accounts,
200
+ unavailable: new Set([...input.excluded, ...cooldowns.map((cooldown) => cooldown.accountId)]),
201
+ readings,
202
+ pin: snapshot.pin?.accountId,
203
+ route: input.route,
204
+ nowMs,
205
+ maxAgeMs,
206
+ reservePercent
207
+ });
208
+ const account = decision === void 0 ? void 0 : accountsById.get(decision.accountId);
209
+ if (decision === void 0 || account === void 0) {
210
+ yield* Effect.logWarning("pool: every account is unavailable");
211
+ return Stream.fail(exhaustedError({
212
+ active: cooldowns,
213
+ all: [...cooldowns, ...input.recorded.filter((cooldown) => !cooldowns.includes(cooldown))],
214
+ lastError: input.lastError
215
+ }));
216
+ }
217
+ yield* updatePin({
218
+ route: input.route,
219
+ decision,
220
+ snapshot,
221
+ cooldowns,
222
+ excluded: input.excluded,
223
+ nowMs
224
+ });
225
+ yield* Effect.annotateCurrentSpan({
226
+ account: account.id,
227
+ source: decision._tag
228
+ });
229
+ return attempt({
230
+ request: input.request,
231
+ route: input.route,
232
+ account,
233
+ excluded: input.excluded,
234
+ recorded: input.recorded
235
+ });
236
+ }).pipe(Effect.withSpan("pool.pick", { attributes: {
237
+ pool: poolId,
238
+ route: input.route
239
+ } }), Effect.annotateLogs({
240
+ pool: poolId,
241
+ route: input.route
242
+ })));
243
+ const provider = { stream: (request) => {
244
+ const route = poolRoute(request.model);
245
+ return route === "" ? Stream.fail(new LLMError({
246
+ cause: "validation_error",
247
+ message: `Pool ${poolId} needs a model id`,
248
+ retryable: false
249
+ })) : pick({
250
+ request,
251
+ route,
252
+ excluded: /* @__PURE__ */ new Set(),
253
+ recorded: [],
254
+ lastError: void 0
255
+ });
256
+ } };
257
+ const recordReading = (accountId, snapshot) => accountsById.has(accountId) ? store.putReading(PoolStoredReading.make({
258
+ accountId,
259
+ snapshot
260
+ })) : Effect.fail(new PoolUnknownAccountError({
261
+ poolId,
262
+ accountId
263
+ }));
264
+ return {
265
+ id: poolId,
266
+ provider,
267
+ recordReading,
268
+ state: Effect.gen(function* () {
269
+ const nowMs = yield* Clock.currentTimeMillis;
270
+ const stored = yield* store.state({
271
+ poolId,
272
+ accountIds
273
+ });
274
+ return {
275
+ poolId,
276
+ pins: stored.pins,
277
+ accounts: accounts.map((account) => ({
278
+ id: account.id,
279
+ priority: account.priority,
280
+ cooldown: stored.cooldowns.find((cooldown) => cooldown.accountId === account.id && isCooldownActive(cooldown, nowMs)),
281
+ reading: stored.readings.find((reading) => reading.accountId === account.id)?.snapshot
282
+ }))
283
+ };
284
+ })
285
+ };
286
+ });
287
+ //#endregion
288
+ export { defaultPoolMaxAgeMs, makePool, poolAuthCooldownMs, poolFailureCooldown, poolMaxCooldownMs, poolQuotaRetryAfterFloorMs };
289
+
290
+ //# sourceMappingURL=pool.mjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"pool.mjs","names":[],"sources":["../src/pool.ts"],"sourcesContent":["import { Clock, Effect, Match, Predicate, Ref, Stream } from 'effect'\nimport {\n LLMError,\n type LLMEvent,\n type LLMProvider,\n type LLMProviderError,\n type LLMRequest\n} from '@yolk-sdk/agent/loop'\nimport { ProviderErrorInfo } from '@yolk-sdk/agent/protocol'\nimport type { ProviderSubscriptionUsageSnapshot } from '@yolk-sdk/agent/providers/subscription-usage'\nimport {\n PoolAuthCooldown,\n PoolConfigError,\n type PoolCooldown,\n PoolPin,\n PoolQuotaCooldown,\n type PoolStoreError,\n PoolUnknownAccountError\n} from './model.ts'\nimport {\n isCooldownActive,\n type PoolCandidate,\n type PoolDecision,\n type PoolReading,\n poolReadingFromSnapshot,\n poolRoute,\n rankPool,\n staleQuotaCooldowns\n} from './rank.ts'\nimport { type PoolStoreApi, type PoolStoreSnapshot, PoolStoredReading } from './store.ts'\n\ntype LLMProviderService = typeof LLMProvider.Service\n\n/** One subscription account of a pool: its own provider, built by the host with its credentials. */\nexport type PoolAccount = {\n /** Unique within the store; cooldowns and readings are keyed by it. */\n readonly id: string\n /** Breaks ranking ties and orders fill-first (higher first). Default 0. */\n readonly priority?: number\n readonly provider: LLMProviderService\n}\n\nexport type PoolOptions = {\n /** Pins are keyed by pool id and route. */\n readonly id: string\n /** Accounts of one provider; requests must suit every one of them. */\n readonly accounts: ReadonlyArray<PoolAccount>\n readonly store: PoolStoreApi\n /** How old a reading may be and still rank accounts. Default 30 minutes. */\n readonly maxAgeMs?: number\n /** Quota left untouched on every window, in percent (0 to below 100). Default 0. */\n readonly reservePercent?: number\n}\n\n/** One account as the host UI sees it. */\nexport type PoolAccountState = {\n readonly id: string\n readonly priority: number\n /** The account's cooldown while it is active. */\n readonly cooldown: PoolCooldown | undefined\n readonly reading: ProviderSubscriptionUsageSnapshot | undefined\n}\n\nexport type PoolState = {\n readonly poolId: string\n /** The account serving each route. */\n readonly pins: ReadonlyArray<PoolPin>\n readonly accounts: ReadonlyArray<PoolAccountState>\n}\n\nexport type Pool = {\n readonly id: string\n /** The pooled provider: provide it as the loop's `LLMProvider`, innermost under host wrappers. */\n readonly provider: LLMProviderService\n /** Records an account's usage snapshot; hosts poll usage on their own schedule. */\n readonly recordReading: (\n accountId: string,\n snapshot: ProviderSubscriptionUsageSnapshot\n ) => Effect.Effect<void, PoolUnknownAccountError | PoolStoreError>\n /** Pins, active cooldowns, and readings for host UIs. */\n readonly state: Effect.Effect<PoolState, PoolStoreError>\n}\n\n/** Default `maxAgeMs`: readings older than 30 minutes no longer rank accounts. */\nexport const defaultPoolMaxAgeMs = 30 * 60_000\n\n/**\n * A `rate_limit` whose `Retry-After` reaches this is a quota lockout, not a transient limit\n * (CLIProxyAPI `expiring-first` `QuotaCooldownFloor`).\n */\nexport const poolQuotaRetryAfterFloorMs = 30 * 60_000\n\n/** The longest cooldown the pool records; a later or non-finite hint is not trusted. */\nexport const poolMaxCooldownMs = 366 * 24 * 60 * 60_000\n\n/** How long an account whose credentials were rejected sits out. */\nexport const poolAuthCooldownMs = 30 * 60_000\n\ntype ValidAccount = PoolCandidate & { readonly provider: LLMProviderService }\n\nconst configError = (reason: PoolConfigError['reason'], message: string) =>\n Effect.fail(new PoolConfigError({ reason, message }))\n\n/**\n * The cooldown an account-scoped failure puts its account in, or undefined when the failure is not\n * the account's (the loop's retry handles those): a `rate_limit` with a quota reset or a\n * `Retry-After` of at least `poolQuotaRetryAfterFloorMs` (quota), or `auth`.\n */\nexport const poolFailureCooldown = (\n error: LLMError,\n accountId: string,\n nowMs: number\n): PoolCooldown | undefined => {\n const info = error.provider\n\n if (info === undefined) {\n return undefined\n }\n\n if (info.kind === 'auth') {\n return PoolAuthCooldown.make({\n accountId,\n startedAtMs: nowMs,\n untilMs: nowMs + poolAuthCooldownMs\n })\n }\n\n if (info.kind !== 'rate_limit') {\n return undefined\n }\n\n const trusted = (untilMs: number) =>\n Number.isFinite(untilMs) && untilMs > nowMs && untilMs - nowMs <= poolMaxCooldownMs\n\n if (info.quotaResetsAtMs !== undefined && trusted(info.quotaResetsAtMs)) {\n return PoolQuotaCooldown.make({ accountId, startedAtMs: nowMs, untilMs: info.quotaResetsAtMs })\n }\n\n if (\n info.retryAfterMs !== undefined &&\n info.retryAfterMs >= poolQuotaRetryAfterFloorMs &&\n trusted(nowMs + info.retryAfterMs)\n ) {\n return PoolQuotaCooldown.make({\n accountId,\n startedAtMs: nowMs,\n untilMs: nowMs + info.retryAfterMs\n })\n }\n\n return undefined\n}\n\nconst iso = (ms: number) => {\n const date = new Date(ms)\n\n return Number.isFinite(date.getTime()) ? date.toISOString() : String(ms)\n}\n\nconst minUntil = (cooldowns: ReadonlyArray<PoolCooldown>) =>\n cooldowns.reduce<number | undefined>(\n (earliest, cooldown) =>\n earliest === undefined || cooldown.untilMs < earliest ? cooldown.untilMs : earliest,\n undefined\n )\n\nconst emptySnapshot: PoolStoreSnapshot = { pin: undefined, cooldowns: [], readings: [] }\n\n/**\n * Pools several subscription accounts of one provider behind one `LLMProvider` (ADR 0011).\n *\n * Each request goes to the account whose longest usage window resets soonest and still has quota\n * (expiring-first), or, without fresh readings, stays on the route's pinned account. An\n * account-scoped failure before the first event (an exhausted quota, rejected credentials) cools the\n * account down and the same request moves to the next account; every other failure, and any failure\n * after an event, reaches the caller unchanged. With every account out it fails `rate_limit`\n * (non-retryable) carrying the earliest quota reset, or `auth` when only auth cooldowns remain.\n *\n * Store failures never fail a request: reads fall back to stateless fill-first and writes are\n * logged.\n */\nexport const makePool = (options: PoolOptions): Effect.Effect<Pool, PoolConfigError> =>\n Effect.gen(function* () {\n const poolId = options.id.trim()\n\n if (poolId === '') {\n return yield* configError('invalid_pool_id', 'Pool id must be a non-empty string')\n }\n\n if (options.accounts.length === 0) {\n return yield* configError('no_accounts', `Pool ${poolId} needs at least one account`)\n }\n\n const maxAgeMs = options.maxAgeMs ?? defaultPoolMaxAgeMs\n const reservePercent = options.reservePercent ?? 0\n\n if (!Number.isFinite(maxAgeMs) || maxAgeMs <= 0) {\n return yield* configError('invalid_max_age', 'maxAgeMs must be a positive number')\n }\n\n if (!Number.isFinite(reservePercent) || reservePercent < 0 || reservePercent >= 100) {\n return yield* configError(\n 'invalid_reserve_percent',\n 'reservePercent must be at least 0 and below 100'\n )\n }\n\n const accounts: Array<ValidAccount> = []\n\n for (const account of options.accounts) {\n const id = account.id.trim()\n const priority = account.priority ?? 0\n\n if (id === '' || id !== account.id) {\n return yield* configError(\n 'invalid_account_id',\n 'Account ids must be non-empty without surrounding whitespace'\n )\n }\n\n if (accounts.some(existing => existing.id === id)) {\n return yield* configError('duplicate_account_id', `Duplicate account id: ${id}`)\n }\n\n if (!Number.isFinite(priority)) {\n return yield* configError('invalid_priority', `Account ${id} has a non-finite priority`)\n }\n\n accounts.push({ id, priority, provider: account.provider })\n }\n\n const accountIds = accounts.map(account => account.id)\n const accountsById = new Map(accounts.map(account => [account.id, account]))\n const store = options.store\n\n const loadSnapshot = (route: string) =>\n store\n .load({ poolId, route, accountIds })\n .pipe(\n Effect.catch(error =>\n Effect.logWarning('pool: store load failed; routing without pool state').pipe(\n Effect.annotateLogs({ operation: error.operation }),\n Effect.as(emptySnapshot)\n )\n )\n )\n\n // Host store messages may carry connection details: log the operation only.\n const logStoreFailure = (operation: PoolStoreError['operation']) => () =>\n Effect.logWarning(`pool: store ${operation} failed`)\n\n const readingsOf = (snapshot: PoolStoreSnapshot) => {\n const readings = new Map<string, PoolReading>()\n\n for (const stored of snapshot.readings) {\n const reading = poolReadingFromSnapshot(stored.snapshot)\n\n if (reading !== undefined) {\n readings.set(stored.accountId, reading)\n }\n }\n\n return readings\n }\n\n const moveReason = (input: {\n readonly decision: PoolDecision\n readonly from: string\n readonly cooldowns: ReadonlyArray<PoolCooldown>\n readonly excluded: ReadonlySet<string>\n }) =>\n Match.value(input.decision).pipe(\n Match.tag('Quota', decision =>\n decision.resetsAtMs === undefined\n ? 'quota ranking, window not started'\n : `quota ranking, window resets ${iso(decision.resetsAtMs)}`\n ),\n Match.orElse(() => {\n const cooldown = input.cooldowns.find(current => current.accountId === input.from)\n\n if (cooldown !== undefined) {\n return Match.value(cooldown).pipe(\n Match.tag(\n 'PoolQuotaCooldown',\n current => `quota cooldown until ${iso(current.untilMs)}`\n ),\n Match.tag(\n 'PoolAuthCooldown',\n current => `auth cooldown until ${iso(current.untilMs)}`\n ),\n Match.exhaustive\n )\n }\n\n if (input.excluded.has(input.from)) {\n return 'failed this request'\n }\n\n return accountsById.has(input.from)\n ? 'a fresh reading shows it used up'\n : 'not configured'\n })\n )\n\n const updatePin = (input: {\n readonly route: string\n readonly decision: PoolDecision\n readonly snapshot: PoolStoreSnapshot\n readonly cooldowns: ReadonlyArray<PoolCooldown>\n readonly excluded: ReadonlySet<string>\n readonly nowMs: number\n }) => {\n const current = input.snapshot.pin?.accountId\n const next = input.decision.accountId\n\n if (current === next) {\n return Effect.void\n }\n\n const pin = PoolPin.make({\n poolId,\n route: input.route,\n accountId: next,\n pinnedAtMs: input.nowMs\n })\n\n const log =\n current === undefined\n ? Effect.logDebug('pool: route pinned').pipe(Effect.annotateLogs({ to: next }))\n : Effect.logInfo(\n `pool: pin moved: ${moveReason({\n decision: input.decision,\n from: current,\n cooldowns: input.cooldowns,\n excluded: input.excluded\n })}`\n ).pipe(Effect.annotateLogs({ from: current, to: next }))\n\n return store.movePin({ expected: current, pin }).pipe(\n Effect.flatMap(written => (written ? log : Effect.void)),\n Effect.catch(logStoreFailure('movePin'))\n )\n }\n\n const exhaustedError = (input: {\n /** Cooldowns still active at the final pick. */\n readonly active: ReadonlyArray<PoolCooldown>\n /** Every cooldown that put an account out, including ones this request recorded that ran out. */\n readonly all: ReadonlyArray<PoolCooldown>\n readonly lastError: LLMError | undefined\n }) => {\n const isQuota = (cooldown: PoolCooldown) => Predicate.isTagged(cooldown, 'PoolQuotaCooldown')\n const quotaResetsAtMs = minUntil(input.active.filter(isQuota))\n const until = minUntil(input.active)\n // The vendor id of the last failure; the pool id when no account was tried in this request.\n const provider = input.lastError?.provider?.provider ?? poolId\n\n const message =\n until === undefined\n ? `Every account of pool ${poolId} is unavailable`\n : `Every account of pool ${poolId} is unavailable until ${iso(until)}`\n\n if (!input.all.some(isQuota)) {\n return new LLMError({\n cause: 'provider_error',\n message,\n retryable: false,\n provider: ProviderErrorInfo.make({ provider, kind: 'auth' })\n })\n }\n\n return new LLMError({\n cause: 'rate_limit',\n message,\n retryable: false,\n provider:\n quotaResetsAtMs === undefined\n ? ProviderErrorInfo.make({ provider, kind: 'rate_limit' })\n : ProviderErrorInfo.make({ provider, kind: 'rate_limit', quotaResetsAtMs })\n })\n }\n\n const attempt = (input: {\n readonly request: LLMRequest\n readonly route: string\n readonly account: ValidAccount\n readonly excluded: ReadonlySet<string>\n readonly recorded: ReadonlyArray<PoolCooldown>\n }): Stream.Stream<LLMEvent, LLMProviderError> =>\n Stream.unwrap(\n Ref.make(false).pipe(\n Effect.map(emitted =>\n input.account.provider.stream(input.request).pipe(\n Stream.tap(() => Ref.set(emitted, true)),\n Stream.catchTags({\n LLMError: error =>\n Stream.unwrap(\n Effect.gen(function* () {\n const nowMs = yield* Clock.currentTimeMillis\n const cooldown = poolFailureCooldown(error, input.account.id, nowMs)\n\n if ((yield* Ref.get(emitted)) || cooldown === undefined) {\n return Stream.fail(error)\n }\n\n yield* store\n .putCooldown(cooldown)\n .pipe(Effect.catch(logStoreFailure('putCooldown')))\n yield* Effect.logInfo(\n `pool: failover after ${Predicate.isTagged(cooldown, 'PoolQuotaCooldown') ? 'quota exhaustion' : 'auth failure'}, cooldown until ${iso(cooldown.untilMs)}`\n ).pipe(Effect.annotateLogs({ account: input.account.id }))\n\n return pick({\n request: input.request,\n route: input.route,\n excluded: new Set([...input.excluded, input.account.id]),\n recorded: [...input.recorded, cooldown],\n lastError: error\n })\n })\n )\n })\n )\n )\n )\n )\n\n const pick = (input: {\n readonly request: LLMRequest\n readonly route: string\n readonly excluded: ReadonlySet<string>\n readonly recorded: ReadonlyArray<PoolCooldown>\n readonly lastError: LLMError | undefined\n }): Stream.Stream<LLMEvent, LLMProviderError> =>\n Stream.unwrap(\n Effect.gen(function* () {\n const nowMs = yield* Clock.currentTimeMillis\n const snapshot = yield* loadSnapshot(input.route)\n const readings = readingsOf(snapshot)\n\n const stale = staleQuotaCooldowns({\n cooldowns: snapshot.cooldowns,\n readings,\n nowMs,\n maxAgeMs\n })\n\n yield* Effect.forEach(\n stale,\n cooldown =>\n store\n .clearCooldown({ accountId: cooldown.accountId, startedAtMs: cooldown.startedAtMs })\n .pipe(\n Effect.flatMap(cleared =>\n cleared\n ? Effect.logInfo(\n `pool: cleared cooldown (until ${iso(cooldown.untilMs)}): a newer reading shows quota left`\n ).pipe(Effect.annotateLogs({ account: cooldown.accountId }))\n : Effect.void\n ),\n Effect.catch(logStoreFailure('clearCooldown'))\n ),\n { discard: true }\n )\n\n const cooldowns = [\n ...snapshot.cooldowns.filter(\n cooldown => isCooldownActive(cooldown, nowMs) && !stale.includes(cooldown)\n ),\n ...input.recorded.filter(cooldown => isCooldownActive(cooldown, nowMs))\n ]\n\n const unavailable = new Set([\n ...input.excluded,\n ...cooldowns.map(cooldown => cooldown.accountId)\n ])\n\n const decision = rankPool({\n accounts,\n unavailable,\n readings,\n pin: snapshot.pin?.accountId,\n route: input.route,\n nowMs,\n maxAgeMs,\n reservePercent\n })\n\n const account = decision === undefined ? undefined : accountsById.get(decision.accountId)\n\n if (decision === undefined || account === undefined) {\n yield* Effect.logWarning('pool: every account is unavailable')\n\n return Stream.fail(\n exhaustedError({\n active: cooldowns,\n // Cooldowns recorded by this request decide the kind even if they already ran out.\n all: [\n ...cooldowns,\n ...input.recorded.filter(cooldown => !cooldowns.includes(cooldown))\n ],\n lastError: input.lastError\n })\n )\n }\n\n yield* updatePin({\n route: input.route,\n decision,\n snapshot,\n cooldowns,\n excluded: input.excluded,\n nowMs\n })\n yield* Effect.annotateCurrentSpan({ account: account.id, source: decision._tag })\n\n return attempt({\n request: input.request,\n route: input.route,\n account,\n excluded: input.excluded,\n recorded: input.recorded\n })\n }).pipe(\n Effect.withSpan('pool.pick', { attributes: { pool: poolId, route: input.route } }),\n Effect.annotateLogs({ pool: poolId, route: input.route })\n )\n )\n\n const provider: LLMProviderService = {\n stream: request => {\n const route = poolRoute(request.model)\n\n return route === ''\n ? Stream.fail(\n new LLMError({\n cause: 'validation_error',\n message: `Pool ${poolId} needs a model id`,\n retryable: false\n })\n )\n : pick({ request, route, excluded: new Set(), recorded: [], lastError: undefined })\n }\n }\n\n const recordReading = (accountId: string, snapshot: ProviderSubscriptionUsageSnapshot) =>\n accountsById.has(accountId)\n ? store.putReading(PoolStoredReading.make({ accountId, snapshot }))\n : Effect.fail(new PoolUnknownAccountError({ poolId, accountId }))\n\n const state = Effect.gen(function* () {\n const nowMs = yield* Clock.currentTimeMillis\n const stored = yield* store.state({ poolId, accountIds })\n\n return {\n poolId,\n pins: stored.pins,\n accounts: accounts.map(account => ({\n id: account.id,\n priority: account.priority,\n cooldown: stored.cooldowns.find(\n cooldown => cooldown.accountId === account.id && isCooldownActive(cooldown, nowMs)\n ),\n reading: stored.readings.find(reading => reading.accountId === account.id)?.snapshot\n }))\n }\n })\n\n return { id: poolId, provider, recordReading, state }\n })\n"],"mappings":";;;;;;;;AAoFA,MAAa,sBAAsB,KAAK;;;;;AAMxC,MAAa,6BAA6B,KAAK;;AAG/C,MAAa,oBAAoB,MAAM,KAAK,KAAK;;AAGjD,MAAa,qBAAqB,KAAK;AAIvC,MAAM,eAAe,QAAmC,YACtD,OAAO,KAAK,IAAI,gBAAgB;CAAE;CAAQ;AAAQ,CAAC,CAAC;;;;;;AAOtD,MAAa,uBACX,OACA,WACA,UAC6B;CAC7B,MAAM,OAAO,MAAM;CAEnB,IAAI,SAAS,KAAA,GACX;CAGF,IAAI,KAAK,SAAS,QAChB,OAAO,iBAAiB,KAAK;EAC3B;EACA,aAAa;EACb,SAAS,QAAQ;CACnB,CAAC;CAGH,IAAI,KAAK,SAAS,cAChB;CAGF,MAAM,WAAW,YACf,OAAO,SAAS,OAAO,KAAK,UAAU,SAAS,UAAU,SAAA;CAE3D,IAAI,KAAK,oBAAoB,KAAA,KAAa,QAAQ,KAAK,eAAe,GACpE,OAAO,kBAAkB,KAAK;EAAE;EAAW,aAAa;EAAO,SAAS,KAAK;CAAgB,CAAC;CAGhG,IACE,KAAK,iBAAiB,KAAA,KACtB,KAAK,gBAAA,QACL,QAAQ,QAAQ,KAAK,YAAY,GAEjC,OAAO,kBAAkB,KAAK;EAC5B;EACA,aAAa;EACb,SAAS,QAAQ,KAAK;CACxB,CAAC;AAIL;AAEA,MAAM,OAAO,OAAe;CAC1B,MAAM,OAAO,IAAI,KAAK,EAAE;CAExB,OAAO,OAAO,SAAS,KAAK,QAAQ,CAAC,IAAI,KAAK,YAAY,IAAI,OAAO,EAAE;AACzE;AAEA,MAAM,YAAY,cAChB,UAAU,QACP,UAAU,aACT,aAAa,KAAA,KAAa,SAAS,UAAU,WAAW,SAAS,UAAU,UAC7E,KAAA,CACF;AAEF,MAAM,gBAAmC;CAAE,KAAK,KAAA;CAAW,WAAW,CAAC;CAAG,UAAU,CAAC;AAAE;;;;;;;;;;;;;;AAevF,MAAa,YAAY,YACvB,OAAO,IAAI,aAAa;CACtB,MAAM,SAAS,QAAQ,GAAG,KAAK;CAE/B,IAAI,WAAW,IACb,OAAO,OAAO,YAAY,mBAAmB,oCAAoC;CAGnF,IAAI,QAAQ,SAAS,WAAW,GAC9B,OAAO,OAAO,YAAY,eAAe,QAAQ,OAAO,4BAA4B;CAGtF,MAAM,WAAW,QAAQ,YAAA;CACzB,MAAM,iBAAiB,QAAQ,kBAAkB;CAEjD,IAAI,CAAC,OAAO,SAAS,QAAQ,KAAK,YAAY,GAC5C,OAAO,OAAO,YAAY,mBAAmB,oCAAoC;CAGnF,IAAI,CAAC,OAAO,SAAS,cAAc,KAAK,iBAAiB,KAAK,kBAAkB,KAC9E,OAAO,OAAO,YACZ,2BACA,iDACF;CAGF,MAAM,WAAgC,CAAC;CAEvC,KAAK,MAAM,WAAW,QAAQ,UAAU;EACtC,MAAM,KAAK,QAAQ,GAAG,KAAK;EAC3B,MAAM,WAAW,QAAQ,YAAY;EAErC,IAAI,OAAO,MAAM,OAAO,QAAQ,IAC9B,OAAO,OAAO,YACZ,sBACA,8DACF;EAGF,IAAI,SAAS,MAAK,aAAY,SAAS,OAAO,EAAE,GAC9C,OAAO,OAAO,YAAY,wBAAwB,yBAAyB,IAAI;EAGjF,IAAI,CAAC,OAAO,SAAS,QAAQ,GAC3B,OAAO,OAAO,YAAY,oBAAoB,WAAW,GAAG,2BAA2B;EAGzF,SAAS,KAAK;GAAE;GAAI;GAAU,UAAU,QAAQ;EAAS,CAAC;CAC5D;CAEA,MAAM,aAAa,SAAS,KAAI,YAAW,QAAQ,EAAE;CACrD,MAAM,eAAe,IAAI,IAAI,SAAS,KAAI,YAAW,CAAC,QAAQ,IAAI,OAAO,CAAC,CAAC;CAC3E,MAAM,QAAQ,QAAQ;CAEtB,MAAM,gBAAgB,UACpB,MACG,KAAK;EAAE;EAAQ;EAAO;CAAW,CAAC,EAClC,KACC,OAAO,OAAM,UACX,OAAO,WAAW,qDAAqD,EAAE,KACvE,OAAO,aAAa,EAAE,WAAW,MAAM,UAAU,CAAC,GAClD,OAAO,GAAG,aAAa,CACzB,CACF,CACF;CAGJ,MAAM,mBAAmB,oBACvB,OAAO,WAAW,eAAe,UAAU,QAAQ;CAErD,MAAM,cAAc,aAAgC;EAClD,MAAM,2BAAW,IAAI,IAAyB;EAE9C,KAAK,MAAM,UAAU,SAAS,UAAU;GACtC,MAAM,UAAU,wBAAwB,OAAO,QAAQ;GAEvD,IAAI,YAAY,KAAA,GACd,SAAS,IAAI,OAAO,WAAW,OAAO;EAE1C;EAEA,OAAO;CACT;CAEA,MAAM,cAAc,UAMlB,MAAM,MAAM,MAAM,QAAQ,EAAE,KAC1B,MAAM,IAAI,UAAS,aACjB,SAAS,eAAe,KAAA,IACpB,sCACA,gCAAgC,IAAI,SAAS,UAAU,GAC7D,GACA,MAAM,aAAa;EACjB,MAAM,WAAW,MAAM,UAAU,MAAK,YAAW,QAAQ,cAAc,MAAM,IAAI;EAEjF,IAAI,aAAa,KAAA,GACf,OAAO,MAAM,MAAM,QAAQ,EAAE,KAC3B,MAAM,IACJ,sBACA,YAAW,wBAAwB,IAAI,QAAQ,OAAO,GACxD,GACA,MAAM,IACJ,qBACA,YAAW,uBAAuB,IAAI,QAAQ,OAAO,GACvD,GACA,MAAM,UACR;EAGF,IAAI,MAAM,SAAS,IAAI,MAAM,IAAI,GAC/B,OAAO;EAGT,OAAO,aAAa,IAAI,MAAM,IAAI,IAC9B,qCACA;CACN,CAAC,CACH;CAEF,MAAM,aAAa,UAOb;EACJ,MAAM,UAAU,MAAM,SAAS,KAAK;EACpC,MAAM,OAAO,MAAM,SAAS;EAE5B,IAAI,YAAY,MACd,OAAO,OAAO;EAGhB,MAAM,MAAM,QAAQ,KAAK;GACvB;GACA,OAAO,MAAM;GACb,WAAW;GACX,YAAY,MAAM;EACpB,CAAC;EAED,MAAM,MACJ,YAAY,KAAA,IACR,OAAO,SAAS,oBAAoB,EAAE,KAAK,OAAO,aAAa,EAAE,IAAI,KAAK,CAAC,CAAC,IAC5E,OAAO,QACL,oBAAoB,WAAW;GAC7B,UAAU,MAAM;GAChB,MAAM;GACN,WAAW,MAAM;GACjB,UAAU,MAAM;EAClB,CAAC,GACH,EAAE,KAAK,OAAO,aAAa;GAAE,MAAM;GAAS,IAAI;EAAK,CAAC,CAAC;EAE7D,OAAO,MAAM,QAAQ;GAAE,UAAU;GAAS;EAAI,CAAC,EAAE,KAC/C,OAAO,SAAQ,YAAY,UAAU,MAAM,OAAO,IAAK,GACvD,OAAO,MAAM,gBAAgB,SAAS,CAAC,CACzC;CACF;CAEA,MAAM,kBAAkB,UAMlB;EACJ,MAAM,WAAW,aAA2B,UAAU,SAAS,UAAU,mBAAmB;EAC5F,MAAM,kBAAkB,SAAS,MAAM,OAAO,OAAO,OAAO,CAAC;EAC7D,MAAM,QAAQ,SAAS,MAAM,MAAM;EAEnC,MAAM,WAAW,MAAM,WAAW,UAAU,YAAY;EAExD,MAAM,UACJ,UAAU,KAAA,IACN,yBAAyB,OAAO,mBAChC,yBAAyB,OAAO,wBAAwB,IAAI,KAAK;EAEvE,IAAI,CAAC,MAAM,IAAI,KAAK,OAAO,GACzB,OAAO,IAAI,SAAS;GAClB,OAAO;GACP;GACA,WAAW;GACX,UAAU,kBAAkB,KAAK;IAAE;IAAU,MAAM;GAAO,CAAC;EAC7D,CAAC;EAGH,OAAO,IAAI,SAAS;GAClB,OAAO;GACP;GACA,WAAW;GACX,UACE,oBAAoB,KAAA,IAChB,kBAAkB,KAAK;IAAE;IAAU,MAAM;GAAa,CAAC,IACvD,kBAAkB,KAAK;IAAE;IAAU,MAAM;IAAc;GAAgB,CAAC;EAChF,CAAC;CACH;CAEA,MAAM,WAAW,UAOf,OAAO,OACL,IAAI,KAAK,KAAK,EAAE,KACd,OAAO,KAAI,YACT,MAAM,QAAQ,SAAS,OAAO,MAAM,OAAO,EAAE,KAC3C,OAAO,UAAU,IAAI,IAAI,SAAS,IAAI,CAAC,GACvC,OAAO,UAAU,EACf,WAAU,UACR,OAAO,OACL,OAAO,IAAI,aAAa;EACtB,MAAM,QAAQ,OAAO,MAAM;EAC3B,MAAM,WAAW,oBAAoB,OAAO,MAAM,QAAQ,IAAI,KAAK;EAEnE,KAAK,OAAO,IAAI,IAAI,OAAO,MAAM,aAAa,KAAA,GAC5C,OAAO,OAAO,KAAK,KAAK;EAG1B,OAAO,MACJ,YAAY,QAAQ,EACpB,KAAK,OAAO,MAAM,gBAAgB,aAAa,CAAC,CAAC;EACpD,OAAO,OAAO,QACZ,wBAAwB,UAAU,SAAS,UAAU,mBAAmB,IAAI,qBAAqB,eAAe,mBAAmB,IAAI,SAAS,OAAO,GACzJ,EAAE,KAAK,OAAO,aAAa,EAAE,SAAS,MAAM,QAAQ,GAAG,CAAC,CAAC;EAEzD,OAAO,KAAK;GACV,SAAS,MAAM;GACf,OAAO,MAAM;GACb,UAAU,IAAI,IAAI,CAAC,GAAG,MAAM,UAAU,MAAM,QAAQ,EAAE,CAAC;GACvD,UAAU,CAAC,GAAG,MAAM,UAAU,QAAQ;GACtC,WAAW;EACb,CAAC;CACH,CAAC,CACH,EACJ,CAAC,CACH,CACF,CACF,CACF;CAEF,MAAM,QAAQ,UAOZ,OAAO,OACL,OAAO,IAAI,aAAa;EACtB,MAAM,QAAQ,OAAO,MAAM;EAC3B,MAAM,WAAW,OAAO,aAAa,MAAM,KAAK;EAChD,MAAM,WAAW,WAAW,QAAQ;EAEpC,MAAM,QAAQ,oBAAoB;GAChC,WAAW,SAAS;GACpB;GACA;GACA;EACF,CAAC;EAED,OAAO,OAAO,QACZ,QACA,aACE,MACG,cAAc;GAAE,WAAW,SAAS;GAAW,aAAa,SAAS;EAAY,CAAC,EAClF,KACC,OAAO,SAAQ,YACb,UACI,OAAO,QACL,iCAAiC,IAAI,SAAS,OAAO,EAAE,oCACzD,EAAE,KAAK,OAAO,aAAa,EAAE,SAAS,SAAS,UAAU,CAAC,CAAC,IAC3D,OAAO,IACb,GACA,OAAO,MAAM,gBAAgB,eAAe,CAAC,CAC/C,GACJ,EAAE,SAAS,KAAK,CAClB;EAEA,MAAM,YAAY,CAChB,GAAG,SAAS,UAAU,QACpB,aAAY,iBAAiB,UAAU,KAAK,KAAK,CAAC,MAAM,SAAS,QAAQ,CAC3E,GACA,GAAG,MAAM,SAAS,QAAO,aAAY,iBAAiB,UAAU,KAAK,CAAC,CACxE;EAOA,MAAM,WAAW,SAAS;GACxB;GACA,aAAA,IAPsB,IAAI,CAC1B,GAAG,MAAM,UACT,GAAG,UAAU,KAAI,aAAY,SAAS,SAAS,CACjD,CAIY;GACV;GACA,KAAK,SAAS,KAAK;GACnB,OAAO,MAAM;GACb;GACA;GACA;EACF,CAAC;EAED,MAAM,UAAU,aAAa,KAAA,IAAY,KAAA,IAAY,aAAa,IAAI,SAAS,SAAS;EAExF,IAAI,aAAa,KAAA,KAAa,YAAY,KAAA,GAAW;GACnD,OAAO,OAAO,WAAW,oCAAoC;GAE7D,OAAO,OAAO,KACZ,eAAe;IACb,QAAQ;IAER,KAAK,CACH,GAAG,WACH,GAAG,MAAM,SAAS,QAAO,aAAY,CAAC,UAAU,SAAS,QAAQ,CAAC,CACpE;IACA,WAAW,MAAM;GACnB,CAAC,CACH;EACF;EAEA,OAAO,UAAU;GACf,OAAO,MAAM;GACb;GACA;GACA;GACA,UAAU,MAAM;GAChB;EACF,CAAC;EACD,OAAO,OAAO,oBAAoB;GAAE,SAAS,QAAQ;GAAI,QAAQ,SAAS;EAAK,CAAC;EAEhF,OAAO,QAAQ;GACb,SAAS,MAAM;GACf,OAAO,MAAM;GACb;GACA,UAAU,MAAM;GAChB,UAAU,MAAM;EAClB,CAAC;CACH,CAAC,EAAE,KACD,OAAO,SAAS,aAAa,EAAE,YAAY;EAAE,MAAM;EAAQ,OAAO,MAAM;CAAM,EAAE,CAAC,GACjF,OAAO,aAAa;EAAE,MAAM;EAAQ,OAAO,MAAM;CAAM,CAAC,CAC1D,CACF;CAEF,MAAM,WAA+B,EACnC,SAAQ,YAAW;EACjB,MAAM,QAAQ,UAAU,QAAQ,KAAK;EAErC,OAAO,UAAU,KACb,OAAO,KACL,IAAI,SAAS;GACX,OAAO;GACP,SAAS,QAAQ,OAAO;GACxB,WAAW;EACb,CAAC,CACH,IACA,KAAK;GAAE;GAAS;GAAO,0BAAU,IAAI,IAAI;GAAG,UAAU,CAAC;GAAG,WAAW,KAAA;EAAU,CAAC;CACtF,EACF;CAEA,MAAM,iBAAiB,WAAmB,aACxC,aAAa,IAAI,SAAS,IACtB,MAAM,WAAW,kBAAkB,KAAK;EAAE;EAAW;CAAS,CAAC,CAAC,IAChE,OAAO,KAAK,IAAI,wBAAwB;EAAE;EAAQ;CAAU,CAAC,CAAC;CAoBpE,OAAO;EAAE,IAAI;EAAQ;EAAU;EAAe,OAlBhC,OAAO,IAAI,aAAa;GACpC,MAAM,QAAQ,OAAO,MAAM;GAC3B,MAAM,SAAS,OAAO,MAAM,MAAM;IAAE;IAAQ;GAAW,CAAC;GAExD,OAAO;IACL;IACA,MAAM,OAAO;IACb,UAAU,SAAS,KAAI,aAAY;KACjC,IAAI,QAAQ;KACZ,UAAU,QAAQ;KAClB,UAAU,OAAO,UAAU,MACzB,aAAY,SAAS,cAAc,QAAQ,MAAM,iBAAiB,UAAU,KAAK,CACnF;KACA,SAAS,OAAO,SAAS,MAAK,YAAW,QAAQ,cAAc,QAAQ,EAAE,GAAG;IAC9E,EAAE;GACJ;EACF,CAEkD;CAAE;AACtD,CAAC"}
@@ -0,0 +1,164 @@
1
+ import { PoolCooldown } from "./model.mjs";
2
+ import { Data } from "effect";
3
+ import { ProviderSubscriptionUsageSnapshot } from "@yolk-sdk/agent/providers/subscription-usage";
4
+
5
+ //#region src/rank.d.ts
6
+ /** One usage window of a reading, as the ranking reads it. */
7
+ type PoolWindow = {
8
+ /** Window length; absent when the provider's window has no fixed length (a billing month). */readonly durationMinutes?: number;
9
+ readonly usedPercent: number; /** When the window resets; absent when it has not started. */
10
+ readonly resetsAtMs?: number; /** Lowercase model substrings the window limits; empty for every model. */
11
+ readonly models: ReadonlyArray<string>;
12
+ };
13
+ /** The latest quota reading of one account. */
14
+ type PoolReading = {
15
+ readonly fetchedAtMs: number;
16
+ readonly windows: ReadonlyArray<PoolWindow>;
17
+ };
18
+ /** A configured account as the ranking sees it. */
19
+ type PoolCandidate = {
20
+ readonly id: string;
21
+ readonly priority: number;
22
+ };
23
+ /**
24
+ * The route a request pins: its model id, trimmed and lowercased. Reasoning effort is a separate
25
+ * request field and never part of the route.
26
+ */
27
+ declare const poolRoute: (model: string) => string;
28
+ /**
29
+ * A usage snapshot as the ranking reads it; an unreadable `fetchedAt` yields no reading. Windows
30
+ * without a length still block a used-up account but never rank it. A window that reports only
31
+ * `resetsAfterSeconds` resets that long after the reading.
32
+ */
33
+ declare const poolReadingFromSnapshot: (snapshot: ProviderSubscriptionUsageSnapshot) => PoolReading | undefined;
34
+ /** Why the ranking chose an account. */
35
+ type PoolDecision = Data.TaggedEnum<{
36
+ /** Fresh readings ranked it first; `resetsAtMs` is its longest window's reset, if started. */Quota: {
37
+ readonly accountId: string;
38
+ readonly resetsAtMs: number | undefined;
39
+ }; /** No candidate is usable by a fresh reading; the route's pin is still available. */
40
+ Sticky: {
41
+ readonly accountId: string;
42
+ }; /** No candidate is usable by a fresh reading and no pin is: highest priority, then lowest id. */
43
+ FillFirst: {
44
+ readonly accountId: string;
45
+ };
46
+ }>;
47
+ declare const PoolDecision: {
48
+ readonly Quota: Data.TaggedEnum.ConstructorFrom<{
49
+ readonly _tag: "Quota";
50
+ readonly accountId: string;
51
+ readonly resetsAtMs: number | undefined;
52
+ }, "_tag">;
53
+ readonly Sticky: Data.TaggedEnum.ConstructorFrom<{
54
+ readonly _tag: "Sticky";
55
+ readonly accountId: string;
56
+ }, "_tag">;
57
+ readonly FillFirst: Data.TaggedEnum.ConstructorFrom<{
58
+ readonly _tag: "FillFirst";
59
+ readonly accountId: string;
60
+ }, "_tag">;
61
+ readonly $is: <Tag extends "Quota" | "Sticky" | "FillFirst">(tag: Tag) => (u: unknown) => u is Extract<{
62
+ readonly _tag: "Quota";
63
+ readonly accountId: string;
64
+ readonly resetsAtMs: number | undefined;
65
+ }, {
66
+ readonly _tag: Tag;
67
+ }> | Extract<{
68
+ readonly _tag: "Sticky";
69
+ readonly accountId: string;
70
+ }, {
71
+ readonly _tag: Tag;
72
+ }> | Extract<{
73
+ readonly _tag: "FillFirst";
74
+ readonly accountId: string;
75
+ }, {
76
+ readonly _tag: Tag;
77
+ }>;
78
+ readonly $match: {
79
+ <Cases extends {
80
+ readonly Quota: (args: {
81
+ readonly _tag: "Quota";
82
+ readonly accountId: string;
83
+ readonly resetsAtMs: number | undefined;
84
+ }) => any;
85
+ readonly Sticky: (args: {
86
+ readonly _tag: "Sticky";
87
+ readonly accountId: string;
88
+ }) => any;
89
+ readonly FillFirst: (args: {
90
+ readonly _tag: "FillFirst";
91
+ readonly accountId: string;
92
+ }) => any;
93
+ }>(cases: Cases): (value: {
94
+ readonly _tag: "Quota";
95
+ readonly accountId: string;
96
+ readonly resetsAtMs: number | undefined;
97
+ } | {
98
+ readonly _tag: "Sticky";
99
+ readonly accountId: string;
100
+ } | {
101
+ readonly _tag: "FillFirst";
102
+ readonly accountId: string;
103
+ }) => import("effect/Unify").Unify<ReturnType<Cases["Quota" | "Sticky" | "FillFirst"]>>;
104
+ <Cases extends {
105
+ readonly Quota: (args: {
106
+ readonly _tag: "Quota";
107
+ readonly accountId: string;
108
+ readonly resetsAtMs: number | undefined;
109
+ }) => any;
110
+ readonly Sticky: (args: {
111
+ readonly _tag: "Sticky";
112
+ readonly accountId: string;
113
+ }) => any;
114
+ readonly FillFirst: (args: {
115
+ readonly _tag: "FillFirst";
116
+ readonly accountId: string;
117
+ }) => any;
118
+ }>(value: {
119
+ readonly _tag: "Quota";
120
+ readonly accountId: string;
121
+ readonly resetsAtMs: number | undefined;
122
+ } | {
123
+ readonly _tag: "Sticky";
124
+ readonly accountId: string;
125
+ } | {
126
+ readonly _tag: "FillFirst";
127
+ readonly accountId: string;
128
+ }, cases: Cases): import("effect/Unify").Unify<ReturnType<Cases["Quota" | "Sticky" | "FillFirst"]>>;
129
+ };
130
+ };
131
+ type PoolRankInput = {
132
+ /** Configured accounts. */readonly accounts: ReadonlyArray<PoolCandidate>; /** Accounts that cannot serve now: an active cooldown, or already tried by this request. */
133
+ readonly unavailable: ReadonlySet<string>;
134
+ readonly readings: ReadonlyMap<string, PoolReading>; /** The account the route is pinned to, if any. */
135
+ readonly pin: string | undefined;
136
+ readonly route: string;
137
+ readonly nowMs: number;
138
+ readonly maxAgeMs: number;
139
+ readonly reservePercent: number;
140
+ };
141
+ /**
142
+ * Expiring-first: the available account whose longest applicable window resets soonest and still
143
+ * has quota. When no available account is usable by a fresh reading (within `maxAgeMs`), it routes
144
+ * stickily: the pin while it is available, else fill-first, preferring accounts that no fresh
145
+ * reading shows used up; those serve only when nothing else is left, so the provider's own answer
146
+ * decides. Undefined when no account is available.
147
+ */
148
+ declare const rankPool: (input: PoolRankInput) => PoolDecision | undefined;
149
+ /** Whether a cooldown still keeps its account out at `nowMs`. */
150
+ declare const isCooldownActive: (cooldown: PoolCooldown, nowMs: number) => boolean;
151
+ /**
152
+ * Active quota cooldowns that a fresh reading taken after they began shows to be over (a limit
153
+ * reset was spent, or the provider reset early). A reading from before the lockout never clears
154
+ * one, and auth cooldowns are never cleared by readings.
155
+ */
156
+ declare const staleQuotaCooldowns: (input: {
157
+ readonly cooldowns: ReadonlyArray<PoolCooldown>;
158
+ readonly readings: ReadonlyMap<string, PoolReading>;
159
+ readonly nowMs: number;
160
+ readonly maxAgeMs: number;
161
+ }) => ReadonlyArray<PoolCooldown>;
162
+ //#endregion
163
+ export { PoolCandidate, PoolDecision, PoolRankInput, PoolReading, PoolWindow, isCooldownActive, poolReadingFromSnapshot, poolRoute, rankPool, staleQuotaCooldowns };
164
+ //# sourceMappingURL=rank.d.mts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"rank.d.mts","names":[],"sources":["../src/rank.ts"],"mappings":";;;;;;KAKY,UAAA;EAAA,uGAED,eAAA;EAAA,SACA,WAAA,UAIqB;EAAA,SAFrB,UAAA,WAFA;EAAA,SAIA,MAAA,EAAQ,aAAa;AAAA;;KAMpB,WAAA;EAAA,SACD,WAAA;EAAA,SACA,OAAA,EAAS,aAAa,CAAC,UAAA;AAAA;;KAItB,aAAA;EAAA,SACD,EAAA;EAAA,SACA,QAAQ;AAAA;;;AANyB;AAI5C;cASa,SAAA,GAAa,KAAa;;;AAPpB;AAOnB;;cAOa,uBAAA,GACX,QAAA,EAAU,iCAAA,KACT,WAAW;;KAsCF,YAAA,GAAe,IAAA,CAAK,UAAU;EAxC7B,8FA0CX,KAAA;IAAA,SAAkB,SAAA;IAAA,SAA4B,UAAA;EAAA,GAzC9C;EA2CA,MAAA;IAAA,SAAmB,SAAA;EAAA,GAJT;EAMV,SAAA;IAAA,SAAsB,SAAA;EAAA;AAAA;AAAA,cAGX,YAAA;EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KAED,aAAA;sCAED,QAAA,EAAU,aAAA,CAAc,aAAA;WAExB,WAAA,EAAa,WAAA;EAAA,SACb,QAAA,EAAU,WAAA,SAAoB,WAAA;WAE9B,GAAA;EAAA,SACA,KAAA;EAAA,SACA,KAAA;EAAA,SACA,QAAA;EAAA,SACA,cAAA;AAAA;;;;;;;;cAkHE,QAAA,GAAY,KAAA,EAAO,aAAA,KAAgB,YAAY;;cA+C/C,gBAAA,GAAoB,QAAA,EAAU,YAAY,EAAE,KAAA;;;;;;cAqB5C,mBAAA,GAAuB,KAAA;EAAA,SACzB,SAAA,EAAW,aAAA,CAAc,YAAA;EAAA,SACzB,QAAA,EAAU,WAAA,SAAoB,WAAA;EAAA,SAC9B,KAAA;EAAA,SACA,QAAA;AAAA,MACP,aAAA,CAAc,YAAA"}
package/dist/rank.mjs ADDED
@@ -0,0 +1,139 @@
1
+ import { Data, Predicate } from "effect";
2
+ //#region src/rank.ts
3
+ /**
4
+ * The route a request pins: its model id, trimmed and lowercased. Reasoning effort is a separate
5
+ * request field and never part of the route.
6
+ */
7
+ const poolRoute = (model) => model.trim().toLowerCase();
8
+ /**
9
+ * A usage snapshot as the ranking reads it; an unreadable `fetchedAt` yields no reading. Windows
10
+ * without a length still block a used-up account but never rank it. A window that reports only
11
+ * `resetsAfterSeconds` resets that long after the reading.
12
+ */
13
+ const poolReadingFromSnapshot = (snapshot) => {
14
+ const fetchedAtMs = Date.parse(snapshot.fetchedAt);
15
+ if (!Number.isFinite(fetchedAtMs)) return;
16
+ return {
17
+ fetchedAtMs,
18
+ windows: Array.from(snapshot.windows).map((window) => {
19
+ const resetsAtMs = window.resetsAt !== void 0 ? Date.parse(window.resetsAt) : window.resetsAfterSeconds !== void 0 ? fetchedAtMs + window.resetsAfterSeconds * 1e3 : NaN;
20
+ const models = (window.models ?? []).flatMap((model) => {
21
+ const normalized = model.trim().toLowerCase();
22
+ return normalized === "" ? [] : [normalized];
23
+ });
24
+ const fields = {
25
+ usedPercent: window.usedPercent,
26
+ models
27
+ };
28
+ if (window.windowDurationMinutes !== void 0) fields.durationMinutes = window.windowDurationMinutes;
29
+ if (Number.isFinite(resetsAtMs)) fields.resetsAtMs = resetsAtMs;
30
+ return fields;
31
+ })
32
+ };
33
+ };
34
+ const PoolDecision = Data.taggedEnum();
35
+ const appliesTo = (window, route) => window.models.length === 0 || window.models.some((model) => route.includes(model));
36
+ const hasStarted = (window, nowMs) => window.resetsAtMs !== void 0 && window.resetsAtMs > nowMs;
37
+ /**
38
+ * The reset of the account's longest window with a length that applies to the route (undefined
39
+ * when it has not started, or no applicable window has a length). Unusable when no window applies
40
+ * or an applicable started window is used up to the reserve; a window whose reset has passed counts
41
+ * as fresh, whatever the reading said.
42
+ */
43
+ const accountRank = (reading, route, nowMs, reservePercent) => {
44
+ let usable = false;
45
+ let longest = 0;
46
+ let resetsAtMs;
47
+ for (const window of reading.windows) {
48
+ if (!appliesTo(window, route)) continue;
49
+ const started = hasStarted(window, nowMs);
50
+ if (started && window.usedPercent >= 100 - reservePercent) return { kind: "exhausted" };
51
+ usable = true;
52
+ if (window.durationMinutes === void 0) continue;
53
+ if (window.durationMinutes > longest) {
54
+ longest = window.durationMinutes;
55
+ resetsAtMs = started ? window.resetsAtMs : void 0;
56
+ } else if (window.durationMinutes === longest && started && window.resetsAtMs !== void 0 && (resetsAtMs === void 0 || window.resetsAtMs < resetsAtMs)) resetsAtMs = window.resetsAtMs;
57
+ }
58
+ return usable ? {
59
+ kind: "usable",
60
+ resetsAtMs
61
+ } : { kind: "unknown" };
62
+ };
63
+ /** Code-unit id order, so ties never depend on locale. */
64
+ const idBefore = (a, b) => a < b;
65
+ /**
66
+ * Started windows by soonest reset, then windows that have not started; ties by higher priority,
67
+ * then lower id.
68
+ */
69
+ const ranksBefore = (a, b) => {
70
+ if (a.resetsAtMs !== void 0 && b.resetsAtMs !== void 0) {
71
+ if (a.resetsAtMs !== b.resetsAtMs) return a.resetsAtMs < b.resetsAtMs;
72
+ } else if (a.resetsAtMs !== void 0 || b.resetsAtMs !== void 0) return a.resetsAtMs !== void 0;
73
+ if (a.candidate.priority !== b.candidate.priority) return a.candidate.priority > b.candidate.priority;
74
+ return idBefore(a.candidate.id, b.candidate.id);
75
+ };
76
+ /** Highest priority, then lowest id. */
77
+ const fillFirst = (candidates) => {
78
+ let best;
79
+ for (const candidate of candidates) if (best === void 0 || candidate.priority > best.priority || candidate.priority === best.priority && idBefore(candidate.id, best.id)) best = candidate;
80
+ return best;
81
+ };
82
+ /**
83
+ * Expiring-first: the available account whose longest applicable window resets soonest and still
84
+ * has quota. When no available account is usable by a fresh reading (within `maxAgeMs`), it routes
85
+ * stickily: the pin while it is available, else fill-first, preferring accounts that no fresh
86
+ * reading shows used up; those serve only when nothing else is left, so the provider's own answer
87
+ * decides. Undefined when no account is available.
88
+ */
89
+ const rankPool = (input) => {
90
+ const candidates = input.accounts.filter((account) => !input.unavailable.has(account.id));
91
+ const exhausted = /* @__PURE__ */ new Set();
92
+ let best;
93
+ for (const candidate of candidates) {
94
+ const reading = input.readings.get(candidate.id);
95
+ if (reading === void 0 || input.nowMs - reading.fetchedAtMs > input.maxAgeMs) continue;
96
+ const rank = accountRank(reading, input.route, input.nowMs, input.reservePercent);
97
+ if (rank.kind === "exhausted") exhausted.add(candidate.id);
98
+ if (rank.kind !== "usable") continue;
99
+ const next = {
100
+ candidate,
101
+ resetsAtMs: rank.resetsAtMs
102
+ };
103
+ if (best === void 0 || ranksBefore(next, best)) best = next;
104
+ }
105
+ if (best !== void 0) return PoolDecision.Quota({
106
+ accountId: best.candidate.id,
107
+ resetsAtMs: best.resetsAtMs
108
+ });
109
+ const unexhausted = candidates.filter((candidate) => !exhausted.has(candidate.id));
110
+ const fallback = unexhausted.length > 0 ? unexhausted : candidates;
111
+ if (input.pin !== void 0 && fallback.some((candidate) => candidate.id === input.pin)) return PoolDecision.Sticky({ accountId: input.pin });
112
+ const first = fillFirst(fallback);
113
+ return first === void 0 ? void 0 : PoolDecision.FillFirst({ accountId: first.id });
114
+ };
115
+ /** Whether a cooldown still keeps its account out at `nowMs`. */
116
+ const isCooldownActive = (cooldown, nowMs) => cooldown.untilMs > nowMs;
117
+ /**
118
+ * Every account-wide window of the reading has quota left or has reset; model-scoped windows block
119
+ * only their models through ranking, never the whole account. A reading without account-wide
120
+ * windows proves nothing.
121
+ */
122
+ const hasQuotaLeft = (reading, nowMs) => {
123
+ const accountWide = reading.windows.filter((window) => window.models.length === 0);
124
+ return accountWide.length > 0 && accountWide.every((window) => !hasStarted(window, nowMs) || window.usedPercent < 100);
125
+ };
126
+ /**
127
+ * Active quota cooldowns that a fresh reading taken after they began shows to be over (a limit
128
+ * reset was spent, or the provider reset early). A reading from before the lockout never clears
129
+ * one, and auth cooldowns are never cleared by readings.
130
+ */
131
+ const staleQuotaCooldowns = (input) => input.cooldowns.filter((cooldown) => {
132
+ if (!Predicate.isTagged(cooldown, "PoolQuotaCooldown") || !isCooldownActive(cooldown, input.nowMs)) return false;
133
+ const reading = input.readings.get(cooldown.accountId);
134
+ return reading !== void 0 && input.nowMs - reading.fetchedAtMs <= input.maxAgeMs && reading.fetchedAtMs > cooldown.startedAtMs && hasQuotaLeft(reading, input.nowMs);
135
+ });
136
+ //#endregion
137
+ export { PoolDecision, isCooldownActive, poolReadingFromSnapshot, poolRoute, rankPool, staleQuotaCooldowns };
138
+
139
+ //# sourceMappingURL=rank.mjs.map