@yolk-sdk/pool 0.0.0-stage → 0.1.0-canary.118
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +123 -2
- package/dist/index.d.mts +4 -0
- package/dist/index.mjs +4 -0
- package/dist/model.d.mts +58 -0
- package/dist/model.d.mts.map +1 -0
- package/dist/model.mjs +68 -0
- package/dist/model.mjs.map +1 -0
- package/dist/pool.d.mts +73 -0
- package/dist/pool.d.mts.map +1 -0
- package/dist/pool.mjs +290 -0
- package/dist/pool.mjs.map +1 -0
- package/dist/rank.d.mts +164 -0
- package/dist/rank.d.mts.map +1 -0
- package/dist/rank.mjs +139 -0
- package/dist/rank.mjs.map +1 -0
- package/dist/store.d.mts +89 -0
- package/dist/store.d.mts.map +1 -0
- package/dist/store.mjs +253 -0
- package/dist/store.mjs.map +1 -0
- package/package.json +59 -3
- package/src/index.ts +32 -0
- package/src/model.ts +81 -0
- package/src/pool.ts +570 -0
- package/src/rank.ts +307 -0
- package/src/store.ts +392 -0
package/src/pool.ts
ADDED
|
@@ -0,0 +1,570 @@
|
|
|
1
|
+
import { Clock, Effect, Match, Predicate, Ref, Stream } from 'effect'
|
|
2
|
+
import {
|
|
3
|
+
LLMError,
|
|
4
|
+
type LLMEvent,
|
|
5
|
+
type LLMProvider,
|
|
6
|
+
type LLMProviderError,
|
|
7
|
+
type LLMRequest
|
|
8
|
+
} from '@yolk-sdk/agent/loop'
|
|
9
|
+
import { ProviderErrorInfo } from '@yolk-sdk/agent/protocol'
|
|
10
|
+
import type { ProviderSubscriptionUsageSnapshot } from '@yolk-sdk/agent/providers/subscription-usage'
|
|
11
|
+
import {
|
|
12
|
+
PoolAuthCooldown,
|
|
13
|
+
PoolConfigError,
|
|
14
|
+
type PoolCooldown,
|
|
15
|
+
PoolPin,
|
|
16
|
+
PoolQuotaCooldown,
|
|
17
|
+
type PoolStoreError,
|
|
18
|
+
PoolUnknownAccountError
|
|
19
|
+
} from './model.ts'
|
|
20
|
+
import {
|
|
21
|
+
isCooldownActive,
|
|
22
|
+
type PoolCandidate,
|
|
23
|
+
type PoolDecision,
|
|
24
|
+
type PoolReading,
|
|
25
|
+
poolReadingFromSnapshot,
|
|
26
|
+
poolRoute,
|
|
27
|
+
rankPool,
|
|
28
|
+
staleQuotaCooldowns
|
|
29
|
+
} from './rank.ts'
|
|
30
|
+
import { type PoolStoreApi, type PoolStoreSnapshot, PoolStoredReading } from './store.ts'
|
|
31
|
+
|
|
32
|
+
type LLMProviderService = typeof LLMProvider.Service
|
|
33
|
+
|
|
34
|
+
/** One subscription account of a pool: its own provider, built by the host with its credentials. */
|
|
35
|
+
export type PoolAccount = {
|
|
36
|
+
/** Unique within the store; cooldowns and readings are keyed by it. */
|
|
37
|
+
readonly id: string
|
|
38
|
+
/** Breaks ranking ties and orders fill-first (higher first). Default 0. */
|
|
39
|
+
readonly priority?: number
|
|
40
|
+
readonly provider: LLMProviderService
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export type PoolOptions = {
|
|
44
|
+
/** Pins are keyed by pool id and route. */
|
|
45
|
+
readonly id: string
|
|
46
|
+
/** Accounts of one provider; requests must suit every one of them. */
|
|
47
|
+
readonly accounts: ReadonlyArray<PoolAccount>
|
|
48
|
+
readonly store: PoolStoreApi
|
|
49
|
+
/** How old a reading may be and still rank accounts. Default 30 minutes. */
|
|
50
|
+
readonly maxAgeMs?: number
|
|
51
|
+
/** Quota left untouched on every window, in percent (0 to below 100). Default 0. */
|
|
52
|
+
readonly reservePercent?: number
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** One account as the host UI sees it. */
|
|
56
|
+
export type PoolAccountState = {
|
|
57
|
+
readonly id: string
|
|
58
|
+
readonly priority: number
|
|
59
|
+
/** The account's cooldown while it is active. */
|
|
60
|
+
readonly cooldown: PoolCooldown | undefined
|
|
61
|
+
readonly reading: ProviderSubscriptionUsageSnapshot | undefined
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export type PoolState = {
|
|
65
|
+
readonly poolId: string
|
|
66
|
+
/** The account serving each route. */
|
|
67
|
+
readonly pins: ReadonlyArray<PoolPin>
|
|
68
|
+
readonly accounts: ReadonlyArray<PoolAccountState>
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export type Pool = {
|
|
72
|
+
readonly id: string
|
|
73
|
+
/** The pooled provider: provide it as the loop's `LLMProvider`, innermost under host wrappers. */
|
|
74
|
+
readonly provider: LLMProviderService
|
|
75
|
+
/** Records an account's usage snapshot; hosts poll usage on their own schedule. */
|
|
76
|
+
readonly recordReading: (
|
|
77
|
+
accountId: string,
|
|
78
|
+
snapshot: ProviderSubscriptionUsageSnapshot
|
|
79
|
+
) => Effect.Effect<void, PoolUnknownAccountError | PoolStoreError>
|
|
80
|
+
/** Pins, active cooldowns, and readings for host UIs. */
|
|
81
|
+
readonly state: Effect.Effect<PoolState, PoolStoreError>
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Default `maxAgeMs`: readings older than 30 minutes no longer rank accounts. */
|
|
85
|
+
export const defaultPoolMaxAgeMs = 30 * 60_000
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* A `rate_limit` whose `Retry-After` reaches this is a quota lockout, not a transient limit
|
|
89
|
+
* (CLIProxyAPI `expiring-first` `QuotaCooldownFloor`).
|
|
90
|
+
*/
|
|
91
|
+
export const poolQuotaRetryAfterFloorMs = 30 * 60_000
|
|
92
|
+
|
|
93
|
+
/** The longest cooldown the pool records; a later or non-finite hint is not trusted. */
|
|
94
|
+
export const poolMaxCooldownMs = 366 * 24 * 60 * 60_000
|
|
95
|
+
|
|
96
|
+
/** How long an account whose credentials were rejected sits out. */
|
|
97
|
+
export const poolAuthCooldownMs = 30 * 60_000
|
|
98
|
+
|
|
99
|
+
type ValidAccount = PoolCandidate & { readonly provider: LLMProviderService }
|
|
100
|
+
|
|
101
|
+
const configError = (reason: PoolConfigError['reason'], message: string) =>
|
|
102
|
+
Effect.fail(new PoolConfigError({ reason, message }))
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* The cooldown an account-scoped failure puts its account in, or undefined when the failure is not
|
|
106
|
+
* the account's (the loop's retry handles those): a `rate_limit` with a quota reset or a
|
|
107
|
+
* `Retry-After` of at least `poolQuotaRetryAfterFloorMs` (quota), or `auth`.
|
|
108
|
+
*/
|
|
109
|
+
export const poolFailureCooldown = (
|
|
110
|
+
error: LLMError,
|
|
111
|
+
accountId: string,
|
|
112
|
+
nowMs: number
|
|
113
|
+
): PoolCooldown | undefined => {
|
|
114
|
+
const info = error.provider
|
|
115
|
+
|
|
116
|
+
if (info === undefined) {
|
|
117
|
+
return undefined
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
if (info.kind === 'auth') {
|
|
121
|
+
return PoolAuthCooldown.make({
|
|
122
|
+
accountId,
|
|
123
|
+
startedAtMs: nowMs,
|
|
124
|
+
untilMs: nowMs + poolAuthCooldownMs
|
|
125
|
+
})
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
if (info.kind !== 'rate_limit') {
|
|
129
|
+
return undefined
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
const trusted = (untilMs: number) =>
|
|
133
|
+
Number.isFinite(untilMs) && untilMs > nowMs && untilMs - nowMs <= poolMaxCooldownMs
|
|
134
|
+
|
|
135
|
+
if (info.quotaResetsAtMs !== undefined && trusted(info.quotaResetsAtMs)) {
|
|
136
|
+
return PoolQuotaCooldown.make({ accountId, startedAtMs: nowMs, untilMs: info.quotaResetsAtMs })
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
if (
|
|
140
|
+
info.retryAfterMs !== undefined &&
|
|
141
|
+
info.retryAfterMs >= poolQuotaRetryAfterFloorMs &&
|
|
142
|
+
trusted(nowMs + info.retryAfterMs)
|
|
143
|
+
) {
|
|
144
|
+
return PoolQuotaCooldown.make({
|
|
145
|
+
accountId,
|
|
146
|
+
startedAtMs: nowMs,
|
|
147
|
+
untilMs: nowMs + info.retryAfterMs
|
|
148
|
+
})
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
return undefined
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
const iso = (ms: number) => {
|
|
155
|
+
const date = new Date(ms)
|
|
156
|
+
|
|
157
|
+
return Number.isFinite(date.getTime()) ? date.toISOString() : String(ms)
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
const minUntil = (cooldowns: ReadonlyArray<PoolCooldown>) =>
|
|
161
|
+
cooldowns.reduce<number | undefined>(
|
|
162
|
+
(earliest, cooldown) =>
|
|
163
|
+
earliest === undefined || cooldown.untilMs < earliest ? cooldown.untilMs : earliest,
|
|
164
|
+
undefined
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
const emptySnapshot: PoolStoreSnapshot = { pin: undefined, cooldowns: [], readings: [] }
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* Pools several subscription accounts of one provider behind one `LLMProvider` (ADR 0011).
|
|
171
|
+
*
|
|
172
|
+
* Each request goes to the account whose longest usage window resets soonest and still has quota
|
|
173
|
+
* (expiring-first), or, without fresh readings, stays on the route's pinned account. An
|
|
174
|
+
* account-scoped failure before the first event (an exhausted quota, rejected credentials) cools the
|
|
175
|
+
* account down and the same request moves to the next account; every other failure, and any failure
|
|
176
|
+
* after an event, reaches the caller unchanged. With every account out it fails `rate_limit`
|
|
177
|
+
* (non-retryable) carrying the earliest quota reset, or `auth` when only auth cooldowns remain.
|
|
178
|
+
*
|
|
179
|
+
* Store failures never fail a request: reads fall back to stateless fill-first and writes are
|
|
180
|
+
* logged.
|
|
181
|
+
*/
|
|
182
|
+
export const makePool = (options: PoolOptions): Effect.Effect<Pool, PoolConfigError> =>
|
|
183
|
+
Effect.gen(function* () {
|
|
184
|
+
const poolId = options.id.trim()
|
|
185
|
+
|
|
186
|
+
if (poolId === '') {
|
|
187
|
+
return yield* configError('invalid_pool_id', 'Pool id must be a non-empty string')
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
if (options.accounts.length === 0) {
|
|
191
|
+
return yield* configError('no_accounts', `Pool ${poolId} needs at least one account`)
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
const maxAgeMs = options.maxAgeMs ?? defaultPoolMaxAgeMs
|
|
195
|
+
const reservePercent = options.reservePercent ?? 0
|
|
196
|
+
|
|
197
|
+
if (!Number.isFinite(maxAgeMs) || maxAgeMs <= 0) {
|
|
198
|
+
return yield* configError('invalid_max_age', 'maxAgeMs must be a positive number')
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
if (!Number.isFinite(reservePercent) || reservePercent < 0 || reservePercent >= 100) {
|
|
202
|
+
return yield* configError(
|
|
203
|
+
'invalid_reserve_percent',
|
|
204
|
+
'reservePercent must be at least 0 and below 100'
|
|
205
|
+
)
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
const accounts: Array<ValidAccount> = []
|
|
209
|
+
|
|
210
|
+
for (const account of options.accounts) {
|
|
211
|
+
const id = account.id.trim()
|
|
212
|
+
const priority = account.priority ?? 0
|
|
213
|
+
|
|
214
|
+
if (id === '' || id !== account.id) {
|
|
215
|
+
return yield* configError(
|
|
216
|
+
'invalid_account_id',
|
|
217
|
+
'Account ids must be non-empty without surrounding whitespace'
|
|
218
|
+
)
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
if (accounts.some(existing => existing.id === id)) {
|
|
222
|
+
return yield* configError('duplicate_account_id', `Duplicate account id: ${id}`)
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
if (!Number.isFinite(priority)) {
|
|
226
|
+
return yield* configError('invalid_priority', `Account ${id} has a non-finite priority`)
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
accounts.push({ id, priority, provider: account.provider })
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
const accountIds = accounts.map(account => account.id)
|
|
233
|
+
const accountsById = new Map(accounts.map(account => [account.id, account]))
|
|
234
|
+
const store = options.store
|
|
235
|
+
|
|
236
|
+
const loadSnapshot = (route: string) =>
|
|
237
|
+
store
|
|
238
|
+
.load({ poolId, route, accountIds })
|
|
239
|
+
.pipe(
|
|
240
|
+
Effect.catch(error =>
|
|
241
|
+
Effect.logWarning('pool: store load failed; routing without pool state').pipe(
|
|
242
|
+
Effect.annotateLogs({ operation: error.operation }),
|
|
243
|
+
Effect.as(emptySnapshot)
|
|
244
|
+
)
|
|
245
|
+
)
|
|
246
|
+
)
|
|
247
|
+
|
|
248
|
+
// Host store messages may carry connection details: log the operation only.
|
|
249
|
+
const logStoreFailure = (operation: PoolStoreError['operation']) => () =>
|
|
250
|
+
Effect.logWarning(`pool: store ${operation} failed`)
|
|
251
|
+
|
|
252
|
+
const readingsOf = (snapshot: PoolStoreSnapshot) => {
|
|
253
|
+
const readings = new Map<string, PoolReading>()
|
|
254
|
+
|
|
255
|
+
for (const stored of snapshot.readings) {
|
|
256
|
+
const reading = poolReadingFromSnapshot(stored.snapshot)
|
|
257
|
+
|
|
258
|
+
if (reading !== undefined) {
|
|
259
|
+
readings.set(stored.accountId, reading)
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
return readings
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
const moveReason = (input: {
|
|
267
|
+
readonly decision: PoolDecision
|
|
268
|
+
readonly from: string
|
|
269
|
+
readonly cooldowns: ReadonlyArray<PoolCooldown>
|
|
270
|
+
readonly excluded: ReadonlySet<string>
|
|
271
|
+
}) =>
|
|
272
|
+
Match.value(input.decision).pipe(
|
|
273
|
+
Match.tag('Quota', decision =>
|
|
274
|
+
decision.resetsAtMs === undefined
|
|
275
|
+
? 'quota ranking, window not started'
|
|
276
|
+
: `quota ranking, window resets ${iso(decision.resetsAtMs)}`
|
|
277
|
+
),
|
|
278
|
+
Match.orElse(() => {
|
|
279
|
+
const cooldown = input.cooldowns.find(current => current.accountId === input.from)
|
|
280
|
+
|
|
281
|
+
if (cooldown !== undefined) {
|
|
282
|
+
return Match.value(cooldown).pipe(
|
|
283
|
+
Match.tag(
|
|
284
|
+
'PoolQuotaCooldown',
|
|
285
|
+
current => `quota cooldown until ${iso(current.untilMs)}`
|
|
286
|
+
),
|
|
287
|
+
Match.tag(
|
|
288
|
+
'PoolAuthCooldown',
|
|
289
|
+
current => `auth cooldown until ${iso(current.untilMs)}`
|
|
290
|
+
),
|
|
291
|
+
Match.exhaustive
|
|
292
|
+
)
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
if (input.excluded.has(input.from)) {
|
|
296
|
+
return 'failed this request'
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
return accountsById.has(input.from)
|
|
300
|
+
? 'a fresh reading shows it used up'
|
|
301
|
+
: 'not configured'
|
|
302
|
+
})
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
const updatePin = (input: {
|
|
306
|
+
readonly route: string
|
|
307
|
+
readonly decision: PoolDecision
|
|
308
|
+
readonly snapshot: PoolStoreSnapshot
|
|
309
|
+
readonly cooldowns: ReadonlyArray<PoolCooldown>
|
|
310
|
+
readonly excluded: ReadonlySet<string>
|
|
311
|
+
readonly nowMs: number
|
|
312
|
+
}) => {
|
|
313
|
+
const current = input.snapshot.pin?.accountId
|
|
314
|
+
const next = input.decision.accountId
|
|
315
|
+
|
|
316
|
+
if (current === next) {
|
|
317
|
+
return Effect.void
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
const pin = PoolPin.make({
|
|
321
|
+
poolId,
|
|
322
|
+
route: input.route,
|
|
323
|
+
accountId: next,
|
|
324
|
+
pinnedAtMs: input.nowMs
|
|
325
|
+
})
|
|
326
|
+
|
|
327
|
+
const log =
|
|
328
|
+
current === undefined
|
|
329
|
+
? Effect.logDebug('pool: route pinned').pipe(Effect.annotateLogs({ to: next }))
|
|
330
|
+
: Effect.logInfo(
|
|
331
|
+
`pool: pin moved: ${moveReason({
|
|
332
|
+
decision: input.decision,
|
|
333
|
+
from: current,
|
|
334
|
+
cooldowns: input.cooldowns,
|
|
335
|
+
excluded: input.excluded
|
|
336
|
+
})}`
|
|
337
|
+
).pipe(Effect.annotateLogs({ from: current, to: next }))
|
|
338
|
+
|
|
339
|
+
return store.movePin({ expected: current, pin }).pipe(
|
|
340
|
+
Effect.flatMap(written => (written ? log : Effect.void)),
|
|
341
|
+
Effect.catch(logStoreFailure('movePin'))
|
|
342
|
+
)
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
const exhaustedError = (input: {
|
|
346
|
+
/** Cooldowns still active at the final pick. */
|
|
347
|
+
readonly active: ReadonlyArray<PoolCooldown>
|
|
348
|
+
/** Every cooldown that put an account out, including ones this request recorded that ran out. */
|
|
349
|
+
readonly all: ReadonlyArray<PoolCooldown>
|
|
350
|
+
readonly lastError: LLMError | undefined
|
|
351
|
+
}) => {
|
|
352
|
+
const isQuota = (cooldown: PoolCooldown) => Predicate.isTagged(cooldown, 'PoolQuotaCooldown')
|
|
353
|
+
const quotaResetsAtMs = minUntil(input.active.filter(isQuota))
|
|
354
|
+
const until = minUntil(input.active)
|
|
355
|
+
// The vendor id of the last failure; the pool id when no account was tried in this request.
|
|
356
|
+
const provider = input.lastError?.provider?.provider ?? poolId
|
|
357
|
+
|
|
358
|
+
const message =
|
|
359
|
+
until === undefined
|
|
360
|
+
? `Every account of pool ${poolId} is unavailable`
|
|
361
|
+
: `Every account of pool ${poolId} is unavailable until ${iso(until)}`
|
|
362
|
+
|
|
363
|
+
if (!input.all.some(isQuota)) {
|
|
364
|
+
return new LLMError({
|
|
365
|
+
cause: 'provider_error',
|
|
366
|
+
message,
|
|
367
|
+
retryable: false,
|
|
368
|
+
provider: ProviderErrorInfo.make({ provider, kind: 'auth' })
|
|
369
|
+
})
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
return new LLMError({
|
|
373
|
+
cause: 'rate_limit',
|
|
374
|
+
message,
|
|
375
|
+
retryable: false,
|
|
376
|
+
provider:
|
|
377
|
+
quotaResetsAtMs === undefined
|
|
378
|
+
? ProviderErrorInfo.make({ provider, kind: 'rate_limit' })
|
|
379
|
+
: ProviderErrorInfo.make({ provider, kind: 'rate_limit', quotaResetsAtMs })
|
|
380
|
+
})
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
const attempt = (input: {
|
|
384
|
+
readonly request: LLMRequest
|
|
385
|
+
readonly route: string
|
|
386
|
+
readonly account: ValidAccount
|
|
387
|
+
readonly excluded: ReadonlySet<string>
|
|
388
|
+
readonly recorded: ReadonlyArray<PoolCooldown>
|
|
389
|
+
}): Stream.Stream<LLMEvent, LLMProviderError> =>
|
|
390
|
+
Stream.unwrap(
|
|
391
|
+
Ref.make(false).pipe(
|
|
392
|
+
Effect.map(emitted =>
|
|
393
|
+
input.account.provider.stream(input.request).pipe(
|
|
394
|
+
Stream.tap(() => Ref.set(emitted, true)),
|
|
395
|
+
Stream.catchTags({
|
|
396
|
+
LLMError: error =>
|
|
397
|
+
Stream.unwrap(
|
|
398
|
+
Effect.gen(function* () {
|
|
399
|
+
const nowMs = yield* Clock.currentTimeMillis
|
|
400
|
+
const cooldown = poolFailureCooldown(error, input.account.id, nowMs)
|
|
401
|
+
|
|
402
|
+
if ((yield* Ref.get(emitted)) || cooldown === undefined) {
|
|
403
|
+
return Stream.fail(error)
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
yield* store
|
|
407
|
+
.putCooldown(cooldown)
|
|
408
|
+
.pipe(Effect.catch(logStoreFailure('putCooldown')))
|
|
409
|
+
yield* Effect.logInfo(
|
|
410
|
+
`pool: failover after ${Predicate.isTagged(cooldown, 'PoolQuotaCooldown') ? 'quota exhaustion' : 'auth failure'}, cooldown until ${iso(cooldown.untilMs)}`
|
|
411
|
+
).pipe(Effect.annotateLogs({ account: input.account.id }))
|
|
412
|
+
|
|
413
|
+
return pick({
|
|
414
|
+
request: input.request,
|
|
415
|
+
route: input.route,
|
|
416
|
+
excluded: new Set([...input.excluded, input.account.id]),
|
|
417
|
+
recorded: [...input.recorded, cooldown],
|
|
418
|
+
lastError: error
|
|
419
|
+
})
|
|
420
|
+
})
|
|
421
|
+
)
|
|
422
|
+
})
|
|
423
|
+
)
|
|
424
|
+
)
|
|
425
|
+
)
|
|
426
|
+
)
|
|
427
|
+
|
|
428
|
+
const pick = (input: {
|
|
429
|
+
readonly request: LLMRequest
|
|
430
|
+
readonly route: string
|
|
431
|
+
readonly excluded: ReadonlySet<string>
|
|
432
|
+
readonly recorded: ReadonlyArray<PoolCooldown>
|
|
433
|
+
readonly lastError: LLMError | undefined
|
|
434
|
+
}): Stream.Stream<LLMEvent, LLMProviderError> =>
|
|
435
|
+
Stream.unwrap(
|
|
436
|
+
Effect.gen(function* () {
|
|
437
|
+
const nowMs = yield* Clock.currentTimeMillis
|
|
438
|
+
const snapshot = yield* loadSnapshot(input.route)
|
|
439
|
+
const readings = readingsOf(snapshot)
|
|
440
|
+
|
|
441
|
+
const stale = staleQuotaCooldowns({
|
|
442
|
+
cooldowns: snapshot.cooldowns,
|
|
443
|
+
readings,
|
|
444
|
+
nowMs,
|
|
445
|
+
maxAgeMs
|
|
446
|
+
})
|
|
447
|
+
|
|
448
|
+
yield* Effect.forEach(
|
|
449
|
+
stale,
|
|
450
|
+
cooldown =>
|
|
451
|
+
store
|
|
452
|
+
.clearCooldown({ accountId: cooldown.accountId, startedAtMs: cooldown.startedAtMs })
|
|
453
|
+
.pipe(
|
|
454
|
+
Effect.flatMap(cleared =>
|
|
455
|
+
cleared
|
|
456
|
+
? Effect.logInfo(
|
|
457
|
+
`pool: cleared cooldown (until ${iso(cooldown.untilMs)}): a newer reading shows quota left`
|
|
458
|
+
).pipe(Effect.annotateLogs({ account: cooldown.accountId }))
|
|
459
|
+
: Effect.void
|
|
460
|
+
),
|
|
461
|
+
Effect.catch(logStoreFailure('clearCooldown'))
|
|
462
|
+
),
|
|
463
|
+
{ discard: true }
|
|
464
|
+
)
|
|
465
|
+
|
|
466
|
+
const cooldowns = [
|
|
467
|
+
...snapshot.cooldowns.filter(
|
|
468
|
+
cooldown => isCooldownActive(cooldown, nowMs) && !stale.includes(cooldown)
|
|
469
|
+
),
|
|
470
|
+
...input.recorded.filter(cooldown => isCooldownActive(cooldown, nowMs))
|
|
471
|
+
]
|
|
472
|
+
|
|
473
|
+
const unavailable = new Set([
|
|
474
|
+
...input.excluded,
|
|
475
|
+
...cooldowns.map(cooldown => cooldown.accountId)
|
|
476
|
+
])
|
|
477
|
+
|
|
478
|
+
const decision = rankPool({
|
|
479
|
+
accounts,
|
|
480
|
+
unavailable,
|
|
481
|
+
readings,
|
|
482
|
+
pin: snapshot.pin?.accountId,
|
|
483
|
+
route: input.route,
|
|
484
|
+
nowMs,
|
|
485
|
+
maxAgeMs,
|
|
486
|
+
reservePercent
|
|
487
|
+
})
|
|
488
|
+
|
|
489
|
+
const account = decision === undefined ? undefined : accountsById.get(decision.accountId)
|
|
490
|
+
|
|
491
|
+
if (decision === undefined || account === undefined) {
|
|
492
|
+
yield* Effect.logWarning('pool: every account is unavailable')
|
|
493
|
+
|
|
494
|
+
return Stream.fail(
|
|
495
|
+
exhaustedError({
|
|
496
|
+
active: cooldowns,
|
|
497
|
+
// Cooldowns recorded by this request decide the kind even if they already ran out.
|
|
498
|
+
all: [
|
|
499
|
+
...cooldowns,
|
|
500
|
+
...input.recorded.filter(cooldown => !cooldowns.includes(cooldown))
|
|
501
|
+
],
|
|
502
|
+
lastError: input.lastError
|
|
503
|
+
})
|
|
504
|
+
)
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
yield* updatePin({
|
|
508
|
+
route: input.route,
|
|
509
|
+
decision,
|
|
510
|
+
snapshot,
|
|
511
|
+
cooldowns,
|
|
512
|
+
excluded: input.excluded,
|
|
513
|
+
nowMs
|
|
514
|
+
})
|
|
515
|
+
yield* Effect.annotateCurrentSpan({ account: account.id, source: decision._tag })
|
|
516
|
+
|
|
517
|
+
return attempt({
|
|
518
|
+
request: input.request,
|
|
519
|
+
route: input.route,
|
|
520
|
+
account,
|
|
521
|
+
excluded: input.excluded,
|
|
522
|
+
recorded: input.recorded
|
|
523
|
+
})
|
|
524
|
+
}).pipe(
|
|
525
|
+
Effect.withSpan('pool.pick', { attributes: { pool: poolId, route: input.route } }),
|
|
526
|
+
Effect.annotateLogs({ pool: poolId, route: input.route })
|
|
527
|
+
)
|
|
528
|
+
)
|
|
529
|
+
|
|
530
|
+
const provider: LLMProviderService = {
|
|
531
|
+
stream: request => {
|
|
532
|
+
const route = poolRoute(request.model)
|
|
533
|
+
|
|
534
|
+
return route === ''
|
|
535
|
+
? Stream.fail(
|
|
536
|
+
new LLMError({
|
|
537
|
+
cause: 'validation_error',
|
|
538
|
+
message: `Pool ${poolId} needs a model id`,
|
|
539
|
+
retryable: false
|
|
540
|
+
})
|
|
541
|
+
)
|
|
542
|
+
: pick({ request, route, excluded: new Set(), recorded: [], lastError: undefined })
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
const recordReading = (accountId: string, snapshot: ProviderSubscriptionUsageSnapshot) =>
|
|
547
|
+
accountsById.has(accountId)
|
|
548
|
+
? store.putReading(PoolStoredReading.make({ accountId, snapshot }))
|
|
549
|
+
: Effect.fail(new PoolUnknownAccountError({ poolId, accountId }))
|
|
550
|
+
|
|
551
|
+
const state = Effect.gen(function* () {
|
|
552
|
+
const nowMs = yield* Clock.currentTimeMillis
|
|
553
|
+
const stored = yield* store.state({ poolId, accountIds })
|
|
554
|
+
|
|
555
|
+
return {
|
|
556
|
+
poolId,
|
|
557
|
+
pins: stored.pins,
|
|
558
|
+
accounts: accounts.map(account => ({
|
|
559
|
+
id: account.id,
|
|
560
|
+
priority: account.priority,
|
|
561
|
+
cooldown: stored.cooldowns.find(
|
|
562
|
+
cooldown => cooldown.accountId === account.id && isCooldownActive(cooldown, nowMs)
|
|
563
|
+
),
|
|
564
|
+
reading: stored.readings.find(reading => reading.accountId === account.id)?.snapshot
|
|
565
|
+
}))
|
|
566
|
+
}
|
|
567
|
+
})
|
|
568
|
+
|
|
569
|
+
return { id: poolId, provider, recordReading, state }
|
|
570
|
+
})
|