@yolk-sdk/pool 0.0.0-stage → 0.1.0-canary.118

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/pool.ts ADDED
@@ -0,0 +1,570 @@
1
+ import { Clock, Effect, Match, Predicate, Ref, Stream } from 'effect'
2
+ import {
3
+ LLMError,
4
+ type LLMEvent,
5
+ type LLMProvider,
6
+ type LLMProviderError,
7
+ type LLMRequest
8
+ } from '@yolk-sdk/agent/loop'
9
+ import { ProviderErrorInfo } from '@yolk-sdk/agent/protocol'
10
+ import type { ProviderSubscriptionUsageSnapshot } from '@yolk-sdk/agent/providers/subscription-usage'
11
+ import {
12
+ PoolAuthCooldown,
13
+ PoolConfigError,
14
+ type PoolCooldown,
15
+ PoolPin,
16
+ PoolQuotaCooldown,
17
+ type PoolStoreError,
18
+ PoolUnknownAccountError
19
+ } from './model.ts'
20
+ import {
21
+ isCooldownActive,
22
+ type PoolCandidate,
23
+ type PoolDecision,
24
+ type PoolReading,
25
+ poolReadingFromSnapshot,
26
+ poolRoute,
27
+ rankPool,
28
+ staleQuotaCooldowns
29
+ } from './rank.ts'
30
+ import { type PoolStoreApi, type PoolStoreSnapshot, PoolStoredReading } from './store.ts'
31
+
32
+ type LLMProviderService = typeof LLMProvider.Service
33
+
34
+ /** One subscription account of a pool: its own provider, built by the host with its credentials. */
35
+ export type PoolAccount = {
36
+ /** Unique within the store; cooldowns and readings are keyed by it. */
37
+ readonly id: string
38
+ /** Breaks ranking ties and orders fill-first (higher first). Default 0. */
39
+ readonly priority?: number
40
+ readonly provider: LLMProviderService
41
+ }
42
+
43
+ export type PoolOptions = {
44
+ /** Pins are keyed by pool id and route. */
45
+ readonly id: string
46
+ /** Accounts of one provider; requests must suit every one of them. */
47
+ readonly accounts: ReadonlyArray<PoolAccount>
48
+ readonly store: PoolStoreApi
49
+ /** How old a reading may be and still rank accounts. Default 30 minutes. */
50
+ readonly maxAgeMs?: number
51
+ /** Quota left untouched on every window, in percent (0 to below 100). Default 0. */
52
+ readonly reservePercent?: number
53
+ }
54
+
55
+ /** One account as the host UI sees it. */
56
+ export type PoolAccountState = {
57
+ readonly id: string
58
+ readonly priority: number
59
+ /** The account's cooldown while it is active. */
60
+ readonly cooldown: PoolCooldown | undefined
61
+ readonly reading: ProviderSubscriptionUsageSnapshot | undefined
62
+ }
63
+
64
+ export type PoolState = {
65
+ readonly poolId: string
66
+ /** The account serving each route. */
67
+ readonly pins: ReadonlyArray<PoolPin>
68
+ readonly accounts: ReadonlyArray<PoolAccountState>
69
+ }
70
+
71
+ export type Pool = {
72
+ readonly id: string
73
+ /** The pooled provider: provide it as the loop's `LLMProvider`, innermost under host wrappers. */
74
+ readonly provider: LLMProviderService
75
+ /** Records an account's usage snapshot; hosts poll usage on their own schedule. */
76
+ readonly recordReading: (
77
+ accountId: string,
78
+ snapshot: ProviderSubscriptionUsageSnapshot
79
+ ) => Effect.Effect<void, PoolUnknownAccountError | PoolStoreError>
80
+ /** Pins, active cooldowns, and readings for host UIs. */
81
+ readonly state: Effect.Effect<PoolState, PoolStoreError>
82
+ }
83
+
84
+ /** Default `maxAgeMs`: readings older than 30 minutes no longer rank accounts. */
85
+ export const defaultPoolMaxAgeMs = 30 * 60_000
86
+
87
+ /**
88
+ * A `rate_limit` whose `Retry-After` reaches this is a quota lockout, not a transient limit
89
+ * (CLIProxyAPI `expiring-first` `QuotaCooldownFloor`).
90
+ */
91
+ export const poolQuotaRetryAfterFloorMs = 30 * 60_000
92
+
93
+ /** The longest cooldown the pool records; a later or non-finite hint is not trusted. */
94
+ export const poolMaxCooldownMs = 366 * 24 * 60 * 60_000
95
+
96
+ /** How long an account whose credentials were rejected sits out. */
97
+ export const poolAuthCooldownMs = 30 * 60_000
98
+
99
+ type ValidAccount = PoolCandidate & { readonly provider: LLMProviderService }
100
+
101
+ const configError = (reason: PoolConfigError['reason'], message: string) =>
102
+ Effect.fail(new PoolConfigError({ reason, message }))
103
+
104
+ /**
105
+ * The cooldown an account-scoped failure puts its account in, or undefined when the failure is not
106
+ * the account's (the loop's retry handles those): a `rate_limit` with a quota reset or a
107
+ * `Retry-After` of at least `poolQuotaRetryAfterFloorMs` (quota), or `auth`.
108
+ */
109
+ export const poolFailureCooldown = (
110
+ error: LLMError,
111
+ accountId: string,
112
+ nowMs: number
113
+ ): PoolCooldown | undefined => {
114
+ const info = error.provider
115
+
116
+ if (info === undefined) {
117
+ return undefined
118
+ }
119
+
120
+ if (info.kind === 'auth') {
121
+ return PoolAuthCooldown.make({
122
+ accountId,
123
+ startedAtMs: nowMs,
124
+ untilMs: nowMs + poolAuthCooldownMs
125
+ })
126
+ }
127
+
128
+ if (info.kind !== 'rate_limit') {
129
+ return undefined
130
+ }
131
+
132
+ const trusted = (untilMs: number) =>
133
+ Number.isFinite(untilMs) && untilMs > nowMs && untilMs - nowMs <= poolMaxCooldownMs
134
+
135
+ if (info.quotaResetsAtMs !== undefined && trusted(info.quotaResetsAtMs)) {
136
+ return PoolQuotaCooldown.make({ accountId, startedAtMs: nowMs, untilMs: info.quotaResetsAtMs })
137
+ }
138
+
139
+ if (
140
+ info.retryAfterMs !== undefined &&
141
+ info.retryAfterMs >= poolQuotaRetryAfterFloorMs &&
142
+ trusted(nowMs + info.retryAfterMs)
143
+ ) {
144
+ return PoolQuotaCooldown.make({
145
+ accountId,
146
+ startedAtMs: nowMs,
147
+ untilMs: nowMs + info.retryAfterMs
148
+ })
149
+ }
150
+
151
+ return undefined
152
+ }
153
+
154
+ const iso = (ms: number) => {
155
+ const date = new Date(ms)
156
+
157
+ return Number.isFinite(date.getTime()) ? date.toISOString() : String(ms)
158
+ }
159
+
160
+ const minUntil = (cooldowns: ReadonlyArray<PoolCooldown>) =>
161
+ cooldowns.reduce<number | undefined>(
162
+ (earliest, cooldown) =>
163
+ earliest === undefined || cooldown.untilMs < earliest ? cooldown.untilMs : earliest,
164
+ undefined
165
+ )
166
+
167
+ const emptySnapshot: PoolStoreSnapshot = { pin: undefined, cooldowns: [], readings: [] }
168
+
169
+ /**
170
+ * Pools several subscription accounts of one provider behind one `LLMProvider` (ADR 0011).
171
+ *
172
+ * Each request goes to the account whose longest usage window resets soonest and still has quota
173
+ * (expiring-first), or, without fresh readings, stays on the route's pinned account. An
174
+ * account-scoped failure before the first event (an exhausted quota, rejected credentials) cools the
175
+ * account down and the same request moves to the next account; every other failure, and any failure
176
+ * after an event, reaches the caller unchanged. With every account out it fails `rate_limit`
177
+ * (non-retryable) carrying the earliest quota reset, or `auth` when only auth cooldowns remain.
178
+ *
179
+ * Store failures never fail a request: reads fall back to stateless fill-first and writes are
180
+ * logged.
181
+ */
182
+ export const makePool = (options: PoolOptions): Effect.Effect<Pool, PoolConfigError> =>
183
+ Effect.gen(function* () {
184
+ const poolId = options.id.trim()
185
+
186
+ if (poolId === '') {
187
+ return yield* configError('invalid_pool_id', 'Pool id must be a non-empty string')
188
+ }
189
+
190
+ if (options.accounts.length === 0) {
191
+ return yield* configError('no_accounts', `Pool ${poolId} needs at least one account`)
192
+ }
193
+
194
+ const maxAgeMs = options.maxAgeMs ?? defaultPoolMaxAgeMs
195
+ const reservePercent = options.reservePercent ?? 0
196
+
197
+ if (!Number.isFinite(maxAgeMs) || maxAgeMs <= 0) {
198
+ return yield* configError('invalid_max_age', 'maxAgeMs must be a positive number')
199
+ }
200
+
201
+ if (!Number.isFinite(reservePercent) || reservePercent < 0 || reservePercent >= 100) {
202
+ return yield* configError(
203
+ 'invalid_reserve_percent',
204
+ 'reservePercent must be at least 0 and below 100'
205
+ )
206
+ }
207
+
208
+ const accounts: Array<ValidAccount> = []
209
+
210
+ for (const account of options.accounts) {
211
+ const id = account.id.trim()
212
+ const priority = account.priority ?? 0
213
+
214
+ if (id === '' || id !== account.id) {
215
+ return yield* configError(
216
+ 'invalid_account_id',
217
+ 'Account ids must be non-empty without surrounding whitespace'
218
+ )
219
+ }
220
+
221
+ if (accounts.some(existing => existing.id === id)) {
222
+ return yield* configError('duplicate_account_id', `Duplicate account id: ${id}`)
223
+ }
224
+
225
+ if (!Number.isFinite(priority)) {
226
+ return yield* configError('invalid_priority', `Account ${id} has a non-finite priority`)
227
+ }
228
+
229
+ accounts.push({ id, priority, provider: account.provider })
230
+ }
231
+
232
+ const accountIds = accounts.map(account => account.id)
233
+ const accountsById = new Map(accounts.map(account => [account.id, account]))
234
+ const store = options.store
235
+
236
+ const loadSnapshot = (route: string) =>
237
+ store
238
+ .load({ poolId, route, accountIds })
239
+ .pipe(
240
+ Effect.catch(error =>
241
+ Effect.logWarning('pool: store load failed; routing without pool state').pipe(
242
+ Effect.annotateLogs({ operation: error.operation }),
243
+ Effect.as(emptySnapshot)
244
+ )
245
+ )
246
+ )
247
+
248
+ // Host store messages may carry connection details: log the operation only.
249
+ const logStoreFailure = (operation: PoolStoreError['operation']) => () =>
250
+ Effect.logWarning(`pool: store ${operation} failed`)
251
+
252
+ const readingsOf = (snapshot: PoolStoreSnapshot) => {
253
+ const readings = new Map<string, PoolReading>()
254
+
255
+ for (const stored of snapshot.readings) {
256
+ const reading = poolReadingFromSnapshot(stored.snapshot)
257
+
258
+ if (reading !== undefined) {
259
+ readings.set(stored.accountId, reading)
260
+ }
261
+ }
262
+
263
+ return readings
264
+ }
265
+
266
+ const moveReason = (input: {
267
+ readonly decision: PoolDecision
268
+ readonly from: string
269
+ readonly cooldowns: ReadonlyArray<PoolCooldown>
270
+ readonly excluded: ReadonlySet<string>
271
+ }) =>
272
+ Match.value(input.decision).pipe(
273
+ Match.tag('Quota', decision =>
274
+ decision.resetsAtMs === undefined
275
+ ? 'quota ranking, window not started'
276
+ : `quota ranking, window resets ${iso(decision.resetsAtMs)}`
277
+ ),
278
+ Match.orElse(() => {
279
+ const cooldown = input.cooldowns.find(current => current.accountId === input.from)
280
+
281
+ if (cooldown !== undefined) {
282
+ return Match.value(cooldown).pipe(
283
+ Match.tag(
284
+ 'PoolQuotaCooldown',
285
+ current => `quota cooldown until ${iso(current.untilMs)}`
286
+ ),
287
+ Match.tag(
288
+ 'PoolAuthCooldown',
289
+ current => `auth cooldown until ${iso(current.untilMs)}`
290
+ ),
291
+ Match.exhaustive
292
+ )
293
+ }
294
+
295
+ if (input.excluded.has(input.from)) {
296
+ return 'failed this request'
297
+ }
298
+
299
+ return accountsById.has(input.from)
300
+ ? 'a fresh reading shows it used up'
301
+ : 'not configured'
302
+ })
303
+ )
304
+
305
+ const updatePin = (input: {
306
+ readonly route: string
307
+ readonly decision: PoolDecision
308
+ readonly snapshot: PoolStoreSnapshot
309
+ readonly cooldowns: ReadonlyArray<PoolCooldown>
310
+ readonly excluded: ReadonlySet<string>
311
+ readonly nowMs: number
312
+ }) => {
313
+ const current = input.snapshot.pin?.accountId
314
+ const next = input.decision.accountId
315
+
316
+ if (current === next) {
317
+ return Effect.void
318
+ }
319
+
320
+ const pin = PoolPin.make({
321
+ poolId,
322
+ route: input.route,
323
+ accountId: next,
324
+ pinnedAtMs: input.nowMs
325
+ })
326
+
327
+ const log =
328
+ current === undefined
329
+ ? Effect.logDebug('pool: route pinned').pipe(Effect.annotateLogs({ to: next }))
330
+ : Effect.logInfo(
331
+ `pool: pin moved: ${moveReason({
332
+ decision: input.decision,
333
+ from: current,
334
+ cooldowns: input.cooldowns,
335
+ excluded: input.excluded
336
+ })}`
337
+ ).pipe(Effect.annotateLogs({ from: current, to: next }))
338
+
339
+ return store.movePin({ expected: current, pin }).pipe(
340
+ Effect.flatMap(written => (written ? log : Effect.void)),
341
+ Effect.catch(logStoreFailure('movePin'))
342
+ )
343
+ }
344
+
345
+ const exhaustedError = (input: {
346
+ /** Cooldowns still active at the final pick. */
347
+ readonly active: ReadonlyArray<PoolCooldown>
348
+ /** Every cooldown that put an account out, including ones this request recorded that ran out. */
349
+ readonly all: ReadonlyArray<PoolCooldown>
350
+ readonly lastError: LLMError | undefined
351
+ }) => {
352
+ const isQuota = (cooldown: PoolCooldown) => Predicate.isTagged(cooldown, 'PoolQuotaCooldown')
353
+ const quotaResetsAtMs = minUntil(input.active.filter(isQuota))
354
+ const until = minUntil(input.active)
355
+ // The vendor id of the last failure; the pool id when no account was tried in this request.
356
+ const provider = input.lastError?.provider?.provider ?? poolId
357
+
358
+ const message =
359
+ until === undefined
360
+ ? `Every account of pool ${poolId} is unavailable`
361
+ : `Every account of pool ${poolId} is unavailable until ${iso(until)}`
362
+
363
+ if (!input.all.some(isQuota)) {
364
+ return new LLMError({
365
+ cause: 'provider_error',
366
+ message,
367
+ retryable: false,
368
+ provider: ProviderErrorInfo.make({ provider, kind: 'auth' })
369
+ })
370
+ }
371
+
372
+ return new LLMError({
373
+ cause: 'rate_limit',
374
+ message,
375
+ retryable: false,
376
+ provider:
377
+ quotaResetsAtMs === undefined
378
+ ? ProviderErrorInfo.make({ provider, kind: 'rate_limit' })
379
+ : ProviderErrorInfo.make({ provider, kind: 'rate_limit', quotaResetsAtMs })
380
+ })
381
+ }
382
+
383
+ const attempt = (input: {
384
+ readonly request: LLMRequest
385
+ readonly route: string
386
+ readonly account: ValidAccount
387
+ readonly excluded: ReadonlySet<string>
388
+ readonly recorded: ReadonlyArray<PoolCooldown>
389
+ }): Stream.Stream<LLMEvent, LLMProviderError> =>
390
+ Stream.unwrap(
391
+ Ref.make(false).pipe(
392
+ Effect.map(emitted =>
393
+ input.account.provider.stream(input.request).pipe(
394
+ Stream.tap(() => Ref.set(emitted, true)),
395
+ Stream.catchTags({
396
+ LLMError: error =>
397
+ Stream.unwrap(
398
+ Effect.gen(function* () {
399
+ const nowMs = yield* Clock.currentTimeMillis
400
+ const cooldown = poolFailureCooldown(error, input.account.id, nowMs)
401
+
402
+ if ((yield* Ref.get(emitted)) || cooldown === undefined) {
403
+ return Stream.fail(error)
404
+ }
405
+
406
+ yield* store
407
+ .putCooldown(cooldown)
408
+ .pipe(Effect.catch(logStoreFailure('putCooldown')))
409
+ yield* Effect.logInfo(
410
+ `pool: failover after ${Predicate.isTagged(cooldown, 'PoolQuotaCooldown') ? 'quota exhaustion' : 'auth failure'}, cooldown until ${iso(cooldown.untilMs)}`
411
+ ).pipe(Effect.annotateLogs({ account: input.account.id }))
412
+
413
+ return pick({
414
+ request: input.request,
415
+ route: input.route,
416
+ excluded: new Set([...input.excluded, input.account.id]),
417
+ recorded: [...input.recorded, cooldown],
418
+ lastError: error
419
+ })
420
+ })
421
+ )
422
+ })
423
+ )
424
+ )
425
+ )
426
+ )
427
+
428
+ const pick = (input: {
429
+ readonly request: LLMRequest
430
+ readonly route: string
431
+ readonly excluded: ReadonlySet<string>
432
+ readonly recorded: ReadonlyArray<PoolCooldown>
433
+ readonly lastError: LLMError | undefined
434
+ }): Stream.Stream<LLMEvent, LLMProviderError> =>
435
+ Stream.unwrap(
436
+ Effect.gen(function* () {
437
+ const nowMs = yield* Clock.currentTimeMillis
438
+ const snapshot = yield* loadSnapshot(input.route)
439
+ const readings = readingsOf(snapshot)
440
+
441
+ const stale = staleQuotaCooldowns({
442
+ cooldowns: snapshot.cooldowns,
443
+ readings,
444
+ nowMs,
445
+ maxAgeMs
446
+ })
447
+
448
+ yield* Effect.forEach(
449
+ stale,
450
+ cooldown =>
451
+ store
452
+ .clearCooldown({ accountId: cooldown.accountId, startedAtMs: cooldown.startedAtMs })
453
+ .pipe(
454
+ Effect.flatMap(cleared =>
455
+ cleared
456
+ ? Effect.logInfo(
457
+ `pool: cleared cooldown (until ${iso(cooldown.untilMs)}): a newer reading shows quota left`
458
+ ).pipe(Effect.annotateLogs({ account: cooldown.accountId }))
459
+ : Effect.void
460
+ ),
461
+ Effect.catch(logStoreFailure('clearCooldown'))
462
+ ),
463
+ { discard: true }
464
+ )
465
+
466
+ const cooldowns = [
467
+ ...snapshot.cooldowns.filter(
468
+ cooldown => isCooldownActive(cooldown, nowMs) && !stale.includes(cooldown)
469
+ ),
470
+ ...input.recorded.filter(cooldown => isCooldownActive(cooldown, nowMs))
471
+ ]
472
+
473
+ const unavailable = new Set([
474
+ ...input.excluded,
475
+ ...cooldowns.map(cooldown => cooldown.accountId)
476
+ ])
477
+
478
+ const decision = rankPool({
479
+ accounts,
480
+ unavailable,
481
+ readings,
482
+ pin: snapshot.pin?.accountId,
483
+ route: input.route,
484
+ nowMs,
485
+ maxAgeMs,
486
+ reservePercent
487
+ })
488
+
489
+ const account = decision === undefined ? undefined : accountsById.get(decision.accountId)
490
+
491
+ if (decision === undefined || account === undefined) {
492
+ yield* Effect.logWarning('pool: every account is unavailable')
493
+
494
+ return Stream.fail(
495
+ exhaustedError({
496
+ active: cooldowns,
497
+ // Cooldowns recorded by this request decide the kind even if they already ran out.
498
+ all: [
499
+ ...cooldowns,
500
+ ...input.recorded.filter(cooldown => !cooldowns.includes(cooldown))
501
+ ],
502
+ lastError: input.lastError
503
+ })
504
+ )
505
+ }
506
+
507
+ yield* updatePin({
508
+ route: input.route,
509
+ decision,
510
+ snapshot,
511
+ cooldowns,
512
+ excluded: input.excluded,
513
+ nowMs
514
+ })
515
+ yield* Effect.annotateCurrentSpan({ account: account.id, source: decision._tag })
516
+
517
+ return attempt({
518
+ request: input.request,
519
+ route: input.route,
520
+ account,
521
+ excluded: input.excluded,
522
+ recorded: input.recorded
523
+ })
524
+ }).pipe(
525
+ Effect.withSpan('pool.pick', { attributes: { pool: poolId, route: input.route } }),
526
+ Effect.annotateLogs({ pool: poolId, route: input.route })
527
+ )
528
+ )
529
+
530
+ const provider: LLMProviderService = {
531
+ stream: request => {
532
+ const route = poolRoute(request.model)
533
+
534
+ return route === ''
535
+ ? Stream.fail(
536
+ new LLMError({
537
+ cause: 'validation_error',
538
+ message: `Pool ${poolId} needs a model id`,
539
+ retryable: false
540
+ })
541
+ )
542
+ : pick({ request, route, excluded: new Set(), recorded: [], lastError: undefined })
543
+ }
544
+ }
545
+
546
+ const recordReading = (accountId: string, snapshot: ProviderSubscriptionUsageSnapshot) =>
547
+ accountsById.has(accountId)
548
+ ? store.putReading(PoolStoredReading.make({ accountId, snapshot }))
549
+ : Effect.fail(new PoolUnknownAccountError({ poolId, accountId }))
550
+
551
+ const state = Effect.gen(function* () {
552
+ const nowMs = yield* Clock.currentTimeMillis
553
+ const stored = yield* store.state({ poolId, accountIds })
554
+
555
+ return {
556
+ poolId,
557
+ pins: stored.pins,
558
+ accounts: accounts.map(account => ({
559
+ id: account.id,
560
+ priority: account.priority,
561
+ cooldown: stored.cooldowns.find(
562
+ cooldown => cooldown.accountId === account.id && isCooldownActive(cooldown, nowMs)
563
+ ),
564
+ reading: stored.readings.find(reading => reading.accountId === account.id)?.snapshot
565
+ }))
566
+ }
567
+ })
568
+
569
+ return { id: poolId, provider, recordReading, state }
570
+ })