@qadanm/opencode-smart-router 0.0.0-stage → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +118 -0
- package/CONTRIBUTING.md +96 -0
- package/LICENSE +21 -0
- package/README.md +341 -2
- package/SECURITY.md +174 -0
- package/config.example.json +108 -0
- package/dist/allowance.js +487 -0
- package/dist/gui-experimental.cjs +1 -0
- package/dist/index.js +11111 -0
- package/dist/npm/index.js +40 -0
- package/dist/tui.js +587 -0
- package/docs/ARCHITECTURE.md +104 -0
- package/docs/DEBUGGING-GUI.md +70 -0
- package/docs/INSTALL.md +120 -0
- package/docs/LIMITATIONS.md +213 -0
- package/docs/MODELS.md +124 -0
- package/docs/OPENDESIGN-INTEGRATION.md +635 -0
- package/docs/QA.md +142 -0
- package/docs/SHARED-CORE.md +72 -0
- package/install +75 -0
- package/package.json +92 -4
- package/scripts/install.sh +324 -0
- package/scripts/uninstall.sh +91 -0
- package/src/allowance.ts +135 -0
- package/src/core/classify/classifier.ts +305 -0
- package/src/core/config-writer.ts +167 -0
- package/src/core/config.ts +1030 -0
- package/src/core/context/strategy.ts +188 -0
- package/src/core/context-health.ts +39 -0
- package/src/core/failures.ts +398 -0
- package/src/core/history/opencode.ts +173 -0
- package/src/core/models/protocol.ts +22 -0
- package/src/core/models/registry.ts +203 -0
- package/src/core/models/seed.ts +771 -0
- package/src/core/offline/mode.ts +321 -0
- package/src/core/plan-page.ts +390 -0
- package/src/core/plans/catalog.ts +481 -0
- package/src/core/policy/calibration.ts +259 -0
- package/src/core/policy/eligibility.ts +578 -0
- package/src/core/providers/enablement.ts +143 -0
- package/src/core/providers/opendesign-credits.ts +206 -0
- package/src/core/providers/opendesign-models.ts +628 -0
- package/src/core/providers/opendesign.ts +474 -0
- package/src/core/quota/calibration.ts +166 -0
- package/src/core/quota/commandcode.ts +471 -0
- package/src/core/quota/credentials.ts +490 -0
- package/src/core/quota/ledger.ts +1273 -0
- package/src/core/quota/opencode-go.ts +148 -0
- package/src/core/quota/opendesign.ts +1043 -0
- package/src/core/quota/reconcile.ts +155 -0
- package/src/core/quota/report.ts +261 -0
- package/src/core/quota/service.ts +273 -0
- package/src/core/reprompt.ts +48 -0
- package/src/core/routing/engine.ts +919 -0
- package/src/core/types.ts +825 -0
- package/src/core/ui.ts +236 -0
- package/src/core/util.ts +114 -0
- package/src/gui/index.tsx +1670 -0
- package/src/gui/viewmodel.ts +456 -0
- package/src/index.ts +48 -0
- package/src/rpc.ts +204 -0
- package/src/server/controller.ts +3281 -0
- package/src/server/route-tracker.ts +60 -0
- package/src/server/shared-core.ts +671 -0
- package/src/tui/index.tsx +168 -0
|
@@ -0,0 +1,1273 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
FailureClass,
|
|
3
|
+
Ledger,
|
|
4
|
+
Metrics,
|
|
5
|
+
ModelMetrics,
|
|
6
|
+
ModelQuotaSnapshot,
|
|
7
|
+
ModelRecord,
|
|
8
|
+
ModelRef,
|
|
9
|
+
ProviderQuotaSnapshot,
|
|
10
|
+
ProviderSettings,
|
|
11
|
+
RoutingDecision,
|
|
12
|
+
SessionState,
|
|
13
|
+
UsageBucket,
|
|
14
|
+
UsageEvent,
|
|
15
|
+
UsageRollup,
|
|
16
|
+
} from "../types.js"
|
|
17
|
+
import { clamp, keyOfRef, modelKey, monthKey, nowId, pct, safeError } from "../util.js"
|
|
18
|
+
|
|
19
|
+
export const LEDGER_VERSION = 1 as const
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* How many provider-scoped failures in a row, with no successful request in
|
|
23
|
+
* between, before the whole provider is treated as not serving.
|
|
24
|
+
*
|
|
25
|
+
* A transient blip is ridden out by the host's own retry and by the owner
|
|
26
|
+
* saying "continue", because a couple of failures in a row is normal during a
|
|
27
|
+
* gateway hiccup. Only a sustained failure, where every attempt fails and
|
|
28
|
+
* nothing succeeds in between, is an outage worth stepping back from.
|
|
29
|
+
*/
|
|
30
|
+
export const PROVIDER_DOWN_STREAK = 5
|
|
31
|
+
|
|
32
|
+
/** How long the provider is held back once it is judged to be down. */
|
|
33
|
+
export const PROVIDER_BACKOFF_MS = 3 * 60_000
|
|
34
|
+
const MAX_USAGE_EVENTS = 2_000
|
|
35
|
+
const MAX_DECISIONS = 120
|
|
36
|
+
const MAX_SESSIONS = 250
|
|
37
|
+
/**
|
|
38
|
+
* How long raw per-request events are kept before they are folded into hourly
|
|
39
|
+
* buckets. Raw events are what the per-request detail is built from; beyond
|
|
40
|
+
* this horizon the aggregates are all that any consumer needs, and they cost
|
|
41
|
+
* three orders of magnitude less space.
|
|
42
|
+
*/
|
|
43
|
+
const RAW_EVENT_HORIZON_MS = 48 * 60 * 60 * 1000
|
|
44
|
+
/** Retention for the hourly aggregates. 400 days covers a yearly comparison. */
|
|
45
|
+
const ROLLUP_RETENTION_MS = 400 * 24 * 60 * 60 * 1000
|
|
46
|
+
/** Upper bound on hourly buckets kept in the ledger. */
|
|
47
|
+
const MAX_BUCKETS = 400 * 24 * 4
|
|
48
|
+
const RETENTION_MS = 40 * 24 * 60 * 60 * 1000
|
|
49
|
+
const FIVE_HOURS_MS = 5 * 60 * 60 * 1000
|
|
50
|
+
const WEEK_MS = 7 * 24 * 60 * 60 * 1000
|
|
51
|
+
const MONTH_MS = 30 * 24 * 60 * 60 * 1000
|
|
52
|
+
const HOUR_MS = 60 * 60 * 1000
|
|
53
|
+
|
|
54
|
+
const emptyFailureCounts = (): Record<FailureClass, number> => ({
|
|
55
|
+
context_overflow: 0,
|
|
56
|
+
rate_limit: 0,
|
|
57
|
+
network: 0,
|
|
58
|
+
auth: 0,
|
|
59
|
+
provider_outage: 0,
|
|
60
|
+
invalid_request: 0,
|
|
61
|
+
image_rejected: 0,
|
|
62
|
+
tool: 0,
|
|
63
|
+
capability: 0,
|
|
64
|
+
interrupted: 0,
|
|
65
|
+
quota_exhausted: 0,
|
|
66
|
+
prompt_rejected: 0,
|
|
67
|
+
unknown: 0,
|
|
68
|
+
})
|
|
69
|
+
|
|
70
|
+
export const emptyMetrics = (now = Date.now()): Metrics => ({
|
|
71
|
+
startedAt: now,
|
|
72
|
+
requests: 0,
|
|
73
|
+
inputTokens: 0,
|
|
74
|
+
cacheReadTokens: 0,
|
|
75
|
+
cacheWriteTokens: 0,
|
|
76
|
+
outputTokens: 0,
|
|
77
|
+
estimatedUsd: 0,
|
|
78
|
+
switches: 0,
|
|
79
|
+
escalations: 0,
|
|
80
|
+
retries: 0,
|
|
81
|
+
preventedPremium: 0,
|
|
82
|
+
manualOverrides: 0,
|
|
83
|
+
dryRunDecisions: 0,
|
|
84
|
+
premiumRequests: 0,
|
|
85
|
+
premiumUsd: 0,
|
|
86
|
+
latencyMsTotal: 0,
|
|
87
|
+
latencySamples: 0,
|
|
88
|
+
failures: emptyFailureCounts(),
|
|
89
|
+
byModel: {},
|
|
90
|
+
timeouts: 0,
|
|
91
|
+
})
|
|
92
|
+
|
|
93
|
+
const emptyModelMetrics = (): ModelMetrics => ({
|
|
94
|
+
requests: 0,
|
|
95
|
+
input: 0,
|
|
96
|
+
output: 0,
|
|
97
|
+
cacheRead: 0,
|
|
98
|
+
cacheWrite: 0,
|
|
99
|
+
usd: 0,
|
|
100
|
+
failures: 0,
|
|
101
|
+
outputTokensPerSecond: 0,
|
|
102
|
+
latencyMsTotal: 0,
|
|
103
|
+
latencySamples: 0,
|
|
104
|
+
timeoutFailures: 0,
|
|
105
|
+
})
|
|
106
|
+
|
|
107
|
+
export const emptyLedger = (now = Date.now()): Ledger => ({
|
|
108
|
+
version: LEDGER_VERSION,
|
|
109
|
+
usage: [],
|
|
110
|
+
premiumSpend: [],
|
|
111
|
+
sessions: {},
|
|
112
|
+
decisions: [],
|
|
113
|
+
metrics: emptyMetrics(now),
|
|
114
|
+
cooldowns: {},
|
|
115
|
+
providerCooldowns: {},
|
|
116
|
+
updatedAt: now,
|
|
117
|
+
})
|
|
118
|
+
|
|
119
|
+
const isRecord = (value: unknown): value is Record<string, unknown> =>
|
|
120
|
+
typeof value === "object" && value !== null && !Array.isArray(value)
|
|
121
|
+
|
|
122
|
+
const finite = (value: unknown, fallback = 0) => (typeof value === "number" && Number.isFinite(value) ? value : fallback)
|
|
123
|
+
|
|
124
|
+
const asModelRef = (value: unknown): ModelRef | undefined => {
|
|
125
|
+
if (!isRecord(value)) return undefined
|
|
126
|
+
if (typeof value.providerID !== "string" || typeof value.id !== "string") return undefined
|
|
127
|
+
return {
|
|
128
|
+
providerID: value.providerID,
|
|
129
|
+
id: value.id,
|
|
130
|
+
...(typeof value.variant === "string" ? { variant: value.variant } : {}),
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
const asUsageEvent = (value: unknown): UsageEvent | undefined => {
|
|
135
|
+
if (!isRecord(value)) return undefined
|
|
136
|
+
if (typeof value.providerID !== "string" || typeof value.modelID !== "string") return undefined
|
|
137
|
+
return {
|
|
138
|
+
id: typeof value.id === "string" ? value.id : nowId("evt"),
|
|
139
|
+
at: finite(value.at, Date.now()),
|
|
140
|
+
providerID: value.providerID,
|
|
141
|
+
modelID: value.modelID,
|
|
142
|
+
...(typeof value.sessionID === "string" ? { sessionID: value.sessionID } : {}),
|
|
143
|
+
usd: Math.max(0, finite(value.usd)),
|
|
144
|
+
input: Math.max(0, finite(value.input)),
|
|
145
|
+
output: Math.max(0, finite(value.output)),
|
|
146
|
+
cacheRead: Math.max(0, finite(value.cacheRead)),
|
|
147
|
+
cacheWrite: Math.max(0, finite(value.cacheWrite)),
|
|
148
|
+
source: value.source === "bootstrap" || value.source === "manual" ? value.source : "step",
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
const asUsageBucket = (value: unknown): UsageBucket | undefined => {
|
|
153
|
+
if (!isRecord(value)) return undefined
|
|
154
|
+
if (typeof value.providerID !== "string" || typeof value.modelID !== "string") return undefined
|
|
155
|
+
const hour = finite(value.hour)
|
|
156
|
+
if (hour <= 0) return undefined
|
|
157
|
+
return {
|
|
158
|
+
hour: Math.floor(hour / HOUR_MS) * HOUR_MS,
|
|
159
|
+
providerID: value.providerID,
|
|
160
|
+
modelID: value.modelID,
|
|
161
|
+
periodID: typeof value.periodID === "string" ? value.periodID : "",
|
|
162
|
+
requests: Math.max(0, finite(value.requests)),
|
|
163
|
+
input: Math.max(0, finite(value.input)),
|
|
164
|
+
output: Math.max(0, finite(value.output)),
|
|
165
|
+
cacheRead: Math.max(0, finite(value.cacheRead)),
|
|
166
|
+
cacheWrite: Math.max(0, finite(value.cacheWrite)),
|
|
167
|
+
usd: Math.max(0, finite(value.usd)),
|
|
168
|
+
failures: Math.max(0, finite(value.failures)),
|
|
169
|
+
latencyMsTotal: Math.max(0, finite(value.latencyMsTotal)),
|
|
170
|
+
latencySamples: Math.max(0, finite(value.latencySamples)),
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
const bucketKeyOf = (bucket: { hour: number; providerID: string; modelID: string; periodID: string }) =>
|
|
175
|
+
`${bucket.hour}:${bucket.providerID}:${bucket.modelID}:${bucket.periodID}`
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Fold an event into an hourly bucket. `periodID` separates consecutive
|
|
179
|
+
* billing periods, so usage from a previous plan period is never re-read as
|
|
180
|
+
* if it had been spent against the current one's limits.
|
|
181
|
+
*/
|
|
182
|
+
const foldEventInto = (
|
|
183
|
+
buckets: Map<string, UsageBucket>,
|
|
184
|
+
event: UsageEvent,
|
|
185
|
+
periodID: string,
|
|
186
|
+
): void => {
|
|
187
|
+
const hour = Math.floor(event.at / HOUR_MS) * HOUR_MS
|
|
188
|
+
const key = bucketKeyOf({ hour, providerID: event.providerID, modelID: event.modelID, periodID })
|
|
189
|
+
const existing = buckets.get(key)
|
|
190
|
+
if (existing) {
|
|
191
|
+
existing.requests += 1
|
|
192
|
+
existing.input += event.input
|
|
193
|
+
existing.output += event.output
|
|
194
|
+
existing.cacheRead += event.cacheRead
|
|
195
|
+
existing.cacheWrite += event.cacheWrite
|
|
196
|
+
existing.usd += event.usd
|
|
197
|
+
return
|
|
198
|
+
}
|
|
199
|
+
buckets.set(key, {
|
|
200
|
+
hour,
|
|
201
|
+
providerID: event.providerID,
|
|
202
|
+
modelID: event.modelID,
|
|
203
|
+
periodID,
|
|
204
|
+
requests: 1,
|
|
205
|
+
input: event.input,
|
|
206
|
+
output: event.output,
|
|
207
|
+
cacheRead: event.cacheRead,
|
|
208
|
+
cacheWrite: event.cacheWrite,
|
|
209
|
+
usd: event.usd,
|
|
210
|
+
failures: 0,
|
|
211
|
+
latencyMsTotal: 0,
|
|
212
|
+
latencySamples: 0,
|
|
213
|
+
})
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
const asSessionState = (value: unknown): SessionState | undefined => { if (!isRecord(value) || typeof value.sessionID !== "string") return undefined
|
|
217
|
+
return {
|
|
218
|
+
sessionID: value.sessionID,
|
|
219
|
+
...(typeof value.parentID === "string" ? { parentID: value.parentID } : {}),
|
|
220
|
+
...(typeof value.projectID === "string" ? { projectID: value.projectID } : {}),
|
|
221
|
+
...(typeof value.directory === "string" ? { directory: value.directory } : {}),
|
|
222
|
+
manualOverride: value.manualOverride === true,
|
|
223
|
+
manualModel: asModelRef(value.manualModel),
|
|
224
|
+
pinnedModel: asModelRef(value.pinnedModel),
|
|
225
|
+
premiumOneShot: value.premiumOneShot === true,
|
|
226
|
+
premiumOneShotAt: typeof value.premiumOneShotAt === "number" ? value.premiumOneShotAt : undefined,
|
|
227
|
+
paused: value.paused === true,
|
|
228
|
+
lastModel: asModelRef(value.lastModel),
|
|
229
|
+
lastRoutedModel: asModelRef(value.lastRoutedModel),
|
|
230
|
+
lastDecisionID: typeof value.lastDecisionID === "string" ? value.lastDecisionID : undefined,
|
|
231
|
+
contextTokens: Math.max(0, finite(value.contextTokens)),
|
|
232
|
+
cacheReadTokens: Math.max(0, finite(value.cacheReadTokens)),
|
|
233
|
+
stepCount: Math.max(0, finite(value.stepCount)),
|
|
234
|
+
status: value.status === "busy" || value.status === "idle" ? value.status : "unknown",
|
|
235
|
+
failures: Array.isArray(value.failures)
|
|
236
|
+
? value.failures
|
|
237
|
+
.filter(isRecord)
|
|
238
|
+
.slice(-12)
|
|
239
|
+
.map((failure) => ({
|
|
240
|
+
class: (typeof failure.class === "string" ? failure.class : "unknown") as FailureClass,
|
|
241
|
+
at: finite(failure.at),
|
|
242
|
+
message: typeof failure.message === "string" ? failure.message.slice(0, 300) : "",
|
|
243
|
+
}))
|
|
244
|
+
: [],
|
|
245
|
+
retries: Math.max(0, finite(value.retries)),
|
|
246
|
+
createdAt: finite(value.createdAt, Date.now()),
|
|
247
|
+
updatedAt: finite(value.updatedAt, Date.now()),
|
|
248
|
+
...(typeof value.title === "string" ? { title: value.title.slice(0, 200) } : {}),
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
export const hydrateLedger = (value: unknown): { ledger: Ledger; warnings: string[] } => {
|
|
253
|
+
const warnings: string[] = []
|
|
254
|
+
const now = Date.now()
|
|
255
|
+
if (!isRecord(value)) return { ledger: emptyLedger(now), warnings: ["No saved ledger found"] }
|
|
256
|
+
if (value.version !== LEDGER_VERSION) warnings.push("Saved ledger version differs; mixing retained data")
|
|
257
|
+
|
|
258
|
+
const metrics = isRecord(value.metrics) ? value.metrics : {}
|
|
259
|
+
const byModel: Metrics["byModel"] = {}
|
|
260
|
+
if (isRecord(metrics.byModel)) {
|
|
261
|
+
for (const [key, entry] of Object.entries(metrics.byModel)) {
|
|
262
|
+
if (!isRecord(entry)) continue
|
|
263
|
+
byModel[key] = {
|
|
264
|
+
requests: Math.max(0, finite(entry.requests)),
|
|
265
|
+
input: Math.max(0, finite(entry.input)),
|
|
266
|
+
output: Math.max(0, finite(entry.output)),
|
|
267
|
+
cacheRead: Math.max(0, finite(entry.cacheRead)),
|
|
268
|
+
cacheWrite: Math.max(0, finite(entry.cacheWrite)),
|
|
269
|
+
usd: Math.max(0, finite(entry.usd)),
|
|
270
|
+
failures: Math.max(0, finite(entry.failures)),
|
|
271
|
+
outputTokensPerSecond: Math.max(0, finite(entry.outputTokensPerSecond)),
|
|
272
|
+
latencyMsTotal: Math.max(0, finite(entry.latencyMsTotal)),
|
|
273
|
+
latencySamples: Math.max(0, finite(entry.latencySamples)),
|
|
274
|
+
timeoutFailures: Math.max(0, finite(entry.timeoutFailures)),
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
const cooldowns: Record<string, number> = {}
|
|
280
|
+
if (isRecord(value.cooldowns)) {
|
|
281
|
+
for (const [key, until] of Object.entries(value.cooldowns)) {
|
|
282
|
+
const parsed = finite(until)
|
|
283
|
+
if (parsed > 0) cooldowns[key] = parsed
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
const providerCooldowns: Record<string, number> = {}
|
|
287
|
+
if (isRecord(value.providerCooldowns)) {
|
|
288
|
+
for (const [key, until] of Object.entries(value.providerCooldowns)) {
|
|
289
|
+
const parsed = finite(until)
|
|
290
|
+
if (parsed > 0) providerCooldowns[key] = parsed
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
const sessions: Record<string, SessionState> = {}
|
|
295
|
+
if (isRecord(value.sessions)) { for (const entry of Object.values(value.sessions)) {
|
|
296
|
+
const session = asSessionState(entry)
|
|
297
|
+
if (session) sessions[session.sessionID] = session
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
const decisions: RoutingDecision[] = []
|
|
302
|
+
if (Array.isArray(value.decisions)) {
|
|
303
|
+
for (const entry of value.decisions) {
|
|
304
|
+
if (isRecord(entry) && typeof entry.id === "string" && typeof entry.at === "number") {
|
|
305
|
+
decisions.push(entry as unknown as RoutingDecision)
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
const usage: UsageEvent[] = []
|
|
311
|
+
if (Array.isArray(value.usage)) {
|
|
312
|
+
for (const entry of value.usage) {
|
|
313
|
+
const event = asUsageEvent(entry)
|
|
314
|
+
if (event) usage.push(event)
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
const buckets: UsageBucket[] = []
|
|
319
|
+
const rollupInput = isRecord(value.usageRollup) ? value.usageRollup : undefined
|
|
320
|
+
if (rollupInput && Array.isArray(rollupInput.buckets)) {
|
|
321
|
+
for (const entry of rollupInput.buckets) {
|
|
322
|
+
const bucket = asUsageBucket(entry)
|
|
323
|
+
if (bucket) buckets.push(bucket)
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/**
|
|
328
|
+
* A ledger written before aggregates existed has more raw events than the
|
|
329
|
+
* cap, and truncating it on load would discard exactly the history the
|
|
330
|
+
* aggregates were introduced to keep. Fold the overflow into buckets here so
|
|
331
|
+
* the first load after an upgrade preserves every token.
|
|
332
|
+
*/
|
|
333
|
+
let loadTruncated: UsageEvent[] | undefined
|
|
334
|
+
if (usage.length > MAX_USAGE_EVENTS && buckets.length === 0) {
|
|
335
|
+
const byKey = new Map<string, UsageBucket>()
|
|
336
|
+
const overflow = usage.length - MAX_USAGE_EVENTS
|
|
337
|
+
for (const event of usage.slice(0, overflow)) {
|
|
338
|
+
foldEventInto(byKey, event, monthKey(event.at))
|
|
339
|
+
}
|
|
340
|
+
buckets.push(...byKey.values())
|
|
341
|
+
loadTruncated = usage.slice(overflow)
|
|
342
|
+
warnings.push(
|
|
343
|
+
`Folded ${overflow} older usage events into hourly aggregates on load instead of discarding them`,
|
|
344
|
+
)
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
const failures = emptyFailureCounts()
|
|
348
|
+
if (isRecord(metrics.failures)) {
|
|
349
|
+
for (const key of Object.keys(failures) as FailureClass[]) {
|
|
350
|
+
failures[key] = Math.max(0, finite(metrics.failures[key]))
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
const ledger: Ledger = {
|
|
355
|
+
version: LEDGER_VERSION,
|
|
356
|
+
bootstrappedAt: typeof value.bootstrappedAt === "number" ? value.bootstrappedAt : undefined,
|
|
357
|
+
bootstrapSource: typeof value.bootstrapSource === "string" ? value.bootstrapSource : undefined,
|
|
358
|
+
cooldowns,
|
|
359
|
+
providerCooldowns,
|
|
360
|
+
usage: loadTruncated ? loadTruncated.slice(-MAX_USAGE_EVENTS) : usage.slice(-MAX_USAGE_EVENTS),
|
|
361
|
+
...(buckets.length > 0
|
|
362
|
+
? {
|
|
363
|
+
usageRollup: {
|
|
364
|
+
version: 1 as const,
|
|
365
|
+
buckets,
|
|
366
|
+
recent: usage.slice(-MAX_USAGE_EVENTS),
|
|
367
|
+
updatedAt: finite(rollupInput?.updatedAt, now),
|
|
368
|
+
},
|
|
369
|
+
}
|
|
370
|
+
: {}),
|
|
371
|
+
premiumSpend: Array.isArray(value.premiumSpend)
|
|
372
|
+
? value.premiumSpend
|
|
373
|
+
.filter(isRecord)
|
|
374
|
+
.map((entry) => ({ month: String(entry.month ?? ""), usd: Math.max(0, finite(entry.usd)) }))
|
|
375
|
+
.filter((entry) => entry.month)
|
|
376
|
+
: [],
|
|
377
|
+
sessions,
|
|
378
|
+
decisions: decisions.slice(0, MAX_DECISIONS),
|
|
379
|
+
metrics: {
|
|
380
|
+
startedAt: finite(metrics.startedAt, now),
|
|
381
|
+
requests: Math.max(0, finite(metrics.requests)),
|
|
382
|
+
inputTokens: Math.max(0, finite(metrics.inputTokens)),
|
|
383
|
+
cacheReadTokens: Math.max(0, finite(metrics.cacheReadTokens)),
|
|
384
|
+
cacheWriteTokens: Math.max(0, finite(metrics.cacheWriteTokens)),
|
|
385
|
+
outputTokens: Math.max(0, finite(metrics.outputTokens)),
|
|
386
|
+
estimatedUsd: Math.max(0, finite(metrics.estimatedUsd)),
|
|
387
|
+
switches: Math.max(0, finite(metrics.switches)),
|
|
388
|
+
escalations: Math.max(0, finite(metrics.escalations)),
|
|
389
|
+
retries: Math.max(0, finite(metrics.retries)),
|
|
390
|
+
preventedPremium: Math.max(0, finite(metrics.preventedPremium)),
|
|
391
|
+
manualOverrides: Math.max(0, finite(metrics.manualOverrides)),
|
|
392
|
+
dryRunDecisions: Math.max(0, finite(metrics.dryRunDecisions)),
|
|
393
|
+
premiumRequests: Math.max(0, finite(metrics.premiumRequests)),
|
|
394
|
+
premiumUsd: Math.max(0, finite(metrics.premiumUsd)),
|
|
395
|
+
latencyMsTotal: Math.max(0, finite(metrics.latencyMsTotal)),
|
|
396
|
+
latencySamples: Math.max(0, finite(metrics.latencySamples)),
|
|
397
|
+
lastLatencyMs: typeof metrics.lastLatencyMs === "number" ? metrics.lastLatencyMs : undefined,
|
|
398
|
+
failures,
|
|
399
|
+
byModel,
|
|
400
|
+
timeouts: Math.max(0, finite(metrics.timeouts)),
|
|
401
|
+
},
|
|
402
|
+
updatedAt: now,
|
|
403
|
+
}
|
|
404
|
+
return { ledger, warnings }
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
export interface StepUsage {
|
|
408
|
+
readonly sessionID: string
|
|
409
|
+
readonly model: ModelRef
|
|
410
|
+
readonly cost: number
|
|
411
|
+
readonly tokens: {
|
|
412
|
+
readonly input: number
|
|
413
|
+
readonly output: number
|
|
414
|
+
readonly reasoning?: number
|
|
415
|
+
readonly cache: { readonly read: number; readonly write: number }
|
|
416
|
+
}
|
|
417
|
+
readonly at?: number
|
|
418
|
+
readonly latencyMs?: number
|
|
419
|
+
readonly premium?: boolean
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
export interface BootstrapSession {
|
|
423
|
+
readonly id: string
|
|
424
|
+
readonly title?: string
|
|
425
|
+
readonly model?: ModelRef
|
|
426
|
+
readonly cost: number
|
|
427
|
+
readonly tokens?: {
|
|
428
|
+
readonly input: number
|
|
429
|
+
readonly output: number
|
|
430
|
+
readonly cache: { readonly read: number; readonly write: number }
|
|
431
|
+
}
|
|
432
|
+
readonly updatedAt: number
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
export interface ModelUsageTotals {
|
|
436
|
+
readonly monthlyUsd: number
|
|
437
|
+
readonly fiveHourUsd: number
|
|
438
|
+
readonly weeklyUsd: number
|
|
439
|
+
readonly input: number
|
|
440
|
+
readonly output: number
|
|
441
|
+
readonly cacheRead: number
|
|
442
|
+
readonly requests: number
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
/**
|
|
446
|
+
* Local ledger for observed usage. Provider-reported totals, when available,
|
|
447
|
+
* are kept in the provider snapshot; this ledger supplies per-model estimates
|
|
448
|
+
* and the fallback when a provider has no readable billing endpoint.
|
|
449
|
+
*/
|
|
450
|
+
export class LedgerStore {
|
|
451
|
+
private ledger: Ledger
|
|
452
|
+
private dirty = false
|
|
453
|
+
/**
|
|
454
|
+
* Consecutive provider-scoped failures with no successful request in
|
|
455
|
+
* between, per provider. A short streak is a transient blip that a retry
|
|
456
|
+
* rides over; a long one means the provider is genuinely not serving.
|
|
457
|
+
* Ephemeral on purpose: it is only meaningful across back-to-back attempts.
|
|
458
|
+
*/
|
|
459
|
+
private readonly providerFaultStreak = new Map<string, number>()
|
|
460
|
+
|
|
461
|
+
constructor(initial?: Ledger) {
|
|
462
|
+
this.ledger = initial ?? emptyLedger()
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
static hydrate(value: unknown) {
|
|
466
|
+
return hydrateLedger(value)
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
get data(): Readonly<Ledger> {
|
|
470
|
+
return this.ledger
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
reset() {
|
|
474
|
+
this.ledger = emptyLedger()
|
|
475
|
+
this.dirty = true
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
get dirtyFlag() {
|
|
479
|
+
return this.dirty
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
markClean() {
|
|
483
|
+
this.dirty = false
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
private touch() {
|
|
487
|
+
this.ledger.updatedAt = Date.now()
|
|
488
|
+
this.dirty = true
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
ensureSession(sessionID: string, parentID?: string, title?: string): SessionState {
|
|
492
|
+
const existing = this.ledger.sessions[sessionID]
|
|
493
|
+
if (existing) {
|
|
494
|
+
if (parentID && !existing.parentID) (existing as { parentID?: string }).parentID = parentID
|
|
495
|
+
if (title && !existing.title) existing.title = title
|
|
496
|
+
return existing
|
|
497
|
+
}
|
|
498
|
+
const now = Date.now()
|
|
499
|
+
const session: SessionState = {
|
|
500
|
+
sessionID,
|
|
501
|
+
...(parentID ? { parentID } : {}),
|
|
502
|
+
manualOverride: false,
|
|
503
|
+
premiumOneShot: false,
|
|
504
|
+
paused: false,
|
|
505
|
+
contextTokens: 0,
|
|
506
|
+
cacheReadTokens: 0,
|
|
507
|
+
stepCount: 0,
|
|
508
|
+
status: "unknown",
|
|
509
|
+
failures: [],
|
|
510
|
+
retries: 0,
|
|
511
|
+
createdAt: now,
|
|
512
|
+
updatedAt: now,
|
|
513
|
+
...(title ? { title } : {}),
|
|
514
|
+
}
|
|
515
|
+
this.ledger.sessions[sessionID] = session
|
|
516
|
+
this.touch()
|
|
517
|
+
return session
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
getSession(sessionID: string): SessionState | undefined {
|
|
521
|
+
return this.ledger.sessions[sessionID]
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
patchSession(sessionID: string, patch: Partial<SessionState>): SessionState {
|
|
525
|
+
const session = this.ensureSession(sessionID)
|
|
526
|
+
Object.assign(session, patch, { updatedAt: Date.now() })
|
|
527
|
+
this.touch()
|
|
528
|
+
return session
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
setManual(sessionID: string, model: ModelRef) {
|
|
532
|
+
const session = this.ensureSession(sessionID)
|
|
533
|
+
session.manualOverride = true
|
|
534
|
+
session.manualModel = model
|
|
535
|
+
session.updatedAt = Date.now()
|
|
536
|
+
this.ledger.metrics.manualOverrides += 1
|
|
537
|
+
this.touch()
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
clearManual(sessionID: string) {
|
|
541
|
+
const session = this.ensureSession(sessionID)
|
|
542
|
+
session.manualOverride = false
|
|
543
|
+
session.manualModel = undefined
|
|
544
|
+
session.updatedAt = Date.now()
|
|
545
|
+
this.touch()
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
grantPremiumOneShot(sessionID: string) {
|
|
549
|
+
const session = this.ensureSession(sessionID)
|
|
550
|
+
session.premiumOneShot = true
|
|
551
|
+
session.premiumOneShotAt = Date.now()
|
|
552
|
+
session.updatedAt = Date.now()
|
|
553
|
+
this.touch()
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
consumePremiumOneShot(sessionID: string) {
|
|
557
|
+
const session = this.ensureSession(sessionID)
|
|
558
|
+
session.premiumOneShot = false
|
|
559
|
+
session.premiumOneShotAt = undefined
|
|
560
|
+
session.updatedAt = Date.now()
|
|
561
|
+
this.touch()
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
premiumSpent(month = monthKey()) {
|
|
565
|
+
return this.ledger.premiumSpend.find((entry) => entry.month === month)?.usd ?? 0
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
private addPremiumSpend(usd: number) {
|
|
569
|
+
if (usd <= 0) return
|
|
570
|
+
const month = monthKey()
|
|
571
|
+
const existing = this.ledger.premiumSpend.find((entry) => entry.month === month)
|
|
572
|
+
if (existing) existing.usd += usd
|
|
573
|
+
else this.ledger.premiumSpend.push({ month, usd })
|
|
574
|
+
this.ledger.metrics.premiumUsd += usd
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
recordUsage(input: StepUsage) {
|
|
578
|
+
const at = input.at ?? Date.now()
|
|
579
|
+
const event: UsageEvent = {
|
|
580
|
+
id: nowId("use"),
|
|
581
|
+
at,
|
|
582
|
+
providerID: input.model.providerID,
|
|
583
|
+
modelID: input.model.id,
|
|
584
|
+
sessionID: input.sessionID,
|
|
585
|
+
usd: Math.max(0, input.cost),
|
|
586
|
+
input: Math.max(0, input.tokens.input),
|
|
587
|
+
output: Math.max(0, input.tokens.output),
|
|
588
|
+
cacheRead: Math.max(0, input.tokens.cache.read),
|
|
589
|
+
cacheWrite: Math.max(0, input.tokens.cache.write),
|
|
590
|
+
source: "step",
|
|
591
|
+
}
|
|
592
|
+
this.ledger.usage.push(event)
|
|
593
|
+
this.ledger.metrics.requests += 1
|
|
594
|
+
this.ledger.metrics.inputTokens += event.input
|
|
595
|
+
this.ledger.metrics.outputTokens += event.output
|
|
596
|
+
this.ledger.metrics.cacheReadTokens += event.cacheRead
|
|
597
|
+
this.ledger.metrics.cacheWriteTokens += event.cacheWrite
|
|
598
|
+
this.ledger.metrics.estimatedUsd += event.usd
|
|
599
|
+
if (input.premium) this.ledger.metrics.premiumRequests += 1
|
|
600
|
+
if (input.latencyMs !== undefined && Number.isFinite(input.latencyMs)) {
|
|
601
|
+
this.ledger.metrics.latencySamples += 1
|
|
602
|
+
this.ledger.metrics.latencyMsTotal += Math.max(0, input.latencyMs)
|
|
603
|
+
this.ledger.metrics.lastLatencyMs = Math.max(0, input.latencyMs)
|
|
604
|
+
}
|
|
605
|
+
const key = modelKey(input.model.providerID, input.model.id)
|
|
606
|
+
const modelMetrics = (this.ledger.metrics.byModel[key] ??= emptyModelMetrics())
|
|
607
|
+
modelMetrics.requests += 1
|
|
608
|
+
modelMetrics.input += event.input
|
|
609
|
+
modelMetrics.output += event.output
|
|
610
|
+
modelMetrics.cacheRead += event.cacheRead
|
|
611
|
+
modelMetrics.cacheWrite += event.cacheWrite
|
|
612
|
+
modelMetrics.usd += event.usd
|
|
613
|
+
if (input.latencyMs !== undefined && Number.isFinite(input.latencyMs) && input.latencyMs > 0) {
|
|
614
|
+
const latency = Math.max(0, input.latencyMs)
|
|
615
|
+
modelMetrics.latencySamples += 1
|
|
616
|
+
modelMetrics.latencyMsTotal += latency
|
|
617
|
+
// Generation speed is the only figure that separates a slow provider
|
|
618
|
+
// from a model that simply reasons for longer: dividing by the tokens
|
|
619
|
+
// actually produced normalizes for reasoning depth.
|
|
620
|
+
if (event.output > 0) {
|
|
621
|
+
const seconds = latency / 1000
|
|
622
|
+
const instant = event.output / Math.max(0.001, seconds)
|
|
623
|
+
// A cumulative mean keeps the metric stable without storing history.
|
|
624
|
+
const previous = modelMetrics.outputTokensPerSecond
|
|
625
|
+
modelMetrics.outputTokensPerSecond = previous > 0 ? previous * 0.7 + instant * 0.3 : instant
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
if (input.premium) this.addPremiumSpend(event.usd)
|
|
629
|
+
|
|
630
|
+
const session = this.ensureSession(input.sessionID)
|
|
631
|
+
session.lastModel = input.model
|
|
632
|
+
session.contextTokens = event.input + event.cacheRead + event.cacheWrite
|
|
633
|
+
session.cacheReadTokens = event.cacheRead
|
|
634
|
+
session.stepCount += 1
|
|
635
|
+
session.updatedAt = at
|
|
636
|
+
this.prune(at)
|
|
637
|
+
this.touch()
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
recordDecision(decision: RoutingDecision) {
|
|
641
|
+
this.ledger.decisions.unshift(decision)
|
|
642
|
+
this.ledger.decisions = this.ledger.decisions.slice(0, MAX_DECISIONS)
|
|
643
|
+
if (decision.switched) this.ledger.metrics.switches += 1
|
|
644
|
+
if (decision.dryRun) this.ledger.metrics.dryRunDecisions += 1
|
|
645
|
+
if (decision.needsPremiumApproval) this.ledger.metrics.preventedPremium += 1
|
|
646
|
+
if (decision.chosen?.providerID && decision.chosen) {
|
|
647
|
+
const chosen = decision.candidates.find(
|
|
648
|
+
(candidate) =>
|
|
649
|
+
candidate.model.providerID === decision.chosen?.providerID && candidate.model.modelID === decision.chosen?.id,
|
|
650
|
+
)
|
|
651
|
+
const current = decision.candidates.find(
|
|
652
|
+
(candidate) =>
|
|
653
|
+
candidate.model.providerID === decision.current?.providerID && candidate.model.modelID === decision.current?.id,
|
|
654
|
+
)
|
|
655
|
+
if (chosen && current && chosen.tier > current.tier) this.ledger.metrics.escalations += 1
|
|
656
|
+
}
|
|
657
|
+
const session = this.ensureSession(decision.sessionID)
|
|
658
|
+
session.lastDecisionID = decision.id
|
|
659
|
+
if (decision.chosen) session.lastRoutedModel = decision.chosen
|
|
660
|
+
session.updatedAt = decision.at
|
|
661
|
+
this.touch()
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
recordFailure(
|
|
665
|
+
sessionID: string,
|
|
666
|
+
model: ModelRef | undefined,
|
|
667
|
+
failureClass: FailureClass,
|
|
668
|
+
message: string,
|
|
669
|
+
options: {
|
|
670
|
+
readonly countAgainstModel?: boolean
|
|
671
|
+
readonly cooldownMs?: number
|
|
672
|
+
readonly providerScoped?: boolean
|
|
673
|
+
/** The gateway refused for capacity reasons; holds back the provider and this endpoint. */
|
|
674
|
+
readonly saturation?: boolean
|
|
675
|
+
} = {},
|
|
676
|
+
) {
|
|
677
|
+
this.ledger.metrics.failures[failureClass] = (this.ledger.metrics.failures[failureClass] ?? 0) + 1
|
|
678
|
+
|
|
679
|
+
// Provider-scoped faults feed the provider streak. A billing-scope fault
|
|
680
|
+
// and a saturated gateway both mean the provider, not the model, is the
|
|
681
|
+
// thing failing, so both count here.
|
|
682
|
+
if ((options.providerScoped || options.saturation) && model) {
|
|
683
|
+
const streak = (this.providerFaultStreak.get(model.providerID) ?? 0) + 1
|
|
684
|
+
this.providerFaultStreak.set(model.providerID, streak)
|
|
685
|
+
if (streak >= PROVIDER_DOWN_STREAK) {
|
|
686
|
+
const until = Date.now() + PROVIDER_BACKOFF_MS
|
|
687
|
+
this.ledger.providerCooldowns = {
|
|
688
|
+
...(this.ledger.providerCooldowns ?? {}),
|
|
689
|
+
[model.providerID]: Math.max(this.ledger.providerCooldowns?.[model.providerID] ?? 0, until),
|
|
690
|
+
}
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
// A billing-scope fault happens in the provider's billing path before any
|
|
695
|
+
// model is selected, so cooling the model that happened to be serving
|
|
696
|
+
// would be arbitrary blame. Stop here and let the streak handle it.
|
|
697
|
+
if (options.providerScoped && !options.saturation && model) {
|
|
698
|
+
const session = this.ensureSession(sessionID)
|
|
699
|
+
session.failures = [...session.failures, { class: failureClass, at: Date.now(), message: safeError(message) }].slice(-12)
|
|
700
|
+
session.updatedAt = Date.now()
|
|
701
|
+
this.touch()
|
|
702
|
+
return
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
// A saturated gateway is a different case again: the endpoint really did
|
|
706
|
+
// refuse this request, so it must be held back at once rather than after
|
|
707
|
+
// the provider streak reaches five. But the refusal says nothing about how
|
|
708
|
+
// capable the model is, so it is kept out of the reliability record that
|
|
709
|
+
// drives the success estimate. Holding back and blaming are separate.
|
|
710
|
+
if (options.saturation && model) {
|
|
711
|
+
this.armModelCooldown(model, options.cooldownMs ?? 0, 1)
|
|
712
|
+
const session = this.ensureSession(sessionID)
|
|
713
|
+
session.failures = [...session.failures, { class: failureClass, at: Date.now(), message: safeError(message) }].slice(-12)
|
|
714
|
+
session.updatedAt = Date.now()
|
|
715
|
+
this.touch()
|
|
716
|
+
return
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
// Only a provider fault counts against a model's reliability. A user
|
|
720
|
+
// interrupt must never make the router distrust a healthy model.
|
|
721
|
+
if (model && options.countAgainstModel !== false) {
|
|
722
|
+
const key = modelKey(model.providerID, model.id)
|
|
723
|
+
const metrics = (this.ledger.metrics.byModel[key] ??= emptyModelMetrics())
|
|
724
|
+
metrics.failures += 1
|
|
725
|
+
if (failureClass === "context_overflow" || failureClass === "rate_limit") {
|
|
726
|
+
metrics.timeoutFailures += 1
|
|
727
|
+
this.ledger.metrics.timeouts += 1
|
|
728
|
+
}
|
|
729
|
+
if ((options.cooldownMs ?? 0) > 0) {
|
|
730
|
+
// Repeated faults extend the cooldown, so a model that keeps dropping
|
|
731
|
+
// streams is avoided for longer each time.
|
|
732
|
+
this.armModelCooldown(model, options.cooldownMs ?? 0, Math.min(4, (metrics.failures % 4) + 1))
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
const session = this.ensureSession(sessionID)
|
|
736
|
+
session.failures = [...session.failures, { class: failureClass, at: Date.now(), message: safeError(message) }].slice(-12)
|
|
737
|
+
session.updatedAt = Date.now()
|
|
738
|
+
this.touch()
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
/** Active failure cooldowns, keyed by model key. */
|
|
742
|
+
cooldowns(): Readonly<Record<string, number>> {
|
|
743
|
+
return this.ledger.cooldowns ?? {}
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
/**
|
|
747
|
+
* Hold one model back from routing without touching its reliability record.
|
|
748
|
+
*
|
|
749
|
+
* These are separate concerns and were previously welded together: a model
|
|
750
|
+
* could only be cooled by also being judged less reliable. A gateway
|
|
751
|
+
* refusing for capacity is proof that this endpoint is not serving, and no
|
|
752
|
+
* evidence at all about how well the model reasons.
|
|
753
|
+
*/
|
|
754
|
+
private armModelCooldown(model: ModelRef, cooldownMs: number, strikes: number) {
|
|
755
|
+
if (cooldownMs <= 0) return
|
|
756
|
+
const key = modelKey(model.providerID, model.id)
|
|
757
|
+
const previous = this.ledger.cooldowns?.[key] ?? 0
|
|
758
|
+
const until = Date.now() + cooldownMs * Math.max(1, strikes)
|
|
759
|
+
this.ledger.cooldowns = { ...(this.ledger.cooldowns ?? {}), [key]: Math.max(previous, until) }
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
/**
|
|
763
|
+
* Hold one model back from routing for the given window, without touching
|
|
764
|
+
* its reliability record or its failure history.
|
|
765
|
+
*
|
|
766
|
+
* The prompt-rejection failover uses this: the model that rejected a
|
|
767
|
+
* request is healthy, the request shape was not, and the next prompt should
|
|
768
|
+
* route elsewhere while the shape stays what it is.
|
|
769
|
+
*/
|
|
770
|
+
holdModel(model: ModelRef, cooldownMs: number): void {
|
|
771
|
+
this.armModelCooldown(model, cooldownMs, 1)
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
/** Active provider outage backoffs, keyed by provider id. */
|
|
775
|
+
providerCooldowns(): Readonly<Record<string, number>> {
|
|
776
|
+
return this.ledger.providerCooldowns ?? {}
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
/**
|
|
780
|
+
* Hold a provider back until a given time because its allowance is spent.
|
|
781
|
+
*
|
|
782
|
+
* Separate from the streak-based backoff in `recordFailure`: a spent
|
|
783
|
+
* allowance is not a fault that might clear in a minute, it is a fact that
|
|
784
|
+
* holds until the window resets. Never shortens an existing cooldown, so a
|
|
785
|
+
* shorter guess cannot undo a longer one.
|
|
786
|
+
*/
|
|
787
|
+
exhaustProvider(providerID: string, until: number): void {
|
|
788
|
+
const current = this.ledger.providerCooldowns?.[providerID] ?? 0
|
|
789
|
+
if (until <= current) return
|
|
790
|
+
this.ledger.providerCooldowns = { ...(this.ledger.providerCooldowns ?? {}), [providerID]: until }
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
isProviderCoolingDown(providerID: string, now = Date.now()): boolean {
|
|
794
|
+
const until = this.ledger.providerCooldowns?.[providerID]
|
|
795
|
+
return typeof until === "number" && until > now
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
/**
|
|
799
|
+
* A successful request proves the provider is serving again, so the
|
|
800
|
+
* consecutive-failure streak starts over.
|
|
801
|
+
*/
|
|
802
|
+
recordProviderSuccess(providerID: string) {
|
|
803
|
+
if (this.providerFaultStreak.size > 0) this.providerFaultStreak.delete(providerID)
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
isCoolingDown(modelKeyValue: string, now = Date.now()): boolean {
|
|
807
|
+
const until = this.ledger.cooldowns?.[modelKeyValue]
|
|
808
|
+
return typeof until === "number" && until > now
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
recordRetry(sessionID: string) {
|
|
812
|
+
this.ledger.metrics.retries += 1
|
|
813
|
+
const session = this.ensureSession(sessionID)
|
|
814
|
+
session.retries += 1
|
|
815
|
+
session.updatedAt = Date.now()
|
|
816
|
+
this.touch()
|
|
817
|
+
}
|
|
818
|
+
|
|
819
|
+
bootstrap(sessions: ReadonlyArray<BootstrapSession>, input: { days: number; limit: number; now?: number }) {
|
|
820
|
+
if (this.ledger.bootstrappedAt) return { added: 0, skipped: true }
|
|
821
|
+
const now = input.now ?? Date.now()
|
|
822
|
+
const cutoff = now - input.days * 24 * 60 * 60 * 1000
|
|
823
|
+
const eligible = sessions
|
|
824
|
+
.filter((session) => session.cost > 0 && session.updatedAt >= cutoff && session.model)
|
|
825
|
+
.sort((a, b) => b.updatedAt - a.updatedAt)
|
|
826
|
+
.slice(0, Math.max(0, input.limit))
|
|
827
|
+
|
|
828
|
+
for (const session of eligible) {
|
|
829
|
+
const model = session.model
|
|
830
|
+
if (!model) continue
|
|
831
|
+
this.ledger.usage.push({
|
|
832
|
+
id: nowId("boot"),
|
|
833
|
+
at: Math.min(now, session.updatedAt),
|
|
834
|
+
providerID: model.providerID,
|
|
835
|
+
modelID: model.id,
|
|
836
|
+
sessionID: session.id,
|
|
837
|
+
usd: Math.max(0, session.cost),
|
|
838
|
+
input: Math.max(0, session.tokens?.input ?? 0),
|
|
839
|
+
output: Math.max(0, session.tokens?.output ?? 0),
|
|
840
|
+
cacheRead: Math.max(0, session.tokens?.cache.read ?? 0),
|
|
841
|
+
cacheWrite: Math.max(0, session.tokens?.cache.write ?? 0),
|
|
842
|
+
source: "bootstrap",
|
|
843
|
+
})
|
|
844
|
+
}
|
|
845
|
+
|
|
846
|
+
this.ledger.bootstrappedAt = now
|
|
847
|
+
this.ledger.bootstrapSource = `Recent sessions (${eligible.length}) attributed to each session's selected model; approximate when a session switched models.`
|
|
848
|
+
this.prune(now)
|
|
849
|
+
this.touch()
|
|
850
|
+
return { added: eligible.length, skipped: false }
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
/**
|
|
854
|
+
* Seed the ledger from OpenCode's own message history.
|
|
855
|
+
*
|
|
856
|
+
* Better than `bootstrap` in three ways, all of which matter for a fixed
|
|
857
|
+
* seven-day window:
|
|
858
|
+
*
|
|
859
|
+
* - it is per MESSAGE, so each event carries its real timestamp and lands in
|
|
860
|
+
* the correct window instead of being smeared onto the session's last
|
|
861
|
+
* update time
|
|
862
|
+
* - it records the model that actually served each request, not the session's
|
|
863
|
+
* currently selected model
|
|
864
|
+
* - it can be re-read, because the database keeps growing and a one-shot
|
|
865
|
+
* import stops being true the moment a new session runs
|
|
866
|
+
*
|
|
867
|
+
* Rows already present are skipped by id, so re-reading is idempotent.
|
|
868
|
+
*/
|
|
869
|
+
importHistory(
|
|
870
|
+
entries: ReadonlyArray<{
|
|
871
|
+
readonly id: string
|
|
872
|
+
readonly at: number
|
|
873
|
+
readonly providerID: string
|
|
874
|
+
readonly modelID: string
|
|
875
|
+
readonly cost: number
|
|
876
|
+
readonly input: number
|
|
877
|
+
readonly output: number
|
|
878
|
+
readonly cacheRead: number
|
|
879
|
+
readonly cacheWrite: number
|
|
880
|
+
}>,
|
|
881
|
+
input: { readonly now?: number; readonly source?: string } = {},
|
|
882
|
+
): { added: number; skipped: number } {
|
|
883
|
+
const now = input.now ?? Date.now()
|
|
884
|
+
const existing = new Set(this.ledger.usage.map((event) => event.id))
|
|
885
|
+
let added = 0
|
|
886
|
+
let skipped = 0
|
|
887
|
+
for (const entry of entries) {
|
|
888
|
+
const id = `hist:${entry.id}`
|
|
889
|
+
if (existing.has(id)) {
|
|
890
|
+
skipped += 1
|
|
891
|
+
continue
|
|
892
|
+
}
|
|
893
|
+
this.ledger.usage.push({
|
|
894
|
+
id,
|
|
895
|
+
at: Math.min(now, entry.at),
|
|
896
|
+
providerID: entry.providerID,
|
|
897
|
+
modelID: entry.modelID,
|
|
898
|
+
usd: Math.max(0, entry.cost),
|
|
899
|
+
input: Math.max(0, entry.input),
|
|
900
|
+
output: Math.max(0, entry.output),
|
|
901
|
+
cacheRead: Math.max(0, entry.cacheRead),
|
|
902
|
+
cacheWrite: Math.max(0, entry.cacheWrite),
|
|
903
|
+
source: "bootstrap",
|
|
904
|
+
})
|
|
905
|
+
existing.add(id)
|
|
906
|
+
added += 1
|
|
907
|
+
}
|
|
908
|
+
if (added > 0) {
|
|
909
|
+
this.ledger.bootstrappedAt = now
|
|
910
|
+
this.ledger.bootstrapSource =
|
|
911
|
+
input.source ??
|
|
912
|
+
`Imported ${added} messages from OpenCode history, each attributed to the model that actually served it.`
|
|
913
|
+
this.prune(now)
|
|
914
|
+
this.touch()
|
|
915
|
+
}
|
|
916
|
+
return { added, skipped }
|
|
917
|
+
}
|
|
918
|
+
|
|
919
|
+
totalsForModel(providerID: string, modelID: string, input: { now?: number; periodStart?: number } = {}): ModelUsageTotals {
|
|
920
|
+
const now = input.now ?? Date.now()
|
|
921
|
+
const periodStart = input.periodStart ?? now - MONTH_MS
|
|
922
|
+
const fiveHourCutoff = now - FIVE_HOURS_MS
|
|
923
|
+
const weekCutoff = now - WEEK_MS
|
|
924
|
+
let monthlyUsd = 0
|
|
925
|
+
let fiveHourUsd = 0
|
|
926
|
+
let weeklyUsd = 0
|
|
927
|
+
let inputTokens = 0
|
|
928
|
+
let output = 0
|
|
929
|
+
let cacheRead = 0
|
|
930
|
+
let requests = 0
|
|
931
|
+
for (const event of this.ledger.usage) {
|
|
932
|
+
if (event.providerID !== providerID || event.modelID !== modelID) continue
|
|
933
|
+
if (event.at >= periodStart) {
|
|
934
|
+
monthlyUsd += event.usd
|
|
935
|
+
inputTokens += event.input
|
|
936
|
+
output += event.output
|
|
937
|
+
cacheRead += event.cacheRead
|
|
938
|
+
requests += 1
|
|
939
|
+
}
|
|
940
|
+
if (event.at >= fiveHourCutoff) fiveHourUsd += event.usd
|
|
941
|
+
if (event.at >= weekCutoff) weeklyUsd += event.usd
|
|
942
|
+
}
|
|
943
|
+
return { monthlyUsd, fiveHourUsd, weeklyUsd, input: inputTokens, output, cacheRead, requests }
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
modelQuotaSnapshots(
|
|
947
|
+
providerID: string,
|
|
948
|
+
models: ReadonlyArray<ModelRecord>,
|
|
949
|
+
provider: ProviderQuotaSnapshot | undefined,
|
|
950
|
+
settings: ProviderSettings,
|
|
951
|
+
now = Date.now(),
|
|
952
|
+
): Record<string, ModelQuotaSnapshot> {
|
|
953
|
+
const periodStart = provider?.periodStart ?? (provider?.monthly.resetsAt ? provider.monthly.resetsAt - MONTH_MS : now - MONTH_MS)
|
|
954
|
+
const result: Record<string, ModelQuotaSnapshot> = {}
|
|
955
|
+
for (const model of models) {
|
|
956
|
+
if (model.providerID !== providerID) continue
|
|
957
|
+
const totals = this.totalsForModel(providerID, model.modelID, { now, periodStart })
|
|
958
|
+
const monthlyCap = model.monthlyAllowanceUsd > 0 ? model.monthlyAllowanceUsd : 0
|
|
959
|
+
const fiveHourCap = monthlyCap > 0 ? monthlyCap * 0.2 : 0
|
|
960
|
+
const weeklyCap = monthlyCap > 0 ? monthlyCap * 0.5 : 0
|
|
961
|
+
const monthlyUsedPct = monthlyCap > 0 ? pct(totals.monthlyUsd, monthlyCap) : 0
|
|
962
|
+
const fiveHourUsedPct = fiveHourCap > 0 ? pct(totals.fiveHourUsd, fiveHourCap) : 0
|
|
963
|
+
const weeklyUsedPct = weeklyCap > 0 ? pct(totals.weeklyUsd, weeklyCap) : 0
|
|
964
|
+
let blocked = false
|
|
965
|
+
let blockReason: string | undefined
|
|
966
|
+
if (monthlyCap > 0 && monthlyUsedPct >= 100) {
|
|
967
|
+
blocked = true
|
|
968
|
+
blockReason = "Estimated monthly model allowance exhausted"
|
|
969
|
+
} else if (fiveHourCap > 0 && fiveHourUsedPct >= 100) {
|
|
970
|
+
blocked = true
|
|
971
|
+
blockReason = "Estimated five-hour model allowance exhausted"
|
|
972
|
+
} else if (weeklyCap > 0 && weeklyUsedPct >= 100) {
|
|
973
|
+
blocked = true
|
|
974
|
+
blockReason = "Estimated weekly model allowance exhausted"
|
|
975
|
+
}
|
|
976
|
+
const key = modelKey(providerID, model.modelID)
|
|
977
|
+
result[key] = {
|
|
978
|
+
model: { providerID, modelID: model.modelID },
|
|
979
|
+
monthlyAllowanceUsd: monthlyCap,
|
|
980
|
+
monthlyUsedUsd: totals.monthlyUsd,
|
|
981
|
+
monthlyRemainingUsd: Math.max(0, monthlyCap - totals.monthlyUsd),
|
|
982
|
+
monthlyUsedPct,
|
|
983
|
+
fiveHourUsedUsd: totals.fiveHourUsd,
|
|
984
|
+
fiveHourCapUsd: fiveHourCap,
|
|
985
|
+
fiveHourUsedPct,
|
|
986
|
+
weeklyUsedUsd: totals.weeklyUsd,
|
|
987
|
+
weeklyCapUsd: weeklyCap,
|
|
988
|
+
weeklyUsedPct,
|
|
989
|
+
blocked,
|
|
990
|
+
...(blockReason ? { blockReason } : {}),
|
|
991
|
+
source: "estimate",
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
return result
|
|
995
|
+
}
|
|
996
|
+
|
|
997
|
+
attachModelQuotas(
|
|
998
|
+
provider: ProviderQuotaSnapshot,
|
|
999
|
+
models: ReadonlyArray<ModelRecord>,
|
|
1000
|
+
settings: ProviderSettings,
|
|
1001
|
+
now = Date.now(),
|
|
1002
|
+
): ProviderQuotaSnapshot {
|
|
1003
|
+
return {
|
|
1004
|
+
...provider,
|
|
1005
|
+
models: this.modelQuotaSnapshots(provider.providerID, models, provider, settings, now),
|
|
1006
|
+
}
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
snapshotMetrics(): Metrics {
|
|
1010
|
+
return structuredClone(this.ledger.metrics)
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
/**
|
|
1014
|
+
* Raw usage events, used by providers whose allowance must be modelled from
|
|
1015
|
+
* local activity rather than read from a provider endpoint.
|
|
1016
|
+
*/
|
|
1017
|
+
/**
|
|
1018
|
+
* Usage events over a window, including history that has been folded.
|
|
1019
|
+
*
|
|
1020
|
+
* This returns the raw event array when no window is given, which is the
|
|
1021
|
+
* cheapest thing to do and what callers that only need recent detail want.
|
|
1022
|
+
*
|
|
1023
|
+
* A windowed call must not return the raw array alone. Once usage overflows
|
|
1024
|
+
* the raw cap the oldest events are folded into hourly buckets and leave that
|
|
1025
|
+
* array, so a raw-only answer shrinks as history ages. Anything estimating a
|
|
1026
|
+
* weekly or monthly allowance from usage would then under-report spend on a
|
|
1027
|
+
* busy account, which is the opposite of failing safe: the busier the week,
|
|
1028
|
+
* the more the estimate says is left.
|
|
1029
|
+
*
|
|
1030
|
+
* Buckets are expanded back into synthetic events carrying the hour they
|
|
1031
|
+
* belong to. Attribution is per model, so pool assignment still works, but
|
|
1032
|
+
* the individual request detail inside a folded hour is gone by design.
|
|
1033
|
+
*/
|
|
1034
|
+
usageEvents(window?: { since: number; until?: number }): ReadonlyArray<UsageEvent> {
|
|
1035
|
+
if (!window) return this.ledger.usage
|
|
1036
|
+
const { since } = window
|
|
1037
|
+
const until = window.until ?? Date.now()
|
|
1038
|
+
const raw = this.ledger.usage.filter((event) => event.at >= since && event.at <= until)
|
|
1039
|
+
const buckets = this.ledger.usageRollup?.buckets
|
|
1040
|
+
if (!buckets || buckets.length === 0) return raw
|
|
1041
|
+
|
|
1042
|
+
const folded: UsageEvent[] = []
|
|
1043
|
+
for (const bucket of buckets) {
|
|
1044
|
+
if (bucket.hour < since || bucket.hour > until) continue
|
|
1045
|
+
folded.push({
|
|
1046
|
+
id: `folded_${bucket.hour}_${bucket.providerID}_${bucket.modelID}`,
|
|
1047
|
+
at: bucket.hour,
|
|
1048
|
+
providerID: bucket.providerID,
|
|
1049
|
+
modelID: bucket.modelID,
|
|
1050
|
+
usd: bucket.usd,
|
|
1051
|
+
input: bucket.input,
|
|
1052
|
+
output: bucket.output,
|
|
1053
|
+
cacheRead: bucket.cacheRead,
|
|
1054
|
+
cacheWrite: bucket.cacheWrite,
|
|
1055
|
+
source: "step",
|
|
1056
|
+
})
|
|
1057
|
+
}
|
|
1058
|
+
// A bucket and a raw event never cover the same request: folding removes
|
|
1059
|
+
// the event from the raw array as it creates the bucket.
|
|
1060
|
+
return [...folded, ...raw].sort((a, b) => a.at - b.at)
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
/**
|
|
1064
|
+
* Fold aged usage events into hourly buckets instead of discarding them.
|
|
1065
|
+
*
|
|
1066
|
+
* The previous implementation truncated the event array, so a heavy
|
|
1067
|
+
* afternoon silently erased the start of the week and no weekly or monthly
|
|
1068
|
+
* forecast was possible. Folding keeps every token accounted for at a tiny
|
|
1069
|
+
* fraction of the storage cost.
|
|
1070
|
+
*/
|
|
1071
|
+
private foldUsage(now: number) {
|
|
1072
|
+
const rollup: UsageRollup = (this.ledger.usageRollup ??= {
|
|
1073
|
+
version: 1,
|
|
1074
|
+
buckets: [],
|
|
1075
|
+
recent: [],
|
|
1076
|
+
updatedAt: now,
|
|
1077
|
+
})
|
|
1078
|
+
const byKey = new Map<string, UsageBucket>()
|
|
1079
|
+
for (const bucket of rollup.buckets) byKey.set(bucketKeyOf(bucket), bucket)
|
|
1080
|
+
|
|
1081
|
+
const rawCutoff = now - RAW_EVENT_HORIZON_MS
|
|
1082
|
+
const kept: UsageEvent[] = []
|
|
1083
|
+
// When the raw array is above the cap, the oldest events are folded even
|
|
1084
|
+
// if they are still inside the horizon. That keeps the in-memory array
|
|
1085
|
+
// bounded without losing a single token from the totals.
|
|
1086
|
+
const overflow = Math.max(0, this.ledger.usage.length - MAX_USAGE_EVENTS * 0.8)
|
|
1087
|
+
let folded = 0
|
|
1088
|
+
for (const event of this.ledger.usage) {
|
|
1089
|
+
const shouldFold = event.at < rawCutoff || folded < overflow
|
|
1090
|
+
if (!shouldFold) {
|
|
1091
|
+
kept.push(event)
|
|
1092
|
+
continue
|
|
1093
|
+
}
|
|
1094
|
+
folded += 1
|
|
1095
|
+
foldEventInto(byKey, event, this.periodIDFor(event.at))
|
|
1096
|
+
}
|
|
1097
|
+
this.ledger.usage = kept
|
|
1098
|
+
|
|
1099
|
+
const bucketCutoff = now - ROLLUP_RETENTION_MS
|
|
1100
|
+
const buckets = [...byKey.values()].filter((bucket) => bucket.hour >= bucketCutoff)
|
|
1101
|
+
buckets.sort((a, b) => a.hour - b.hour)
|
|
1102
|
+
rollup.buckets =
|
|
1103
|
+
buckets.length > MAX_BUCKETS ? buckets.slice(buckets.length - MAX_BUCKETS) : buckets
|
|
1104
|
+
rollup.recent = kept
|
|
1105
|
+
rollup.updatedAt = now
|
|
1106
|
+
}
|
|
1107
|
+
|
|
1108
|
+
/**
|
|
1109
|
+
* Billing period key for an instant.
|
|
1110
|
+
*
|
|
1111
|
+
* This is a calendar month boundary, which is exact for providers whose
|
|
1112
|
+
* allowance resets per billing cycle and a deliberate approximation for
|
|
1113
|
+
* providers whose period starts on a subscription anniversary. It exists to
|
|
1114
|
+
* keep two *different* billing periods from being summed as one, which is
|
|
1115
|
+
* the failure that matters here.
|
|
1116
|
+
*/
|
|
1117
|
+
periodIDFor(at: number): string {
|
|
1118
|
+
return monthKey(at)
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
/** Totals over an arbitrary window, combining folded buckets and raw events. */
|
|
1122
|
+
aggregate(input: {
|
|
1123
|
+
providerID?: string
|
|
1124
|
+
modelID?: string
|
|
1125
|
+
since: number
|
|
1126
|
+
until?: number
|
|
1127
|
+
}): {
|
|
1128
|
+
since: number
|
|
1129
|
+
until: number
|
|
1130
|
+
requests: number
|
|
1131
|
+
input: number
|
|
1132
|
+
output: number
|
|
1133
|
+
cacheRead: number
|
|
1134
|
+
cacheWrite: number
|
|
1135
|
+
usd: number
|
|
1136
|
+
failures: number
|
|
1137
|
+
latencyMsTotal: number
|
|
1138
|
+
latencySamples: number
|
|
1139
|
+
} {
|
|
1140
|
+
const until = input.until ?? Date.now()
|
|
1141
|
+
const totals = {
|
|
1142
|
+
requests: 0,
|
|
1143
|
+
input: 0,
|
|
1144
|
+
output: 0,
|
|
1145
|
+
cacheRead: 0,
|
|
1146
|
+
cacheWrite: 0,
|
|
1147
|
+
usd: 0,
|
|
1148
|
+
failures: 0,
|
|
1149
|
+
latencyMsTotal: 0,
|
|
1150
|
+
latencySamples: 0,
|
|
1151
|
+
}
|
|
1152
|
+
const matches = (providerID: string, modelID: string) =>
|
|
1153
|
+
(!input.providerID || providerID === input.providerID) && (!input.modelID || modelID === input.modelID)
|
|
1154
|
+
|
|
1155
|
+
for (const bucket of this.ledger.usageRollup?.buckets ?? []) {
|
|
1156
|
+
if (bucket.hour < input.since || bucket.hour > until) continue
|
|
1157
|
+
if (!matches(bucket.providerID, bucket.modelID)) continue
|
|
1158
|
+
totals.requests += bucket.requests
|
|
1159
|
+
totals.input += bucket.input
|
|
1160
|
+
totals.output += bucket.output
|
|
1161
|
+
totals.cacheRead += bucket.cacheRead
|
|
1162
|
+
totals.cacheWrite += bucket.cacheWrite
|
|
1163
|
+
totals.usd += bucket.usd
|
|
1164
|
+
totals.failures += bucket.failures
|
|
1165
|
+
totals.latencyMsTotal += bucket.latencyMsTotal
|
|
1166
|
+
totals.latencySamples += bucket.latencySamples
|
|
1167
|
+
}
|
|
1168
|
+
for (const event of this.ledger.usage) {
|
|
1169
|
+
if (event.at < input.since || event.at > until) continue
|
|
1170
|
+
if (!matches(event.providerID, event.modelID)) continue
|
|
1171
|
+
totals.requests += 1
|
|
1172
|
+
totals.input += event.input
|
|
1173
|
+
totals.output += event.output
|
|
1174
|
+
totals.cacheRead += event.cacheRead
|
|
1175
|
+
totals.cacheWrite += event.cacheWrite
|
|
1176
|
+
totals.usd += event.usd
|
|
1177
|
+
}
|
|
1178
|
+
return { ...totals, since: input.since, until }
|
|
1179
|
+
}
|
|
1180
|
+
|
|
1181
|
+
/** Usage grouped by provider and model over a window. */
|
|
1182
|
+
usageByModel(input: { since: number; until?: number }): Array<{
|
|
1183
|
+
providerID: string
|
|
1184
|
+
modelID: string
|
|
1185
|
+
requests: number
|
|
1186
|
+
input: number
|
|
1187
|
+
output: number
|
|
1188
|
+
cacheRead: number
|
|
1189
|
+
cacheWrite: number
|
|
1190
|
+
usd: number
|
|
1191
|
+
}> {
|
|
1192
|
+
const grouped = new Map<string, {
|
|
1193
|
+
providerID: string
|
|
1194
|
+
modelID: string
|
|
1195
|
+
requests: number
|
|
1196
|
+
input: number
|
|
1197
|
+
output: number
|
|
1198
|
+
cacheRead: number
|
|
1199
|
+
cacheWrite: number
|
|
1200
|
+
usd: number
|
|
1201
|
+
}>()
|
|
1202
|
+
const until = input.until ?? Date.now()
|
|
1203
|
+
const add = (
|
|
1204
|
+
providerID: string,
|
|
1205
|
+
modelID: string,
|
|
1206
|
+
row: { requests: number; input: number; output: number; cacheRead: number; cacheWrite: number; usd: number },
|
|
1207
|
+
) => {
|
|
1208
|
+
const key = modelKey(providerID, modelID)
|
|
1209
|
+
let entry = grouped.get(key)
|
|
1210
|
+
if (!entry) {
|
|
1211
|
+
entry = {
|
|
1212
|
+
providerID,
|
|
1213
|
+
modelID,
|
|
1214
|
+
requests: 0,
|
|
1215
|
+
input: 0,
|
|
1216
|
+
output: 0,
|
|
1217
|
+
cacheRead: 0,
|
|
1218
|
+
cacheWrite: 0,
|
|
1219
|
+
usd: 0,
|
|
1220
|
+
}
|
|
1221
|
+
grouped.set(key, entry)
|
|
1222
|
+
}
|
|
1223
|
+
entry.requests += row.requests
|
|
1224
|
+
entry.input += row.input
|
|
1225
|
+
entry.output += row.output
|
|
1226
|
+
entry.cacheRead += row.cacheRead
|
|
1227
|
+
entry.cacheWrite += row.cacheWrite
|
|
1228
|
+
entry.usd += row.usd
|
|
1229
|
+
}
|
|
1230
|
+
for (const bucket of this.ledger.usageRollup?.buckets ?? []) {
|
|
1231
|
+
if (bucket.hour < input.since || bucket.hour > until) continue
|
|
1232
|
+
add(bucket.providerID, bucket.modelID, bucket)
|
|
1233
|
+
}
|
|
1234
|
+
for (const event of this.ledger.usage) {
|
|
1235
|
+
if (event.at < input.since || event.at > until) continue
|
|
1236
|
+
add(event.providerID, event.modelID, { ...event, requests: 1 })
|
|
1237
|
+
}
|
|
1238
|
+
return [...grouped.values()].sort((a, b) => b.usd - a.usd)
|
|
1239
|
+
}
|
|
1240
|
+
|
|
1241
|
+
prune(now = Date.now()) {
|
|
1242
|
+
this.foldUsage(now)
|
|
1243
|
+
if (this.ledger.decisions.length > MAX_DECISIONS) this.ledger.decisions = this.ledger.decisions.slice(0, MAX_DECISIONS)
|
|
1244
|
+
const sessions = Object.values(this.ledger.sessions)
|
|
1245
|
+
if (sessions.length > MAX_SESSIONS) {
|
|
1246
|
+
sessions
|
|
1247
|
+
.sort((a, b) => a.updatedAt - b.updatedAt)
|
|
1248
|
+
.slice(0, sessions.length - MAX_SESSIONS)
|
|
1249
|
+
.forEach((session) => delete this.ledger.sessions[session.sessionID])
|
|
1250
|
+
}
|
|
1251
|
+
}
|
|
1252
|
+
|
|
1253
|
+
export(): Ledger {
|
|
1254
|
+
this.prune()
|
|
1255
|
+
return structuredClone(this.ledger)
|
|
1256
|
+
}
|
|
1257
|
+
|
|
1258
|
+
get lastDecision(): RoutingDecision | undefined {
|
|
1259
|
+
return this.ledger.decisions[0]
|
|
1260
|
+
}
|
|
1261
|
+
|
|
1262
|
+
decisions(limit = 10): RoutingDecision[] {
|
|
1263
|
+
return this.ledger.decisions.slice(0, clamp(limit, 1, MAX_DECISIONS))
|
|
1264
|
+
}
|
|
1265
|
+
|
|
1266
|
+
modelMetrics(key: string) {
|
|
1267
|
+
return this.ledger.metrics.byModel[key]
|
|
1268
|
+
}
|
|
1269
|
+
|
|
1270
|
+
sessionModelKey(sessionID: string) {
|
|
1271
|
+
return keyOfRef(this.ledger.sessions[sessionID]?.lastModel)
|
|
1272
|
+
}
|
|
1273
|
+
}
|