@velum-labs/routekit-daemon 0.13.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -11,6 +11,19 @@ export declare class CallAttributionStore implements ProvenanceSink {
11
11
  ttlMs?: number;
12
12
  now?: () => number;
13
13
  });
14
+ configureBudget(options: {
15
+ limit: number;
16
+ ttlMs: number;
17
+ }): void;
18
+ budget(): {
19
+ limit: number;
20
+ ttlMs: number;
21
+ };
14
22
  onModelCall(modelCall: ModelCallRecord): void;
15
23
  get(callId: string): RouteKitCallInspection | undefined;
24
+ /** Snapshot retained inspections after TTL prune (insertion order). */
25
+ list(): RouteKitCallInspection[];
26
+ /** True when the store has dropped records due to the capacity budget. */
27
+ truncated(): boolean;
28
+ size(): number;
16
29
  }
@@ -93,11 +93,27 @@ export class CallAttributionStore {
93
93
  #limit;
94
94
  #ttlMs;
95
95
  #now;
96
+ #evicted = false;
96
97
  constructor(options = {}) {
97
98
  this.#limit = options.limit ?? DEFAULT_CALL_ATTRIBUTION_LIMIT;
98
99
  this.#ttlMs = options.ttlMs ?? DEFAULT_CALL_ATTRIBUTION_TTL_MS;
99
100
  this.#now = options.now ?? Date.now;
100
101
  }
102
+ configureBudget(options) {
103
+ this.#limit = options.limit;
104
+ this.#ttlMs = options.ttlMs;
105
+ this.#prune(this.#now());
106
+ while (this.#entries.size > this.#limit) {
107
+ const oldest = this.#entries.keys().next().value;
108
+ if (oldest === undefined)
109
+ break;
110
+ this.#entries.delete(oldest);
111
+ this.#evicted = true;
112
+ }
113
+ }
114
+ budget() {
115
+ return { limit: this.#limit, ttlMs: this.#ttlMs };
116
+ }
101
117
  onModelCall(modelCall) {
102
118
  const inspection = callInspection(modelCall);
103
119
  if (inspection === undefined)
@@ -111,12 +127,26 @@ export class CallAttributionStore {
111
127
  if (oldest === undefined)
112
128
  break;
113
129
  this.#entries.delete(oldest);
130
+ this.#evicted = true;
114
131
  }
115
132
  }
116
133
  get(callId) {
117
134
  this.#prune(this.#now());
118
135
  return this.#entries.get(callId)?.inspection;
119
136
  }
137
+ /** Snapshot retained inspections after TTL prune (insertion order). */
138
+ list() {
139
+ this.#prune(this.#now());
140
+ return [...this.#entries.values()].map((entry) => entry.inspection);
141
+ }
142
+ /** True when the store has dropped records due to the capacity budget. */
143
+ truncated() {
144
+ return this.#evicted;
145
+ }
146
+ size() {
147
+ this.#prune(this.#now());
148
+ return this.#entries.size;
149
+ }
120
150
  #prune(now) {
121
151
  for (const [callId, entry] of this.#entries) {
122
152
  if (now - entry.insertedAt <= this.#ttlMs)
package/dist/index.js CHANGED
@@ -8,10 +8,11 @@
8
8
  */
9
9
  import { chmodSync, existsSync, lstatSync, mkdirSync, readFileSync, rmSync } from "node:fs";
10
10
  import { basename, dirname, join } from "node:path";
11
- import { CLIPROXY_API_KEY_ENV, CLIPROXY_BASE_URL_ENV, accountStoreEntries, cliproxyAuthDirectory, cliproxyAccountEntries, cliproxyAccountMatchesKind, cliproxyApiKey, cliproxyBaseUrl, cliproxyCredentialValid, defaultSubscriptionAccountDirectory, RateLimitTracker, removeCliproxyAccount, removeSubscriptionAccount, renameSubscriptionAccount, sanitizeSubscriptionLabel } from "@velum-labs/routekit-accounts";
11
+ import { AccountActivityCoordinator, CLIPROXY_API_KEY_ENV, CLIPROXY_BASE_URL_ENV, accountStoreEntries, cliproxyAuthDirectory, cliproxyAccountEntries, cliproxyAccountMatchesKind, cliproxyApiKey, cliproxyBaseUrl, cliproxyCredentialValid, defaultSubscriptionAccountDirectory, RateLimitTracker, removeCliproxyAccount, removeSubscriptionAccount, renameSubscriptionAccount, sanitizeSubscriptionLabel, subscriptionAccountIdentity } from "@velum-labs/routekit-accounts";
12
12
  import { configuredProviderIds, globalRouterConfigPath, parseRouterConfigDocument, routekitHome, writeRouterConfig } from "@velum-labs/routekit-config";
13
13
  import { createRouteKitControlHandler, ROUTEKIT_CONTROL_CAPABILITY } from "@velum-labs/routekit-control";
14
14
  import { startSwitchingGatewayProxy } from "@velum-labs/routekit-gateway";
15
+ import { resolveLeaderboardConfig } from "@velum-labs/routekit-gateway";
15
16
  import { PROVIDERS, accountKindForCliproxyAuthType, resolveAccountConnector } from "@velum-labs/routekit-registry";
16
17
  import { startRouter } from "@velum-labs/routekit-router";
17
18
  import { acquireLifecycleLock, CONTROL_PROTOCOL_VERSION, ControlClient, ControlError, createPortlessSession, createServiceRecordStore, createTokenStore, encodeJoinCredential, extendCleanupGrace, generateControlToken, nextServiceGeneration, processIdentity, registerCleanup, SERVICE_HOME_MODE, startControlServer, supervisorFromEnv, writeFileAtomic } from "@velum-labs/routekit-runtime";
@@ -19,7 +20,8 @@ import { createConsentManager } from "@velum-labs/routekit-telemetry-core";
19
20
  import { parse as parseYaml, stringify as stringifyYaml } from "yaml";
20
21
  import { createCliproxySidecar } from "./cliproxy-sidecar.js";
21
22
  import { cleanupAccountTransaction, markAccountTransactionCommitted, prepareAccountTransaction, recoverAccountTransactions, rollbackAccountTransaction } from "./account-transaction.js";
22
- import { CallAttributionStore } from "./call-attribution-store.js";
23
+ import { CallAttributionStore, callInspection } from "./call-attribution-store.js";
24
+ import { aggregateInspections, buildLeaderboardResult, LeaderboardRollupStore } from "./leaderboard.js";
23
25
  export const ROUTEKIT_DAEMON_KIND = "daemon";
24
26
  export const ROUTEKIT_PRODUCT = "routekit";
25
27
  function dataTokenPath(home) {
@@ -275,6 +277,7 @@ export async function startRouteKitDaemon(options) {
275
277
  let portless;
276
278
  let sidecarRef;
277
279
  let activeRouter;
280
+ let accountActivity;
278
281
  let record;
279
282
  let closed = false;
280
283
  let draining = false;
@@ -317,7 +320,39 @@ export async function startRouteKitDaemon(options) {
317
320
  writeRevisions(home, revisions);
318
321
  const sidecar = createCliproxySidecar({ env });
319
322
  sidecarRef = sidecar;
320
- const callAttributions = new CallAttributionStore();
323
+ let leaderboardConfig = resolveLeaderboardConfig(currentConfig);
324
+ const callAttributions = new CallAttributionStore({
325
+ limit: leaderboardConfig.liveLimit,
326
+ ttlMs: leaderboardConfig.liveTtlHours * 60 * 60 * 1_000
327
+ });
328
+ const leaderboardRollups = new LeaderboardRollupStore({
329
+ home,
330
+ config: leaderboardConfig
331
+ });
332
+ // Independent of leaderboard durable rollups: last-selection only.
333
+ mkdirSync(join(home, "usage"), { recursive: true, mode: 0o700 });
334
+ accountActivity = new AccountActivityCoordinator({
335
+ statePath: join(home, "usage", "account-activity.v1.json")
336
+ });
337
+ const applyLeaderboardConfig = (config) => {
338
+ leaderboardConfig = resolveLeaderboardConfig(config);
339
+ callAttributions.configureBudget({
340
+ limit: leaderboardConfig.liveLimit,
341
+ ttlMs: leaderboardConfig.liveTtlHours * 60 * 60 * 1_000
342
+ });
343
+ leaderboardRollups.configure({
344
+ durable: leaderboardConfig.durable,
345
+ durableRetentionDays: leaderboardConfig.durableRetentionDays
346
+ });
347
+ };
348
+ const provenance = {
349
+ onModelCall(record) {
350
+ callAttributions.onModelCall(record);
351
+ const inspection = callInspection(record);
352
+ if (inspection !== undefined)
353
+ leaderboardRollups.record(inspection);
354
+ }
355
+ };
321
356
  const wantsCliproxySidecar = (config) => config.providers["cliproxy"] !== undefined;
322
357
  // Router generations reach the managed sidecar with its own ingress key
323
358
  // and configured listen address; resolved per generation so state created
@@ -339,7 +374,8 @@ export async function startRouteKitDaemon(options) {
339
374
  host: "127.0.0.1",
340
375
  port: 0,
341
376
  env: routerEnv(),
342
- provenance: callAttributions,
377
+ provenance,
378
+ activity: accountActivity,
343
379
  drainGraceMs
344
380
  });
345
381
  await sidecar.reconcile(wantsCliproxySidecar(currentConfig));
@@ -413,6 +449,7 @@ export async function startRouteKitDaemon(options) {
413
449
  currentConfig = nextConfig;
414
450
  currentDocument = input.write ? readFileSync(configPath, "utf8") : nextDocument;
415
451
  revisions = nextRevisions;
452
+ applyLeaderboardConfig(currentConfig);
416
453
  if (previousRouter !== undefined) {
417
454
  try {
418
455
  if (previousTarget !== undefined) {
@@ -600,6 +637,46 @@ export async function startRouteKitDaemon(options) {
600
637
  }
601
638
  return inspection;
602
639
  },
640
+ "calls.leaderboard": async (params) => {
641
+ const by = params.by ?? "principal";
642
+ const sort = params.sort ?? "cost";
643
+ const limit = params.limit ?? 20;
644
+ const window = params.window ?? "live";
645
+ const nowIso = new Date().toISOString();
646
+ if (window === "live") {
647
+ const inspections = callAttributions.list();
648
+ const aggregated = aggregateInspections(inspections, { by, sort, limit });
649
+ return buildLeaderboardResult({
650
+ by,
651
+ sort,
652
+ source: "live",
653
+ windowStart: aggregated.windowStart ?? nowIso,
654
+ windowEnd: aggregated.windowEnd ?? nowIso,
655
+ sampleSize: aggregated.sampleSize,
656
+ truncated: callAttributions.truncated(),
657
+ budget: leaderboardConfig,
658
+ rows: aggregated.rows
659
+ });
660
+ }
661
+ if (!leaderboardConfig.durable) {
662
+ throw new ControlError({
663
+ code: "bad_request",
664
+ message: "durable leaderboard rollups are disabled; set leaderboard.durable: true in router.yaml"
665
+ });
666
+ }
667
+ const aggregated = leaderboardRollups.query({ by, sort, limit, window });
668
+ return buildLeaderboardResult({
669
+ by,
670
+ sort,
671
+ source: "durable",
672
+ windowStart: aggregated.windowStart,
673
+ windowEnd: aggregated.windowEnd,
674
+ sampleSize: aggregated.sampleSize,
675
+ truncated: false,
676
+ budget: leaderboardConfig,
677
+ rows: aggregated.rows
678
+ });
679
+ },
603
680
  "accounts.list": async () => ({
604
681
  accounts: accountEntries(env).map((entry) => {
605
682
  if (entry.connector === "native")
@@ -618,6 +695,7 @@ export async function startRouteKitDaemon(options) {
618
695
  return {
619
696
  accounts: entries.map((entry) => {
620
697
  if (entry.connector === "cliproxy") {
698
+ const ready = entry.credentialValid && cliproxyConfigured && cliproxyReachable;
621
699
  return {
622
700
  subscriptionKind: entry.subscriptionKind,
623
701
  label: entry.label,
@@ -625,8 +703,11 @@ export async function startRouteKitDaemon(options) {
625
703
  ...(entry.localOnly === true ? { localOnly: true } : {}),
626
704
  credentialValid: entry.credentialValid,
627
705
  configured: cliproxyConfigured,
628
- relayOpen: entry.credentialValid && cliproxyConfigured && cliproxyReachable,
629
- active: entry.credentialValid && cliproxyConfigured && cliproxyReachable,
706
+ relayOpen: ready,
707
+ serving: false,
708
+ inFlight: 0,
709
+ lastSelected: false,
710
+ active: false,
630
711
  models: []
631
712
  };
632
713
  }
@@ -642,7 +723,13 @@ export async function startRouteKitDaemon(options) {
642
723
  configured: currentConfig.providers[entry.subscriptionKind] !== undefined,
643
724
  relayOpen: member?.relayReady === true &&
644
725
  currentConfig.providers[entry.subscriptionKind] !== undefined,
645
- active: member?.active ?? false,
726
+ serving: member?.serving ?? false,
727
+ inFlight: member?.inFlight ?? 0,
728
+ ...(member?.lastSelectedAt !== undefined
729
+ ? { lastSelectedAt: member.lastSelectedAt }
730
+ : {}),
731
+ lastSelected: member?.lastSelected ?? false,
732
+ active: member?.lastSelected ?? false,
646
733
  models: member?.models ?? [],
647
734
  ...(member?.limits !== undefined ? { limits: member.limits } : {})
648
735
  };
@@ -956,11 +1043,12 @@ export async function startRouteKitDaemon(options) {
956
1043
  const nextConfig = disableProvider
957
1044
  ? parseConfigDocument(nextDocument)
958
1045
  : currentConfig;
1046
+ const activityPath = join(home, "usage", "account-activity.v1.json");
959
1047
  const transaction = prepareAccountTransaction({
960
1048
  home,
961
1049
  configPath,
962
- accountPaths: [nativePath],
963
- accountRoots: [activeNativeDirectory],
1050
+ accountPaths: [nativePath, activityPath],
1051
+ accountRoots: [activeNativeDirectory, home],
964
1052
  kind: nativeKind,
965
1053
  provider: nativeKind,
966
1054
  labels: [params.label]
@@ -976,7 +1064,10 @@ export async function startRouteKitDaemon(options) {
976
1064
  write: disableProvider,
977
1065
  configRevision: disableProvider,
978
1066
  accountRevision: true,
979
- beforeSwap: () => markAccountTransactionCommitted(transaction)
1067
+ beforeSwap: () => {
1068
+ accountActivity.remove(subscriptionAccountIdentity(nativeKind, params.label));
1069
+ markAccountTransactionCommitted(transaction);
1070
+ }
980
1071
  });
981
1072
  try {
982
1073
  cleanupAccountTransaction(transaction);
@@ -989,6 +1080,7 @@ export async function startRouteKitDaemon(options) {
989
1080
  const rollbackFailures = [];
990
1081
  try {
991
1082
  rollbackAccountTransaction(transaction, home);
1083
+ accountActivity?.reload();
992
1084
  }
993
1085
  catch (rollbackError) {
994
1086
  rollbackFailures.push(rollbackError);
@@ -1057,6 +1149,7 @@ export async function startRouteKitDaemon(options) {
1057
1149
  const sourcePath = join(directory, `${params.source}.json`);
1058
1150
  const targetPath = join(directory, `${params.target}.json`);
1059
1151
  const trackerPath = join(directory, ".state.json");
1152
+ const activityPath = join(home, "usage", "account-activity.v1.json");
1060
1153
  if (!existsSync(sourcePath)) {
1061
1154
  throw new ControlError({
1062
1155
  code: "not_found",
@@ -1082,8 +1175,8 @@ export async function startRouteKitDaemon(options) {
1082
1175
  const transaction = prepareAccountTransaction({
1083
1176
  home,
1084
1177
  configPath,
1085
- accountPaths: [sourcePath, targetPath, trackerPath],
1086
- accountRoots: [directory],
1178
+ accountPaths: [sourcePath, targetPath, trackerPath, activityPath],
1179
+ accountRoots: [directory, home],
1087
1180
  kind,
1088
1181
  provider: kind,
1089
1182
  labels: [params.source, params.target]
@@ -1097,7 +1190,10 @@ export async function startRouteKitDaemon(options) {
1097
1190
  await replaceRouter(currentConfig, currentDocument, {
1098
1191
  write: false,
1099
1192
  accountRevision: true,
1100
- beforeSwap: () => markAccountTransactionCommitted(transaction)
1193
+ beforeSwap: () => {
1194
+ accountActivity.rename(subscriptionAccountIdentity(kind, params.source), subscriptionAccountIdentity(kind, params.target));
1195
+ markAccountTransactionCommitted(transaction);
1196
+ }
1101
1197
  });
1102
1198
  try {
1103
1199
  cleanupAccountTransaction(transaction);
@@ -1110,6 +1206,7 @@ export async function startRouteKitDaemon(options) {
1110
1206
  const rollbackFailures = [];
1111
1207
  try {
1112
1208
  rollbackAccountTransaction(transaction, home);
1209
+ accountActivity?.reload();
1113
1210
  }
1114
1211
  catch (rollbackError) {
1115
1212
  rollbackFailures.push(rollbackError);
@@ -1138,6 +1235,43 @@ export async function startRouteKitDaemon(options) {
1138
1235
  "accounts.usage": async (_params, context) => {
1139
1236
  return await activeRouter.usage(context.signal);
1140
1237
  },
1238
+ "accounts.redeemReset": async (params, context) => {
1239
+ try {
1240
+ const result = await activeRouter.redeemReset({
1241
+ kind: params.kind,
1242
+ label: params.label,
1243
+ ...(params.creditId !== undefined ? { creditId: params.creditId } : {}),
1244
+ ...(params.redeemRequestId !== undefined
1245
+ ? { redeemRequestId: params.redeemRequestId }
1246
+ : {})
1247
+ }, context.signal);
1248
+ return {
1249
+ ok: result.ok,
1250
+ code: result.code,
1251
+ kind: "codex",
1252
+ label: result.label,
1253
+ redeemRequestId: result.redeemRequestId,
1254
+ ...(result.creditId !== undefined ? { creditId: result.creditId } : {}),
1255
+ ...(result.windowsReset !== undefined
1256
+ ? { windowsReset: result.windowsReset }
1257
+ : {}),
1258
+ usage: result.usage
1259
+ };
1260
+ }
1261
+ catch (error) {
1262
+ const message = error instanceof Error ? error.message : String(error);
1263
+ if (message.includes("is not enrolled") || message.includes("no redeemable")) {
1264
+ throw new ControlError({ code: "not_found", message });
1265
+ }
1266
+ if (message.includes("does not support") ||
1267
+ message.includes("no codex account pool") ||
1268
+ message.includes("creditId must not be empty") ||
1269
+ message.includes("account label is required")) {
1270
+ throw new ControlError({ code: "bad_request", message });
1271
+ }
1272
+ throw new ControlError({ code: "internal", message });
1273
+ }
1274
+ },
1141
1275
  "telemetry.get": async () => ({ enabled: telemetry.resolve().enabled }),
1142
1276
  "telemetry.set": async (params) => {
1143
1277
  await serializeMutation(async () => {
@@ -1262,7 +1396,7 @@ export async function startRouteKitDaemon(options) {
1262
1396
  token: issued.token,
1263
1397
  ...(issued.plane === "control"
1264
1398
  ? {
1265
- joinToken: encodeJoinCredential({
1399
+ joinCredential: encodeJoinCredential({
1266
1400
  publicRecordPath: daemonPublicRecordPath(home),
1267
1401
  token: issued.token
1268
1402
  })
@@ -1364,6 +1498,7 @@ export async function startRouteKitDaemon(options) {
1364
1498
  lifecycle = "draining";
1365
1499
  await proxy?.drain(drainGraceMs);
1366
1500
  await activeRouter?.close();
1501
+ accountActivity?.close();
1367
1502
  await sidecar.close();
1368
1503
  await control?.close();
1369
1504
  if (portless?.enabled)
@@ -1400,6 +1535,7 @@ export async function startRouteKitDaemon(options) {
1400
1535
  catch (error) {
1401
1536
  await proxy?.close();
1402
1537
  await activeRouter?.close();
1538
+ accountActivity?.close();
1403
1539
  await sidecarRef?.close();
1404
1540
  await control?.close();
1405
1541
  if (portless?.enabled)
@@ -0,0 +1,52 @@
1
+ import type { RouteKitCallInspection, RouteKitControlParams, RouteKitLeaderboard } from "@velum-labs/routekit-control";
2
+ import type { LeaderboardConfig } from "@velum-labs/routekit-gateway";
3
+ export type LeaderboardDimension = RouteKitLeaderboard["by"];
4
+ export type LeaderboardSort = RouteKitLeaderboard["sort"];
5
+ export type LeaderboardWindow = NonNullable<RouteKitControlParams["calls.leaderboard"]["window"]>;
6
+ export declare const LEADERBOARD_ROLLUP_VERSION: 1;
7
+ export declare const LEADERBOARD_ROLLUP_RELATIVE_PATH: string;
8
+ export declare function aggregateInspections(inspections: readonly RouteKitCallInspection[], options: {
9
+ by: LeaderboardDimension;
10
+ sort: LeaderboardSort;
11
+ limit: number;
12
+ }): {
13
+ rows: RouteKitLeaderboard["rows"];
14
+ sampleSize: number;
15
+ windowStart?: string;
16
+ windowEnd?: string;
17
+ };
18
+ export declare class LeaderboardRollupStore {
19
+ #private;
20
+ constructor(options: {
21
+ home: string;
22
+ config: LeaderboardConfig;
23
+ now?: () => number;
24
+ flushDelayMs?: number;
25
+ });
26
+ configure(config: Pick<LeaderboardConfig, "durable" | "durableRetentionDays">): void;
27
+ record(inspection: RouteKitCallInspection): void;
28
+ flush(): void;
29
+ query(options: {
30
+ by: LeaderboardDimension;
31
+ sort: LeaderboardSort;
32
+ limit: number;
33
+ window: Exclude<LeaderboardWindow, "live">;
34
+ }): {
35
+ rows: RouteKitLeaderboard["rows"];
36
+ sampleSize: number;
37
+ windowStart: string;
38
+ windowEnd: string;
39
+ };
40
+ path(): string;
41
+ }
42
+ export declare function buildLeaderboardResult(input: {
43
+ by: LeaderboardDimension;
44
+ sort: LeaderboardSort;
45
+ source: RouteKitLeaderboard["source"];
46
+ windowStart: string;
47
+ windowEnd: string;
48
+ sampleSize: number;
49
+ truncated: boolean;
50
+ budget: LeaderboardConfig;
51
+ rows: RouteKitLeaderboard["rows"];
52
+ }): RouteKitLeaderboard;
@@ -0,0 +1,404 @@
1
+ /**
2
+ * Usage leaderboard aggregation over live call attribution and optional
3
+ * durable hourly rollups under `$ROUTEKIT_HOME/usage/`.
4
+ */
5
+ import { chmodSync, existsSync, mkdirSync, readFileSync } from "node:fs";
6
+ import { dirname, join } from "node:path";
7
+ import { writeFileAtomic } from "@velum-labs/routekit-runtime";
8
+ export const LEADERBOARD_ROLLUP_VERSION = 1;
9
+ export const LEADERBOARD_ROLLUP_RELATIVE_PATH = join("usage", "leaderboard-rollups.v1.json");
10
+ const WINDOW_MS = {
11
+ "1h": 60 * 60 * 1_000,
12
+ "24h": 24 * 60 * 60 * 1_000,
13
+ "7d": 7 * 24 * 60 * 60 * 1_000
14
+ };
15
+ function emptyCounters(key, label) {
16
+ return {
17
+ key,
18
+ ...(label !== undefined ? { label } : {}),
19
+ requests: 0,
20
+ success: 0,
21
+ error: 0,
22
+ tokensIn: 0,
23
+ tokensOut: 0,
24
+ tokensTotal: 0,
25
+ estimateUsd: 0,
26
+ unknownCostCount: 0,
27
+ unknownUsageCount: 0,
28
+ latencyMsSum: 0,
29
+ latencyMsCount: 0
30
+ };
31
+ }
32
+ function dimensionKey(inspection, by) {
33
+ switch (by) {
34
+ case "principal": {
35
+ const tokenId = inspection.principal?.tokenId;
36
+ if (tokenId === undefined)
37
+ return undefined;
38
+ return {
39
+ key: tokenId,
40
+ ...(inspection.principal?.label !== undefined
41
+ ? { label: inspection.principal.label }
42
+ : {})
43
+ };
44
+ }
45
+ case "model":
46
+ return { key: inspection.effectiveModel };
47
+ case "provider":
48
+ return { key: inspection.provider };
49
+ }
50
+ }
51
+ function addInspection(bucket, inspection, keepSamples) {
52
+ bucket.requests += 1;
53
+ if (inspection.status === "succeeded")
54
+ bucket.success += 1;
55
+ else
56
+ bucket.error += 1;
57
+ const usage = inspection.usage;
58
+ if (usage === undefined || inspection.cost.unknownUsage) {
59
+ bucket.unknownUsageCount += 1;
60
+ }
61
+ else {
62
+ bucket.tokensIn += usage.prompt_tokens ?? 0;
63
+ bucket.tokensOut += usage.completion_tokens ?? 0;
64
+ bucket.tokensTotal +=
65
+ usage.total_tokens ??
66
+ (usage.prompt_tokens ?? 0) + (usage.completion_tokens ?? 0);
67
+ }
68
+ if (inspection.cost.estimateUsd !== undefined && !inspection.cost.unknownCost) {
69
+ bucket.estimateUsd += inspection.cost.estimateUsd;
70
+ }
71
+ else if (inspection.cost.unknownCost || inspection.cost.estimateUsd === undefined) {
72
+ bucket.unknownCostCount += 1;
73
+ }
74
+ const latencyMs = inspection.timing.latencyMs;
75
+ if (latencyMs !== undefined) {
76
+ bucket.latencyMsSum += latencyMs;
77
+ bucket.latencyMsCount += 1;
78
+ if (keepSamples) {
79
+ bucket.latencySamples ??= [];
80
+ bucket.latencySamples.push(latencyMs);
81
+ }
82
+ }
83
+ if (bucket.label === undefined &&
84
+ inspection.principal?.label !== undefined &&
85
+ bucket.key === inspection.principal.tokenId) {
86
+ bucket.label = inspection.principal.label;
87
+ }
88
+ }
89
+ function mergeCounters(target, source) {
90
+ target.requests += source.requests;
91
+ target.success += source.success;
92
+ target.error += source.error;
93
+ target.tokensIn += source.tokensIn;
94
+ target.tokensOut += source.tokensOut;
95
+ target.tokensTotal += source.tokensTotal;
96
+ target.estimateUsd += source.estimateUsd;
97
+ target.unknownCostCount += source.unknownCostCount;
98
+ target.unknownUsageCount += source.unknownUsageCount;
99
+ target.latencyMsSum += source.latencyMsSum;
100
+ target.latencyMsCount += source.latencyMsCount;
101
+ if (target.label === undefined && source.label !== undefined) {
102
+ target.label = source.label;
103
+ }
104
+ }
105
+ function percentile(sorted, p) {
106
+ if (sorted.length === 0)
107
+ return undefined;
108
+ if (sorted.length === 1)
109
+ return sorted[0];
110
+ const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1));
111
+ return sorted[index];
112
+ }
113
+ function sortValue(bucket, sort) {
114
+ switch (sort) {
115
+ case "cost":
116
+ return bucket.estimateUsd;
117
+ case "requests":
118
+ return bucket.requests;
119
+ case "tokens":
120
+ return bucket.tokensTotal;
121
+ case "errors":
122
+ return bucket.error;
123
+ case "latency":
124
+ return bucket.latencyMsCount === 0
125
+ ? 0
126
+ : bucket.latencyMsSum / bucket.latencyMsCount;
127
+ }
128
+ }
129
+ function toRow(bucket, rank) {
130
+ const samples = bucket.latencySamples?.slice().sort((a, b) => a - b);
131
+ const avg = bucket.latencyMsCount > 0
132
+ ? bucket.latencyMsSum / bucket.latencyMsCount
133
+ : undefined;
134
+ return {
135
+ rank,
136
+ key: bucket.key,
137
+ ...(bucket.label !== undefined ? { label: bucket.label } : {}),
138
+ requests: bucket.requests,
139
+ success: bucket.success,
140
+ error: bucket.error,
141
+ tokensIn: bucket.tokensIn,
142
+ tokensOut: bucket.tokensOut,
143
+ tokensTotal: bucket.tokensTotal,
144
+ ...(bucket.estimateUsd > 0 || bucket.unknownCostCount === 0
145
+ ? { estimateUsd: bucket.estimateUsd }
146
+ : {}),
147
+ unknownCostCount: bucket.unknownCostCount,
148
+ unknownUsageCount: bucket.unknownUsageCount,
149
+ ...(avg !== undefined ? { latencyMsAvg: avg } : {}),
150
+ ...(samples !== undefined
151
+ ? {
152
+ ...(percentile(samples, 50) !== undefined
153
+ ? { latencyMsP50: percentile(samples, 50) }
154
+ : {}),
155
+ ...(percentile(samples, 95) !== undefined
156
+ ? { latencyMsP95: percentile(samples, 95) }
157
+ : {})
158
+ }
159
+ : {})
160
+ };
161
+ }
162
+ export function aggregateInspections(inspections, options) {
163
+ const groups = new Map();
164
+ let windowStart;
165
+ let windowEnd;
166
+ for (const inspection of inspections) {
167
+ const dim = dimensionKey(inspection, options.by);
168
+ if (dim === undefined)
169
+ continue;
170
+ let bucket = groups.get(dim.key);
171
+ if (bucket === undefined) {
172
+ bucket = emptyCounters(dim.key, dim.label);
173
+ groups.set(dim.key, bucket);
174
+ }
175
+ addInspection(bucket, inspection, true);
176
+ const started = inspection.timing.startedAt;
177
+ if (windowStart === undefined || started < windowStart)
178
+ windowStart = started;
179
+ const finished = inspection.timing.finishedAt ?? started;
180
+ if (windowEnd === undefined || finished > windowEnd)
181
+ windowEnd = finished;
182
+ }
183
+ const ranked = [...groups.values()].sort((a, b) => {
184
+ const delta = sortValue(b, options.sort) - sortValue(a, options.sort);
185
+ if (delta !== 0)
186
+ return delta;
187
+ return a.key.localeCompare(b.key);
188
+ });
189
+ return {
190
+ rows: ranked.slice(0, options.limit).map((bucket, index) => toRow(bucket, index + 1)),
191
+ sampleSize: inspections.length,
192
+ ...(windowStart !== undefined ? { windowStart } : {}),
193
+ ...(windowEnd !== undefined ? { windowEnd } : {})
194
+ };
195
+ }
196
+ function hourFloor(iso) {
197
+ const date = new Date(iso);
198
+ if (Number.isNaN(date.getTime())) {
199
+ const fallback = new Date();
200
+ fallback.setUTCMinutes(0, 0, 0);
201
+ return fallback.toISOString();
202
+ }
203
+ date.setUTCMinutes(0, 0, 0);
204
+ return date.toISOString();
205
+ }
206
+ function serializeBucket(bucket) {
207
+ const { latencySamples: _samples, ...rest } = bucket;
208
+ return rest;
209
+ }
210
+ export class LeaderboardRollupStore {
211
+ #path;
212
+ #now;
213
+ #flushDelayMs;
214
+ #enabled;
215
+ #retentionDays;
216
+ #file;
217
+ #timer;
218
+ #dirty = false;
219
+ constructor(options) {
220
+ this.#path = join(options.home, LEADERBOARD_ROLLUP_RELATIVE_PATH);
221
+ this.#now = options.now ?? Date.now;
222
+ this.#flushDelayMs = options.flushDelayMs ?? 250;
223
+ this.#enabled = options.config.durable;
224
+ this.#retentionDays = options.config.durableRetentionDays;
225
+ this.#file = this.#load();
226
+ }
227
+ configure(config) {
228
+ this.#enabled = config.durable;
229
+ this.#retentionDays = config.durableRetentionDays;
230
+ this.#file.retentionDays = this.#retentionDays;
231
+ this.#prune(this.#now());
232
+ if (this.#enabled)
233
+ this.#scheduleFlush();
234
+ }
235
+ record(inspection) {
236
+ if (!this.#enabled)
237
+ return;
238
+ const hour = hourFloor(inspection.timing.startedAt);
239
+ let bucket = this.#file.buckets.find((entry) => entry.hour === hour);
240
+ if (bucket === undefined) {
241
+ bucket = { hour, byPrincipal: {}, byModel: {}, byProvider: {} };
242
+ this.#file.buckets.push(bucket);
243
+ this.#file.buckets.sort((a, b) => a.hour.localeCompare(b.hour));
244
+ }
245
+ for (const by of ["principal", "model", "provider"]) {
246
+ const dim = dimensionKey(inspection, by);
247
+ if (dim === undefined)
248
+ continue;
249
+ const map = by === "principal"
250
+ ? bucket.byPrincipal
251
+ : by === "model"
252
+ ? bucket.byModel
253
+ : bucket.byProvider;
254
+ let counters = map[dim.key];
255
+ if (counters === undefined) {
256
+ counters = emptyCounters(dim.key, dim.label);
257
+ map[dim.key] = counters;
258
+ }
259
+ addInspection(counters, inspection, false);
260
+ map[dim.key] = serializeBucket(counters);
261
+ }
262
+ this.#file.updatedAt = new Date(this.#now()).toISOString();
263
+ this.#prune(this.#now());
264
+ this.#dirty = true;
265
+ this.#scheduleFlush();
266
+ }
267
+ flush() {
268
+ if (this.#timer !== undefined) {
269
+ clearTimeout(this.#timer);
270
+ this.#timer = undefined;
271
+ }
272
+ if (!this.#dirty)
273
+ return;
274
+ this.#write();
275
+ }
276
+ query(options) {
277
+ this.flush();
278
+ const endMs = this.#now();
279
+ const startMs = endMs - WINDOW_MS[options.window];
280
+ const startIso = new Date(startMs).toISOString();
281
+ const endIso = new Date(endMs).toISOString();
282
+ const groups = new Map();
283
+ let sampleSize = 0;
284
+ for (const hour of this.#file.buckets) {
285
+ if (hour.hour < hourFloor(startIso))
286
+ continue;
287
+ if (hour.hour > endIso)
288
+ continue;
289
+ const map = options.by === "principal"
290
+ ? hour.byPrincipal
291
+ : options.by === "model"
292
+ ? hour.byModel
293
+ : hour.byProvider;
294
+ for (const counters of Object.values(map)) {
295
+ sampleSize += counters.requests;
296
+ let bucket = groups.get(counters.key);
297
+ if (bucket === undefined) {
298
+ bucket = emptyCounters(counters.key, counters.label);
299
+ groups.set(counters.key, bucket);
300
+ }
301
+ mergeCounters(bucket, counters);
302
+ }
303
+ }
304
+ const ranked = [...groups.values()].sort((a, b) => {
305
+ const delta = sortValue(b, options.sort) - sortValue(a, options.sort);
306
+ if (delta !== 0)
307
+ return delta;
308
+ return a.key.localeCompare(b.key);
309
+ });
310
+ return {
311
+ rows: ranked.slice(0, options.limit).map((bucket, index) => toRow(bucket, index + 1)),
312
+ sampleSize,
313
+ windowStart: startIso,
314
+ windowEnd: endIso
315
+ };
316
+ }
317
+ path() {
318
+ return this.#path;
319
+ }
320
+ #load() {
321
+ if (!existsSync(this.#path)) {
322
+ return {
323
+ version: LEADERBOARD_ROLLUP_VERSION,
324
+ updatedAt: new Date(this.#now()).toISOString(),
325
+ retentionDays: this.#retentionDays,
326
+ buckets: []
327
+ };
328
+ }
329
+ try {
330
+ const parsed = JSON.parse(readFileSync(this.#path, "utf8"));
331
+ if (parsed.version !== LEADERBOARD_ROLLUP_VERSION || !Array.isArray(parsed.buckets)) {
332
+ return {
333
+ version: LEADERBOARD_ROLLUP_VERSION,
334
+ updatedAt: new Date(this.#now()).toISOString(),
335
+ retentionDays: this.#retentionDays,
336
+ buckets: []
337
+ };
338
+ }
339
+ return {
340
+ version: LEADERBOARD_ROLLUP_VERSION,
341
+ updatedAt: typeof parsed.updatedAt === "string"
342
+ ? parsed.updatedAt
343
+ : new Date(this.#now()).toISOString(),
344
+ retentionDays: this.#retentionDays,
345
+ buckets: parsed.buckets
346
+ };
347
+ }
348
+ catch {
349
+ return {
350
+ version: LEADERBOARD_ROLLUP_VERSION,
351
+ updatedAt: new Date(this.#now()).toISOString(),
352
+ retentionDays: this.#retentionDays,
353
+ buckets: []
354
+ };
355
+ }
356
+ }
357
+ #prune(nowMs) {
358
+ const cutoff = new Date(nowMs - this.#retentionDays * 24 * 60 * 60 * 1_000);
359
+ cutoff.setUTCMinutes(0, 0, 0);
360
+ const cutoffIso = cutoff.toISOString();
361
+ const before = this.#file.buckets.length;
362
+ this.#file.buckets = this.#file.buckets.filter((bucket) => bucket.hour >= cutoffIso);
363
+ if (this.#file.buckets.length !== before)
364
+ this.#dirty = true;
365
+ }
366
+ #scheduleFlush() {
367
+ if (this.#timer !== undefined)
368
+ return;
369
+ this.#timer = setTimeout(() => {
370
+ this.#timer = undefined;
371
+ if (this.#dirty)
372
+ this.#write();
373
+ }, this.#flushDelayMs);
374
+ this.#timer.unref?.();
375
+ }
376
+ #write() {
377
+ mkdirSync(dirname(this.#path), { recursive: true, mode: 0o700 });
378
+ writeFileAtomic(this.#path, `${JSON.stringify(this.#file, null, 2)}\n`, {
379
+ mode: 0o600
380
+ });
381
+ chmodSync(this.#path, 0o600);
382
+ this.#dirty = false;
383
+ }
384
+ }
385
+ export function buildLeaderboardResult(input) {
386
+ return {
387
+ by: input.by,
388
+ sort: input.sort,
389
+ source: input.source,
390
+ window: {
391
+ start: input.windowStart,
392
+ end: input.windowEnd
393
+ },
394
+ sampleSize: input.sampleSize,
395
+ truncated: input.truncated,
396
+ budget: {
397
+ liveLimit: input.budget.liveLimit,
398
+ liveTtlHours: input.budget.liveTtlHours,
399
+ durable: input.budget.durable,
400
+ durableRetentionDays: input.budget.durableRetentionDays
401
+ },
402
+ rows: input.rows
403
+ };
404
+ }
@@ -67,7 +67,19 @@ test("call attribution store evicts by capacity and expiry", () => {
67
67
  store.onModelCall(modelCall("call_3"));
68
68
  assert.equal(store.get("call_1"), undefined);
69
69
  assert.ok(store.get("call_2"));
70
+ assert.equal(store.truncated(), true);
71
+ assert.equal(store.list().length, 2);
70
72
  now = 111;
71
73
  assert.equal(store.get("call_2"), undefined);
72
74
  assert.ok(store.get("call_3"));
73
75
  });
76
+ test("call attribution store applies a tighter live budget on configure", () => {
77
+ const store = new CallAttributionStore({ limit: 3, ttlMs: 60_000 });
78
+ store.onModelCall(modelCall("call_1"));
79
+ store.onModelCall(modelCall("call_2"));
80
+ store.onModelCall(modelCall("call_3"));
81
+ store.configureBudget({ limit: 1, ttlMs: 60_000 });
82
+ assert.equal(store.size(), 1);
83
+ assert.equal(store.list()[0]?.callId, "call_3");
84
+ assert.deepEqual(store.budget(), { limit: 1, ttlMs: 60_000 });
85
+ });
@@ -264,6 +264,19 @@ test("singleton daemon exposes authenticated control and a stable reloadable dat
264
264
  assert.equal(embeddingInspection.provider, "openai");
265
265
  assert.equal(embeddingInspection.billingMode, "api_key");
266
266
  await assert.rejects(client.call("calls.inspect", { callId: "model_call_missing" }), (error) => error instanceof ControlError && error.code === "not_found");
267
+ const leaderboard = await client.call("calls.leaderboard", {
268
+ by: "provider",
269
+ sort: "requests",
270
+ limit: 5,
271
+ window: "live"
272
+ });
273
+ assert.equal(leaderboard.by, "provider");
274
+ assert.equal(leaderboard.source, "live");
275
+ assert.ok(leaderboard.sampleSize >= 1);
276
+ assert.ok(leaderboard.rows.some((row) => row.key === "openai"));
277
+ await assert.rejects(client.call("calls.leaderboard", { window: "24h" }), (error) => error instanceof ControlError &&
278
+ error.code === "bad_request" &&
279
+ /durable leaderboard rollups are disabled/.test(error.message));
267
280
  await assert.rejects(client.call("config.update", {
268
281
  expectedRevision: snapshot.revision,
269
282
  document: "providers: {}\n"
@@ -311,6 +324,155 @@ test("singleton daemon exposes authenticated control and a stable reloadable dat
311
324
  rmSync(root, { recursive: true, force: true });
312
325
  }
313
326
  });
327
+ test("daemon account activity persists last selection independently of leaderboard rollups", async () => {
328
+ const root = mkdtempSync(join(tmpdir(), "routekit-daemon-activity-"));
329
+ const stateHome = join(root, "state");
330
+ const configPath = join(root, "router.yaml");
331
+ const accountsDirectory = join(stateHome, "subscriptions", "codex");
332
+ const activityPath = join(stateHome, "usage", "account-activity.v1.json");
333
+ mkdirSync(accountsDirectory, { recursive: true, mode: 0o700 });
334
+ mkdirSync(join(stateHome, "usage"), { recursive: true, mode: 0o700 });
335
+ writeFileSync(join(accountsDirectory, "work.json"), `${JSON.stringify(nativeCredential("codex"))}\n`, { mode: 0o600 });
336
+ writeFileSync(activityPath, `${JSON.stringify({
337
+ version: 1,
338
+ sequence: 3,
339
+ accounts: [
340
+ {
341
+ identity: "codex:work",
342
+ lastSelectedAt: 1_700_000_000_000,
343
+ sequence: 3
344
+ }
345
+ ]
346
+ })}\n`, { mode: 0o600 });
347
+ writeFileSync(configPath, [
348
+ "providers:",
349
+ " codex: {}",
350
+ "defaultModel: codex/gpt-test-model",
351
+ "leaderboard:",
352
+ " durable: false",
353
+ ""
354
+ ].join("\n"));
355
+ try {
356
+ await withMockNativeDiscovery("codex", async () => {
357
+ const daemon = await startRouteKitDaemon({
358
+ packageVersion: "1.2.3",
359
+ stateHome,
360
+ configPath,
361
+ port: 0,
362
+ portless: false,
363
+ env: {
364
+ HOME: root,
365
+ ROUTEKIT_HOME: stateHome,
366
+ ROUTEKIT_PORTLESS: "0"
367
+ }
368
+ });
369
+ try {
370
+ const client = new RouteKitControlClient({
371
+ url: daemon.record.url,
372
+ token: daemon.record.controlToken
373
+ });
374
+ const status = await client.call("accounts.status", {});
375
+ const usage = await client.call("accounts.usage", {});
376
+ assert.equal(status.accounts[0]?.lastSelected, true);
377
+ assert.equal(status.accounts[0]?.active, true);
378
+ assert.equal(status.accounts[0]?.serving, false);
379
+ assert.equal(status.accounts[0]?.inFlight, 0);
380
+ assert.equal(status.accounts[0]?.lastSelectedAt, 1_700_000_000_000);
381
+ assert.equal(usage.accountSets[0]?.members[0]?.lastSelected, true);
382
+ assert.equal(usage.accountSets[0]?.members[0]?.active, true);
383
+ assert.equal(usage.accountSets[0]?.members[0]?.lastSelectedAt, 1_700_000_000_000);
384
+ assert.equal(existsSync(join(stateHome, "usage", "leaderboard-rollups.v1.json")), false);
385
+ await client.call("accounts.rename", { kind: "codex", source: "work", target: "personal" }, { idempotencyKey: "activity-rename" });
386
+ const renamedStatus = await client.call("accounts.status", {});
387
+ assert.equal(renamedStatus.accounts[0]?.label, "personal");
388
+ assert.equal(renamedStatus.accounts[0]?.lastSelected, true);
389
+ assert.equal(renamedStatus.accounts[0]?.lastSelectedAt, 1_700_000_000_000);
390
+ const persisted = JSON.parse(readFileSync(activityPath, "utf8"));
391
+ assert.deepEqual(persisted.accounts.map((account) => account.identity), ["codex:personal"]);
392
+ const beforeReload = await client.call("daemon.status", {});
393
+ const reloaded = await client.call("config.update", {
394
+ expectedRevision: (await client.call("config.get", {})).revision,
395
+ document: [
396
+ "providers:",
397
+ " codex:",
398
+ " strategy: sticky",
399
+ "defaultModel: codex/gpt-test-model",
400
+ "leaderboard:",
401
+ " durable: false",
402
+ ""
403
+ ].join("\n")
404
+ });
405
+ assert.ok(reloaded.revision > beforeReload.configRevision);
406
+ const afterReload = await client.call("accounts.status", {});
407
+ assert.equal(afterReload.accounts[0]?.label, "personal");
408
+ assert.equal(afterReload.accounts[0]?.lastSelected, true);
409
+ assert.equal(afterReload.accounts[0]?.lastSelectedAt, 1_700_000_000_000);
410
+ assert.equal(afterReload.accounts[0]?.serving, false);
411
+ assert.equal(afterReload.accounts[0]?.inFlight, 0);
412
+ await assert.rejects(client.call("calls.leaderboard", { window: "24h" }), (error) => error instanceof ControlError &&
413
+ error.code === "bad_request" &&
414
+ /durable leaderboard rollups are disabled/.test(error.message));
415
+ const removed = await client.call("accounts.remove", { kind: "codex", label: "personal" }, { idempotencyKey: "activity-remove" });
416
+ assert.equal(removed.removed, true);
417
+ const afterRemove = JSON.parse(readFileSync(activityPath, "utf8"));
418
+ assert.deepEqual(afterRemove.accounts, []);
419
+ }
420
+ finally {
421
+ await daemon.close();
422
+ }
423
+ writeFileSync(join(accountsDirectory, "personal.json"), `${JSON.stringify(nativeCredential("codex"))}\n`, { mode: 0o600 });
424
+ writeFileSync(activityPath, `${JSON.stringify({
425
+ version: 1,
426
+ sequence: 4,
427
+ accounts: [
428
+ {
429
+ identity: "codex:personal",
430
+ lastSelectedAt: 1_700_000_000_000,
431
+ sequence: 4
432
+ }
433
+ ]
434
+ })}\n`, { mode: 0o600 });
435
+ writeFileSync(configPath, [
436
+ "providers:",
437
+ " codex: {}",
438
+ "defaultModel: codex/gpt-test-model",
439
+ "leaderboard:",
440
+ " durable: false",
441
+ ""
442
+ ].join("\n"));
443
+ const restarted = await startRouteKitDaemon({
444
+ packageVersion: "1.2.3",
445
+ stateHome,
446
+ configPath,
447
+ port: 0,
448
+ portless: false,
449
+ env: {
450
+ HOME: root,
451
+ ROUTEKIT_HOME: stateHome,
452
+ ROUTEKIT_PORTLESS: "0"
453
+ }
454
+ });
455
+ try {
456
+ const client = new RouteKitControlClient({
457
+ url: restarted.record.url,
458
+ token: restarted.record.controlToken
459
+ });
460
+ const status = await client.call("accounts.status", {});
461
+ assert.equal(status.accounts[0]?.label, "personal");
462
+ assert.equal(status.accounts[0]?.lastSelected, true);
463
+ assert.equal(status.accounts[0]?.lastSelectedAt, 1_700_000_000_000);
464
+ assert.equal(status.accounts[0]?.serving, false);
465
+ assert.equal(status.accounts[0]?.inFlight, 0);
466
+ }
467
+ finally {
468
+ await restarted.close();
469
+ }
470
+ });
471
+ }
472
+ finally {
473
+ rmSync(root, { recursive: true, force: true });
474
+ }
475
+ });
314
476
  for (const kind of ["claude-code", "codex"]) {
315
477
  test(`native ${kind} account rename preserves routing and usage state`, async () => {
316
478
  const root = mkdtempSync(join(tmpdir(), `routekit-daemon-rename-${kind}-`));
@@ -824,7 +986,10 @@ test("daemon owns the cliproxy sidecar: spawn, restart, account routing, shutdow
824
986
  credentialValid: true,
825
987
  configured: true,
826
988
  relayOpen: true,
827
- active: true,
989
+ serving: false,
990
+ inFlight: 0,
991
+ lastSelected: false,
992
+ active: false,
828
993
  models: []
829
994
  }
830
995
  ]);
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,258 @@
1
+ import assert from "node:assert/strict";
2
+ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
3
+ import { tmpdir } from "node:os";
4
+ import { join } from "node:path";
5
+ import test from "node:test";
6
+ import { AccountActivityCoordinator } from "@velum-labs/routekit-accounts";
7
+ import { aggregateInspections, LeaderboardRollupStore } from "../leaderboard.js";
8
+ function inspection(input) {
9
+ return {
10
+ callId: input.callId,
11
+ status: input.status ?? "succeeded",
12
+ effectiveModel: input.model ?? "openai/gpt-5.5",
13
+ provider: input.provider ?? "openai",
14
+ billingMode: "api_key",
15
+ ...(input.principal !== undefined ? { principal: input.principal } : {}),
16
+ retries: { attempts: 1, total: 0, accountFailovers: 0 },
17
+ usage: {
18
+ prompt_tokens: input.tokens ?? 10,
19
+ completion_tokens: 5,
20
+ total_tokens: (input.tokens ?? 10) + 5
21
+ },
22
+ cost: {
23
+ ...(input.cost !== undefined ? { estimateUsd: input.cost } : {}),
24
+ unknownUsage: false,
25
+ unknownCost: input.unknownCost ?? false
26
+ },
27
+ timing: {
28
+ startedAt: input.startedAt ?? "2026-07-27T10:00:00.000Z",
29
+ finishedAt: input.startedAt ?? "2026-07-27T10:00:01.000Z",
30
+ latencyMs: input.latencyMs ?? 1_000
31
+ }
32
+ };
33
+ }
34
+ test("aggregateInspections ranks principals by cost and keeps unknown cost visible", () => {
35
+ const result = aggregateInspections([
36
+ inspection({
37
+ callId: "a",
38
+ principal: { tokenId: "tok_a", label: "alice" },
39
+ cost: 0.5
40
+ }),
41
+ inspection({
42
+ callId: "b",
43
+ principal: { tokenId: "tok_b", label: "bob" },
44
+ cost: 1.25
45
+ }),
46
+ inspection({
47
+ callId: "c",
48
+ principal: { tokenId: "tok_a", label: "alice" },
49
+ unknownCost: true
50
+ }),
51
+ inspection({
52
+ callId: "d",
53
+ model: "anthropic/claude-sonnet-4-5",
54
+ provider: "anthropic",
55
+ cost: 9
56
+ })
57
+ ], { by: "principal", sort: "cost", limit: 10 });
58
+ assert.equal(result.rows.length, 2);
59
+ assert.equal(result.rows[0]?.key, "tok_b");
60
+ assert.equal(result.rows[0]?.estimateUsd, 1.25);
61
+ assert.equal(result.rows[1]?.key, "tok_a");
62
+ assert.equal(result.rows[1]?.estimateUsd, 0.5);
63
+ assert.equal(result.rows[1]?.unknownCostCount, 1);
64
+ assert.equal(result.sampleSize, 4);
65
+ });
66
+ test("aggregateInspections can rank by model and sort by requests", () => {
67
+ const result = aggregateInspections([
68
+ inspection({ callId: "1", model: "openai/a", cost: 1 }),
69
+ inspection({ callId: "2", model: "openai/a", cost: 1 }),
70
+ inspection({ callId: "3", model: "openai/b", cost: 5 })
71
+ ], { by: "model", sort: "requests", limit: 1 });
72
+ assert.equal(result.rows.length, 1);
73
+ assert.equal(result.rows[0]?.key, "openai/a");
74
+ assert.equal(result.rows[0]?.requests, 2);
75
+ });
76
+ test("LeaderboardRollupStore persists hourly buckets across reload and prunes old hours", () => {
77
+ const home = mkdtempSync(join(tmpdir(), "routekit-leaderboard-"));
78
+ let now = Date.parse("2026-07-27T12:30:00.000Z");
79
+ try {
80
+ const store = new LeaderboardRollupStore({
81
+ home,
82
+ config: {
83
+ liveLimit: 1000,
84
+ liveTtlHours: 24,
85
+ durable: true,
86
+ durableRetentionDays: 1
87
+ },
88
+ now: () => now,
89
+ flushDelayMs: 0
90
+ });
91
+ store.record(inspection({
92
+ callId: "old",
93
+ principal: { tokenId: "tok_old", label: "old" },
94
+ cost: 1,
95
+ startedAt: "2026-07-25T10:15:00.000Z"
96
+ }));
97
+ store.record(inspection({
98
+ callId: "new",
99
+ principal: { tokenId: "tok_new", label: "new" },
100
+ cost: 2,
101
+ startedAt: "2026-07-27T11:15:00.000Z"
102
+ }));
103
+ store.flush();
104
+ const raw = JSON.parse(readFileSync(store.path(), "utf8"));
105
+ assert.equal(raw.buckets.length, 1);
106
+ assert.equal(raw.buckets[0]?.hour, "2026-07-27T11:00:00.000Z");
107
+ const reloaded = new LeaderboardRollupStore({
108
+ home,
109
+ config: {
110
+ liveLimit: 1000,
111
+ liveTtlHours: 24,
112
+ durable: true,
113
+ durableRetentionDays: 14
114
+ },
115
+ now: () => now,
116
+ flushDelayMs: 0
117
+ });
118
+ const board = reloaded.query({
119
+ by: "principal",
120
+ sort: "cost",
121
+ limit: 10,
122
+ window: "24h"
123
+ });
124
+ assert.equal(board.rows.length, 1);
125
+ assert.equal(board.rows[0]?.key, "tok_new");
126
+ assert.equal(board.rows[0]?.estimateUsd, 2);
127
+ }
128
+ finally {
129
+ rmSync(home, { recursive: true, force: true });
130
+ }
131
+ });
132
+ test("LeaderboardRollupStore ignores records while durable is disabled", () => {
133
+ const home = mkdtempSync(join(tmpdir(), "routekit-leaderboard-off-"));
134
+ try {
135
+ const store = new LeaderboardRollupStore({
136
+ home,
137
+ config: {
138
+ liveLimit: 1000,
139
+ liveTtlHours: 24,
140
+ durable: false,
141
+ durableRetentionDays: 14
142
+ },
143
+ flushDelayMs: 0
144
+ });
145
+ store.record(inspection({
146
+ callId: "x",
147
+ principal: { tokenId: "tok", label: "x" },
148
+ cost: 1
149
+ }));
150
+ store.flush();
151
+ assert.equal(store.query({
152
+ by: "principal",
153
+ sort: "cost",
154
+ limit: 10,
155
+ window: "24h"
156
+ }).rows.length, 0);
157
+ }
158
+ finally {
159
+ rmSync(home, { recursive: true, force: true });
160
+ }
161
+ });
162
+ test("provider leaderboard aggregates completed calls across accounts under one provider", () => {
163
+ // Account identity is not a leaderboard dimension; two accounts under
164
+ // the same provider roll into one provider row from completed calls only.
165
+ const result = aggregateInspections([
166
+ inspection({
167
+ callId: "work-1",
168
+ provider: "codex",
169
+ model: "codex/gpt-5.5",
170
+ cost: 1,
171
+ tokens: 20
172
+ }),
173
+ inspection({
174
+ callId: "personal-1",
175
+ provider: "codex",
176
+ model: "codex/gpt-5.5",
177
+ cost: 2,
178
+ tokens: 30
179
+ }),
180
+ inspection({
181
+ callId: "claude-1",
182
+ provider: "claude-code",
183
+ model: "claude-code/claude-opus",
184
+ cost: 5,
185
+ tokens: 10
186
+ })
187
+ ], { by: "provider", sort: "requests", limit: 10 });
188
+ assert.equal(result.rows.length, 2);
189
+ assert.equal(result.rows[0]?.key, "codex");
190
+ assert.equal(result.rows[0]?.requests, 2);
191
+ assert.equal(result.rows[0]?.tokensTotal, 60);
192
+ assert.equal(result.rows[0]?.estimateUsd, 3);
193
+ assert.equal(result.rows[1]?.key, "claude-code");
194
+ assert.equal(result.rows[1]?.requests, 1);
195
+ });
196
+ test("account activity persistence stays independent of leaderboard rollups", () => {
197
+ const home = mkdtempSync(join(tmpdir(), "routekit-leaderboard-activity-"));
198
+ const usageDirectory = join(home, "usage");
199
+ mkdirSync(usageDirectory, { recursive: true, mode: 0o700 });
200
+ const activityPath = join(usageDirectory, "account-activity.v1.json");
201
+ try {
202
+ const rollups = new LeaderboardRollupStore({
203
+ home,
204
+ config: {
205
+ liveLimit: 1000,
206
+ liveTtlHours: 24,
207
+ durable: false,
208
+ durableRetentionDays: 14
209
+ },
210
+ flushDelayMs: 0
211
+ });
212
+ const activity = new AccountActivityCoordinator({
213
+ statePath: activityPath,
214
+ persistDebounceMs: 0,
215
+ now: () => 1_700_000_000_000
216
+ });
217
+ const before = aggregateInspections([
218
+ inspection({
219
+ callId: "a",
220
+ provider: "codex",
221
+ principal: { tokenId: "tok", label: "alice" },
222
+ cost: 1.5
223
+ })
224
+ ], { by: "principal", sort: "cost", limit: 10 });
225
+ const release = activity.beginAttempt("codex:work");
226
+ activity.flush();
227
+ release();
228
+ rollups.record(inspection({
229
+ callId: "ignored-while-disabled",
230
+ principal: { tokenId: "tok", label: "alice" },
231
+ cost: 99
232
+ }));
233
+ rollups.flush();
234
+ const after = aggregateInspections([
235
+ inspection({
236
+ callId: "a",
237
+ provider: "codex",
238
+ principal: { tokenId: "tok", label: "alice" },
239
+ cost: 1.5
240
+ })
241
+ ], { by: "principal", sort: "cost", limit: 10 });
242
+ assert.deepEqual(after, before);
243
+ assert.equal(existsSync(activityPath), true);
244
+ assert.equal(existsSync(rollups.path()), false);
245
+ assert.equal(activity.snapshot("codex:work").lastSelected, true);
246
+ assert.equal(activity.snapshot("codex:work").inFlight, 0);
247
+ assert.equal(rollups.query({
248
+ by: "principal",
249
+ sort: "cost",
250
+ limit: 10,
251
+ window: "24h"
252
+ }).rows.length, 0);
253
+ activity.close();
254
+ }
255
+ finally {
256
+ rmSync(home, { recursive: true, force: true });
257
+ }
258
+ });
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@velum-labs/routekit-daemon",
3
3
  "private": false,
4
- "version": "0.13.0",
4
+ "version": "0.15.0",
5
5
  "description": "Singleton RouteKit control daemon and stable model gateway.",
6
6
  "repository": {
7
7
  "type": "git",
@@ -34,14 +34,14 @@
34
34
  ],
35
35
  "dependencies": {
36
36
  "yaml": "2.9.0",
37
- "@velum-labs/routekit-control": "0.13.0",
38
- "@velum-labs/routekit-gateway": "0.13.0",
39
- "@velum-labs/routekit-registry": "0.13.0",
40
- "@velum-labs/routekit-router": "0.13.0",
41
- "@velum-labs/routekit-config": "0.13.0",
42
- "@velum-labs/routekit-accounts": "0.13.0",
43
- "@velum-labs/routekit-telemetry-core": "0.13.0",
44
- "@velum-labs/routekit-runtime": "0.13.0"
37
+ "@velum-labs/routekit-accounts": "0.15.0",
38
+ "@velum-labs/routekit-config": "0.15.0",
39
+ "@velum-labs/routekit-control": "0.15.0",
40
+ "@velum-labs/routekit-gateway": "0.15.0",
41
+ "@velum-labs/routekit-registry": "0.15.0",
42
+ "@velum-labs/routekit-router": "0.15.0",
43
+ "@velum-labs/routekit-runtime": "0.15.0",
44
+ "@velum-labs/routekit-telemetry-core": "0.15.0"
45
45
  },
46
46
  "scripts": {
47
47
  "build": "tsc -b",