@velum-labs/routekit-daemon 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/call-attribution-store.d.ts +13 -0
- package/dist/call-attribution-store.js +30 -0
- package/dist/index.js +150 -14
- package/dist/leaderboard.d.ts +52 -0
- package/dist/leaderboard.js +404 -0
- package/dist/test/call-attribution-store.test.js +12 -0
- package/dist/test/daemon.test.js +166 -1
- package/dist/test/leaderboard.test.d.ts +1 -0
- package/dist/test/leaderboard.test.js +258 -0
- package/package.json +9 -9
|
@@ -11,6 +11,19 @@ export declare class CallAttributionStore implements ProvenanceSink {
|
|
|
11
11
|
ttlMs?: number;
|
|
12
12
|
now?: () => number;
|
|
13
13
|
});
|
|
14
|
+
configureBudget(options: {
|
|
15
|
+
limit: number;
|
|
16
|
+
ttlMs: number;
|
|
17
|
+
}): void;
|
|
18
|
+
budget(): {
|
|
19
|
+
limit: number;
|
|
20
|
+
ttlMs: number;
|
|
21
|
+
};
|
|
14
22
|
onModelCall(modelCall: ModelCallRecord): void;
|
|
15
23
|
get(callId: string): RouteKitCallInspection | undefined;
|
|
24
|
+
/** Snapshot retained inspections after TTL prune (insertion order). */
|
|
25
|
+
list(): RouteKitCallInspection[];
|
|
26
|
+
/** True when the store has dropped records due to the capacity budget. */
|
|
27
|
+
truncated(): boolean;
|
|
28
|
+
size(): number;
|
|
16
29
|
}
|
|
@@ -93,11 +93,27 @@ export class CallAttributionStore {
|
|
|
93
93
|
#limit;
|
|
94
94
|
#ttlMs;
|
|
95
95
|
#now;
|
|
96
|
+
#evicted = false;
|
|
96
97
|
constructor(options = {}) {
|
|
97
98
|
this.#limit = options.limit ?? DEFAULT_CALL_ATTRIBUTION_LIMIT;
|
|
98
99
|
this.#ttlMs = options.ttlMs ?? DEFAULT_CALL_ATTRIBUTION_TTL_MS;
|
|
99
100
|
this.#now = options.now ?? Date.now;
|
|
100
101
|
}
|
|
102
|
+
configureBudget(options) {
|
|
103
|
+
this.#limit = options.limit;
|
|
104
|
+
this.#ttlMs = options.ttlMs;
|
|
105
|
+
this.#prune(this.#now());
|
|
106
|
+
while (this.#entries.size > this.#limit) {
|
|
107
|
+
const oldest = this.#entries.keys().next().value;
|
|
108
|
+
if (oldest === undefined)
|
|
109
|
+
break;
|
|
110
|
+
this.#entries.delete(oldest);
|
|
111
|
+
this.#evicted = true;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
budget() {
|
|
115
|
+
return { limit: this.#limit, ttlMs: this.#ttlMs };
|
|
116
|
+
}
|
|
101
117
|
onModelCall(modelCall) {
|
|
102
118
|
const inspection = callInspection(modelCall);
|
|
103
119
|
if (inspection === undefined)
|
|
@@ -111,12 +127,26 @@ export class CallAttributionStore {
|
|
|
111
127
|
if (oldest === undefined)
|
|
112
128
|
break;
|
|
113
129
|
this.#entries.delete(oldest);
|
|
130
|
+
this.#evicted = true;
|
|
114
131
|
}
|
|
115
132
|
}
|
|
116
133
|
get(callId) {
|
|
117
134
|
this.#prune(this.#now());
|
|
118
135
|
return this.#entries.get(callId)?.inspection;
|
|
119
136
|
}
|
|
137
|
+
/** Snapshot retained inspections after TTL prune (insertion order). */
|
|
138
|
+
list() {
|
|
139
|
+
this.#prune(this.#now());
|
|
140
|
+
return [...this.#entries.values()].map((entry) => entry.inspection);
|
|
141
|
+
}
|
|
142
|
+
/** True when the store has dropped records due to the capacity budget. */
|
|
143
|
+
truncated() {
|
|
144
|
+
return this.#evicted;
|
|
145
|
+
}
|
|
146
|
+
size() {
|
|
147
|
+
this.#prune(this.#now());
|
|
148
|
+
return this.#entries.size;
|
|
149
|
+
}
|
|
120
150
|
#prune(now) {
|
|
121
151
|
for (const [callId, entry] of this.#entries) {
|
|
122
152
|
if (now - entry.insertedAt <= this.#ttlMs)
|
package/dist/index.js
CHANGED
|
@@ -8,10 +8,11 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import { chmodSync, existsSync, lstatSync, mkdirSync, readFileSync, rmSync } from "node:fs";
|
|
10
10
|
import { basename, dirname, join } from "node:path";
|
|
11
|
-
import { CLIPROXY_API_KEY_ENV, CLIPROXY_BASE_URL_ENV, accountStoreEntries, cliproxyAuthDirectory, cliproxyAccountEntries, cliproxyAccountMatchesKind, cliproxyApiKey, cliproxyBaseUrl, cliproxyCredentialValid, defaultSubscriptionAccountDirectory, RateLimitTracker, removeCliproxyAccount, removeSubscriptionAccount, renameSubscriptionAccount, sanitizeSubscriptionLabel } from "@velum-labs/routekit-accounts";
|
|
11
|
+
import { AccountActivityCoordinator, CLIPROXY_API_KEY_ENV, CLIPROXY_BASE_URL_ENV, accountStoreEntries, cliproxyAuthDirectory, cliproxyAccountEntries, cliproxyAccountMatchesKind, cliproxyApiKey, cliproxyBaseUrl, cliproxyCredentialValid, defaultSubscriptionAccountDirectory, RateLimitTracker, removeCliproxyAccount, removeSubscriptionAccount, renameSubscriptionAccount, sanitizeSubscriptionLabel, subscriptionAccountIdentity } from "@velum-labs/routekit-accounts";
|
|
12
12
|
import { configuredProviderIds, globalRouterConfigPath, parseRouterConfigDocument, routekitHome, writeRouterConfig } from "@velum-labs/routekit-config";
|
|
13
13
|
import { createRouteKitControlHandler, ROUTEKIT_CONTROL_CAPABILITY } from "@velum-labs/routekit-control";
|
|
14
14
|
import { startSwitchingGatewayProxy } from "@velum-labs/routekit-gateway";
|
|
15
|
+
import { resolveLeaderboardConfig } from "@velum-labs/routekit-gateway";
|
|
15
16
|
import { PROVIDERS, accountKindForCliproxyAuthType, resolveAccountConnector } from "@velum-labs/routekit-registry";
|
|
16
17
|
import { startRouter } from "@velum-labs/routekit-router";
|
|
17
18
|
import { acquireLifecycleLock, CONTROL_PROTOCOL_VERSION, ControlClient, ControlError, createPortlessSession, createServiceRecordStore, createTokenStore, encodeJoinCredential, extendCleanupGrace, generateControlToken, nextServiceGeneration, processIdentity, registerCleanup, SERVICE_HOME_MODE, startControlServer, supervisorFromEnv, writeFileAtomic } from "@velum-labs/routekit-runtime";
|
|
@@ -19,7 +20,8 @@ import { createConsentManager } from "@velum-labs/routekit-telemetry-core";
|
|
|
19
20
|
import { parse as parseYaml, stringify as stringifyYaml } from "yaml";
|
|
20
21
|
import { createCliproxySidecar } from "./cliproxy-sidecar.js";
|
|
21
22
|
import { cleanupAccountTransaction, markAccountTransactionCommitted, prepareAccountTransaction, recoverAccountTransactions, rollbackAccountTransaction } from "./account-transaction.js";
|
|
22
|
-
import { CallAttributionStore } from "./call-attribution-store.js";
|
|
23
|
+
import { CallAttributionStore, callInspection } from "./call-attribution-store.js";
|
|
24
|
+
import { aggregateInspections, buildLeaderboardResult, LeaderboardRollupStore } from "./leaderboard.js";
|
|
23
25
|
export const ROUTEKIT_DAEMON_KIND = "daemon";
|
|
24
26
|
export const ROUTEKIT_PRODUCT = "routekit";
|
|
25
27
|
function dataTokenPath(home) {
|
|
@@ -275,6 +277,7 @@ export async function startRouteKitDaemon(options) {
|
|
|
275
277
|
let portless;
|
|
276
278
|
let sidecarRef;
|
|
277
279
|
let activeRouter;
|
|
280
|
+
let accountActivity;
|
|
278
281
|
let record;
|
|
279
282
|
let closed = false;
|
|
280
283
|
let draining = false;
|
|
@@ -317,7 +320,39 @@ export async function startRouteKitDaemon(options) {
|
|
|
317
320
|
writeRevisions(home, revisions);
|
|
318
321
|
const sidecar = createCliproxySidecar({ env });
|
|
319
322
|
sidecarRef = sidecar;
|
|
320
|
-
|
|
323
|
+
let leaderboardConfig = resolveLeaderboardConfig(currentConfig);
|
|
324
|
+
const callAttributions = new CallAttributionStore({
|
|
325
|
+
limit: leaderboardConfig.liveLimit,
|
|
326
|
+
ttlMs: leaderboardConfig.liveTtlHours * 60 * 60 * 1_000
|
|
327
|
+
});
|
|
328
|
+
const leaderboardRollups = new LeaderboardRollupStore({
|
|
329
|
+
home,
|
|
330
|
+
config: leaderboardConfig
|
|
331
|
+
});
|
|
332
|
+
// Independent of leaderboard durable rollups: last-selection only.
|
|
333
|
+
mkdirSync(join(home, "usage"), { recursive: true, mode: 0o700 });
|
|
334
|
+
accountActivity = new AccountActivityCoordinator({
|
|
335
|
+
statePath: join(home, "usage", "account-activity.v1.json")
|
|
336
|
+
});
|
|
337
|
+
const applyLeaderboardConfig = (config) => {
|
|
338
|
+
leaderboardConfig = resolveLeaderboardConfig(config);
|
|
339
|
+
callAttributions.configureBudget({
|
|
340
|
+
limit: leaderboardConfig.liveLimit,
|
|
341
|
+
ttlMs: leaderboardConfig.liveTtlHours * 60 * 60 * 1_000
|
|
342
|
+
});
|
|
343
|
+
leaderboardRollups.configure({
|
|
344
|
+
durable: leaderboardConfig.durable,
|
|
345
|
+
durableRetentionDays: leaderboardConfig.durableRetentionDays
|
|
346
|
+
});
|
|
347
|
+
};
|
|
348
|
+
const provenance = {
|
|
349
|
+
onModelCall(record) {
|
|
350
|
+
callAttributions.onModelCall(record);
|
|
351
|
+
const inspection = callInspection(record);
|
|
352
|
+
if (inspection !== undefined)
|
|
353
|
+
leaderboardRollups.record(inspection);
|
|
354
|
+
}
|
|
355
|
+
};
|
|
321
356
|
const wantsCliproxySidecar = (config) => config.providers["cliproxy"] !== undefined;
|
|
322
357
|
// Router generations reach the managed sidecar with its own ingress key
|
|
323
358
|
// and configured listen address; resolved per generation so state created
|
|
@@ -339,7 +374,8 @@ export async function startRouteKitDaemon(options) {
|
|
|
339
374
|
host: "127.0.0.1",
|
|
340
375
|
port: 0,
|
|
341
376
|
env: routerEnv(),
|
|
342
|
-
provenance
|
|
377
|
+
provenance,
|
|
378
|
+
activity: accountActivity,
|
|
343
379
|
drainGraceMs
|
|
344
380
|
});
|
|
345
381
|
await sidecar.reconcile(wantsCliproxySidecar(currentConfig));
|
|
@@ -413,6 +449,7 @@ export async function startRouteKitDaemon(options) {
|
|
|
413
449
|
currentConfig = nextConfig;
|
|
414
450
|
currentDocument = input.write ? readFileSync(configPath, "utf8") : nextDocument;
|
|
415
451
|
revisions = nextRevisions;
|
|
452
|
+
applyLeaderboardConfig(currentConfig);
|
|
416
453
|
if (previousRouter !== undefined) {
|
|
417
454
|
try {
|
|
418
455
|
if (previousTarget !== undefined) {
|
|
@@ -600,6 +637,46 @@ export async function startRouteKitDaemon(options) {
|
|
|
600
637
|
}
|
|
601
638
|
return inspection;
|
|
602
639
|
},
|
|
640
|
+
"calls.leaderboard": async (params) => {
|
|
641
|
+
const by = params.by ?? "principal";
|
|
642
|
+
const sort = params.sort ?? "cost";
|
|
643
|
+
const limit = params.limit ?? 20;
|
|
644
|
+
const window = params.window ?? "live";
|
|
645
|
+
const nowIso = new Date().toISOString();
|
|
646
|
+
if (window === "live") {
|
|
647
|
+
const inspections = callAttributions.list();
|
|
648
|
+
const aggregated = aggregateInspections(inspections, { by, sort, limit });
|
|
649
|
+
return buildLeaderboardResult({
|
|
650
|
+
by,
|
|
651
|
+
sort,
|
|
652
|
+
source: "live",
|
|
653
|
+
windowStart: aggregated.windowStart ?? nowIso,
|
|
654
|
+
windowEnd: aggregated.windowEnd ?? nowIso,
|
|
655
|
+
sampleSize: aggregated.sampleSize,
|
|
656
|
+
truncated: callAttributions.truncated(),
|
|
657
|
+
budget: leaderboardConfig,
|
|
658
|
+
rows: aggregated.rows
|
|
659
|
+
});
|
|
660
|
+
}
|
|
661
|
+
if (!leaderboardConfig.durable) {
|
|
662
|
+
throw new ControlError({
|
|
663
|
+
code: "bad_request",
|
|
664
|
+
message: "durable leaderboard rollups are disabled; set leaderboard.durable: true in router.yaml"
|
|
665
|
+
});
|
|
666
|
+
}
|
|
667
|
+
const aggregated = leaderboardRollups.query({ by, sort, limit, window });
|
|
668
|
+
return buildLeaderboardResult({
|
|
669
|
+
by,
|
|
670
|
+
sort,
|
|
671
|
+
source: "durable",
|
|
672
|
+
windowStart: aggregated.windowStart,
|
|
673
|
+
windowEnd: aggregated.windowEnd,
|
|
674
|
+
sampleSize: aggregated.sampleSize,
|
|
675
|
+
truncated: false,
|
|
676
|
+
budget: leaderboardConfig,
|
|
677
|
+
rows: aggregated.rows
|
|
678
|
+
});
|
|
679
|
+
},
|
|
603
680
|
"accounts.list": async () => ({
|
|
604
681
|
accounts: accountEntries(env).map((entry) => {
|
|
605
682
|
if (entry.connector === "native")
|
|
@@ -618,6 +695,7 @@ export async function startRouteKitDaemon(options) {
|
|
|
618
695
|
return {
|
|
619
696
|
accounts: entries.map((entry) => {
|
|
620
697
|
if (entry.connector === "cliproxy") {
|
|
698
|
+
const ready = entry.credentialValid && cliproxyConfigured && cliproxyReachable;
|
|
621
699
|
return {
|
|
622
700
|
subscriptionKind: entry.subscriptionKind,
|
|
623
701
|
label: entry.label,
|
|
@@ -625,8 +703,11 @@ export async function startRouteKitDaemon(options) {
|
|
|
625
703
|
...(entry.localOnly === true ? { localOnly: true } : {}),
|
|
626
704
|
credentialValid: entry.credentialValid,
|
|
627
705
|
configured: cliproxyConfigured,
|
|
628
|
-
relayOpen:
|
|
629
|
-
|
|
706
|
+
relayOpen: ready,
|
|
707
|
+
serving: false,
|
|
708
|
+
inFlight: 0,
|
|
709
|
+
lastSelected: false,
|
|
710
|
+
active: false,
|
|
630
711
|
models: []
|
|
631
712
|
};
|
|
632
713
|
}
|
|
@@ -642,7 +723,13 @@ export async function startRouteKitDaemon(options) {
|
|
|
642
723
|
configured: currentConfig.providers[entry.subscriptionKind] !== undefined,
|
|
643
724
|
relayOpen: member?.relayReady === true &&
|
|
644
725
|
currentConfig.providers[entry.subscriptionKind] !== undefined,
|
|
645
|
-
|
|
726
|
+
serving: member?.serving ?? false,
|
|
727
|
+
inFlight: member?.inFlight ?? 0,
|
|
728
|
+
...(member?.lastSelectedAt !== undefined
|
|
729
|
+
? { lastSelectedAt: member.lastSelectedAt }
|
|
730
|
+
: {}),
|
|
731
|
+
lastSelected: member?.lastSelected ?? false,
|
|
732
|
+
active: member?.lastSelected ?? false,
|
|
646
733
|
models: member?.models ?? [],
|
|
647
734
|
...(member?.limits !== undefined ? { limits: member.limits } : {})
|
|
648
735
|
};
|
|
@@ -956,11 +1043,12 @@ export async function startRouteKitDaemon(options) {
|
|
|
956
1043
|
const nextConfig = disableProvider
|
|
957
1044
|
? parseConfigDocument(nextDocument)
|
|
958
1045
|
: currentConfig;
|
|
1046
|
+
const activityPath = join(home, "usage", "account-activity.v1.json");
|
|
959
1047
|
const transaction = prepareAccountTransaction({
|
|
960
1048
|
home,
|
|
961
1049
|
configPath,
|
|
962
|
-
accountPaths: [nativePath],
|
|
963
|
-
accountRoots: [activeNativeDirectory],
|
|
1050
|
+
accountPaths: [nativePath, activityPath],
|
|
1051
|
+
accountRoots: [activeNativeDirectory, home],
|
|
964
1052
|
kind: nativeKind,
|
|
965
1053
|
provider: nativeKind,
|
|
966
1054
|
labels: [params.label]
|
|
@@ -976,7 +1064,10 @@ export async function startRouteKitDaemon(options) {
|
|
|
976
1064
|
write: disableProvider,
|
|
977
1065
|
configRevision: disableProvider,
|
|
978
1066
|
accountRevision: true,
|
|
979
|
-
beforeSwap: () =>
|
|
1067
|
+
beforeSwap: () => {
|
|
1068
|
+
accountActivity.remove(subscriptionAccountIdentity(nativeKind, params.label));
|
|
1069
|
+
markAccountTransactionCommitted(transaction);
|
|
1070
|
+
}
|
|
980
1071
|
});
|
|
981
1072
|
try {
|
|
982
1073
|
cleanupAccountTransaction(transaction);
|
|
@@ -989,6 +1080,7 @@ export async function startRouteKitDaemon(options) {
|
|
|
989
1080
|
const rollbackFailures = [];
|
|
990
1081
|
try {
|
|
991
1082
|
rollbackAccountTransaction(transaction, home);
|
|
1083
|
+
accountActivity?.reload();
|
|
992
1084
|
}
|
|
993
1085
|
catch (rollbackError) {
|
|
994
1086
|
rollbackFailures.push(rollbackError);
|
|
@@ -1057,6 +1149,7 @@ export async function startRouteKitDaemon(options) {
|
|
|
1057
1149
|
const sourcePath = join(directory, `${params.source}.json`);
|
|
1058
1150
|
const targetPath = join(directory, `${params.target}.json`);
|
|
1059
1151
|
const trackerPath = join(directory, ".state.json");
|
|
1152
|
+
const activityPath = join(home, "usage", "account-activity.v1.json");
|
|
1060
1153
|
if (!existsSync(sourcePath)) {
|
|
1061
1154
|
throw new ControlError({
|
|
1062
1155
|
code: "not_found",
|
|
@@ -1082,8 +1175,8 @@ export async function startRouteKitDaemon(options) {
|
|
|
1082
1175
|
const transaction = prepareAccountTransaction({
|
|
1083
1176
|
home,
|
|
1084
1177
|
configPath,
|
|
1085
|
-
accountPaths: [sourcePath, targetPath, trackerPath],
|
|
1086
|
-
accountRoots: [directory],
|
|
1178
|
+
accountPaths: [sourcePath, targetPath, trackerPath, activityPath],
|
|
1179
|
+
accountRoots: [directory, home],
|
|
1087
1180
|
kind,
|
|
1088
1181
|
provider: kind,
|
|
1089
1182
|
labels: [params.source, params.target]
|
|
@@ -1097,7 +1190,10 @@ export async function startRouteKitDaemon(options) {
|
|
|
1097
1190
|
await replaceRouter(currentConfig, currentDocument, {
|
|
1098
1191
|
write: false,
|
|
1099
1192
|
accountRevision: true,
|
|
1100
|
-
beforeSwap: () =>
|
|
1193
|
+
beforeSwap: () => {
|
|
1194
|
+
accountActivity.rename(subscriptionAccountIdentity(kind, params.source), subscriptionAccountIdentity(kind, params.target));
|
|
1195
|
+
markAccountTransactionCommitted(transaction);
|
|
1196
|
+
}
|
|
1101
1197
|
});
|
|
1102
1198
|
try {
|
|
1103
1199
|
cleanupAccountTransaction(transaction);
|
|
@@ -1110,6 +1206,7 @@ export async function startRouteKitDaemon(options) {
|
|
|
1110
1206
|
const rollbackFailures = [];
|
|
1111
1207
|
try {
|
|
1112
1208
|
rollbackAccountTransaction(transaction, home);
|
|
1209
|
+
accountActivity?.reload();
|
|
1113
1210
|
}
|
|
1114
1211
|
catch (rollbackError) {
|
|
1115
1212
|
rollbackFailures.push(rollbackError);
|
|
@@ -1138,6 +1235,43 @@ export async function startRouteKitDaemon(options) {
|
|
|
1138
1235
|
"accounts.usage": async (_params, context) => {
|
|
1139
1236
|
return await activeRouter.usage(context.signal);
|
|
1140
1237
|
},
|
|
1238
|
+
"accounts.redeemReset": async (params, context) => {
|
|
1239
|
+
try {
|
|
1240
|
+
const result = await activeRouter.redeemReset({
|
|
1241
|
+
kind: params.kind,
|
|
1242
|
+
label: params.label,
|
|
1243
|
+
...(params.creditId !== undefined ? { creditId: params.creditId } : {}),
|
|
1244
|
+
...(params.redeemRequestId !== undefined
|
|
1245
|
+
? { redeemRequestId: params.redeemRequestId }
|
|
1246
|
+
: {})
|
|
1247
|
+
}, context.signal);
|
|
1248
|
+
return {
|
|
1249
|
+
ok: result.ok,
|
|
1250
|
+
code: result.code,
|
|
1251
|
+
kind: "codex",
|
|
1252
|
+
label: result.label,
|
|
1253
|
+
redeemRequestId: result.redeemRequestId,
|
|
1254
|
+
...(result.creditId !== undefined ? { creditId: result.creditId } : {}),
|
|
1255
|
+
...(result.windowsReset !== undefined
|
|
1256
|
+
? { windowsReset: result.windowsReset }
|
|
1257
|
+
: {}),
|
|
1258
|
+
usage: result.usage
|
|
1259
|
+
};
|
|
1260
|
+
}
|
|
1261
|
+
catch (error) {
|
|
1262
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1263
|
+
if (message.includes("is not enrolled") || message.includes("no redeemable")) {
|
|
1264
|
+
throw new ControlError({ code: "not_found", message });
|
|
1265
|
+
}
|
|
1266
|
+
if (message.includes("does not support") ||
|
|
1267
|
+
message.includes("no codex account pool") ||
|
|
1268
|
+
message.includes("creditId must not be empty") ||
|
|
1269
|
+
message.includes("account label is required")) {
|
|
1270
|
+
throw new ControlError({ code: "bad_request", message });
|
|
1271
|
+
}
|
|
1272
|
+
throw new ControlError({ code: "internal", message });
|
|
1273
|
+
}
|
|
1274
|
+
},
|
|
1141
1275
|
"telemetry.get": async () => ({ enabled: telemetry.resolve().enabled }),
|
|
1142
1276
|
"telemetry.set": async (params) => {
|
|
1143
1277
|
await serializeMutation(async () => {
|
|
@@ -1262,7 +1396,7 @@ export async function startRouteKitDaemon(options) {
|
|
|
1262
1396
|
token: issued.token,
|
|
1263
1397
|
...(issued.plane === "control"
|
|
1264
1398
|
? {
|
|
1265
|
-
|
|
1399
|
+
joinCredential: encodeJoinCredential({
|
|
1266
1400
|
publicRecordPath: daemonPublicRecordPath(home),
|
|
1267
1401
|
token: issued.token
|
|
1268
1402
|
})
|
|
@@ -1364,6 +1498,7 @@ export async function startRouteKitDaemon(options) {
|
|
|
1364
1498
|
lifecycle = "draining";
|
|
1365
1499
|
await proxy?.drain(drainGraceMs);
|
|
1366
1500
|
await activeRouter?.close();
|
|
1501
|
+
accountActivity?.close();
|
|
1367
1502
|
await sidecar.close();
|
|
1368
1503
|
await control?.close();
|
|
1369
1504
|
if (portless?.enabled)
|
|
@@ -1400,6 +1535,7 @@ export async function startRouteKitDaemon(options) {
|
|
|
1400
1535
|
catch (error) {
|
|
1401
1536
|
await proxy?.close();
|
|
1402
1537
|
await activeRouter?.close();
|
|
1538
|
+
accountActivity?.close();
|
|
1403
1539
|
await sidecarRef?.close();
|
|
1404
1540
|
await control?.close();
|
|
1405
1541
|
if (portless?.enabled)
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type { RouteKitCallInspection, RouteKitControlParams, RouteKitLeaderboard } from "@velum-labs/routekit-control";
|
|
2
|
+
import type { LeaderboardConfig } from "@velum-labs/routekit-gateway";
|
|
3
|
+
export type LeaderboardDimension = RouteKitLeaderboard["by"];
|
|
4
|
+
export type LeaderboardSort = RouteKitLeaderboard["sort"];
|
|
5
|
+
export type LeaderboardWindow = NonNullable<RouteKitControlParams["calls.leaderboard"]["window"]>;
|
|
6
|
+
export declare const LEADERBOARD_ROLLUP_VERSION: 1;
|
|
7
|
+
export declare const LEADERBOARD_ROLLUP_RELATIVE_PATH: string;
|
|
8
|
+
export declare function aggregateInspections(inspections: readonly RouteKitCallInspection[], options: {
|
|
9
|
+
by: LeaderboardDimension;
|
|
10
|
+
sort: LeaderboardSort;
|
|
11
|
+
limit: number;
|
|
12
|
+
}): {
|
|
13
|
+
rows: RouteKitLeaderboard["rows"];
|
|
14
|
+
sampleSize: number;
|
|
15
|
+
windowStart?: string;
|
|
16
|
+
windowEnd?: string;
|
|
17
|
+
};
|
|
18
|
+
export declare class LeaderboardRollupStore {
|
|
19
|
+
#private;
|
|
20
|
+
constructor(options: {
|
|
21
|
+
home: string;
|
|
22
|
+
config: LeaderboardConfig;
|
|
23
|
+
now?: () => number;
|
|
24
|
+
flushDelayMs?: number;
|
|
25
|
+
});
|
|
26
|
+
configure(config: Pick<LeaderboardConfig, "durable" | "durableRetentionDays">): void;
|
|
27
|
+
record(inspection: RouteKitCallInspection): void;
|
|
28
|
+
flush(): void;
|
|
29
|
+
query(options: {
|
|
30
|
+
by: LeaderboardDimension;
|
|
31
|
+
sort: LeaderboardSort;
|
|
32
|
+
limit: number;
|
|
33
|
+
window: Exclude<LeaderboardWindow, "live">;
|
|
34
|
+
}): {
|
|
35
|
+
rows: RouteKitLeaderboard["rows"];
|
|
36
|
+
sampleSize: number;
|
|
37
|
+
windowStart: string;
|
|
38
|
+
windowEnd: string;
|
|
39
|
+
};
|
|
40
|
+
path(): string;
|
|
41
|
+
}
|
|
42
|
+
export declare function buildLeaderboardResult(input: {
|
|
43
|
+
by: LeaderboardDimension;
|
|
44
|
+
sort: LeaderboardSort;
|
|
45
|
+
source: RouteKitLeaderboard["source"];
|
|
46
|
+
windowStart: string;
|
|
47
|
+
windowEnd: string;
|
|
48
|
+
sampleSize: number;
|
|
49
|
+
truncated: boolean;
|
|
50
|
+
budget: LeaderboardConfig;
|
|
51
|
+
rows: RouteKitLeaderboard["rows"];
|
|
52
|
+
}): RouteKitLeaderboard;
|
|
@@ -0,0 +1,404 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Usage leaderboard aggregation over live call attribution and optional
|
|
3
|
+
* durable hourly rollups under `$ROUTEKIT_HOME/usage/`.
|
|
4
|
+
*/
|
|
5
|
+
import { chmodSync, existsSync, mkdirSync, readFileSync } from "node:fs";
|
|
6
|
+
import { dirname, join } from "node:path";
|
|
7
|
+
import { writeFileAtomic } from "@velum-labs/routekit-runtime";
|
|
8
|
+
export const LEADERBOARD_ROLLUP_VERSION = 1;
|
|
9
|
+
export const LEADERBOARD_ROLLUP_RELATIVE_PATH = join("usage", "leaderboard-rollups.v1.json");
|
|
10
|
+
const WINDOW_MS = {
|
|
11
|
+
"1h": 60 * 60 * 1_000,
|
|
12
|
+
"24h": 24 * 60 * 60 * 1_000,
|
|
13
|
+
"7d": 7 * 24 * 60 * 60 * 1_000
|
|
14
|
+
};
|
|
15
|
+
function emptyCounters(key, label) {
|
|
16
|
+
return {
|
|
17
|
+
key,
|
|
18
|
+
...(label !== undefined ? { label } : {}),
|
|
19
|
+
requests: 0,
|
|
20
|
+
success: 0,
|
|
21
|
+
error: 0,
|
|
22
|
+
tokensIn: 0,
|
|
23
|
+
tokensOut: 0,
|
|
24
|
+
tokensTotal: 0,
|
|
25
|
+
estimateUsd: 0,
|
|
26
|
+
unknownCostCount: 0,
|
|
27
|
+
unknownUsageCount: 0,
|
|
28
|
+
latencyMsSum: 0,
|
|
29
|
+
latencyMsCount: 0
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
function dimensionKey(inspection, by) {
|
|
33
|
+
switch (by) {
|
|
34
|
+
case "principal": {
|
|
35
|
+
const tokenId = inspection.principal?.tokenId;
|
|
36
|
+
if (tokenId === undefined)
|
|
37
|
+
return undefined;
|
|
38
|
+
return {
|
|
39
|
+
key: tokenId,
|
|
40
|
+
...(inspection.principal?.label !== undefined
|
|
41
|
+
? { label: inspection.principal.label }
|
|
42
|
+
: {})
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
case "model":
|
|
46
|
+
return { key: inspection.effectiveModel };
|
|
47
|
+
case "provider":
|
|
48
|
+
return { key: inspection.provider };
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
function addInspection(bucket, inspection, keepSamples) {
|
|
52
|
+
bucket.requests += 1;
|
|
53
|
+
if (inspection.status === "succeeded")
|
|
54
|
+
bucket.success += 1;
|
|
55
|
+
else
|
|
56
|
+
bucket.error += 1;
|
|
57
|
+
const usage = inspection.usage;
|
|
58
|
+
if (usage === undefined || inspection.cost.unknownUsage) {
|
|
59
|
+
bucket.unknownUsageCount += 1;
|
|
60
|
+
}
|
|
61
|
+
else {
|
|
62
|
+
bucket.tokensIn += usage.prompt_tokens ?? 0;
|
|
63
|
+
bucket.tokensOut += usage.completion_tokens ?? 0;
|
|
64
|
+
bucket.tokensTotal +=
|
|
65
|
+
usage.total_tokens ??
|
|
66
|
+
(usage.prompt_tokens ?? 0) + (usage.completion_tokens ?? 0);
|
|
67
|
+
}
|
|
68
|
+
if (inspection.cost.estimateUsd !== undefined && !inspection.cost.unknownCost) {
|
|
69
|
+
bucket.estimateUsd += inspection.cost.estimateUsd;
|
|
70
|
+
}
|
|
71
|
+
else if (inspection.cost.unknownCost || inspection.cost.estimateUsd === undefined) {
|
|
72
|
+
bucket.unknownCostCount += 1;
|
|
73
|
+
}
|
|
74
|
+
const latencyMs = inspection.timing.latencyMs;
|
|
75
|
+
if (latencyMs !== undefined) {
|
|
76
|
+
bucket.latencyMsSum += latencyMs;
|
|
77
|
+
bucket.latencyMsCount += 1;
|
|
78
|
+
if (keepSamples) {
|
|
79
|
+
bucket.latencySamples ??= [];
|
|
80
|
+
bucket.latencySamples.push(latencyMs);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
if (bucket.label === undefined &&
|
|
84
|
+
inspection.principal?.label !== undefined &&
|
|
85
|
+
bucket.key === inspection.principal.tokenId) {
|
|
86
|
+
bucket.label = inspection.principal.label;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
function mergeCounters(target, source) {
|
|
90
|
+
target.requests += source.requests;
|
|
91
|
+
target.success += source.success;
|
|
92
|
+
target.error += source.error;
|
|
93
|
+
target.tokensIn += source.tokensIn;
|
|
94
|
+
target.tokensOut += source.tokensOut;
|
|
95
|
+
target.tokensTotal += source.tokensTotal;
|
|
96
|
+
target.estimateUsd += source.estimateUsd;
|
|
97
|
+
target.unknownCostCount += source.unknownCostCount;
|
|
98
|
+
target.unknownUsageCount += source.unknownUsageCount;
|
|
99
|
+
target.latencyMsSum += source.latencyMsSum;
|
|
100
|
+
target.latencyMsCount += source.latencyMsCount;
|
|
101
|
+
if (target.label === undefined && source.label !== undefined) {
|
|
102
|
+
target.label = source.label;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
function percentile(sorted, p) {
|
|
106
|
+
if (sorted.length === 0)
|
|
107
|
+
return undefined;
|
|
108
|
+
if (sorted.length === 1)
|
|
109
|
+
return sorted[0];
|
|
110
|
+
const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1));
|
|
111
|
+
return sorted[index];
|
|
112
|
+
}
|
|
113
|
+
function sortValue(bucket, sort) {
|
|
114
|
+
switch (sort) {
|
|
115
|
+
case "cost":
|
|
116
|
+
return bucket.estimateUsd;
|
|
117
|
+
case "requests":
|
|
118
|
+
return bucket.requests;
|
|
119
|
+
case "tokens":
|
|
120
|
+
return bucket.tokensTotal;
|
|
121
|
+
case "errors":
|
|
122
|
+
return bucket.error;
|
|
123
|
+
case "latency":
|
|
124
|
+
return bucket.latencyMsCount === 0
|
|
125
|
+
? 0
|
|
126
|
+
: bucket.latencyMsSum / bucket.latencyMsCount;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
function toRow(bucket, rank) {
|
|
130
|
+
const samples = bucket.latencySamples?.slice().sort((a, b) => a - b);
|
|
131
|
+
const avg = bucket.latencyMsCount > 0
|
|
132
|
+
? bucket.latencyMsSum / bucket.latencyMsCount
|
|
133
|
+
: undefined;
|
|
134
|
+
return {
|
|
135
|
+
rank,
|
|
136
|
+
key: bucket.key,
|
|
137
|
+
...(bucket.label !== undefined ? { label: bucket.label } : {}),
|
|
138
|
+
requests: bucket.requests,
|
|
139
|
+
success: bucket.success,
|
|
140
|
+
error: bucket.error,
|
|
141
|
+
tokensIn: bucket.tokensIn,
|
|
142
|
+
tokensOut: bucket.tokensOut,
|
|
143
|
+
tokensTotal: bucket.tokensTotal,
|
|
144
|
+
...(bucket.estimateUsd > 0 || bucket.unknownCostCount === 0
|
|
145
|
+
? { estimateUsd: bucket.estimateUsd }
|
|
146
|
+
: {}),
|
|
147
|
+
unknownCostCount: bucket.unknownCostCount,
|
|
148
|
+
unknownUsageCount: bucket.unknownUsageCount,
|
|
149
|
+
...(avg !== undefined ? { latencyMsAvg: avg } : {}),
|
|
150
|
+
...(samples !== undefined
|
|
151
|
+
? {
|
|
152
|
+
...(percentile(samples, 50) !== undefined
|
|
153
|
+
? { latencyMsP50: percentile(samples, 50) }
|
|
154
|
+
: {}),
|
|
155
|
+
...(percentile(samples, 95) !== undefined
|
|
156
|
+
? { latencyMsP95: percentile(samples, 95) }
|
|
157
|
+
: {})
|
|
158
|
+
}
|
|
159
|
+
: {})
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
export function aggregateInspections(inspections, options) {
|
|
163
|
+
const groups = new Map();
|
|
164
|
+
let windowStart;
|
|
165
|
+
let windowEnd;
|
|
166
|
+
for (const inspection of inspections) {
|
|
167
|
+
const dim = dimensionKey(inspection, options.by);
|
|
168
|
+
if (dim === undefined)
|
|
169
|
+
continue;
|
|
170
|
+
let bucket = groups.get(dim.key);
|
|
171
|
+
if (bucket === undefined) {
|
|
172
|
+
bucket = emptyCounters(dim.key, dim.label);
|
|
173
|
+
groups.set(dim.key, bucket);
|
|
174
|
+
}
|
|
175
|
+
addInspection(bucket, inspection, true);
|
|
176
|
+
const started = inspection.timing.startedAt;
|
|
177
|
+
if (windowStart === undefined || started < windowStart)
|
|
178
|
+
windowStart = started;
|
|
179
|
+
const finished = inspection.timing.finishedAt ?? started;
|
|
180
|
+
if (windowEnd === undefined || finished > windowEnd)
|
|
181
|
+
windowEnd = finished;
|
|
182
|
+
}
|
|
183
|
+
const ranked = [...groups.values()].sort((a, b) => {
|
|
184
|
+
const delta = sortValue(b, options.sort) - sortValue(a, options.sort);
|
|
185
|
+
if (delta !== 0)
|
|
186
|
+
return delta;
|
|
187
|
+
return a.key.localeCompare(b.key);
|
|
188
|
+
});
|
|
189
|
+
return {
|
|
190
|
+
rows: ranked.slice(0, options.limit).map((bucket, index) => toRow(bucket, index + 1)),
|
|
191
|
+
sampleSize: inspections.length,
|
|
192
|
+
...(windowStart !== undefined ? { windowStart } : {}),
|
|
193
|
+
...(windowEnd !== undefined ? { windowEnd } : {})
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
function hourFloor(iso) {
|
|
197
|
+
const date = new Date(iso);
|
|
198
|
+
if (Number.isNaN(date.getTime())) {
|
|
199
|
+
const fallback = new Date();
|
|
200
|
+
fallback.setUTCMinutes(0, 0, 0);
|
|
201
|
+
return fallback.toISOString();
|
|
202
|
+
}
|
|
203
|
+
date.setUTCMinutes(0, 0, 0);
|
|
204
|
+
return date.toISOString();
|
|
205
|
+
}
|
|
206
|
+
function serializeBucket(bucket) {
|
|
207
|
+
const { latencySamples: _samples, ...rest } = bucket;
|
|
208
|
+
return rest;
|
|
209
|
+
}
|
|
210
|
+
export class LeaderboardRollupStore {
|
|
211
|
+
#path;
|
|
212
|
+
#now;
|
|
213
|
+
#flushDelayMs;
|
|
214
|
+
#enabled;
|
|
215
|
+
#retentionDays;
|
|
216
|
+
#file;
|
|
217
|
+
#timer;
|
|
218
|
+
#dirty = false;
|
|
219
|
+
constructor(options) {
|
|
220
|
+
this.#path = join(options.home, LEADERBOARD_ROLLUP_RELATIVE_PATH);
|
|
221
|
+
this.#now = options.now ?? Date.now;
|
|
222
|
+
this.#flushDelayMs = options.flushDelayMs ?? 250;
|
|
223
|
+
this.#enabled = options.config.durable;
|
|
224
|
+
this.#retentionDays = options.config.durableRetentionDays;
|
|
225
|
+
this.#file = this.#load();
|
|
226
|
+
}
|
|
227
|
+
configure(config) {
|
|
228
|
+
this.#enabled = config.durable;
|
|
229
|
+
this.#retentionDays = config.durableRetentionDays;
|
|
230
|
+
this.#file.retentionDays = this.#retentionDays;
|
|
231
|
+
this.#prune(this.#now());
|
|
232
|
+
if (this.#enabled)
|
|
233
|
+
this.#scheduleFlush();
|
|
234
|
+
}
|
|
235
|
+
record(inspection) {
|
|
236
|
+
if (!this.#enabled)
|
|
237
|
+
return;
|
|
238
|
+
const hour = hourFloor(inspection.timing.startedAt);
|
|
239
|
+
let bucket = this.#file.buckets.find((entry) => entry.hour === hour);
|
|
240
|
+
if (bucket === undefined) {
|
|
241
|
+
bucket = { hour, byPrincipal: {}, byModel: {}, byProvider: {} };
|
|
242
|
+
this.#file.buckets.push(bucket);
|
|
243
|
+
this.#file.buckets.sort((a, b) => a.hour.localeCompare(b.hour));
|
|
244
|
+
}
|
|
245
|
+
for (const by of ["principal", "model", "provider"]) {
|
|
246
|
+
const dim = dimensionKey(inspection, by);
|
|
247
|
+
if (dim === undefined)
|
|
248
|
+
continue;
|
|
249
|
+
const map = by === "principal"
|
|
250
|
+
? bucket.byPrincipal
|
|
251
|
+
: by === "model"
|
|
252
|
+
? bucket.byModel
|
|
253
|
+
: bucket.byProvider;
|
|
254
|
+
let counters = map[dim.key];
|
|
255
|
+
if (counters === undefined) {
|
|
256
|
+
counters = emptyCounters(dim.key, dim.label);
|
|
257
|
+
map[dim.key] = counters;
|
|
258
|
+
}
|
|
259
|
+
addInspection(counters, inspection, false);
|
|
260
|
+
map[dim.key] = serializeBucket(counters);
|
|
261
|
+
}
|
|
262
|
+
this.#file.updatedAt = new Date(this.#now()).toISOString();
|
|
263
|
+
this.#prune(this.#now());
|
|
264
|
+
this.#dirty = true;
|
|
265
|
+
this.#scheduleFlush();
|
|
266
|
+
}
|
|
267
|
+
flush() {
|
|
268
|
+
if (this.#timer !== undefined) {
|
|
269
|
+
clearTimeout(this.#timer);
|
|
270
|
+
this.#timer = undefined;
|
|
271
|
+
}
|
|
272
|
+
if (!this.#dirty)
|
|
273
|
+
return;
|
|
274
|
+
this.#write();
|
|
275
|
+
}
|
|
276
|
+
query(options) {
|
|
277
|
+
this.flush();
|
|
278
|
+
const endMs = this.#now();
|
|
279
|
+
const startMs = endMs - WINDOW_MS[options.window];
|
|
280
|
+
const startIso = new Date(startMs).toISOString();
|
|
281
|
+
const endIso = new Date(endMs).toISOString();
|
|
282
|
+
const groups = new Map();
|
|
283
|
+
let sampleSize = 0;
|
|
284
|
+
for (const hour of this.#file.buckets) {
|
|
285
|
+
if (hour.hour < hourFloor(startIso))
|
|
286
|
+
continue;
|
|
287
|
+
if (hour.hour > endIso)
|
|
288
|
+
continue;
|
|
289
|
+
const map = options.by === "principal"
|
|
290
|
+
? hour.byPrincipal
|
|
291
|
+
: options.by === "model"
|
|
292
|
+
? hour.byModel
|
|
293
|
+
: hour.byProvider;
|
|
294
|
+
for (const counters of Object.values(map)) {
|
|
295
|
+
sampleSize += counters.requests;
|
|
296
|
+
let bucket = groups.get(counters.key);
|
|
297
|
+
if (bucket === undefined) {
|
|
298
|
+
bucket = emptyCounters(counters.key, counters.label);
|
|
299
|
+
groups.set(counters.key, bucket);
|
|
300
|
+
}
|
|
301
|
+
mergeCounters(bucket, counters);
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
const ranked = [...groups.values()].sort((a, b) => {
|
|
305
|
+
const delta = sortValue(b, options.sort) - sortValue(a, options.sort);
|
|
306
|
+
if (delta !== 0)
|
|
307
|
+
return delta;
|
|
308
|
+
return a.key.localeCompare(b.key);
|
|
309
|
+
});
|
|
310
|
+
return {
|
|
311
|
+
rows: ranked.slice(0, options.limit).map((bucket, index) => toRow(bucket, index + 1)),
|
|
312
|
+
sampleSize,
|
|
313
|
+
windowStart: startIso,
|
|
314
|
+
windowEnd: endIso
|
|
315
|
+
};
|
|
316
|
+
}
|
|
317
|
+
path() {
|
|
318
|
+
return this.#path;
|
|
319
|
+
}
|
|
320
|
+
#load() {
|
|
321
|
+
if (!existsSync(this.#path)) {
|
|
322
|
+
return {
|
|
323
|
+
version: LEADERBOARD_ROLLUP_VERSION,
|
|
324
|
+
updatedAt: new Date(this.#now()).toISOString(),
|
|
325
|
+
retentionDays: this.#retentionDays,
|
|
326
|
+
buckets: []
|
|
327
|
+
};
|
|
328
|
+
}
|
|
329
|
+
try {
|
|
330
|
+
const parsed = JSON.parse(readFileSync(this.#path, "utf8"));
|
|
331
|
+
if (parsed.version !== LEADERBOARD_ROLLUP_VERSION || !Array.isArray(parsed.buckets)) {
|
|
332
|
+
return {
|
|
333
|
+
version: LEADERBOARD_ROLLUP_VERSION,
|
|
334
|
+
updatedAt: new Date(this.#now()).toISOString(),
|
|
335
|
+
retentionDays: this.#retentionDays,
|
|
336
|
+
buckets: []
|
|
337
|
+
};
|
|
338
|
+
}
|
|
339
|
+
return {
|
|
340
|
+
version: LEADERBOARD_ROLLUP_VERSION,
|
|
341
|
+
updatedAt: typeof parsed.updatedAt === "string"
|
|
342
|
+
? parsed.updatedAt
|
|
343
|
+
: new Date(this.#now()).toISOString(),
|
|
344
|
+
retentionDays: this.#retentionDays,
|
|
345
|
+
buckets: parsed.buckets
|
|
346
|
+
};
|
|
347
|
+
}
|
|
348
|
+
catch {
|
|
349
|
+
return {
|
|
350
|
+
version: LEADERBOARD_ROLLUP_VERSION,
|
|
351
|
+
updatedAt: new Date(this.#now()).toISOString(),
|
|
352
|
+
retentionDays: this.#retentionDays,
|
|
353
|
+
buckets: []
|
|
354
|
+
};
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
#prune(nowMs) {
|
|
358
|
+
const cutoff = new Date(nowMs - this.#retentionDays * 24 * 60 * 60 * 1_000);
|
|
359
|
+
cutoff.setUTCMinutes(0, 0, 0);
|
|
360
|
+
const cutoffIso = cutoff.toISOString();
|
|
361
|
+
const before = this.#file.buckets.length;
|
|
362
|
+
this.#file.buckets = this.#file.buckets.filter((bucket) => bucket.hour >= cutoffIso);
|
|
363
|
+
if (this.#file.buckets.length !== before)
|
|
364
|
+
this.#dirty = true;
|
|
365
|
+
}
|
|
366
|
+
#scheduleFlush() {
|
|
367
|
+
if (this.#timer !== undefined)
|
|
368
|
+
return;
|
|
369
|
+
this.#timer = setTimeout(() => {
|
|
370
|
+
this.#timer = undefined;
|
|
371
|
+
if (this.#dirty)
|
|
372
|
+
this.#write();
|
|
373
|
+
}, this.#flushDelayMs);
|
|
374
|
+
this.#timer.unref?.();
|
|
375
|
+
}
|
|
376
|
+
#write() {
|
|
377
|
+
mkdirSync(dirname(this.#path), { recursive: true, mode: 0o700 });
|
|
378
|
+
writeFileAtomic(this.#path, `${JSON.stringify(this.#file, null, 2)}\n`, {
|
|
379
|
+
mode: 0o600
|
|
380
|
+
});
|
|
381
|
+
chmodSync(this.#path, 0o600);
|
|
382
|
+
this.#dirty = false;
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
export function buildLeaderboardResult(input) {
|
|
386
|
+
return {
|
|
387
|
+
by: input.by,
|
|
388
|
+
sort: input.sort,
|
|
389
|
+
source: input.source,
|
|
390
|
+
window: {
|
|
391
|
+
start: input.windowStart,
|
|
392
|
+
end: input.windowEnd
|
|
393
|
+
},
|
|
394
|
+
sampleSize: input.sampleSize,
|
|
395
|
+
truncated: input.truncated,
|
|
396
|
+
budget: {
|
|
397
|
+
liveLimit: input.budget.liveLimit,
|
|
398
|
+
liveTtlHours: input.budget.liveTtlHours,
|
|
399
|
+
durable: input.budget.durable,
|
|
400
|
+
durableRetentionDays: input.budget.durableRetentionDays
|
|
401
|
+
},
|
|
402
|
+
rows: input.rows
|
|
403
|
+
};
|
|
404
|
+
}
|
|
@@ -67,7 +67,19 @@ test("call attribution store evicts by capacity and expiry", () => {
|
|
|
67
67
|
store.onModelCall(modelCall("call_3"));
|
|
68
68
|
assert.equal(store.get("call_1"), undefined);
|
|
69
69
|
assert.ok(store.get("call_2"));
|
|
70
|
+
assert.equal(store.truncated(), true);
|
|
71
|
+
assert.equal(store.list().length, 2);
|
|
70
72
|
now = 111;
|
|
71
73
|
assert.equal(store.get("call_2"), undefined);
|
|
72
74
|
assert.ok(store.get("call_3"));
|
|
73
75
|
});
|
|
76
|
+
test("call attribution store applies a tighter live budget on configure", () => {
|
|
77
|
+
const store = new CallAttributionStore({ limit: 3, ttlMs: 60_000 });
|
|
78
|
+
store.onModelCall(modelCall("call_1"));
|
|
79
|
+
store.onModelCall(modelCall("call_2"));
|
|
80
|
+
store.onModelCall(modelCall("call_3"));
|
|
81
|
+
store.configureBudget({ limit: 1, ttlMs: 60_000 });
|
|
82
|
+
assert.equal(store.size(), 1);
|
|
83
|
+
assert.equal(store.list()[0]?.callId, "call_3");
|
|
84
|
+
assert.deepEqual(store.budget(), { limit: 1, ttlMs: 60_000 });
|
|
85
|
+
});
|
package/dist/test/daemon.test.js
CHANGED
|
@@ -264,6 +264,19 @@ test("singleton daemon exposes authenticated control and a stable reloadable dat
|
|
|
264
264
|
assert.equal(embeddingInspection.provider, "openai");
|
|
265
265
|
assert.equal(embeddingInspection.billingMode, "api_key");
|
|
266
266
|
await assert.rejects(client.call("calls.inspect", { callId: "model_call_missing" }), (error) => error instanceof ControlError && error.code === "not_found");
|
|
267
|
+
const leaderboard = await client.call("calls.leaderboard", {
|
|
268
|
+
by: "provider",
|
|
269
|
+
sort: "requests",
|
|
270
|
+
limit: 5,
|
|
271
|
+
window: "live"
|
|
272
|
+
});
|
|
273
|
+
assert.equal(leaderboard.by, "provider");
|
|
274
|
+
assert.equal(leaderboard.source, "live");
|
|
275
|
+
assert.ok(leaderboard.sampleSize >= 1);
|
|
276
|
+
assert.ok(leaderboard.rows.some((row) => row.key === "openai"));
|
|
277
|
+
await assert.rejects(client.call("calls.leaderboard", { window: "24h" }), (error) => error instanceof ControlError &&
|
|
278
|
+
error.code === "bad_request" &&
|
|
279
|
+
/durable leaderboard rollups are disabled/.test(error.message));
|
|
267
280
|
await assert.rejects(client.call("config.update", {
|
|
268
281
|
expectedRevision: snapshot.revision,
|
|
269
282
|
document: "providers: {}\n"
|
|
@@ -311,6 +324,155 @@ test("singleton daemon exposes authenticated control and a stable reloadable dat
|
|
|
311
324
|
rmSync(root, { recursive: true, force: true });
|
|
312
325
|
}
|
|
313
326
|
});
|
|
327
|
+
test("daemon account activity persists last selection independently of leaderboard rollups", async () => {
|
|
328
|
+
const root = mkdtempSync(join(tmpdir(), "routekit-daemon-activity-"));
|
|
329
|
+
const stateHome = join(root, "state");
|
|
330
|
+
const configPath = join(root, "router.yaml");
|
|
331
|
+
const accountsDirectory = join(stateHome, "subscriptions", "codex");
|
|
332
|
+
const activityPath = join(stateHome, "usage", "account-activity.v1.json");
|
|
333
|
+
mkdirSync(accountsDirectory, { recursive: true, mode: 0o700 });
|
|
334
|
+
mkdirSync(join(stateHome, "usage"), { recursive: true, mode: 0o700 });
|
|
335
|
+
writeFileSync(join(accountsDirectory, "work.json"), `${JSON.stringify(nativeCredential("codex"))}\n`, { mode: 0o600 });
|
|
336
|
+
writeFileSync(activityPath, `${JSON.stringify({
|
|
337
|
+
version: 1,
|
|
338
|
+
sequence: 3,
|
|
339
|
+
accounts: [
|
|
340
|
+
{
|
|
341
|
+
identity: "codex:work",
|
|
342
|
+
lastSelectedAt: 1_700_000_000_000,
|
|
343
|
+
sequence: 3
|
|
344
|
+
}
|
|
345
|
+
]
|
|
346
|
+
})}\n`, { mode: 0o600 });
|
|
347
|
+
writeFileSync(configPath, [
|
|
348
|
+
"providers:",
|
|
349
|
+
" codex: {}",
|
|
350
|
+
"defaultModel: codex/gpt-test-model",
|
|
351
|
+
"leaderboard:",
|
|
352
|
+
" durable: false",
|
|
353
|
+
""
|
|
354
|
+
].join("\n"));
|
|
355
|
+
try {
|
|
356
|
+
await withMockNativeDiscovery("codex", async () => {
|
|
357
|
+
const daemon = await startRouteKitDaemon({
|
|
358
|
+
packageVersion: "1.2.3",
|
|
359
|
+
stateHome,
|
|
360
|
+
configPath,
|
|
361
|
+
port: 0,
|
|
362
|
+
portless: false,
|
|
363
|
+
env: {
|
|
364
|
+
HOME: root,
|
|
365
|
+
ROUTEKIT_HOME: stateHome,
|
|
366
|
+
ROUTEKIT_PORTLESS: "0"
|
|
367
|
+
}
|
|
368
|
+
});
|
|
369
|
+
try {
|
|
370
|
+
const client = new RouteKitControlClient({
|
|
371
|
+
url: daemon.record.url,
|
|
372
|
+
token: daemon.record.controlToken
|
|
373
|
+
});
|
|
374
|
+
const status = await client.call("accounts.status", {});
|
|
375
|
+
const usage = await client.call("accounts.usage", {});
|
|
376
|
+
assert.equal(status.accounts[0]?.lastSelected, true);
|
|
377
|
+
assert.equal(status.accounts[0]?.active, true);
|
|
378
|
+
assert.equal(status.accounts[0]?.serving, false);
|
|
379
|
+
assert.equal(status.accounts[0]?.inFlight, 0);
|
|
380
|
+
assert.equal(status.accounts[0]?.lastSelectedAt, 1_700_000_000_000);
|
|
381
|
+
assert.equal(usage.accountSets[0]?.members[0]?.lastSelected, true);
|
|
382
|
+
assert.equal(usage.accountSets[0]?.members[0]?.active, true);
|
|
383
|
+
assert.equal(usage.accountSets[0]?.members[0]?.lastSelectedAt, 1_700_000_000_000);
|
|
384
|
+
assert.equal(existsSync(join(stateHome, "usage", "leaderboard-rollups.v1.json")), false);
|
|
385
|
+
await client.call("accounts.rename", { kind: "codex", source: "work", target: "personal" }, { idempotencyKey: "activity-rename" });
|
|
386
|
+
const renamedStatus = await client.call("accounts.status", {});
|
|
387
|
+
assert.equal(renamedStatus.accounts[0]?.label, "personal");
|
|
388
|
+
assert.equal(renamedStatus.accounts[0]?.lastSelected, true);
|
|
389
|
+
assert.equal(renamedStatus.accounts[0]?.lastSelectedAt, 1_700_000_000_000);
|
|
390
|
+
const persisted = JSON.parse(readFileSync(activityPath, "utf8"));
|
|
391
|
+
assert.deepEqual(persisted.accounts.map((account) => account.identity), ["codex:personal"]);
|
|
392
|
+
const beforeReload = await client.call("daemon.status", {});
|
|
393
|
+
const reloaded = await client.call("config.update", {
|
|
394
|
+
expectedRevision: (await client.call("config.get", {})).revision,
|
|
395
|
+
document: [
|
|
396
|
+
"providers:",
|
|
397
|
+
" codex:",
|
|
398
|
+
" strategy: sticky",
|
|
399
|
+
"defaultModel: codex/gpt-test-model",
|
|
400
|
+
"leaderboard:",
|
|
401
|
+
" durable: false",
|
|
402
|
+
""
|
|
403
|
+
].join("\n")
|
|
404
|
+
});
|
|
405
|
+
assert.ok(reloaded.revision > beforeReload.configRevision);
|
|
406
|
+
const afterReload = await client.call("accounts.status", {});
|
|
407
|
+
assert.equal(afterReload.accounts[0]?.label, "personal");
|
|
408
|
+
assert.equal(afterReload.accounts[0]?.lastSelected, true);
|
|
409
|
+
assert.equal(afterReload.accounts[0]?.lastSelectedAt, 1_700_000_000_000);
|
|
410
|
+
assert.equal(afterReload.accounts[0]?.serving, false);
|
|
411
|
+
assert.equal(afterReload.accounts[0]?.inFlight, 0);
|
|
412
|
+
await assert.rejects(client.call("calls.leaderboard", { window: "24h" }), (error) => error instanceof ControlError &&
|
|
413
|
+
error.code === "bad_request" &&
|
|
414
|
+
/durable leaderboard rollups are disabled/.test(error.message));
|
|
415
|
+
const removed = await client.call("accounts.remove", { kind: "codex", label: "personal" }, { idempotencyKey: "activity-remove" });
|
|
416
|
+
assert.equal(removed.removed, true);
|
|
417
|
+
const afterRemove = JSON.parse(readFileSync(activityPath, "utf8"));
|
|
418
|
+
assert.deepEqual(afterRemove.accounts, []);
|
|
419
|
+
}
|
|
420
|
+
finally {
|
|
421
|
+
await daemon.close();
|
|
422
|
+
}
|
|
423
|
+
writeFileSync(join(accountsDirectory, "personal.json"), `${JSON.stringify(nativeCredential("codex"))}\n`, { mode: 0o600 });
|
|
424
|
+
writeFileSync(activityPath, `${JSON.stringify({
|
|
425
|
+
version: 1,
|
|
426
|
+
sequence: 4,
|
|
427
|
+
accounts: [
|
|
428
|
+
{
|
|
429
|
+
identity: "codex:personal",
|
|
430
|
+
lastSelectedAt: 1_700_000_000_000,
|
|
431
|
+
sequence: 4
|
|
432
|
+
}
|
|
433
|
+
]
|
|
434
|
+
})}\n`, { mode: 0o600 });
|
|
435
|
+
writeFileSync(configPath, [
|
|
436
|
+
"providers:",
|
|
437
|
+
" codex: {}",
|
|
438
|
+
"defaultModel: codex/gpt-test-model",
|
|
439
|
+
"leaderboard:",
|
|
440
|
+
" durable: false",
|
|
441
|
+
""
|
|
442
|
+
].join("\n"));
|
|
443
|
+
const restarted = await startRouteKitDaemon({
|
|
444
|
+
packageVersion: "1.2.3",
|
|
445
|
+
stateHome,
|
|
446
|
+
configPath,
|
|
447
|
+
port: 0,
|
|
448
|
+
portless: false,
|
|
449
|
+
env: {
|
|
450
|
+
HOME: root,
|
|
451
|
+
ROUTEKIT_HOME: stateHome,
|
|
452
|
+
ROUTEKIT_PORTLESS: "0"
|
|
453
|
+
}
|
|
454
|
+
});
|
|
455
|
+
try {
|
|
456
|
+
const client = new RouteKitControlClient({
|
|
457
|
+
url: restarted.record.url,
|
|
458
|
+
token: restarted.record.controlToken
|
|
459
|
+
});
|
|
460
|
+
const status = await client.call("accounts.status", {});
|
|
461
|
+
assert.equal(status.accounts[0]?.label, "personal");
|
|
462
|
+
assert.equal(status.accounts[0]?.lastSelected, true);
|
|
463
|
+
assert.equal(status.accounts[0]?.lastSelectedAt, 1_700_000_000_000);
|
|
464
|
+
assert.equal(status.accounts[0]?.serving, false);
|
|
465
|
+
assert.equal(status.accounts[0]?.inFlight, 0);
|
|
466
|
+
}
|
|
467
|
+
finally {
|
|
468
|
+
await restarted.close();
|
|
469
|
+
}
|
|
470
|
+
});
|
|
471
|
+
}
|
|
472
|
+
finally {
|
|
473
|
+
rmSync(root, { recursive: true, force: true });
|
|
474
|
+
}
|
|
475
|
+
});
|
|
314
476
|
for (const kind of ["claude-code", "codex"]) {
|
|
315
477
|
test(`native ${kind} account rename preserves routing and usage state`, async () => {
|
|
316
478
|
const root = mkdtempSync(join(tmpdir(), `routekit-daemon-rename-${kind}-`));
|
|
@@ -824,7 +986,10 @@ test("daemon owns the cliproxy sidecar: spawn, restart, account routing, shutdow
|
|
|
824
986
|
credentialValid: true,
|
|
825
987
|
configured: true,
|
|
826
988
|
relayOpen: true,
|
|
827
|
-
|
|
989
|
+
serving: false,
|
|
990
|
+
inFlight: 0,
|
|
991
|
+
lastSelected: false,
|
|
992
|
+
active: false,
|
|
828
993
|
models: []
|
|
829
994
|
}
|
|
830
995
|
]);
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import test from "node:test";
|
|
6
|
+
import { AccountActivityCoordinator } from "@velum-labs/routekit-accounts";
|
|
7
|
+
import { aggregateInspections, LeaderboardRollupStore } from "../leaderboard.js";
|
|
8
|
+
function inspection(input) {
|
|
9
|
+
return {
|
|
10
|
+
callId: input.callId,
|
|
11
|
+
status: input.status ?? "succeeded",
|
|
12
|
+
effectiveModel: input.model ?? "openai/gpt-5.5",
|
|
13
|
+
provider: input.provider ?? "openai",
|
|
14
|
+
billingMode: "api_key",
|
|
15
|
+
...(input.principal !== undefined ? { principal: input.principal } : {}),
|
|
16
|
+
retries: { attempts: 1, total: 0, accountFailovers: 0 },
|
|
17
|
+
usage: {
|
|
18
|
+
prompt_tokens: input.tokens ?? 10,
|
|
19
|
+
completion_tokens: 5,
|
|
20
|
+
total_tokens: (input.tokens ?? 10) + 5
|
|
21
|
+
},
|
|
22
|
+
cost: {
|
|
23
|
+
...(input.cost !== undefined ? { estimateUsd: input.cost } : {}),
|
|
24
|
+
unknownUsage: false,
|
|
25
|
+
unknownCost: input.unknownCost ?? false
|
|
26
|
+
},
|
|
27
|
+
timing: {
|
|
28
|
+
startedAt: input.startedAt ?? "2026-07-27T10:00:00.000Z",
|
|
29
|
+
finishedAt: input.startedAt ?? "2026-07-27T10:00:01.000Z",
|
|
30
|
+
latencyMs: input.latencyMs ?? 1_000
|
|
31
|
+
}
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
test("aggregateInspections ranks principals by cost and keeps unknown cost visible", () => {
|
|
35
|
+
const result = aggregateInspections([
|
|
36
|
+
inspection({
|
|
37
|
+
callId: "a",
|
|
38
|
+
principal: { tokenId: "tok_a", label: "alice" },
|
|
39
|
+
cost: 0.5
|
|
40
|
+
}),
|
|
41
|
+
inspection({
|
|
42
|
+
callId: "b",
|
|
43
|
+
principal: { tokenId: "tok_b", label: "bob" },
|
|
44
|
+
cost: 1.25
|
|
45
|
+
}),
|
|
46
|
+
inspection({
|
|
47
|
+
callId: "c",
|
|
48
|
+
principal: { tokenId: "tok_a", label: "alice" },
|
|
49
|
+
unknownCost: true
|
|
50
|
+
}),
|
|
51
|
+
inspection({
|
|
52
|
+
callId: "d",
|
|
53
|
+
model: "anthropic/claude-sonnet-4-5",
|
|
54
|
+
provider: "anthropic",
|
|
55
|
+
cost: 9
|
|
56
|
+
})
|
|
57
|
+
], { by: "principal", sort: "cost", limit: 10 });
|
|
58
|
+
assert.equal(result.rows.length, 2);
|
|
59
|
+
assert.equal(result.rows[0]?.key, "tok_b");
|
|
60
|
+
assert.equal(result.rows[0]?.estimateUsd, 1.25);
|
|
61
|
+
assert.equal(result.rows[1]?.key, "tok_a");
|
|
62
|
+
assert.equal(result.rows[1]?.estimateUsd, 0.5);
|
|
63
|
+
assert.equal(result.rows[1]?.unknownCostCount, 1);
|
|
64
|
+
assert.equal(result.sampleSize, 4);
|
|
65
|
+
});
|
|
66
|
+
test("aggregateInspections can rank by model and sort by requests", () => {
|
|
67
|
+
const result = aggregateInspections([
|
|
68
|
+
inspection({ callId: "1", model: "openai/a", cost: 1 }),
|
|
69
|
+
inspection({ callId: "2", model: "openai/a", cost: 1 }),
|
|
70
|
+
inspection({ callId: "3", model: "openai/b", cost: 5 })
|
|
71
|
+
], { by: "model", sort: "requests", limit: 1 });
|
|
72
|
+
assert.equal(result.rows.length, 1);
|
|
73
|
+
assert.equal(result.rows[0]?.key, "openai/a");
|
|
74
|
+
assert.equal(result.rows[0]?.requests, 2);
|
|
75
|
+
});
|
|
76
|
+
test("LeaderboardRollupStore persists hourly buckets across reload and prunes old hours", () => {
|
|
77
|
+
const home = mkdtempSync(join(tmpdir(), "routekit-leaderboard-"));
|
|
78
|
+
let now = Date.parse("2026-07-27T12:30:00.000Z");
|
|
79
|
+
try {
|
|
80
|
+
const store = new LeaderboardRollupStore({
|
|
81
|
+
home,
|
|
82
|
+
config: {
|
|
83
|
+
liveLimit: 1000,
|
|
84
|
+
liveTtlHours: 24,
|
|
85
|
+
durable: true,
|
|
86
|
+
durableRetentionDays: 1
|
|
87
|
+
},
|
|
88
|
+
now: () => now,
|
|
89
|
+
flushDelayMs: 0
|
|
90
|
+
});
|
|
91
|
+
store.record(inspection({
|
|
92
|
+
callId: "old",
|
|
93
|
+
principal: { tokenId: "tok_old", label: "old" },
|
|
94
|
+
cost: 1,
|
|
95
|
+
startedAt: "2026-07-25T10:15:00.000Z"
|
|
96
|
+
}));
|
|
97
|
+
store.record(inspection({
|
|
98
|
+
callId: "new",
|
|
99
|
+
principal: { tokenId: "tok_new", label: "new" },
|
|
100
|
+
cost: 2,
|
|
101
|
+
startedAt: "2026-07-27T11:15:00.000Z"
|
|
102
|
+
}));
|
|
103
|
+
store.flush();
|
|
104
|
+
const raw = JSON.parse(readFileSync(store.path(), "utf8"));
|
|
105
|
+
assert.equal(raw.buckets.length, 1);
|
|
106
|
+
assert.equal(raw.buckets[0]?.hour, "2026-07-27T11:00:00.000Z");
|
|
107
|
+
const reloaded = new LeaderboardRollupStore({
|
|
108
|
+
home,
|
|
109
|
+
config: {
|
|
110
|
+
liveLimit: 1000,
|
|
111
|
+
liveTtlHours: 24,
|
|
112
|
+
durable: true,
|
|
113
|
+
durableRetentionDays: 14
|
|
114
|
+
},
|
|
115
|
+
now: () => now,
|
|
116
|
+
flushDelayMs: 0
|
|
117
|
+
});
|
|
118
|
+
const board = reloaded.query({
|
|
119
|
+
by: "principal",
|
|
120
|
+
sort: "cost",
|
|
121
|
+
limit: 10,
|
|
122
|
+
window: "24h"
|
|
123
|
+
});
|
|
124
|
+
assert.equal(board.rows.length, 1);
|
|
125
|
+
assert.equal(board.rows[0]?.key, "tok_new");
|
|
126
|
+
assert.equal(board.rows[0]?.estimateUsd, 2);
|
|
127
|
+
}
|
|
128
|
+
finally {
|
|
129
|
+
rmSync(home, { recursive: true, force: true });
|
|
130
|
+
}
|
|
131
|
+
});
|
|
132
|
+
test("LeaderboardRollupStore ignores records while durable is disabled", () => {
|
|
133
|
+
const home = mkdtempSync(join(tmpdir(), "routekit-leaderboard-off-"));
|
|
134
|
+
try {
|
|
135
|
+
const store = new LeaderboardRollupStore({
|
|
136
|
+
home,
|
|
137
|
+
config: {
|
|
138
|
+
liveLimit: 1000,
|
|
139
|
+
liveTtlHours: 24,
|
|
140
|
+
durable: false,
|
|
141
|
+
durableRetentionDays: 14
|
|
142
|
+
},
|
|
143
|
+
flushDelayMs: 0
|
|
144
|
+
});
|
|
145
|
+
store.record(inspection({
|
|
146
|
+
callId: "x",
|
|
147
|
+
principal: { tokenId: "tok", label: "x" },
|
|
148
|
+
cost: 1
|
|
149
|
+
}));
|
|
150
|
+
store.flush();
|
|
151
|
+
assert.equal(store.query({
|
|
152
|
+
by: "principal",
|
|
153
|
+
sort: "cost",
|
|
154
|
+
limit: 10,
|
|
155
|
+
window: "24h"
|
|
156
|
+
}).rows.length, 0);
|
|
157
|
+
}
|
|
158
|
+
finally {
|
|
159
|
+
rmSync(home, { recursive: true, force: true });
|
|
160
|
+
}
|
|
161
|
+
});
|
|
162
|
+
test("provider leaderboard aggregates completed calls across accounts under one provider", () => {
|
|
163
|
+
// Account identity is not a leaderboard dimension; two accounts under
|
|
164
|
+
// the same provider roll into one provider row from completed calls only.
|
|
165
|
+
const result = aggregateInspections([
|
|
166
|
+
inspection({
|
|
167
|
+
callId: "work-1",
|
|
168
|
+
provider: "codex",
|
|
169
|
+
model: "codex/gpt-5.5",
|
|
170
|
+
cost: 1,
|
|
171
|
+
tokens: 20
|
|
172
|
+
}),
|
|
173
|
+
inspection({
|
|
174
|
+
callId: "personal-1",
|
|
175
|
+
provider: "codex",
|
|
176
|
+
model: "codex/gpt-5.5",
|
|
177
|
+
cost: 2,
|
|
178
|
+
tokens: 30
|
|
179
|
+
}),
|
|
180
|
+
inspection({
|
|
181
|
+
callId: "claude-1",
|
|
182
|
+
provider: "claude-code",
|
|
183
|
+
model: "claude-code/claude-opus",
|
|
184
|
+
cost: 5,
|
|
185
|
+
tokens: 10
|
|
186
|
+
})
|
|
187
|
+
], { by: "provider", sort: "requests", limit: 10 });
|
|
188
|
+
assert.equal(result.rows.length, 2);
|
|
189
|
+
assert.equal(result.rows[0]?.key, "codex");
|
|
190
|
+
assert.equal(result.rows[0]?.requests, 2);
|
|
191
|
+
assert.equal(result.rows[0]?.tokensTotal, 60);
|
|
192
|
+
assert.equal(result.rows[0]?.estimateUsd, 3);
|
|
193
|
+
assert.equal(result.rows[1]?.key, "claude-code");
|
|
194
|
+
assert.equal(result.rows[1]?.requests, 1);
|
|
195
|
+
});
|
|
196
|
+
test("account activity persistence stays independent of leaderboard rollups", () => {
|
|
197
|
+
const home = mkdtempSync(join(tmpdir(), "routekit-leaderboard-activity-"));
|
|
198
|
+
const usageDirectory = join(home, "usage");
|
|
199
|
+
mkdirSync(usageDirectory, { recursive: true, mode: 0o700 });
|
|
200
|
+
const activityPath = join(usageDirectory, "account-activity.v1.json");
|
|
201
|
+
try {
|
|
202
|
+
const rollups = new LeaderboardRollupStore({
|
|
203
|
+
home,
|
|
204
|
+
config: {
|
|
205
|
+
liveLimit: 1000,
|
|
206
|
+
liveTtlHours: 24,
|
|
207
|
+
durable: false,
|
|
208
|
+
durableRetentionDays: 14
|
|
209
|
+
},
|
|
210
|
+
flushDelayMs: 0
|
|
211
|
+
});
|
|
212
|
+
const activity = new AccountActivityCoordinator({
|
|
213
|
+
statePath: activityPath,
|
|
214
|
+
persistDebounceMs: 0,
|
|
215
|
+
now: () => 1_700_000_000_000
|
|
216
|
+
});
|
|
217
|
+
const before = aggregateInspections([
|
|
218
|
+
inspection({
|
|
219
|
+
callId: "a",
|
|
220
|
+
provider: "codex",
|
|
221
|
+
principal: { tokenId: "tok", label: "alice" },
|
|
222
|
+
cost: 1.5
|
|
223
|
+
})
|
|
224
|
+
], { by: "principal", sort: "cost", limit: 10 });
|
|
225
|
+
const release = activity.beginAttempt("codex:work");
|
|
226
|
+
activity.flush();
|
|
227
|
+
release();
|
|
228
|
+
rollups.record(inspection({
|
|
229
|
+
callId: "ignored-while-disabled",
|
|
230
|
+
principal: { tokenId: "tok", label: "alice" },
|
|
231
|
+
cost: 99
|
|
232
|
+
}));
|
|
233
|
+
rollups.flush();
|
|
234
|
+
const after = aggregateInspections([
|
|
235
|
+
inspection({
|
|
236
|
+
callId: "a",
|
|
237
|
+
provider: "codex",
|
|
238
|
+
principal: { tokenId: "tok", label: "alice" },
|
|
239
|
+
cost: 1.5
|
|
240
|
+
})
|
|
241
|
+
], { by: "principal", sort: "cost", limit: 10 });
|
|
242
|
+
assert.deepEqual(after, before);
|
|
243
|
+
assert.equal(existsSync(activityPath), true);
|
|
244
|
+
assert.equal(existsSync(rollups.path()), false);
|
|
245
|
+
assert.equal(activity.snapshot("codex:work").lastSelected, true);
|
|
246
|
+
assert.equal(activity.snapshot("codex:work").inFlight, 0);
|
|
247
|
+
assert.equal(rollups.query({
|
|
248
|
+
by: "principal",
|
|
249
|
+
sort: "cost",
|
|
250
|
+
limit: 10,
|
|
251
|
+
window: "24h"
|
|
252
|
+
}).rows.length, 0);
|
|
253
|
+
activity.close();
|
|
254
|
+
}
|
|
255
|
+
finally {
|
|
256
|
+
rmSync(home, { recursive: true, force: true });
|
|
257
|
+
}
|
|
258
|
+
});
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@velum-labs/routekit-daemon",
|
|
3
3
|
"private": false,
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.15.0",
|
|
5
5
|
"description": "Singleton RouteKit control daemon and stable model gateway.",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
@@ -34,14 +34,14 @@
|
|
|
34
34
|
],
|
|
35
35
|
"dependencies": {
|
|
36
36
|
"yaml": "2.9.0",
|
|
37
|
-
"@velum-labs/routekit-
|
|
38
|
-
"@velum-labs/routekit-
|
|
39
|
-
"@velum-labs/routekit-
|
|
40
|
-
"@velum-labs/routekit-
|
|
41
|
-
"@velum-labs/routekit-
|
|
42
|
-
"@velum-labs/routekit-
|
|
43
|
-
"@velum-labs/routekit-
|
|
44
|
-
"@velum-labs/routekit-
|
|
37
|
+
"@velum-labs/routekit-accounts": "0.15.0",
|
|
38
|
+
"@velum-labs/routekit-config": "0.15.0",
|
|
39
|
+
"@velum-labs/routekit-control": "0.15.0",
|
|
40
|
+
"@velum-labs/routekit-gateway": "0.15.0",
|
|
41
|
+
"@velum-labs/routekit-registry": "0.15.0",
|
|
42
|
+
"@velum-labs/routekit-router": "0.15.0",
|
|
43
|
+
"@velum-labs/routekit-runtime": "0.15.0",
|
|
44
|
+
"@velum-labs/routekit-telemetry-core": "0.15.0"
|
|
45
45
|
},
|
|
46
46
|
"scripts": {
|
|
47
47
|
"build": "tsc -b",
|