omnigateway 0.10.3 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -2
- package/bin/omni.js +440 -301
- package/gateway.js +579 -358
- package/package.json +1 -1
- package/public/assets/{Chip-BeKVtewW.js → Chip-CmL-9rw7.js} +1 -1
- package/public/assets/{CopyValue-NTHhiCuT.js → CopyValue-CTVbU0O_.js} +1 -1
- package/public/assets/{Lamp-C2Q3t5Mn.js → Lamp-DtHKJH6P.js} +1 -1
- package/public/assets/{Rack-sUGRoZUx.js → Rack-_A_3zqxC.js} +1 -1
- package/public/assets/{RequestTable-C2RLARWy.js → RequestTable-Cokk7wKP.js} +1 -1
- package/public/assets/{SummaryDeck-BHXcb_Ji.js → SummaryDeck-C0IbqnLz.js} +1 -1
- package/public/assets/{Toggle-DwKz1aVq.js → Toggle-ChjN-tw1.js} +1 -1
- package/public/assets/{TokenBreakdown-DM-1HvOr.js → TokenBreakdown-BPoXM3JM.js} +1 -1
- package/public/assets/{WindowChart-BsonO5Py.js → WindowChart-BPEsEGFL.js} +1 -1
- package/public/assets/_app.accounts-DKmxz7Tl.js +59 -0
- package/public/assets/{_app.console-BQJMLRv4.js → _app.console-Bkdallia.js} +1 -1
- package/public/assets/{_app.database-BIiajZm6.js → _app.database-DxqN1395.js} +1 -1
- package/public/assets/{_app.index-DagK0Pqp.js → _app.index-BUqor-qH.js} +3 -3
- package/public/assets/{_app.keys-DLAG_bmt.js → _app.keys-DMCPEj1g.js} +1 -1
- package/public/assets/{_app.logs-AyVq3lLb.js → _app.logs-DGQD8ZAq.js} +1 -1
- package/public/assets/{_app.models-CwlkGPrx.js → _app.models-D6SV_GVk.js} +1 -1
- package/public/assets/{_app.plugins._pluginId-BhvEnawY.js → _app.plugins._pluginId-CJ7ObXXe.js} +1 -1
- package/public/assets/{_app.settings-BKA-1YLs.js → _app.settings-BVimpngR.js} +1 -1
- package/public/assets/{_app.usage-C7JEWKZM.js → _app.usage-Y2Nkaw0t.js} +1 -1
- package/public/assets/{chevron-right-S84SGCvk.js → chevron-right-BfnK63GB.js} +1 -1
- package/public/assets/{client-DCvpw0LS.js → client-WgQ5A2PD.js} +1 -1
- package/public/assets/{index-pZbadEun.js → index-BTE8vyBz.js} +3 -3
- package/public/assets/{login-CitmheGz.js → login-CrJK9wRA.js} +1 -1
- package/public/assets/plus-Cv51cK5k.js +1 -0
- package/public/assets/{preload-helper-BC9bKUg1.js → preload-helper-BknjH67x.js} +1 -1
- package/public/assets/{shared-C47BfV_u.js → shared-oTMOLWMT.js} +1 -1
- package/public/assets/{trash-2-DmtElUHr.js → trash-2-CSkf6ME_.js} +1 -1
- package/public/index.html +4 -4
- package/public/assets/_app.accounts-DUYxItf0.js +0 -59
- package/public/assets/plus-E7qLz9CF.js +0 -1
package/gateway.js
CHANGED
|
@@ -24443,13 +24443,13 @@ function prepareArtifact(input2) {
|
|
|
24443
24443
|
error: error61.value
|
|
24444
24444
|
};
|
|
24445
24445
|
const json2 = JSON.stringify(bounded);
|
|
24446
|
-
const size =
|
|
24446
|
+
const size = Buffer.byteLength(json2);
|
|
24447
24447
|
if (size <= MAX_ARTIFACT_BYTES)
|
|
24448
24448
|
return { artifact: bounded, json: json2 };
|
|
24449
24449
|
const marker = omission(size);
|
|
24450
24450
|
const omitted = omitBodies(bounded, marker);
|
|
24451
24451
|
const omittedJson = JSON.stringify(omitted);
|
|
24452
|
-
if (
|
|
24452
|
+
if (Buffer.byteLength(omittedJson) <= MAX_ARTIFACT_BYTES) {
|
|
24453
24453
|
return { artifact: omitted, json: omittedJson };
|
|
24454
24454
|
}
|
|
24455
24455
|
const stripped = { ...omitted, error: marker };
|
|
@@ -24630,6 +24630,7 @@ var FIELD_ORDER = [
|
|
|
24630
24630
|
"sizeBytes",
|
|
24631
24631
|
"host",
|
|
24632
24632
|
"port",
|
|
24633
|
+
"dayOffsetMinutes",
|
|
24633
24634
|
"path",
|
|
24634
24635
|
"supervisor",
|
|
24635
24636
|
"reason"
|
|
@@ -24953,6 +24954,261 @@ function validateRequest(req) {
|
|
|
24953
24954
|
}
|
|
24954
24955
|
return { ...req, messages: cleaned };
|
|
24955
24956
|
}
|
|
24957
|
+
// packages/store/src/sqlite/rollup.ts
|
|
24958
|
+
var HOUR_MS = 3600000;
|
|
24959
|
+
function hourOf(at) {
|
|
24960
|
+
return Math.floor(at / HOUR_MS);
|
|
24961
|
+
}
|
|
24962
|
+
var DAY_MS = 86400000;
|
|
24963
|
+
function hostDayOffsetMinutes() {
|
|
24964
|
+
return -new Date().getTimezoneOffset() || 0;
|
|
24965
|
+
}
|
|
24966
|
+
function startOfDay(at, offsetMinutes) {
|
|
24967
|
+
const offsetMs = offsetMinutes * 60000;
|
|
24968
|
+
return Math.floor((at + offsetMs) / DAY_MS) * DAY_MS - offsetMs;
|
|
24969
|
+
}
|
|
24970
|
+
var UPSERT = `
|
|
24971
|
+
INSERT INTO usage_daily
|
|
24972
|
+
(day, provider, credential_id, requested_model, resolved_model, api_key_id,
|
|
24973
|
+
requests, errors, input_tokens, output_tokens, cache_read_tokens,
|
|
24974
|
+
cache_write_tokens, rtk_saved_tokens, rtk_applied_requests, cost_usd, duration_ms_sum)
|
|
24975
|
+
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
|
24976
|
+
ON CONFLICT (day, provider, credential_id, requested_model, resolved_model, api_key_id)
|
|
24977
|
+
DO UPDATE SET
|
|
24978
|
+
requests = requests + excluded.requests,
|
|
24979
|
+
errors = errors + excluded.errors,
|
|
24980
|
+
input_tokens = input_tokens + excluded.input_tokens,
|
|
24981
|
+
output_tokens = output_tokens + excluded.output_tokens,
|
|
24982
|
+
cache_read_tokens = cache_read_tokens + excluded.cache_read_tokens,
|
|
24983
|
+
cache_write_tokens = cache_write_tokens + excluded.cache_write_tokens,
|
|
24984
|
+
rtk_saved_tokens = rtk_saved_tokens + excluded.rtk_saved_tokens,
|
|
24985
|
+
rtk_applied_requests = rtk_applied_requests + excluded.rtk_applied_requests,
|
|
24986
|
+
cost_usd = cost_usd + excluded.cost_usd,
|
|
24987
|
+
duration_ms_sum = duration_ms_sum + excluded.duration_ms_sum`;
|
|
24988
|
+
function keyOf(log, dayOffsetMinutes) {
|
|
24989
|
+
return [
|
|
24990
|
+
startOfDay(log.at, dayOffsetMinutes),
|
|
24991
|
+
log.resolvedProvider ?? "",
|
|
24992
|
+
log.credentialId ?? "",
|
|
24993
|
+
log.requestedModel,
|
|
24994
|
+
log.resolvedModel ?? "",
|
|
24995
|
+
log.apiKeyId ?? ""
|
|
24996
|
+
];
|
|
24997
|
+
}
|
|
24998
|
+
function countersOf(log) {
|
|
24999
|
+
return {
|
|
25000
|
+
requests: 1,
|
|
25001
|
+
errors: log.status >= 400 ? 1 : 0,
|
|
25002
|
+
inputTokens: log.inputTokens,
|
|
25003
|
+
outputTokens: log.outputTokens,
|
|
25004
|
+
cacheReadTokens: log.cacheReadTokens,
|
|
25005
|
+
cacheWriteTokens: log.cacheWriteTokens,
|
|
25006
|
+
rtkSavedTokens: log.rtkEstimatedTokensSaved,
|
|
25007
|
+
rtkAppliedRequests: log.rtkApplied ? 1 : 0,
|
|
25008
|
+
costUsd: log.costUsd,
|
|
25009
|
+
durationMsSum: log.durationMs
|
|
25010
|
+
};
|
|
25011
|
+
}
|
|
25012
|
+
var BACKFILL_UPSERT = `
|
|
25013
|
+
INSERT INTO usage_daily
|
|
25014
|
+
(day, provider, credential_id, requested_model, resolved_model, api_key_id,
|
|
25015
|
+
requests, errors, input_tokens, output_tokens, cache_read_tokens,
|
|
25016
|
+
cache_write_tokens, cost_usd, duration_ms_sum)
|
|
25017
|
+
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
|
25018
|
+
ON CONFLICT (day, provider, credential_id, requested_model, resolved_model, api_key_id)
|
|
25019
|
+
DO UPDATE SET
|
|
25020
|
+
requests = requests + excluded.requests,
|
|
25021
|
+
errors = errors + excluded.errors,
|
|
25022
|
+
input_tokens = input_tokens + excluded.input_tokens,
|
|
25023
|
+
output_tokens = output_tokens + excluded.output_tokens,
|
|
25024
|
+
cache_read_tokens = cache_read_tokens + excluded.cache_read_tokens,
|
|
25025
|
+
cache_write_tokens = cache_write_tokens + excluded.cache_write_tokens,
|
|
25026
|
+
cost_usd = cost_usd + excluded.cost_usd,
|
|
25027
|
+
duration_ms_sum = duration_ms_sum + excluded.duration_ms_sum`;
|
|
25028
|
+
function upsertPreRtk(db, key, c) {
|
|
25029
|
+
db.run(BACKFILL_UPSERT, [
|
|
25030
|
+
...key,
|
|
25031
|
+
c.requests,
|
|
25032
|
+
c.errors,
|
|
25033
|
+
c.inputTokens,
|
|
25034
|
+
c.outputTokens,
|
|
25035
|
+
c.cacheReadTokens,
|
|
25036
|
+
c.cacheWriteTokens,
|
|
25037
|
+
c.costUsd,
|
|
25038
|
+
c.durationMsSum
|
|
25039
|
+
]);
|
|
25040
|
+
}
|
|
25041
|
+
function upsert(db, key, c) {
|
|
25042
|
+
db.run(UPSERT, [
|
|
25043
|
+
...key,
|
|
25044
|
+
c.requests,
|
|
25045
|
+
c.errors,
|
|
25046
|
+
c.inputTokens,
|
|
25047
|
+
c.outputTokens,
|
|
25048
|
+
c.cacheReadTokens,
|
|
25049
|
+
c.cacheWriteTokens,
|
|
25050
|
+
c.rtkSavedTokens,
|
|
25051
|
+
c.rtkAppliedRequests,
|
|
25052
|
+
c.costUsd,
|
|
25053
|
+
c.durationMsSum
|
|
25054
|
+
]);
|
|
25055
|
+
}
|
|
25056
|
+
function rollupLog(db, log, dayOffsetMinutes) {
|
|
25057
|
+
upsert(db, keyOf(log, dayOffsetMinutes), countersOf(log));
|
|
25058
|
+
}
|
|
25059
|
+
function backfillDaily(db, dayOffsetMinutes) {
|
|
25060
|
+
const groups = new Map;
|
|
25061
|
+
for (const row of db.query(`SELECT at, api_key_id, requested_model, resolved_provider, resolved_model, credential_id,
|
|
25062
|
+
status, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens,
|
|
25063
|
+
duration_ms, cost_usd
|
|
25064
|
+
FROM request_logs`).all()) {
|
|
25065
|
+
const key = [
|
|
25066
|
+
startOfDay(row.at, dayOffsetMinutes),
|
|
25067
|
+
row.resolved_provider ?? "",
|
|
25068
|
+
row.credential_id ?? "",
|
|
25069
|
+
row.requested_model,
|
|
25070
|
+
row.resolved_model ?? "",
|
|
25071
|
+
row.api_key_id ?? ""
|
|
25072
|
+
];
|
|
25073
|
+
const id = key.join("\x00");
|
|
25074
|
+
const seen = groups.get(id);
|
|
25075
|
+
const counters = seen?.counters ?? {
|
|
25076
|
+
requests: 0,
|
|
25077
|
+
errors: 0,
|
|
25078
|
+
inputTokens: 0,
|
|
25079
|
+
outputTokens: 0,
|
|
25080
|
+
cacheReadTokens: 0,
|
|
25081
|
+
cacheWriteTokens: 0,
|
|
25082
|
+
rtkSavedTokens: 0,
|
|
25083
|
+
rtkAppliedRequests: 0,
|
|
25084
|
+
costUsd: 0,
|
|
25085
|
+
durationMsSum: 0
|
|
25086
|
+
};
|
|
25087
|
+
counters.requests += 1;
|
|
25088
|
+
counters.errors += row.status >= 400 ? 1 : 0;
|
|
25089
|
+
counters.inputTokens += row.input_tokens;
|
|
25090
|
+
counters.outputTokens += row.output_tokens;
|
|
25091
|
+
counters.cacheReadTokens += row.cache_read_tokens;
|
|
25092
|
+
counters.cacheWriteTokens += row.cache_write_tokens;
|
|
25093
|
+
counters.costUsd += row.cost_usd;
|
|
25094
|
+
counters.durationMsSum += row.duration_ms;
|
|
25095
|
+
if (seen === undefined)
|
|
25096
|
+
groups.set(id, { key, counters });
|
|
25097
|
+
}
|
|
25098
|
+
for (const group of groups.values())
|
|
25099
|
+
upsertPreRtk(db, group.key, group.counters);
|
|
25100
|
+
return groups.size;
|
|
25101
|
+
}
|
|
25102
|
+
var HOURLY_UPSERT = `
|
|
25103
|
+
INSERT INTO usage_rollup
|
|
25104
|
+
(api_key_id, hour, requests, input_tokens, output_tokens,
|
|
25105
|
+
cache_read_tokens, cache_write_tokens, cost_usd)
|
|
25106
|
+
VALUES (?,?,?,?,?,?,?,?)
|
|
25107
|
+
ON CONFLICT (api_key_id, hour)
|
|
25108
|
+
DO UPDATE SET
|
|
25109
|
+
requests = requests + excluded.requests,
|
|
25110
|
+
input_tokens = input_tokens + excluded.input_tokens,
|
|
25111
|
+
output_tokens = output_tokens + excluded.output_tokens,
|
|
25112
|
+
cache_read_tokens = cache_read_tokens + excluded.cache_read_tokens,
|
|
25113
|
+
cache_write_tokens = cache_write_tokens + excluded.cache_write_tokens,
|
|
25114
|
+
cost_usd = cost_usd + excluded.cost_usd`;
|
|
25115
|
+
function rollupHour(db, log) {
|
|
25116
|
+
const apiKeyId = log.apiKeyId;
|
|
25117
|
+
if (apiKeyId === null)
|
|
25118
|
+
return;
|
|
25119
|
+
db.run(HOURLY_UPSERT, [
|
|
25120
|
+
apiKeyId,
|
|
25121
|
+
hourOf(log.at),
|
|
25122
|
+
1,
|
|
25123
|
+
log.inputTokens,
|
|
25124
|
+
log.outputTokens,
|
|
25125
|
+
log.cacheReadTokens,
|
|
25126
|
+
log.cacheWriteTokens,
|
|
25127
|
+
log.costUsd
|
|
25128
|
+
]);
|
|
25129
|
+
}
|
|
25130
|
+
var REBUILD = `
|
|
25131
|
+
INSERT INTO usage_rollup
|
|
25132
|
+
(api_key_id, hour, requests, input_tokens, output_tokens,
|
|
25133
|
+
cache_read_tokens, cache_write_tokens, cost_usd)
|
|
25134
|
+
SELECT api_key_id,
|
|
25135
|
+
CAST(at / 3600000 AS INTEGER),
|
|
25136
|
+
COUNT(*),
|
|
25137
|
+
COALESCE(SUM(input_tokens), 0),
|
|
25138
|
+
COALESCE(SUM(output_tokens), 0),
|
|
25139
|
+
COALESCE(SUM(cache_read_tokens), 0),
|
|
25140
|
+
COALESCE(SUM(cache_write_tokens), 0),
|
|
25141
|
+
COALESCE(SUM(cost_usd), 0)
|
|
25142
|
+
FROM request_logs
|
|
25143
|
+
WHERE state = 'done' AND api_key_id IS NOT NULL
|
|
25144
|
+
GROUP BY api_key_id, CAST(at / 3600000 AS INTEGER)`;
|
|
25145
|
+
function rebuildRollup(db) {
|
|
25146
|
+
db.run("DELETE FROM usage_rollup");
|
|
25147
|
+
db.run(REBUILD);
|
|
25148
|
+
}
|
|
25149
|
+
function auditRollup(db) {
|
|
25150
|
+
const row = db.query(`WITH truth AS (
|
|
25151
|
+
SELECT api_key_id AS k,
|
|
25152
|
+
CAST(at / 3600000 AS INTEGER) AS h,
|
|
25153
|
+
COUNT(*) AS requests,
|
|
25154
|
+
COALESCE(SUM(input_tokens), 0) AS input_tokens,
|
|
25155
|
+
COALESCE(SUM(output_tokens), 0) AS output_tokens,
|
|
25156
|
+
COALESCE(SUM(cache_read_tokens), 0) AS cache_read_tokens,
|
|
25157
|
+
COALESCE(SUM(cache_write_tokens), 0) AS cache_write_tokens,
|
|
25158
|
+
COALESCE(SUM(cost_usd), 0) AS cost_usd
|
|
25159
|
+
FROM request_logs
|
|
25160
|
+
WHERE state = 'done' AND api_key_id IS NOT NULL
|
|
25161
|
+
GROUP BY k, h
|
|
25162
|
+
)
|
|
25163
|
+
SELECT (SELECT COUNT(*) FROM truth) AS buckets,
|
|
25164
|
+
(SELECT COUNT(*)
|
|
25165
|
+
FROM truth
|
|
25166
|
+
LEFT JOIN usage_rollup r ON r.api_key_id = truth.k AND r.hour = truth.h
|
|
25167
|
+
WHERE r.api_key_id IS NULL
|
|
25168
|
+
OR r.requests <> truth.requests
|
|
25169
|
+
OR r.input_tokens <> truth.input_tokens
|
|
25170
|
+
OR r.output_tokens <> truth.output_tokens
|
|
25171
|
+
OR r.cache_read_tokens <> truth.cache_read_tokens
|
|
25172
|
+
OR r.cache_write_tokens <> truth.cache_write_tokens
|
|
25173
|
+
OR ABS(r.cost_usd - truth.cost_usd) > 1e-9 + ABS(truth.cost_usd) * 1e-9
|
|
25174
|
+
) AS mismatched,
|
|
25175
|
+
(SELECT COUNT(*)
|
|
25176
|
+
FROM usage_rollup r
|
|
25177
|
+
LEFT JOIN truth ON r.api_key_id = truth.k AND r.hour = truth.h
|
|
25178
|
+
WHERE truth.k IS NULL
|
|
25179
|
+
) AS orphans`).get();
|
|
25180
|
+
const buckets = row?.buckets ?? 0;
|
|
25181
|
+
const mismatched = (row?.mismatched ?? 0) + (row?.orphans ?? 0);
|
|
25182
|
+
return { buckets, mismatched, ok: mismatched === 0 };
|
|
25183
|
+
}
|
|
25184
|
+
function backfillRtkUsage(db, dayOffsetMinutes) {
|
|
25185
|
+
const groups = new Map;
|
|
25186
|
+
for (const row of db.query(`SELECT at, api_key_id, requested_model, resolved_provider, resolved_model, credential_id,
|
|
25187
|
+
rtk_applied, rtk_estimated_tokens_saved
|
|
25188
|
+
FROM request_logs
|
|
25189
|
+
WHERE state = 'done'`).all()) {
|
|
25190
|
+
const key = [
|
|
25191
|
+
startOfDay(row.at, dayOffsetMinutes),
|
|
25192
|
+
row.resolved_provider ?? "",
|
|
25193
|
+
row.credential_id ?? "",
|
|
25194
|
+
row.requested_model,
|
|
25195
|
+
row.resolved_model ?? "",
|
|
25196
|
+
row.api_key_id ?? ""
|
|
25197
|
+
];
|
|
25198
|
+
const id = key.join("\x00");
|
|
25199
|
+
const group = groups.get(id) ?? { key, saved: 0, applied: 0 };
|
|
25200
|
+
group.saved += row.rtk_estimated_tokens_saved;
|
|
25201
|
+
group.applied += row.rtk_applied === 1 ? 1 : 0;
|
|
25202
|
+
groups.set(id, group);
|
|
25203
|
+
}
|
|
25204
|
+
for (const { key, saved, applied } of groups.values()) {
|
|
25205
|
+
db.run(`UPDATE usage_daily
|
|
25206
|
+
SET rtk_saved_tokens = ?, rtk_applied_requests = ?
|
|
25207
|
+
WHERE day = ? AND provider = ? AND credential_id = ? AND requested_model = ?
|
|
25208
|
+
AND resolved_model = ? AND api_key_id = ?`, [saved, applied, ...key]);
|
|
25209
|
+
}
|
|
25210
|
+
}
|
|
25211
|
+
|
|
24956
25212
|
// packages/store/src/postgres/db.ts
|
|
24957
25213
|
var {SQL } = globalThis.Bun;
|
|
24958
25214
|
|
|
@@ -25258,11 +25514,74 @@ CREATE TRIGGER api_keys_config_version
|
|
|
25258
25514
|
FOR EACH STATEMENT EXECUTE FUNCTION bump_config_version();
|
|
25259
25515
|
`;
|
|
25260
25516
|
|
|
25517
|
+
// packages/store/src/postgres/migrations/002_health_measurements.sql
|
|
25518
|
+
var _002_health_measurements_default = `-- DEPLOY HAZARD \u2014 read before applying to a fleet.
|
|
25519
|
+
--
|
|
25520
|
+
-- This migration drops two columns that every gateway before it writes on
|
|
25521
|
+
-- the success path (\`UPSERT_HEALTH\` names \`ewma_ttft_ms\` and \`last_used_at\`).
|
|
25522
|
+
-- The first new replica to boot applies it under the migration lock; from that
|
|
25523
|
+
-- moment every OLD replica still running throws on every successful request:
|
|
25524
|
+
-- the upstream call has completed and been billed, the client has received its
|
|
25525
|
+
-- content frames, and then the connection aborts with no terminal frame. The
|
|
25526
|
+
-- \`RollingUpdate\` in \`k8s/deployment.yaml\` does not protect against this \u2014 the
|
|
25527
|
+
-- "surviving" replica is the one that breaks.
|
|
25528
|
+
--
|
|
25529
|
+
-- Required: scale to ONE replica before deploying this release, or accept that
|
|
25530
|
+
-- every old replica fails every successful request until it is replaced.
|
|
25531
|
+
-- Rollback is unsafe: an old image cannot re-add the columns, and running it
|
|
25532
|
+
-- against this schema fails the same way. The only way back is a snapshot
|
|
25533
|
+
-- restore taken before the deploy. Deliberate: kept as a drop, not
|
|
25534
|
+
-- expand/contract, on the operator's decision \u2014 see the write-path spec.
|
|
25535
|
+
--
|
|
25536
|
+
-- \`credential_health\` keeps decisions, not measurements.
|
|
25537
|
+
--
|
|
25538
|
+
-- \`ewma_ttft_ms\` and \`last_used_at\` were written on every successful request,
|
|
25539
|
+
-- and a success that changes nothing else now writes nothing. Both live in the
|
|
25540
|
+
-- gateway's process-local \`loadRegistry\` instead. The console's "last used"
|
|
25541
|
+
-- reads \`request_logs\` through \`usage.lastUsedByCredential\`, served by
|
|
25542
|
+
-- \`idx_request_logs_cred\`.
|
|
25543
|
+
ALTER TABLE credential_health DROP COLUMN ewma_ttft_ms;
|
|
25544
|
+
ALTER TABLE credential_health DROP COLUMN last_used_at;
|
|
25545
|
+
|
|
25546
|
+
-- The statement-level trigger bumped \`config_version\` \u2014 one row, locked until
|
|
25547
|
+
-- commit, by every writer on every replica \u2014 on every health write, and every
|
|
25548
|
+
-- bump makes every replica rebuild its routing snapshot. Only a change to a
|
|
25549
|
+
-- column routing *decides* on needs that; a bare \`consecutive_failures\`
|
|
25550
|
+
-- increment is patched into the held snapshot from the \`healthSaved\` change.
|
|
25551
|
+
--
|
|
25552
|
+
-- The INSERT arm must stay \`FOR EACH ROW\`. The write is an upsert, and a
|
|
25553
|
+
-- statement-level INSERT trigger fires on an upsert whichever path each row
|
|
25554
|
+
-- took, which would bump on every write and make the WHEN clause below moot.
|
|
25555
|
+
-- Row-level \`AFTER INSERT\` fires only for rows actually inserted.
|
|
25556
|
+
DROP TRIGGER credential_health_config_version ON credential_health;
|
|
25557
|
+
|
|
25558
|
+
CREATE TRIGGER credential_health_config_version_upd
|
|
25559
|
+
AFTER UPDATE ON credential_health
|
|
25560
|
+
FOR EACH ROW
|
|
25561
|
+
WHEN (OLD.breaker_state IS DISTINCT FROM NEW.breaker_state
|
|
25562
|
+
OR OLD.rate_limited_until IS DISTINCT FROM NEW.rate_limited_until
|
|
25563
|
+
OR OLD.opened_at IS DISTINCT FROM NEW.opened_at)
|
|
25564
|
+
EXECUTE FUNCTION bump_config_version();
|
|
25565
|
+
|
|
25566
|
+
CREATE TRIGGER credential_health_config_version_ins
|
|
25567
|
+
AFTER INSERT ON credential_health
|
|
25568
|
+
FOR EACH ROW EXECUTE FUNCTION bump_config_version();
|
|
25569
|
+
|
|
25570
|
+
-- DELETE needs no WHEN, so it stays statement-level as in 001: a credential
|
|
25571
|
+
-- with M model rows bumps once, not M times.
|
|
25572
|
+
CREATE TRIGGER credential_health_config_version_del
|
|
25573
|
+
AFTER DELETE ON credential_health
|
|
25574
|
+
FOR EACH STATEMENT EXECUTE FUNCTION bump_config_version();
|
|
25575
|
+
`;
|
|
25576
|
+
|
|
25261
25577
|
// packages/store/src/postgres/db.ts
|
|
25262
25578
|
var num = (v) => Number(v);
|
|
25263
25579
|
var numOrNull = (v) => v === null ? null : Number(v);
|
|
25264
25580
|
var MIGRATION_LOCK = 7140641;
|
|
25265
|
-
var MIGRATIONS = [
|
|
25581
|
+
var MIGRATIONS = [
|
|
25582
|
+
{ id: 1, sql: _001_init_default },
|
|
25583
|
+
{ id: 2, sql: _002_health_measurements_default }
|
|
25584
|
+
];
|
|
25266
25585
|
async function openPg(url2) {
|
|
25267
25586
|
const sql = new SQL({ url: url2 });
|
|
25268
25587
|
const conn = await sql.reserve();
|
|
@@ -25574,31 +25893,25 @@ function toHealth(r) {
|
|
|
25574
25893
|
breakerState: r.breaker_state,
|
|
25575
25894
|
consecutiveFailures: r.consecutive_failures,
|
|
25576
25895
|
openedAt: numOrNull(r.opened_at),
|
|
25577
|
-
rateLimitedUntil: numOrNull(r.rate_limited_until)
|
|
25578
|
-
ewmaTtftMs: r.ewma_ttft_ms,
|
|
25579
|
-
lastUsedAt: numOrNull(r.last_used_at)
|
|
25896
|
+
rateLimitedUntil: numOrNull(r.rate_limited_until)
|
|
25580
25897
|
};
|
|
25581
25898
|
}
|
|
25582
25899
|
var UPSERT_HEALTH = `INSERT INTO credential_health
|
|
25583
25900
|
(credential_id, model, breaker_state, consecutive_failures, opened_at,
|
|
25584
|
-
rate_limited_until
|
|
25585
|
-
VALUES ($1,$2,$3,$4,$5,$6
|
|
25901
|
+
rate_limited_until)
|
|
25902
|
+
VALUES ($1,$2,$3,$4,$5,$6)
|
|
25586
25903
|
ON CONFLICT (credential_id, model) DO UPDATE SET
|
|
25587
25904
|
breaker_state = EXCLUDED.breaker_state,
|
|
25588
25905
|
consecutive_failures = EXCLUDED.consecutive_failures,
|
|
25589
25906
|
opened_at = EXCLUDED.opened_at,
|
|
25590
|
-
rate_limited_until = EXCLUDED.rate_limited_until
|
|
25591
|
-
ewma_ttft_ms = EXCLUDED.ewma_ttft_ms,
|
|
25592
|
-
last_used_at = EXCLUDED.last_used_at`;
|
|
25907
|
+
rate_limited_until = EXCLUDED.rate_limited_until`;
|
|
25593
25908
|
var healthValues = (r) => [
|
|
25594
25909
|
r.credentialId,
|
|
25595
25910
|
r.model,
|
|
25596
25911
|
r.breakerState,
|
|
25597
25912
|
r.consecutiveFailures,
|
|
25598
25913
|
r.openedAt,
|
|
25599
|
-
r.rateLimitedUntil
|
|
25600
|
-
r.ewmaTtftMs,
|
|
25601
|
-
r.lastUsedAt
|
|
25914
|
+
r.rateLimitedUntil
|
|
25602
25915
|
];
|
|
25603
25916
|
function createCredentialRepo(sql, key, emit) {
|
|
25604
25917
|
const open2 = async (v) => v === null ? null : decrypt(key, v);
|
|
@@ -25791,13 +26104,6 @@ function createCredentialRepo(sql, key, emit) {
|
|
|
25791
26104
|
const rows = await sql.unsafe("SELECT * FROM credential_health");
|
|
25792
26105
|
return rows.map(toHealth);
|
|
25793
26106
|
},
|
|
25794
|
-
async saveHealth(rows) {
|
|
25795
|
-
await sql.begin(async (tx) => {
|
|
25796
|
-
for (const r of rows)
|
|
25797
|
-
await tx.unsafe(UPSERT_HEALTH, healthValues(r));
|
|
25798
|
-
});
|
|
25799
|
-
emit({ type: "healthSaved", rows });
|
|
25800
|
-
},
|
|
25801
26107
|
async updateHealth(credentialId, model, apply) {
|
|
25802
26108
|
const written = await sql.begin(async (tx) => {
|
|
25803
26109
|
await tx.unsafe("SELECT pg_advisory_xact_lock(hashtext($1))", [`${credentialId}|${model}`]);
|
|
@@ -26297,229 +26603,6 @@ function isRtkFilterId(value) {
|
|
|
26297
26603
|
return FILTER_IDS.has(value);
|
|
26298
26604
|
}
|
|
26299
26605
|
|
|
26300
|
-
// packages/store/src/sqlite/rollup.ts
|
|
26301
|
-
var HOUR_MS = 3600000;
|
|
26302
|
-
function hourOf(at) {
|
|
26303
|
-
return Math.floor(at / HOUR_MS);
|
|
26304
|
-
}
|
|
26305
|
-
function startOfLocalDay(at) {
|
|
26306
|
-
const day = new Date(at);
|
|
26307
|
-
day.setHours(0, 0, 0, 0);
|
|
26308
|
-
return day.getTime();
|
|
26309
|
-
}
|
|
26310
|
-
var UPSERT = `
|
|
26311
|
-
INSERT INTO usage_daily
|
|
26312
|
-
(day, provider, credential_id, requested_model, resolved_model, api_key_id,
|
|
26313
|
-
requests, errors, input_tokens, output_tokens, cache_read_tokens,
|
|
26314
|
-
cache_write_tokens, rtk_saved_tokens, rtk_applied_requests, cost_usd, duration_ms_sum)
|
|
26315
|
-
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
|
26316
|
-
ON CONFLICT (day, provider, credential_id, requested_model, resolved_model, api_key_id)
|
|
26317
|
-
DO UPDATE SET
|
|
26318
|
-
requests = requests + excluded.requests,
|
|
26319
|
-
errors = errors + excluded.errors,
|
|
26320
|
-
input_tokens = input_tokens + excluded.input_tokens,
|
|
26321
|
-
output_tokens = output_tokens + excluded.output_tokens,
|
|
26322
|
-
cache_read_tokens = cache_read_tokens + excluded.cache_read_tokens,
|
|
26323
|
-
cache_write_tokens = cache_write_tokens + excluded.cache_write_tokens,
|
|
26324
|
-
rtk_saved_tokens = rtk_saved_tokens + excluded.rtk_saved_tokens,
|
|
26325
|
-
rtk_applied_requests = rtk_applied_requests + excluded.rtk_applied_requests,
|
|
26326
|
-
cost_usd = cost_usd + excluded.cost_usd,
|
|
26327
|
-
duration_ms_sum = duration_ms_sum + excluded.duration_ms_sum`;
|
|
26328
|
-
function keyOf(log) {
|
|
26329
|
-
return [
|
|
26330
|
-
startOfLocalDay(log.at),
|
|
26331
|
-
log.resolvedProvider ?? "",
|
|
26332
|
-
log.credentialId ?? "",
|
|
26333
|
-
log.requestedModel,
|
|
26334
|
-
log.resolvedModel ?? "",
|
|
26335
|
-
log.apiKeyId ?? ""
|
|
26336
|
-
];
|
|
26337
|
-
}
|
|
26338
|
-
function countersOf(log) {
|
|
26339
|
-
return {
|
|
26340
|
-
requests: 1,
|
|
26341
|
-
errors: log.status >= 400 ? 1 : 0,
|
|
26342
|
-
inputTokens: log.inputTokens,
|
|
26343
|
-
outputTokens: log.outputTokens,
|
|
26344
|
-
cacheReadTokens: log.cacheReadTokens,
|
|
26345
|
-
cacheWriteTokens: log.cacheWriteTokens,
|
|
26346
|
-
rtkSavedTokens: log.rtkEstimatedTokensSaved,
|
|
26347
|
-
rtkAppliedRequests: log.rtkApplied ? 1 : 0,
|
|
26348
|
-
costUsd: log.costUsd,
|
|
26349
|
-
durationMsSum: log.durationMs
|
|
26350
|
-
};
|
|
26351
|
-
}
|
|
26352
|
-
function upsert(db, key, c) {
|
|
26353
|
-
db.run(UPSERT, [
|
|
26354
|
-
...key,
|
|
26355
|
-
c.requests,
|
|
26356
|
-
c.errors,
|
|
26357
|
-
c.inputTokens,
|
|
26358
|
-
c.outputTokens,
|
|
26359
|
-
c.cacheReadTokens,
|
|
26360
|
-
c.cacheWriteTokens,
|
|
26361
|
-
c.rtkSavedTokens,
|
|
26362
|
-
c.rtkAppliedRequests,
|
|
26363
|
-
c.costUsd,
|
|
26364
|
-
c.durationMsSum
|
|
26365
|
-
]);
|
|
26366
|
-
}
|
|
26367
|
-
function rollupLog(db, log) {
|
|
26368
|
-
upsert(db, keyOf(log), countersOf(log));
|
|
26369
|
-
}
|
|
26370
|
-
function backfillDaily(db) {
|
|
26371
|
-
const groups = new Map;
|
|
26372
|
-
for (const row of db.query(`SELECT at, api_key_id, requested_model, resolved_provider, resolved_model, credential_id,
|
|
26373
|
-
status, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens,
|
|
26374
|
-
duration_ms, cost_usd
|
|
26375
|
-
FROM request_logs`).all()) {
|
|
26376
|
-
const key = [
|
|
26377
|
-
startOfLocalDay(row.at),
|
|
26378
|
-
row.resolved_provider ?? "",
|
|
26379
|
-
row.credential_id ?? "",
|
|
26380
|
-
row.requested_model,
|
|
26381
|
-
row.resolved_model ?? "",
|
|
26382
|
-
row.api_key_id ?? ""
|
|
26383
|
-
];
|
|
26384
|
-
const id = key.join("\x00");
|
|
26385
|
-
const seen = groups.get(id);
|
|
26386
|
-
const counters = seen?.counters ?? {
|
|
26387
|
-
requests: 0,
|
|
26388
|
-
errors: 0,
|
|
26389
|
-
inputTokens: 0,
|
|
26390
|
-
outputTokens: 0,
|
|
26391
|
-
cacheReadTokens: 0,
|
|
26392
|
-
cacheWriteTokens: 0,
|
|
26393
|
-
rtkSavedTokens: 0,
|
|
26394
|
-
rtkAppliedRequests: 0,
|
|
26395
|
-
costUsd: 0,
|
|
26396
|
-
durationMsSum: 0
|
|
26397
|
-
};
|
|
26398
|
-
counters.requests += 1;
|
|
26399
|
-
counters.errors += row.status >= 400 ? 1 : 0;
|
|
26400
|
-
counters.inputTokens += row.input_tokens;
|
|
26401
|
-
counters.outputTokens += row.output_tokens;
|
|
26402
|
-
counters.cacheReadTokens += row.cache_read_tokens;
|
|
26403
|
-
counters.cacheWriteTokens += row.cache_write_tokens;
|
|
26404
|
-
counters.costUsd += row.cost_usd;
|
|
26405
|
-
counters.durationMsSum += row.duration_ms;
|
|
26406
|
-
if (seen === undefined)
|
|
26407
|
-
groups.set(id, { key, counters });
|
|
26408
|
-
}
|
|
26409
|
-
for (const group of groups.values())
|
|
26410
|
-
upsert(db, group.key, group.counters);
|
|
26411
|
-
return groups.size;
|
|
26412
|
-
}
|
|
26413
|
-
var HOURLY_UPSERT = `
|
|
26414
|
-
INSERT INTO usage_rollup
|
|
26415
|
-
(api_key_id, hour, requests, input_tokens, output_tokens,
|
|
26416
|
-
cache_read_tokens, cache_write_tokens, cost_usd)
|
|
26417
|
-
VALUES (?,?,?,?,?,?,?,?)
|
|
26418
|
-
ON CONFLICT (api_key_id, hour)
|
|
26419
|
-
DO UPDATE SET
|
|
26420
|
-
requests = requests + excluded.requests,
|
|
26421
|
-
input_tokens = input_tokens + excluded.input_tokens,
|
|
26422
|
-
output_tokens = output_tokens + excluded.output_tokens,
|
|
26423
|
-
cache_read_tokens = cache_read_tokens + excluded.cache_read_tokens,
|
|
26424
|
-
cache_write_tokens = cache_write_tokens + excluded.cache_write_tokens,
|
|
26425
|
-
cost_usd = cost_usd + excluded.cost_usd`;
|
|
26426
|
-
function rollupHour(db, log) {
|
|
26427
|
-
const apiKeyId = log.apiKeyId;
|
|
26428
|
-
if (apiKeyId === null)
|
|
26429
|
-
return;
|
|
26430
|
-
db.run(HOURLY_UPSERT, [
|
|
26431
|
-
apiKeyId,
|
|
26432
|
-
hourOf(log.at),
|
|
26433
|
-
1,
|
|
26434
|
-
log.inputTokens,
|
|
26435
|
-
log.outputTokens,
|
|
26436
|
-
log.cacheReadTokens,
|
|
26437
|
-
log.cacheWriteTokens,
|
|
26438
|
-
log.costUsd
|
|
26439
|
-
]);
|
|
26440
|
-
}
|
|
26441
|
-
var REBUILD = `
|
|
26442
|
-
INSERT INTO usage_rollup
|
|
26443
|
-
(api_key_id, hour, requests, input_tokens, output_tokens,
|
|
26444
|
-
cache_read_tokens, cache_write_tokens, cost_usd)
|
|
26445
|
-
SELECT api_key_id,
|
|
26446
|
-
CAST(at / 3600000 AS INTEGER),
|
|
26447
|
-
COUNT(*),
|
|
26448
|
-
COALESCE(SUM(input_tokens), 0),
|
|
26449
|
-
COALESCE(SUM(output_tokens), 0),
|
|
26450
|
-
COALESCE(SUM(cache_read_tokens), 0),
|
|
26451
|
-
COALESCE(SUM(cache_write_tokens), 0),
|
|
26452
|
-
COALESCE(SUM(cost_usd), 0)
|
|
26453
|
-
FROM request_logs
|
|
26454
|
-
WHERE state = 'done' AND api_key_id IS NOT NULL
|
|
26455
|
-
GROUP BY api_key_id, CAST(at / 3600000 AS INTEGER)`;
|
|
26456
|
-
function rebuildRollup(db) {
|
|
26457
|
-
db.run("DELETE FROM usage_rollup");
|
|
26458
|
-
db.run(REBUILD);
|
|
26459
|
-
}
|
|
26460
|
-
function auditRollup(db) {
|
|
26461
|
-
const row = db.query(`WITH truth AS (
|
|
26462
|
-
SELECT api_key_id AS k,
|
|
26463
|
-
CAST(at / 3600000 AS INTEGER) AS h,
|
|
26464
|
-
COUNT(*) AS requests,
|
|
26465
|
-
COALESCE(SUM(input_tokens), 0) AS input_tokens,
|
|
26466
|
-
COALESCE(SUM(output_tokens), 0) AS output_tokens,
|
|
26467
|
-
COALESCE(SUM(cache_read_tokens), 0) AS cache_read_tokens,
|
|
26468
|
-
COALESCE(SUM(cache_write_tokens), 0) AS cache_write_tokens,
|
|
26469
|
-
COALESCE(SUM(cost_usd), 0) AS cost_usd
|
|
26470
|
-
FROM request_logs
|
|
26471
|
-
WHERE state = 'done' AND api_key_id IS NOT NULL
|
|
26472
|
-
GROUP BY k, h
|
|
26473
|
-
)
|
|
26474
|
-
SELECT (SELECT COUNT(*) FROM truth) AS buckets,
|
|
26475
|
-
(SELECT COUNT(*)
|
|
26476
|
-
FROM truth
|
|
26477
|
-
LEFT JOIN usage_rollup r ON r.api_key_id = truth.k AND r.hour = truth.h
|
|
26478
|
-
WHERE r.api_key_id IS NULL
|
|
26479
|
-
OR r.requests <> truth.requests
|
|
26480
|
-
OR r.input_tokens <> truth.input_tokens
|
|
26481
|
-
OR r.output_tokens <> truth.output_tokens
|
|
26482
|
-
OR r.cache_read_tokens <> truth.cache_read_tokens
|
|
26483
|
-
OR r.cache_write_tokens <> truth.cache_write_tokens
|
|
26484
|
-
OR ABS(r.cost_usd - truth.cost_usd) > 1e-9 + ABS(truth.cost_usd) * 1e-9
|
|
26485
|
-
) AS mismatched,
|
|
26486
|
-
(SELECT COUNT(*)
|
|
26487
|
-
FROM usage_rollup r
|
|
26488
|
-
LEFT JOIN truth ON r.api_key_id = truth.k AND r.hour = truth.h
|
|
26489
|
-
WHERE truth.k IS NULL
|
|
26490
|
-
) AS orphans`).get();
|
|
26491
|
-
const buckets = row?.buckets ?? 0;
|
|
26492
|
-
const mismatched = (row?.mismatched ?? 0) + (row?.orphans ?? 0);
|
|
26493
|
-
return { buckets, mismatched, ok: mismatched === 0 };
|
|
26494
|
-
}
|
|
26495
|
-
function backfillRtkUsage(db) {
|
|
26496
|
-
const groups = new Map;
|
|
26497
|
-
for (const row of db.query(`SELECT at, api_key_id, requested_model, resolved_provider, resolved_model, credential_id,
|
|
26498
|
-
rtk_applied, rtk_estimated_tokens_saved
|
|
26499
|
-
FROM request_logs
|
|
26500
|
-
WHERE state = 'done'`).all()) {
|
|
26501
|
-
const key = [
|
|
26502
|
-
startOfLocalDay(row.at),
|
|
26503
|
-
row.resolved_provider ?? "",
|
|
26504
|
-
row.credential_id ?? "",
|
|
26505
|
-
row.requested_model,
|
|
26506
|
-
row.resolved_model ?? "",
|
|
26507
|
-
row.api_key_id ?? ""
|
|
26508
|
-
];
|
|
26509
|
-
const id = key.join("\x00");
|
|
26510
|
-
const group = groups.get(id) ?? { key, saved: 0, applied: 0 };
|
|
26511
|
-
group.saved += row.rtk_estimated_tokens_saved;
|
|
26512
|
-
group.applied += row.rtk_applied === 1 ? 1 : 0;
|
|
26513
|
-
groups.set(id, group);
|
|
26514
|
-
}
|
|
26515
|
-
for (const { key, saved, applied } of groups.values()) {
|
|
26516
|
-
db.run(`UPDATE usage_daily
|
|
26517
|
-
SET rtk_saved_tokens = ?, rtk_applied_requests = ?
|
|
26518
|
-
WHERE day = ? AND provider = ? AND credential_id = ? AND requested_model = ?
|
|
26519
|
-
AND resolved_model = ? AND api_key_id = ?`, [saved, applied, ...key]);
|
|
26520
|
-
}
|
|
26521
|
-
}
|
|
26522
|
-
|
|
26523
26606
|
// packages/store/src/postgres/rollup.ts
|
|
26524
26607
|
var DAILY_UPSERT = `
|
|
26525
26608
|
INSERT INTO usage_daily
|
|
@@ -26539,9 +26622,9 @@ var DAILY_UPSERT = `
|
|
|
26539
26622
|
rtk_applied_requests = usage_daily.rtk_applied_requests + EXCLUDED.rtk_applied_requests,
|
|
26540
26623
|
cost_usd = usage_daily.cost_usd + EXCLUDED.cost_usd,
|
|
26541
26624
|
duration_ms_sum = usage_daily.duration_ms_sum + EXCLUDED.duration_ms_sum`;
|
|
26542
|
-
async function rollupLog2(conn, log) {
|
|
26625
|
+
async function rollupLog2(conn, log, dayOffsetMinutes) {
|
|
26543
26626
|
await conn.unsafe(DAILY_UPSERT, [
|
|
26544
|
-
|
|
26627
|
+
startOfDay(log.at, dayOffsetMinutes),
|
|
26545
26628
|
log.resolvedProvider ?? "",
|
|
26546
26629
|
log.credentialId ?? "",
|
|
26547
26630
|
log.requestedModel,
|
|
@@ -26785,7 +26868,7 @@ var sum = (row) => ({
|
|
|
26785
26868
|
tokens: row === undefined ? 0 : num(row.tokens),
|
|
26786
26869
|
costUsd: row === undefined ? 0 : row.cost_usd
|
|
26787
26870
|
});
|
|
26788
|
-
function createUsageRepo(sql, nodeId) {
|
|
26871
|
+
function createUsageRepo(sql, nodeId, dayOffsetMinutes) {
|
|
26789
26872
|
const complete = async (conn, log) => {
|
|
26790
26873
|
const written = (await conn.unsafe(COMPLETE, values(log, "done", nodeId))).count;
|
|
26791
26874
|
if (written === 0)
|
|
@@ -26793,7 +26876,7 @@ function createUsageRepo(sql, nodeId) {
|
|
|
26793
26876
|
const stored = (await conn.unsafe("SELECT * FROM request_logs WHERE id = $1", [log.id]))[0];
|
|
26794
26877
|
if (stored !== undefined) {
|
|
26795
26878
|
const restored = toLog(stored);
|
|
26796
|
-
await rollupLog2(conn, restored);
|
|
26879
|
+
await rollupLog2(conn, restored, dayOffsetMinutes);
|
|
26797
26880
|
await rollupHour2(conn, restored);
|
|
26798
26881
|
}
|
|
26799
26882
|
};
|
|
@@ -26824,6 +26907,10 @@ function createUsageRepo(sql, nodeId) {
|
|
|
26824
26907
|
]) : await sql.unsafe("SELECT * FROM request_logs WHERE api_key_id = $1 ORDER BY at DESC LIMIT $2", [apiKeyId, limit]);
|
|
26825
26908
|
return rows.map(toLog);
|
|
26826
26909
|
},
|
|
26910
|
+
async lastUsedByCredential(credentialId) {
|
|
26911
|
+
const rows = await sql.unsafe("SELECT MAX(at) AS at FROM request_logs WHERE credential_id = $1", [credentialId]);
|
|
26912
|
+
return numOrNull(rows[0]?.at ?? null);
|
|
26913
|
+
},
|
|
26827
26914
|
async scan(cursor, limit) {
|
|
26828
26915
|
const rows = cursor === null ? await sql.unsafe("SELECT * FROM request_logs ORDER BY at, id LIMIT $1", [
|
|
26829
26916
|
limit
|
|
@@ -26886,7 +26973,7 @@ function createUsageRepo(sql, nodeId) {
|
|
|
26886
26973
|
const daily = grain === "daily";
|
|
26887
26974
|
const key = columnFor(grain, q.groupBy);
|
|
26888
26975
|
const split = q.splitBy === undefined ? null : columnFor(grain, q.splitBy);
|
|
26889
|
-
const since = daily ?
|
|
26976
|
+
const since = daily ? startOfDay(q.since, dayOffsetMinutes) : q.since;
|
|
26890
26977
|
const until = q.until ?? Number.MAX_SAFE_INTEGER;
|
|
26891
26978
|
const timeColumn = daily ? "day" : "at";
|
|
26892
26979
|
const scope = q.apiKeyId;
|
|
@@ -26962,6 +27049,7 @@ function redacted(url2) {
|
|
|
26962
27049
|
async function createPostgresStore(opts) {
|
|
26963
27050
|
const logger2 = opts.logger ?? noopLogger;
|
|
26964
27051
|
const nodeId = opts.nodeId ?? crypto.randomUUID();
|
|
27052
|
+
const dayOffsetMinutes = opts.dayOffsetMinutes ?? hostDayOffsetMinutes();
|
|
26965
27053
|
const listeners = new Set;
|
|
26966
27054
|
const emit = (change) => {
|
|
26967
27055
|
for (const listener of listeners) {
|
|
@@ -26993,7 +27081,7 @@ async function createPostgresStore(opts) {
|
|
|
26993
27081
|
credentials: createCredentialRepo(sql, opts.encryptionKey, emit),
|
|
26994
27082
|
config: createConfigRepo(sql, emit),
|
|
26995
27083
|
keys: createKeyRepo(sql, logger2),
|
|
26996
|
-
usage: createUsageRepo(sql, nodeId),
|
|
27084
|
+
usage: createUsageRepo(sql, nodeId, dayOffsetMinutes),
|
|
26997
27085
|
bodies: createBodyRepo(sql, opts.encryptionKey),
|
|
26998
27086
|
maintenance: createMaintenanceRepo(sql, nodeId),
|
|
26999
27087
|
plugins: createPluginRepo(sql),
|
|
@@ -27230,22 +27318,18 @@ function toHealth2(r) {
|
|
|
27230
27318
|
breakerState: r.breaker_state,
|
|
27231
27319
|
consecutiveFailures: r.consecutive_failures,
|
|
27232
27320
|
openedAt: r.opened_at,
|
|
27233
|
-
rateLimitedUntil: r.rate_limited_until
|
|
27234
|
-
ewmaTtftMs: r.ewma_ttft_ms,
|
|
27235
|
-
lastUsedAt: r.last_used_at
|
|
27321
|
+
rateLimitedUntil: r.rate_limited_until
|
|
27236
27322
|
};
|
|
27237
27323
|
}
|
|
27238
27324
|
var UPSERT_HEALTH2 = `INSERT INTO credential_health
|
|
27239
27325
|
(credential_id, model, breaker_state, consecutive_failures, opened_at,
|
|
27240
|
-
rate_limited_until
|
|
27241
|
-
VALUES (
|
|
27326
|
+
rate_limited_until)
|
|
27327
|
+
VALUES (?,?,?,?,?,?)
|
|
27242
27328
|
ON CONFLICT (credential_id, model) DO UPDATE SET
|
|
27243
27329
|
breaker_state = excluded.breaker_state,
|
|
27244
27330
|
consecutive_failures = excluded.consecutive_failures,
|
|
27245
27331
|
opened_at = excluded.opened_at,
|
|
27246
|
-
rate_limited_until = excluded.rate_limited_until
|
|
27247
|
-
ewma_ttft_ms = excluded.ewma_ttft_ms,
|
|
27248
|
-
last_used_at = excluded.last_used_at`;
|
|
27332
|
+
rate_limited_until = excluded.rate_limited_until`;
|
|
27249
27333
|
function createCredentialRepo2(db, key, emit = () => {}) {
|
|
27250
27334
|
const open2 = async (v) => v === null ? null : decrypt(key, v);
|
|
27251
27335
|
const secretsFrom = async (row) => ({
|
|
@@ -27428,22 +27512,13 @@ function createCredentialRepo2(db, key, emit = () => {}) {
|
|
|
27428
27512
|
async listHealth() {
|
|
27429
27513
|
return db.query("SELECT * FROM credential_health").all().map(toHealth2);
|
|
27430
27514
|
},
|
|
27431
|
-
async saveHealth(rows) {
|
|
27432
|
-
const stmt = db.prepare(UPSERT_HEALTH2);
|
|
27433
|
-
db.transaction(() => {
|
|
27434
|
-
for (const r of rows) {
|
|
27435
|
-
stmt.run(r.credentialId, r.model, r.breakerState, r.consecutiveFailures, r.openedAt, r.rateLimitedUntil, r.ewmaTtftMs, r.lastUsedAt);
|
|
27436
|
-
}
|
|
27437
|
-
})();
|
|
27438
|
-
emit({ type: "healthSaved", rows });
|
|
27439
|
-
},
|
|
27440
27515
|
async updateHealth(credentialId, model, apply) {
|
|
27441
27516
|
const current = db.prepare("SELECT * FROM credential_health WHERE credential_id = ? AND model = ?");
|
|
27442
27517
|
const stmt = db.prepare(UPSERT_HEALTH2);
|
|
27443
27518
|
const written = db.transaction(() => {
|
|
27444
27519
|
const row = current.get(credentialId, model);
|
|
27445
27520
|
const next = apply(row === null ? null : toHealth2(row));
|
|
27446
|
-
stmt.run(next.credentialId, next.model, next.breakerState, next.consecutiveFailures, next.openedAt, next.rateLimitedUntil
|
|
27521
|
+
stmt.run(next.credentialId, next.model, next.breakerState, next.consecutiveFailures, next.openedAt, next.rateLimitedUntil);
|
|
27447
27522
|
return next;
|
|
27448
27523
|
})();
|
|
27449
27524
|
emit({ type: "healthSaved", rows: [written] });
|
|
@@ -28006,6 +28081,34 @@ CREATE TABLE nodes (
|
|
|
28006
28081
|
ALTER TABLE credentials ADD COLUMN token_version INTEGER NOT NULL DEFAULT 0;
|
|
28007
28082
|
`;
|
|
28008
28083
|
|
|
28084
|
+
// packages/store/src/sqlite/migrations/013_health_measurements.sql
|
|
28085
|
+
var _013_health_measurements_default = `-- DEPLOY HAZARD \u2014 read before upgrading an install that is still serving.
|
|
28086
|
+
--
|
|
28087
|
+
-- This migration drops two columns that every gateway before it writes on
|
|
28088
|
+
-- the success path (\`UPSERT_HEALTH\` names \`ewma_ttft_ms\` and \`last_used_at\`).
|
|
28089
|
+
-- SQLite is single-process, so the hazard is the rollback, not the rollout: an
|
|
28090
|
+
-- older gateway started against this database fails every successful request
|
|
28091
|
+
-- after the upstream call has completed and been billed \u2014 content frames out,
|
|
28092
|
+
-- then an aborted connection with no terminal frame. Rolling back the binary
|
|
28093
|
+
-- does not roll back the schema. The only way back is a snapshot taken before
|
|
28094
|
+
-- the upgrade. Deliberate: kept as a drop, not expand/contract, on the
|
|
28095
|
+
-- operator's decision \u2014 see the write-path spec.
|
|
28096
|
+
--
|
|
28097
|
+
-- \`credential_health\` keeps decisions, not measurements.
|
|
28098
|
+
--
|
|
28099
|
+
-- \`ewma_ttft_ms\` and \`last_used_at\` were written on every successful request,
|
|
28100
|
+
-- and a success that changes nothing else now writes nothing. Both live in the
|
|
28101
|
+
-- gateway's process-local \`loadRegistry\` instead: latency is a property of
|
|
28102
|
+
-- *this* process's path to the provider, and the round-robin tiebreak reads
|
|
28103
|
+
-- the value the process holds. The console's "last used" reads
|
|
28104
|
+
-- \`request_logs\` through \`usage.lastUsedByCredential\`, served by
|
|
28105
|
+
-- \`idx_request_logs_cred\`.
|
|
28106
|
+
--
|
|
28107
|
+
-- Neither column is indexed or constrained, so \`DROP COLUMN\` is permitted.
|
|
28108
|
+
ALTER TABLE credential_health DROP COLUMN ewma_ttft_ms;
|
|
28109
|
+
ALTER TABLE credential_health DROP COLUMN last_used_at;
|
|
28110
|
+
`;
|
|
28111
|
+
|
|
28009
28112
|
// packages/store/src/sqlite/db.ts
|
|
28010
28113
|
var MIGRATIONS2 = [
|
|
28011
28114
|
{ id: 1, sql: _001_init_default2 },
|
|
@@ -28019,9 +28122,15 @@ var MIGRATIONS2 = [
|
|
|
28019
28122
|
{ id: 9, sql: _009_key_limits_default },
|
|
28020
28123
|
{ id: 10, sql: _010_usage_rollup_default, after: rebuildRollup },
|
|
28021
28124
|
{ id: 11, sql: _011_plugin_migrations_default },
|
|
28022
|
-
{ id: 12, sql: _012_nodes_default }
|
|
28125
|
+
{ id: 12, sql: _012_nodes_default },
|
|
28126
|
+
{ id: 13, sql: _013_health_measurements_default }
|
|
28023
28127
|
];
|
|
28024
|
-
function
|
|
28128
|
+
function statements(sql) {
|
|
28129
|
+
return sql.split(`
|
|
28130
|
+
`).filter((line) => !line.trimStart().startsWith("--")).join(`
|
|
28131
|
+
`).split(/;[ \t]*(?:\n|$)/).map((s) => s.trim()).filter((s) => s.length > 0);
|
|
28132
|
+
}
|
|
28133
|
+
function openDb(path, dayOffsetMinutes = hostDayOffsetMinutes()) {
|
|
28025
28134
|
const db = new Database(path, { create: true });
|
|
28026
28135
|
db.run("PRAGMA journal_mode = WAL");
|
|
28027
28136
|
db.run("PRAGMA synchronous = NORMAL");
|
|
@@ -28033,8 +28142,9 @@ function openDb(path) {
|
|
|
28033
28142
|
if (done.has(m.id))
|
|
28034
28143
|
continue;
|
|
28035
28144
|
db.transaction(() => {
|
|
28036
|
-
|
|
28037
|
-
|
|
28145
|
+
for (const statement of statements(m.sql))
|
|
28146
|
+
db.run(statement);
|
|
28147
|
+
m.after?.(db, dayOffsetMinutes);
|
|
28038
28148
|
db.run("INSERT INTO migrations (id, applied_at) VALUES (?, ?)", [m.id, Date.now()]);
|
|
28039
28149
|
})();
|
|
28040
28150
|
}
|
|
@@ -28451,14 +28561,14 @@ var COMPLETE2 = `INSERT INTO request_logs ${COLUMNS2} VALUES ${PLACEHOLDERS2}
|
|
|
28451
28561
|
rtk_estimated_tokens_saved = excluded.rtk_estimated_tokens_saved,
|
|
28452
28562
|
rtk_filters = excluded.rtk_filters
|
|
28453
28563
|
WHERE request_logs.state = 'pending'`;
|
|
28454
|
-
function createUsageRepo2(db, nodeId) {
|
|
28564
|
+
function createUsageRepo2(db, nodeId, dayOffsetMinutes = hostDayOffsetMinutes()) {
|
|
28455
28565
|
const complete = db.transaction((log) => {
|
|
28456
28566
|
if (db.run(COMPLETE2, values2(log, "done", nodeId)).changes === 0)
|
|
28457
28567
|
return;
|
|
28458
28568
|
const stored = db.query("SELECT * FROM request_logs WHERE id = ?").get(log.id);
|
|
28459
28569
|
if (stored !== null) {
|
|
28460
28570
|
const restored = toLog2(stored);
|
|
28461
|
-
rollupLog(db, restored);
|
|
28571
|
+
rollupLog(db, restored, dayOffsetMinutes);
|
|
28462
28572
|
rollupHour(db, restored);
|
|
28463
28573
|
}
|
|
28464
28574
|
});
|
|
@@ -28508,6 +28618,10 @@ function createUsageRepo2(db, nodeId) {
|
|
|
28508
28618
|
}
|
|
28509
28619
|
return db.query("SELECT * FROM request_logs WHERE api_key_id = ? ORDER BY at DESC LIMIT ?").all(apiKeyId, limit).map(toLog2);
|
|
28510
28620
|
},
|
|
28621
|
+
async lastUsedByCredential(credentialId) {
|
|
28622
|
+
const row = db.query("SELECT MAX(at) AS at FROM request_logs WHERE credential_id = ?").get(credentialId);
|
|
28623
|
+
return row?.at ?? null;
|
|
28624
|
+
},
|
|
28511
28625
|
async scan(cursor, limit) {
|
|
28512
28626
|
const rows = cursor === null ? db.query("SELECT * FROM request_logs ORDER BY at, id LIMIT ?").all(limit) : db.query(`SELECT * FROM request_logs
|
|
28513
28627
|
WHERE at > ? OR (at = ? AND id > ?)
|
|
@@ -28569,7 +28683,7 @@ function createUsageRepo2(db, nodeId) {
|
|
|
28569
28683
|
const daily = grain === "daily";
|
|
28570
28684
|
const key = columnFor2(grain, q.groupBy);
|
|
28571
28685
|
const split = q.splitBy === undefined ? null : columnFor2(grain, q.splitBy);
|
|
28572
|
-
const since = daily ?
|
|
28686
|
+
const since = daily ? startOfDay(q.since, dayOffsetMinutes) : q.since;
|
|
28573
28687
|
const until = q.until ?? Number.MAX_SAFE_INTEGER;
|
|
28574
28688
|
const timeColumn = daily ? "day" : "at";
|
|
28575
28689
|
const scope = q.apiKeyId;
|
|
@@ -28620,6 +28734,7 @@ function createUsageRepo2(db, nodeId) {
|
|
|
28620
28734
|
async function createStore(opts) {
|
|
28621
28735
|
const logger2 = opts.logger ?? noopLogger;
|
|
28622
28736
|
const nodeId = opts.nodeId ?? crypto.randomUUID();
|
|
28737
|
+
const dayOffsetMinutes = opts.dayOffsetMinutes ?? hostDayOffsetMinutes();
|
|
28623
28738
|
const listeners = new Set;
|
|
28624
28739
|
const emit = (change) => {
|
|
28625
28740
|
for (const listener of listeners) {
|
|
@@ -28629,14 +28744,14 @@ async function createStore(opts) {
|
|
|
28629
28744
|
}
|
|
28630
28745
|
};
|
|
28631
28746
|
const open2 = () => {
|
|
28632
|
-
const db = openDb(opts.path);
|
|
28747
|
+
const db = openDb(opts.path, dayOffsetMinutes);
|
|
28633
28748
|
logger2.debug("store opened", { path: opts.path });
|
|
28634
28749
|
return {
|
|
28635
28750
|
db,
|
|
28636
28751
|
credentials: createCredentialRepo2(db, opts.encryptionKey, emit),
|
|
28637
28752
|
config: createConfigRepo2(db, emit),
|
|
28638
28753
|
keys: createKeyRepo2(db, logger2),
|
|
28639
|
-
usage: createUsageRepo2(db, nodeId),
|
|
28754
|
+
usage: createUsageRepo2(db, nodeId, dayOffsetMinutes),
|
|
28640
28755
|
bodies: createBodyRepo2(db, opts.encryptionKey, bodiesDirFor(opts.path)),
|
|
28641
28756
|
maintenance: createMaintenanceRepo2(db, nodeId),
|
|
28642
28757
|
plugins: createPluginRepo2(db)
|
|
@@ -28662,7 +28777,6 @@ async function createStore(opts) {
|
|
|
28662
28777
|
updateSecrets: (id, secrets, expiresAt, expectedVersion) => handle.credentials.updateSecrets(id, secrets, expiresAt, expectedVersion),
|
|
28663
28778
|
remove: (id) => handle.credentials.remove(id),
|
|
28664
28779
|
listHealth: () => handle.credentials.listHealth(),
|
|
28665
|
-
saveHealth: (rows) => handle.credentials.saveHealth(rows),
|
|
28666
28780
|
updateHealth: (id, model, apply) => handle.credentials.updateHealth(id, model, apply),
|
|
28667
28781
|
listQuota: () => handle.credentials.listQuota(),
|
|
28668
28782
|
saveQuota: (rows) => handle.credentials.saveQuota(rows),
|
|
@@ -28697,6 +28811,7 @@ async function createStore(opts) {
|
|
|
28697
28811
|
append: (log) => handle.usage.append(log),
|
|
28698
28812
|
sweepPending: (now) => handle.usage.sweepPending(now),
|
|
28699
28813
|
recent: (limit, apiKeyId) => handle.usage.recent(limit, apiKeyId),
|
|
28814
|
+
lastUsedByCredential: (credentialId) => handle.usage.lastUsedByCredential(credentialId),
|
|
28700
28815
|
scan: (cursor, limit) => handle.usage.scan(cursor, limit),
|
|
28701
28816
|
aggregate: (q) => handle.usage.aggregate(q),
|
|
28702
28817
|
sumSince: (apiKeyId, sinceMs) => handle.usage.sumSince(apiKeyId, sinceMs),
|
|
@@ -29689,6 +29804,7 @@ function toClientLog(log) {
|
|
|
29689
29804
|
var MIN_KEY_LENGTH = 16;
|
|
29690
29805
|
var TRUTHY = new Set(["1", "true", "yes", "on"]);
|
|
29691
29806
|
var DECIMAL_INTEGER = /^\d+$/;
|
|
29807
|
+
var SIGNED_INTEGER = /^-?\d+$/;
|
|
29692
29808
|
function optionalText(value, fallback) {
|
|
29693
29809
|
return value?.trim() || fallback;
|
|
29694
29810
|
}
|
|
@@ -29719,6 +29835,11 @@ function loadConfig(env) {
|
|
|
29719
29835
|
} else if (databaseUrl !== undefined || redisUrl !== undefined) {
|
|
29720
29836
|
throw new Error("OMNI_DATABASE_URL and OMNI_REDIS_URL are cluster-mode settings; set OMNI_CLUSTER_MODE=true or unset them");
|
|
29721
29837
|
}
|
|
29838
|
+
const rawDayOffset = env.OMNI_DAY_OFFSET_MINUTES?.trim();
|
|
29839
|
+
const parsedDayOffset = rawDayOffset === undefined ? NaN : Number(rawDayOffset);
|
|
29840
|
+
const validDayOffset = rawDayOffset !== undefined && SIGNED_INTEGER.test(rawDayOffset) && Number.isInteger(parsedDayOffset) && parsedDayOffset >= -1440 && parsedDayOffset <= 1440;
|
|
29841
|
+
const dayOffsetMinutes = validDayOffset ? parsedDayOffset : hostDayOffsetMinutes();
|
|
29842
|
+
const dayOffsetFallbackFrom = rawDayOffset !== undefined && !validDayOffset ? rawDayOffset : null;
|
|
29722
29843
|
const bodyLoggingAllowed = TRUTHY.has((env.OMNI_BODY_LOGGING_ALLOWED ?? "").trim().toLowerCase());
|
|
29723
29844
|
const rawLogLevel = env.OMNI_LOG_LEVEL?.trim();
|
|
29724
29845
|
const logLevel = parseLogLevel(rawLogLevel);
|
|
@@ -29749,6 +29870,8 @@ function loadConfig(env) {
|
|
|
29749
29870
|
clusterMode,
|
|
29750
29871
|
databaseUrl: clusterMode ? databaseUrl ?? null : null,
|
|
29751
29872
|
redisUrl: clusterMode ? redisUrl ?? null : null,
|
|
29873
|
+
dayOffsetMinutes,
|
|
29874
|
+
dayOffsetFallbackFrom,
|
|
29752
29875
|
metricsToken,
|
|
29753
29876
|
metricsMaxSeries,
|
|
29754
29877
|
otlpEndpoint,
|
|
@@ -36676,6 +36799,7 @@ function eligible(input2) {
|
|
|
36676
36799
|
drop("expired");
|
|
36677
36800
|
continue;
|
|
36678
36801
|
}
|
|
36802
|
+
let probe = false;
|
|
36679
36803
|
const h = snapshot.health.get(healthKey(credential.id, target.model));
|
|
36680
36804
|
if (h !== undefined) {
|
|
36681
36805
|
if (h.rateLimitedUntil !== null && h.rateLimitedUntil > now) {
|
|
@@ -36688,6 +36812,9 @@ function eligible(input2) {
|
|
|
36688
36812
|
drop("breaker:open");
|
|
36689
36813
|
continue;
|
|
36690
36814
|
}
|
|
36815
|
+
probe = true;
|
|
36816
|
+
} else if (h.breakerState === "halfOpen") {
|
|
36817
|
+
probe = true;
|
|
36691
36818
|
}
|
|
36692
36819
|
}
|
|
36693
36820
|
const spent = (snapshot.quota.get(credential.id) ?? []).find((w) => w.limit !== null && w.used >= w.limit && (w.resetsAt === null || w.resetsAt > now));
|
|
@@ -36695,7 +36822,7 @@ function eligible(input2) {
|
|
|
36695
36822
|
drop(`quota:${spent.windowType}`);
|
|
36696
36823
|
continue;
|
|
36697
36824
|
}
|
|
36698
|
-
pairs.push({ credential, target });
|
|
36825
|
+
pairs.push({ credential, target, probe });
|
|
36699
36826
|
}
|
|
36700
36827
|
if (target.credentialId === undefined && providerSeen && !servedAny) {
|
|
36701
36828
|
excluded.push({
|
|
@@ -36772,7 +36899,7 @@ function healthScore(h) {
|
|
|
36772
36899
|
return h?.breakerState === "open" || h?.breakerState === "halfOpen" ? base * 0.5 : base;
|
|
36773
36900
|
}
|
|
36774
36901
|
function score(pairs, input2) {
|
|
36775
|
-
const { snapshot, now, load } = input2;
|
|
36902
|
+
const { snapshot, now, load, recent } = input2;
|
|
36776
36903
|
const w = snapshot.settings.weights;
|
|
36777
36904
|
const tiers = pairs.map((p) => p.target.tier);
|
|
36778
36905
|
const minTier = Math.min(...tiers);
|
|
@@ -36780,9 +36907,10 @@ function score(pairs, input2) {
|
|
|
36780
36907
|
const tokens2 = estimateTokens(input2.request);
|
|
36781
36908
|
const costs = pairs.map((p) => requestCost(p.target, tokens2));
|
|
36782
36909
|
const bestCost = bestPositive(costs);
|
|
36910
|
+
const ewmaOf = (key) => recent?.get(key)?.ewmaTtftMs ?? null;
|
|
36783
36911
|
const bestLatency = bestPositive(pairs.flatMap((p) => {
|
|
36784
|
-
const
|
|
36785
|
-
return
|
|
36912
|
+
const ewma = ewmaOf(healthKey(p.credential.id, p.target.model));
|
|
36913
|
+
return ewma === null ? [] : [ewma];
|
|
36786
36914
|
}));
|
|
36787
36915
|
return pairs.map((pair, i) => {
|
|
36788
36916
|
const key = healthKey(pair.credential.id, pair.target.model);
|
|
@@ -36794,12 +36922,14 @@ function score(pairs, input2) {
|
|
|
36794
36922
|
const quota = quotaHeadroom(pair.credential, snapshot.quota.get(pair.credential.id) ?? [], now, snapshot.settings.quotaPollIntervalMs);
|
|
36795
36923
|
const ownCost = costs[i];
|
|
36796
36924
|
const cost = ownCost <= 0 || bestCost === null ? UNKNOWN : ratio(ownCost, bestCost);
|
|
36797
|
-
const
|
|
36925
|
+
const ewma = ewmaOf(key);
|
|
36926
|
+
const latency = ewma === null || ewma <= 0 || bestLatency === null ? UNKNOWN : ratio(ewma, bestLatency);
|
|
36798
36927
|
const reasons = { tier, health, quota, cost, latency, load: loadTerm };
|
|
36799
36928
|
const base = tier * w.tier + health * w.health + quota * w.quota + cost * w.cost + latency * w.latency + loadTerm * w.load;
|
|
36800
36929
|
return {
|
|
36801
36930
|
credential: pair.credential,
|
|
36802
36931
|
target: pair.target,
|
|
36932
|
+
probe: pair.probe,
|
|
36803
36933
|
score: base * pair.credential.weight * pair.target.weight,
|
|
36804
36934
|
reasons
|
|
36805
36935
|
};
|
|
@@ -36825,7 +36955,6 @@ var PENALTY = {
|
|
|
36825
36955
|
ALL_CANDIDATES_FAILED: "none",
|
|
36826
36956
|
INTERNAL: "none"
|
|
36827
36957
|
};
|
|
36828
|
-
var EWMA_ALPHA = 0.3;
|
|
36829
36958
|
var DEFAULT_RATE_LIMIT_MS = 60000;
|
|
36830
36959
|
var QUOTA_PARK_MS = 3600000;
|
|
36831
36960
|
var MAX_JITTER_MS = 2000;
|
|
@@ -36836,21 +36965,28 @@ function blankHealth(credentialId, model) {
|
|
|
36836
36965
|
breakerState: "closed",
|
|
36837
36966
|
consecutiveFailures: 0,
|
|
36838
36967
|
openedAt: null,
|
|
36839
|
-
rateLimitedUntil: null
|
|
36840
|
-
ewmaTtftMs: null,
|
|
36841
|
-
lastUsedAt: null
|
|
36968
|
+
rateLimitedUntil: null
|
|
36842
36969
|
};
|
|
36843
36970
|
}
|
|
36844
|
-
|
|
36845
|
-
|
|
36971
|
+
var SUCCESS_RESETS = [
|
|
36972
|
+
"breakerState",
|
|
36973
|
+
"consecutiveFailures",
|
|
36974
|
+
"openedAt",
|
|
36975
|
+
"rateLimitedUntil"
|
|
36976
|
+
];
|
|
36977
|
+
function successWouldChange(current) {
|
|
36978
|
+
if (current === undefined)
|
|
36979
|
+
return false;
|
|
36980
|
+
const blank = blankHealth(current.credentialId, current.model);
|
|
36981
|
+
return SUCCESS_RESETS.some((field) => current[field] !== blank[field]);
|
|
36982
|
+
}
|
|
36983
|
+
function recordSuccess(current) {
|
|
36846
36984
|
return {
|
|
36847
36985
|
...current,
|
|
36848
36986
|
breakerState: "closed",
|
|
36849
36987
|
consecutiveFailures: 0,
|
|
36850
36988
|
openedAt: null,
|
|
36851
|
-
rateLimitedUntil: null
|
|
36852
|
-
ewmaTtftMs: ewma,
|
|
36853
|
-
lastUsedAt: opts.now
|
|
36989
|
+
rateLimitedUntil: null
|
|
36854
36990
|
};
|
|
36855
36991
|
}
|
|
36856
36992
|
function recordFailure(current, opts) {
|
|
@@ -36860,7 +36996,7 @@ function recordFailure(current, opts) {
|
|
|
36860
36996
|
if (penalty === "soft") {
|
|
36861
36997
|
const base = opts.code === "QUOTA_EXHAUSTED" ? QUOTA_PARK_MS : opts.retryAfterMs ?? DEFAULT_RATE_LIMIT_MS;
|
|
36862
36998
|
const until = opts.now + base + Math.round((opts.jitter ?? 0) * MAX_JITTER_MS);
|
|
36863
|
-
return { ...current, rateLimitedUntil: until
|
|
36999
|
+
return { ...current, rateLimitedUntil: until };
|
|
36864
37000
|
}
|
|
36865
37001
|
const failures = current.consecutiveFailures + 1;
|
|
36866
37002
|
const open2 = opts.code === "AUTH" || current.breakerState === "halfOpen" || failures >= opts.settings.breakerThreshold;
|
|
@@ -36868,8 +37004,7 @@ function recordFailure(current, opts) {
|
|
|
36868
37004
|
...current,
|
|
36869
37005
|
consecutiveFailures: failures,
|
|
36870
37006
|
breakerState: open2 ? "open" : "closed",
|
|
36871
|
-
openedAt: open2 ? opts.now : current.openedAt
|
|
36872
|
-
lastUsedAt: opts.now
|
|
37007
|
+
openedAt: open2 ? opts.now : current.openedAt
|
|
36873
37008
|
};
|
|
36874
37009
|
}
|
|
36875
37010
|
// packages/router/src/resolve.ts
|
|
@@ -36950,8 +37085,8 @@ function rank(input2) {
|
|
|
36950
37085
|
break;
|
|
36951
37086
|
case "roundRobin": {
|
|
36952
37087
|
const idle = (c) => {
|
|
36953
|
-
const
|
|
36954
|
-
return
|
|
37088
|
+
const at = input2.recent?.get(healthKey(c.credential.id, c.target.model))?.lastReleasedAt;
|
|
37089
|
+
return at === undefined ? Number.POSITIVE_INFINITY : input2.now - at;
|
|
36955
37090
|
};
|
|
36956
37091
|
const spent = (c) => headroom(c) < QUOTA_FLOOR ? 1 : 0;
|
|
36957
37092
|
scored.sort((a, b) => spent(a) - spent(b) || inflight(a) - inflight(b) || idle(b) - idle(a));
|
|
@@ -37107,16 +37242,23 @@ async function createApiKeyCredential(store, input2, logger2 = noopLogger, provi
|
|
|
37107
37242
|
return summarizeCredential(created);
|
|
37108
37243
|
}
|
|
37109
37244
|
async function credentialHealth(deps) {
|
|
37110
|
-
const [health, quota, settings] = await Promise.all([
|
|
37245
|
+
const [health, quota, settings, credentials] = await Promise.all([
|
|
37111
37246
|
deps.store.credentials.listHealth(),
|
|
37112
37247
|
deps.store.credentials.listQuota(),
|
|
37113
|
-
deps.store.config.getSettings()
|
|
37248
|
+
deps.store.config.getSettings(),
|
|
37249
|
+
deps.store.credentials.list()
|
|
37114
37250
|
]);
|
|
37251
|
+
const lastUsed = {};
|
|
37252
|
+
for (const c of credentials) {
|
|
37253
|
+
const at = await deps.store.usage.lastUsedByCredential(c.id);
|
|
37254
|
+
if (at !== null)
|
|
37255
|
+
lastUsed[c.id] = at;
|
|
37256
|
+
}
|
|
37115
37257
|
const burn = burnEstimates(quota, {
|
|
37116
37258
|
now: deps.now(),
|
|
37117
37259
|
pollIntervalMs: settings.quotaPollIntervalMs
|
|
37118
37260
|
});
|
|
37119
|
-
return { health, quota, burn };
|
|
37261
|
+
return { health, quota, burn, lastUsed };
|
|
37120
37262
|
}
|
|
37121
37263
|
async function patchCredential(deps, id, input2) {
|
|
37122
37264
|
const patch = parseOrThrow(credentialPatchSchema, input2);
|
|
@@ -37228,14 +37370,14 @@ function listSnapshots(deps) {
|
|
|
37228
37370
|
}
|
|
37229
37371
|
return snapshots.sort((a, b) => b.createdAt - a.createdAt);
|
|
37230
37372
|
}
|
|
37231
|
-
var
|
|
37373
|
+
var DAY_MS2 = 86400000;
|
|
37232
37374
|
var SNAPSHOT_HEADROOM = 1.1;
|
|
37233
37375
|
function retentionOf(settings) {
|
|
37234
37376
|
return { keepLatest: settings.snapshotKeepLatest, maxAgeDays: settings.snapshotMaxAgeDays };
|
|
37235
37377
|
}
|
|
37236
37378
|
function applyRetention(deps, policy, now) {
|
|
37237
37379
|
const dir = snapshotsDir(deps);
|
|
37238
|
-
const cutoff = now - policy.maxAgeDays *
|
|
37380
|
+
const cutoff = now - policy.maxAgeDays * DAY_MS2;
|
|
37239
37381
|
const snapshots = listSnapshots(deps);
|
|
37240
37382
|
const undo = snapshots.find((snapshot) => snapshot.reason === "preRestore")?.id;
|
|
37241
37383
|
let removed = 0;
|
|
@@ -38490,12 +38632,12 @@ async function accountQuota(deps) {
|
|
|
38490
38632
|
}
|
|
38491
38633
|
|
|
38492
38634
|
// packages/control/src/quota/history.ts
|
|
38493
|
-
var
|
|
38635
|
+
var DAY_MS3 = 24 * 60 * 60 * 1000;
|
|
38494
38636
|
var HOUR_MS3 = 3600000;
|
|
38495
38637
|
async function retainedSpan(deps, input2, maxSpanMs) {
|
|
38496
38638
|
const now = deps.now();
|
|
38497
38639
|
const settings = await deps.store.config.getSettings();
|
|
38498
|
-
const retained = now - settings.logRetentionDays *
|
|
38640
|
+
const retained = now - settings.logRetentionDays * DAY_MS3;
|
|
38499
38641
|
const oldest = maxSpanMs === undefined ? retained : Math.max(retained, now - maxSpanMs);
|
|
38500
38642
|
return {
|
|
38501
38643
|
since: Math.max(optionalNumber(input2.since, oldest), oldest),
|
|
@@ -53559,9 +53701,11 @@ function withReset(headroom, window2, resetAt) {
|
|
|
53559
53701
|
// apps/gateway/src/dispatch/loadRegistry.ts
|
|
53560
53702
|
var PREFIX6 = "load:";
|
|
53561
53703
|
var SLOT_TTL_MS = 300000;
|
|
53562
|
-
|
|
53704
|
+
var EWMA_ALPHA = 0.3;
|
|
53705
|
+
function createLoadRegistry(coord = memoryCoord(), now = Date.now) {
|
|
53563
53706
|
const local = new Map;
|
|
53564
53707
|
const providers = new Map;
|
|
53708
|
+
const history = new Map;
|
|
53565
53709
|
let remote = new Map;
|
|
53566
53710
|
return {
|
|
53567
53711
|
acquire(credentialId, model, provider) {
|
|
@@ -53571,10 +53715,15 @@ function createLoadRegistry(coord = memoryCoord()) {
|
|
|
53571
53715
|
providers.set(provider, (providers.get(provider) ?? 0) + 1);
|
|
53572
53716
|
coord.gauge.acquire(PREFIX6 + key, SLOT_TTL_MS);
|
|
53573
53717
|
let released = false;
|
|
53574
|
-
return () => {
|
|
53718
|
+
return (ttftMs) => {
|
|
53575
53719
|
if (released)
|
|
53576
53720
|
return;
|
|
53577
53721
|
released = true;
|
|
53722
|
+
const prior = history.get(key)?.ewmaTtftMs ?? null;
|
|
53723
|
+
history.set(key, {
|
|
53724
|
+
lastReleasedAt: now(),
|
|
53725
|
+
ewmaTtftMs: ttftMs === undefined || ttftMs === null ? prior : prior === null ? ttftMs : prior * (1 - EWMA_ALPHA) + ttftMs * EWMA_ALPHA
|
|
53726
|
+
});
|
|
53578
53727
|
const next = (local.get(key) ?? 0) - 1;
|
|
53579
53728
|
if (next > 0)
|
|
53580
53729
|
local.set(key, next);
|
|
@@ -53601,6 +53750,9 @@ function createLoadRegistry(coord = memoryCoord()) {
|
|
|
53601
53750
|
localCounts() {
|
|
53602
53751
|
return new Map(providers);
|
|
53603
53752
|
},
|
|
53753
|
+
recent() {
|
|
53754
|
+
return history;
|
|
53755
|
+
},
|
|
53604
53756
|
async refresh() {
|
|
53605
53757
|
const held = await coord.gauge.snapshot(PREFIX6);
|
|
53606
53758
|
const sample = new Map;
|
|
@@ -57826,6 +57978,7 @@ function priceOf(prices, usage, provider, providers = PROVIDER_DESCRIPTORS) {
|
|
|
57826
57978
|
}
|
|
57827
57979
|
|
|
57828
57980
|
// apps/gateway/src/dispatch/index.ts
|
|
57981
|
+
var PROBE_TTL_MS = 180000;
|
|
57829
57982
|
async function dispatch(request2, deps, signal, requestId) {
|
|
57830
57983
|
const logger2 = deps.logger ?? noopLogger;
|
|
57831
57984
|
const startedAt = deps.now();
|
|
@@ -57919,6 +58072,7 @@ async function dispatch(request2, deps, signal, requestId) {
|
|
|
57919
58072
|
now: startedAt,
|
|
57920
58073
|
rand: deps.rand(),
|
|
57921
58074
|
load: deps.loadRegistry.counts(),
|
|
58075
|
+
recent: deps.loadRegistry.recent(),
|
|
57922
58076
|
providers: deps.providers
|
|
57923
58077
|
});
|
|
57924
58078
|
if (deps.trace !== null && deps.trace !== undefined) {
|
|
@@ -57980,8 +58134,11 @@ async function dispatch(request2, deps, signal, requestId) {
|
|
|
57980
58134
|
async function* run() {
|
|
57981
58135
|
try {
|
|
57982
58136
|
let lastError = null;
|
|
58137
|
+
let attempts = 0;
|
|
57983
58138
|
candidateLoop:
|
|
57984
|
-
for (
|
|
58139
|
+
for (const candidate of candidates) {
|
|
58140
|
+
if (attempts >= maxAttempts)
|
|
58141
|
+
break;
|
|
57985
58142
|
try {
|
|
57986
58143
|
checkCancellation();
|
|
57987
58144
|
} catch (error61) {
|
|
@@ -57990,22 +58147,45 @@ async function dispatch(request2, deps, signal, requestId) {
|
|
|
57990
58147
|
lastError = error61 instanceof GatewayError ? error61 : new GatewayError("TIMEOUT", "request deadline exceeded");
|
|
57991
58148
|
break;
|
|
57992
58149
|
}
|
|
57993
|
-
const candidate = candidates[i];
|
|
57994
58150
|
const adoptable = eagerHeld && eager.credentialId === candidate.credential.id && eager.model === candidate.target.model;
|
|
57995
58151
|
if (adoptable)
|
|
57996
58152
|
eagerHeld = false;
|
|
57997
58153
|
const releaseSlot = adoptable ? eager.release : deps.loadRegistry.acquire(candidate.credential.id, candidate.target.model, candidate.target.provider);
|
|
58154
|
+
const successWrites = successWouldChange(snapshot.health.get(healthKey(candidate.credential.id, candidate.target.model)));
|
|
57998
58155
|
const attemptStartedAt = deps.now();
|
|
57999
58156
|
let attemptCode;
|
|
58000
|
-
|
|
58001
|
-
|
|
58002
|
-
attempt: i + 1,
|
|
58003
|
-
provider: candidate.target.provider,
|
|
58004
|
-
model: candidate.target.model,
|
|
58005
|
-
credential_id: candidate.credential.id
|
|
58006
|
-
});
|
|
58007
|
-
}
|
|
58157
|
+
let releaseProbe;
|
|
58158
|
+
let probing = false;
|
|
58008
58159
|
try {
|
|
58160
|
+
if (candidate.probe) {
|
|
58161
|
+
const key = `probe:${healthKey(candidate.credential.id, candidate.target.model)}`;
|
|
58162
|
+
const held = await deps.coord.gauge.acquire(key, PROBE_TTL_MS);
|
|
58163
|
+
releaseProbe = () => deps.coord.gauge.release(key);
|
|
58164
|
+
if (held > 0) {
|
|
58165
|
+
noteDegradations([`excluded:${candidate.credential.id}:breaker:probing`]);
|
|
58166
|
+
logger2.debug("routing candidate excluded", {
|
|
58167
|
+
requestId,
|
|
58168
|
+
credentialId: candidate.credential.id,
|
|
58169
|
+
reason: "breaker:probing"
|
|
58170
|
+
});
|
|
58171
|
+
continue;
|
|
58172
|
+
}
|
|
58173
|
+
await persistHealth(candidate, (current) => {
|
|
58174
|
+
if (current.breakerState === "closed")
|
|
58175
|
+
return current;
|
|
58176
|
+
probing = true;
|
|
58177
|
+
return { ...current, breakerState: "halfOpen" };
|
|
58178
|
+
});
|
|
58179
|
+
}
|
|
58180
|
+
const i = attempts++;
|
|
58181
|
+
if (deps.trace !== null && deps.trace !== undefined) {
|
|
58182
|
+
deps.trace.activeAttempt = addSpan(deps.trace, "dispatch.attempt", 0, attemptStartedAt - startedAt, attemptStartedAt - startedAt, {
|
|
58183
|
+
attempt: i + 1,
|
|
58184
|
+
provider: candidate.target.provider,
|
|
58185
|
+
model: candidate.target.model,
|
|
58186
|
+
credential_id: candidate.credential.id
|
|
58187
|
+
});
|
|
58188
|
+
}
|
|
58009
58189
|
log.attempts = i + 1;
|
|
58010
58190
|
log.credentialId = candidate.credential.id;
|
|
58011
58191
|
log.resolvedProvider = candidate.target.provider;
|
|
@@ -58028,6 +58208,7 @@ async function dispatch(request2, deps, signal, requestId) {
|
|
|
58028
58208
|
log.cacheWriteTokens = 0;
|
|
58029
58209
|
log.ttftMs = null;
|
|
58030
58210
|
let committed = false;
|
|
58211
|
+
let closedAtCommit = false;
|
|
58031
58212
|
let authRefreshRetried = false;
|
|
58032
58213
|
let retrySecrets;
|
|
58033
58214
|
const attemptNow = deps.now();
|
|
@@ -58082,6 +58263,11 @@ async function dispatch(request2, deps, signal, requestId) {
|
|
|
58082
58263
|
for (const buffered of pending)
|
|
58083
58264
|
yield buffered;
|
|
58084
58265
|
pending.length = 0;
|
|
58266
|
+
if (probing) {
|
|
58267
|
+
await persistHealth(candidate, recordSuccess);
|
|
58268
|
+
probing = false;
|
|
58269
|
+
closedAtCommit = true;
|
|
58270
|
+
}
|
|
58085
58271
|
}
|
|
58086
58272
|
if (event.type === "end") {
|
|
58087
58273
|
terminal = true;
|
|
@@ -58139,11 +58325,8 @@ async function dispatch(request2, deps, signal, requestId) {
|
|
|
58139
58325
|
if (!terminal) {
|
|
58140
58326
|
throw new GatewayError("UPSTREAM", "upstream stream ended without a terminal event");
|
|
58141
58327
|
}
|
|
58142
|
-
|
|
58143
|
-
|
|
58144
|
-
now: deps.now(),
|
|
58145
|
-
ttftMs: log.ttftMs
|
|
58146
|
-
}));
|
|
58328
|
+
if (successWrites && !closedAtCommit)
|
|
58329
|
+
await persistHealth(candidate, recordSuccess);
|
|
58147
58330
|
log.status = 200;
|
|
58148
58331
|
log.errorCode = null;
|
|
58149
58332
|
log.durationMs = deps.now() - startedAt;
|
|
@@ -58196,7 +58379,7 @@ async function dispatch(request2, deps, signal, requestId) {
|
|
|
58196
58379
|
...failure.retryAfterMs === undefined ? {} : { retryAfterMs: failure.retryAfterMs },
|
|
58197
58380
|
jitter: deps.rand()
|
|
58198
58381
|
}));
|
|
58199
|
-
if (!committed && RETRYABLE[failure.code] &&
|
|
58382
|
+
if (!committed && RETRYABLE[failure.code] && attempts < maxAttempts) {
|
|
58200
58383
|
logger2.warn("attempt failed; retrying", {
|
|
58201
58384
|
requestId,
|
|
58202
58385
|
provider: candidate.target.provider,
|
|
@@ -58236,14 +58419,16 @@ async function dispatch(request2, deps, signal, requestId) {
|
|
|
58236
58419
|
}
|
|
58237
58420
|
deps.trace.activeAttempt = null;
|
|
58238
58421
|
}
|
|
58239
|
-
releaseSlot();
|
|
58422
|
+
releaseSlot(log.ttftMs);
|
|
58423
|
+
releaseProbe?.();
|
|
58240
58424
|
}
|
|
58241
58425
|
}
|
|
58242
|
-
const
|
|
58426
|
+
const untried = attempts === 0 && lastError === null;
|
|
58427
|
+
const code = lastError?.code === "TIMEOUT" ? "TIMEOUT" : lastError !== null && !RETRYABLE[lastError.code] ? lastError.code : untried ? "NO_CANDIDATES" : "ALL_CANDIDATES_FAILED";
|
|
58243
58428
|
log.status = HTTP_STATUS[code];
|
|
58244
58429
|
log.errorCode = code;
|
|
58245
58430
|
log.durationMs = deps.now() - startedAt;
|
|
58246
|
-
const message = lastError?.message ?? "all candidates failed";
|
|
58431
|
+
const message = lastError?.message ?? (untried ? `no eligible credential for model "${safeToken(request2.model)}"` : "all candidates failed");
|
|
58247
58432
|
reject(lastError?.provider === undefined ? new GatewayError(code, message) : new GatewayError(code, message, {
|
|
58248
58433
|
provider: lastError.provider,
|
|
58249
58434
|
...lastError.gatewayAuthored ? { gatewayAuthored: true } : {}
|
|
@@ -59706,17 +59891,14 @@ async function handle(deps, rateLimiter, surface, request2) {
|
|
|
59706
59891
|
}
|
|
59707
59892
|
function proxyRoutes(deps) {
|
|
59708
59893
|
const logger2 = deps.logger ?? noopLogger;
|
|
59709
|
-
const
|
|
59710
|
-
|
|
59711
|
-
now: deps.now,
|
|
59712
|
-
logger: logger2,
|
|
59713
|
-
...deps.coord === undefined ? {} : { coord: deps.coord }
|
|
59714
|
-
});
|
|
59894
|
+
const coord = deps.coord ?? memoryCoord();
|
|
59895
|
+
const rateLimiter = deps.rateLimiter ?? new ApiKeyRateLimiter({ store: deps.store, now: deps.now, logger: logger2, coord });
|
|
59715
59896
|
const dispatchDeps = {
|
|
59716
59897
|
...deps,
|
|
59717
59898
|
logger: logger2,
|
|
59899
|
+
coord,
|
|
59718
59900
|
snapshots: deps.snapshots ?? createRoutingSnapshotCache(deps.store, logger2),
|
|
59719
|
-
loadRegistry: deps.loadRegistry ?? createLoadRegistry(
|
|
59901
|
+
loadRegistry: deps.loadRegistry ?? createLoadRegistry(coord),
|
|
59720
59902
|
keepaliveMs: deps.keepaliveMs ?? KEEPALIVE_MS
|
|
59721
59903
|
};
|
|
59722
59904
|
return new Elysia().post("/v1/messages", ({ request: request2, server }) => {
|
|
@@ -60660,7 +60842,7 @@ function createRing(limits) {
|
|
|
60660
60842
|
}
|
|
60661
60843
|
|
|
60662
60844
|
// apps/gateway/src/version.ts
|
|
60663
|
-
var VERSION = "0.
|
|
60845
|
+
var VERSION = "0.11.0";
|
|
60664
60846
|
|
|
60665
60847
|
// apps/gateway/src/app.ts
|
|
60666
60848
|
var ADMIN_SESSION_TTL_MS = 12 * 60 * 60 * 1000;
|
|
@@ -60788,7 +60970,7 @@ function createApp(deps) {
|
|
|
60788
60970
|
return new Response(null, { status: 401 });
|
|
60789
60971
|
}
|
|
60790
60972
|
const coordFallback = deps.coordHealthy?.() === false;
|
|
60791
|
-
return new Response(telemetry.scrape(loadRegistry.localCounts?.() ?? new Map, registry2.stats(), coordFallback), { headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8" } });
|
|
60973
|
+
return new Response(telemetry.scrape(loadRegistry.localCounts?.() ?? new Map, registry2.stats(), coordFallback, deps.coordFaults?.() ?? 0), { headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8" } });
|
|
60792
60974
|
})).use(proxyRoutes({
|
|
60793
60975
|
store: deps.store,
|
|
60794
60976
|
snapshots,
|
|
@@ -60917,6 +61099,7 @@ var CHANNEL2 = `${NS}pubsub`;
|
|
|
60917
61099
|
var LOCK_POLL_MS = 50;
|
|
60918
61100
|
var FAULT_LOG_INTERVAL_MS = 30000;
|
|
60919
61101
|
var CONNECT_WAIT_MS = 2000;
|
|
61102
|
+
var COMMAND_TIMEOUT_MS = 1000;
|
|
60920
61103
|
var RECONNECTED2 = "coord:reconnected";
|
|
60921
61104
|
var WINDOW_CLAIM = `
|
|
60922
61105
|
redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', tonumber(ARGV[1]) - tonumber(ARGV[2]))
|
|
@@ -61030,10 +61213,17 @@ var SCRIPTS = [
|
|
|
61030
61213
|
LEASE_ACQUIRE,
|
|
61031
61214
|
PUBLISH_SEQUENCED
|
|
61032
61215
|
];
|
|
61216
|
+
|
|
61217
|
+
class TimedOut extends Error {
|
|
61218
|
+
constructor() {
|
|
61219
|
+
super("command timed out");
|
|
61220
|
+
}
|
|
61221
|
+
}
|
|
61033
61222
|
function redisCoord(deps) {
|
|
61034
61223
|
const logger2 = deps.logger ?? noopLogger;
|
|
61035
61224
|
const now = deps.now ?? (() => Date.now());
|
|
61036
61225
|
const faultInterval = deps.faultLogIntervalMs ?? FAULT_LOG_INTERVAL_MS;
|
|
61226
|
+
const commandTimeout = deps.commandTimeoutMs ?? COMMAND_TIMEOUT_MS;
|
|
61037
61227
|
const fallback = memoryCoord({ now });
|
|
61038
61228
|
const client = new RedisClient(deps.url, {
|
|
61039
61229
|
connectionTimeout: 2000,
|
|
@@ -61047,17 +61237,15 @@ function redisCoord(deps) {
|
|
|
61047
61237
|
const colon = name.indexOf(":");
|
|
61048
61238
|
return `${NS}gidx:${colon === -1 ? name : name.slice(0, colon + 1)}`;
|
|
61049
61239
|
};
|
|
61240
|
+
let faults = 0;
|
|
61050
61241
|
const fault = (error61) => {
|
|
61051
61242
|
healthy = false;
|
|
61243
|
+
faults++;
|
|
61052
61244
|
const at = now();
|
|
61053
61245
|
if (at - lastFault < faultInterval)
|
|
61054
61246
|
return;
|
|
61055
61247
|
lastFault = at;
|
|
61056
|
-
logger2.warn("coordinator unreachable; serving from memory", {
|
|
61057
|
-
coord: "redis",
|
|
61058
|
-
coordFallback: true,
|
|
61059
|
-
reason: describeError(error61, "unknown")
|
|
61060
|
-
});
|
|
61248
|
+
logger2.warn(error61 instanceof TimedOut ? "coordinator slow; serving from memory" : "coordinator unreachable; serving from memory", { coord: "redis", coordFallback: true, reason: describeError(error61, "unknown") });
|
|
61061
61249
|
};
|
|
61062
61250
|
const shas = new Map;
|
|
61063
61251
|
const loadScripts = async () => {
|
|
@@ -61085,22 +61273,39 @@ function redisCoord(deps) {
|
|
|
61085
61273
|
}, (error61) => fault(error61)),
|
|
61086
61274
|
Bun.sleep(CONNECT_WAIT_MS)
|
|
61087
61275
|
]);
|
|
61276
|
+
let lostDebit = false;
|
|
61088
61277
|
const reseedAfterOutage = async () => {
|
|
61089
61278
|
const keys = await scan(client, `${key("b", "")}*`);
|
|
61090
61279
|
if (keys.length > 0)
|
|
61091
61280
|
await client.send("UNLINK", keys);
|
|
61092
61281
|
};
|
|
61282
|
+
const deadline = async (redis) => {
|
|
61283
|
+
let timer;
|
|
61284
|
+
try {
|
|
61285
|
+
return await Promise.race([
|
|
61286
|
+
redis(),
|
|
61287
|
+
new Promise((_resolve, reject) => {
|
|
61288
|
+
timer = setTimeout(() => reject(new TimedOut), commandTimeout);
|
|
61289
|
+
})
|
|
61290
|
+
]);
|
|
61291
|
+
} finally {
|
|
61292
|
+
clearTimeout(timer);
|
|
61293
|
+
}
|
|
61294
|
+
};
|
|
61093
61295
|
const attempt2 = async (redis, instead) => {
|
|
61094
61296
|
try {
|
|
61095
61297
|
await ready;
|
|
61096
61298
|
if (!client.connected)
|
|
61097
61299
|
throw new Error("not connected");
|
|
61098
|
-
const out = await redis
|
|
61099
|
-
if (!healthy && lastFault !== Number.NEGATIVE_INFINITY) {
|
|
61100
|
-
healthy = true;
|
|
61101
|
-
reseedAfterOutage().catch(fault);
|
|
61102
|
-
}
|
|
61300
|
+
const out = await deadline(redis);
|
|
61103
61301
|
healthy = true;
|
|
61302
|
+
if (lostDebit) {
|
|
61303
|
+
lostDebit = false;
|
|
61304
|
+
reseedAfterOutage().catch((error61) => {
|
|
61305
|
+
lostDebit = true;
|
|
61306
|
+
fault(error61);
|
|
61307
|
+
});
|
|
61308
|
+
}
|
|
61104
61309
|
return out;
|
|
61105
61310
|
} catch (error61) {
|
|
61106
61311
|
fault(error61);
|
|
@@ -61146,6 +61351,7 @@ function redisCoord(deps) {
|
|
|
61146
61351
|
subscriber.connect().catch(fault);
|
|
61147
61352
|
return {
|
|
61148
61353
|
healthy: () => healthy,
|
|
61354
|
+
faults: () => faults,
|
|
61149
61355
|
close() {
|
|
61150
61356
|
subscriber.close();
|
|
61151
61357
|
client.close();
|
|
@@ -61192,7 +61398,10 @@ function redisCoord(deps) {
|
|
|
61192
61398
|
add(name, grainMs, windowMs, at, delta2) {
|
|
61193
61399
|
return attempt2(async () => {
|
|
61194
61400
|
await evalScript(BUCKETS_ADD, [key("b", name)], [at, grainMs, windowMs, delta2.requests, delta2.tokens, delta2.costUsd]);
|
|
61195
|
-
}, () =>
|
|
61401
|
+
}, () => {
|
|
61402
|
+
lostDebit = true;
|
|
61403
|
+
return fallback.buckets.add(name, grainMs, windowMs, at, delta2);
|
|
61404
|
+
});
|
|
61196
61405
|
},
|
|
61197
61406
|
sum(name, grainMs, windowMs, at) {
|
|
61198
61407
|
return attempt2(async () => {
|
|
@@ -61231,14 +61440,15 @@ function redisCoord(deps) {
|
|
|
61231
61440
|
async withLock(name, ttlMs, waitMs, fn) {
|
|
61232
61441
|
const full = key("lock", name);
|
|
61233
61442
|
const token = crypto.randomUUID();
|
|
61234
|
-
const
|
|
61443
|
+
const deadline2 = performance.now() + waitMs;
|
|
61235
61444
|
for (;; ) {
|
|
61236
61445
|
const taken = await attempt2(async () => await client.set(full, token, "PX", String(ttlMs), "NX") === "OK", async () => {
|
|
61446
|
+
evalScript(MUTEX_RELEASE, [full], [token]).catch(() => {});
|
|
61237
61447
|
throw new LockUnavailable(name);
|
|
61238
61448
|
});
|
|
61239
61449
|
if (taken)
|
|
61240
61450
|
break;
|
|
61241
|
-
if (performance.now() >=
|
|
61451
|
+
if (performance.now() >= deadline2)
|
|
61242
61452
|
throw new LockUnavailable(name);
|
|
61243
61453
|
await Bun.sleep(LOCK_POLL_MS);
|
|
61244
61454
|
}
|
|
@@ -61274,17 +61484,17 @@ function redisCoord(deps) {
|
|
|
61274
61484
|
}
|
|
61275
61485
|
},
|
|
61276
61486
|
pubsub: {
|
|
61277
|
-
publish(topic, payload) {
|
|
61487
|
+
async publish(topic, payload) {
|
|
61488
|
+
await listened;
|
|
61278
61489
|
return attempt2(async () => {
|
|
61279
|
-
await listened;
|
|
61280
61490
|
if (!listening)
|
|
61281
61491
|
throw new Error("not listening");
|
|
61282
61492
|
await client.publish(CHANNEL2, JSON.stringify({ topic, payload }));
|
|
61283
61493
|
}, () => fallback.pubsub.publish(topic, payload));
|
|
61284
61494
|
},
|
|
61285
|
-
publishSequenced(topic, payload, seqKey) {
|
|
61495
|
+
async publishSequenced(topic, payload, seqKey) {
|
|
61496
|
+
await listened;
|
|
61286
61497
|
return attempt2(async () => {
|
|
61287
|
-
await listened;
|
|
61288
61498
|
if (!listening)
|
|
61289
61499
|
throw new Error("not listening");
|
|
61290
61500
|
return await evalScript(PUBLISH_SEQUENCED, [key("seq", seqKey)], [CHANNEL2, topic, payload]);
|
|
@@ -61378,13 +61588,13 @@ async function underLease(lease, name, ttlMs, fn) {
|
|
|
61378
61588
|
|
|
61379
61589
|
// apps/gateway/src/maintenance.ts
|
|
61380
61590
|
var SWEEP_INTERVAL_MS2 = 60 * 60 * 1000;
|
|
61381
|
-
var
|
|
61591
|
+
var DAY_MS4 = 24 * 60 * 60 * 1000;
|
|
61382
61592
|
var ROLLUP_RETENTION_DAYS = 400;
|
|
61383
61593
|
async function pruneLogs(store, now) {
|
|
61384
61594
|
const settings = await store.config.getSettings();
|
|
61385
|
-
const rawHorizon = now - settings.logRetentionDays *
|
|
61595
|
+
const rawHorizon = now - settings.logRetentionDays * DAY_MS4;
|
|
61386
61596
|
const raw = await store.usage.prune(rawHorizon);
|
|
61387
|
-
const daily = await store.usage.pruneDaily(now - ROLLUP_RETENTION_DAYS *
|
|
61597
|
+
const daily = await store.usage.pruneDaily(now - ROLLUP_RETENTION_DAYS * DAY_MS4);
|
|
61388
61598
|
const quotaSamples = await store.credentials.pruneQuotaSamples(rawHorizon);
|
|
61389
61599
|
const bodies = await store.bodies.prune(rawHorizon);
|
|
61390
61600
|
const bodiesOverCap = await store.bodies.pruneToCap();
|
|
@@ -62147,6 +62357,10 @@ var DEFINITIONS = {
|
|
|
62147
62357
|
help: "Whether this process is using coordination fallback.",
|
|
62148
62358
|
type: "gauge"
|
|
62149
62359
|
},
|
|
62360
|
+
omni_coord_faults_total: {
|
|
62361
|
+
help: "Coordinator calls this process answered from memory, timeouts included.",
|
|
62362
|
+
type: "counter"
|
|
62363
|
+
},
|
|
62150
62364
|
omni_metrics_series_folded_total: {
|
|
62151
62365
|
help: "Metric series folded into api_key_id other.",
|
|
62152
62366
|
type: "counter"
|
|
@@ -62392,13 +62606,14 @@ function createTelemetry(opts) {
|
|
|
62392
62606
|
if (opts.metricsEnabled)
|
|
62393
62607
|
registry2.add("omni_ratelimit_rejected_total", { dimension, window: window2 ?? "none" });
|
|
62394
62608
|
},
|
|
62395
|
-
scrape(localInflight, streams, coordFallback) {
|
|
62609
|
+
scrape(localInflight, streams, coordFallback, coordFaults) {
|
|
62396
62610
|
for (const [provider, value] of localInflight)
|
|
62397
62611
|
registry2.set("omni_inflight", { provider }, value);
|
|
62398
62612
|
registry2.set("omni_stream_connections", {}, streams.connections);
|
|
62399
62613
|
registry2.set("omni_stream_queued", {}, streams.queued);
|
|
62400
62614
|
registry2.set("omni_stream_dropped_total", {}, streams.dropped);
|
|
62401
62615
|
registry2.set("omni_coord_fallback", {}, coordFallback ? 1 : 0);
|
|
62616
|
+
registry2.set("omni_coord_faults_total", {}, coordFaults);
|
|
62402
62617
|
return renderPrometheus(registry2.snapshot());
|
|
62403
62618
|
},
|
|
62404
62619
|
stop() {
|
|
@@ -62450,7 +62665,12 @@ async function main() {
|
|
|
62450
62665
|
if (config2.logLevelFallbackFrom !== null) {
|
|
62451
62666
|
logger2.info("invalid log level; using info", { reason: config2.logLevelFallbackFrom });
|
|
62452
62667
|
}
|
|
62453
|
-
logger2.info("omnigateway booting",
|
|
62668
|
+
logger2.info(config2.dayOffsetFallbackFrom === null ? "omnigateway booting" : "omnigateway booting; invalid day offset, using host offset", {
|
|
62669
|
+
host: config2.host,
|
|
62670
|
+
port: config2.port,
|
|
62671
|
+
dayOffsetMinutes: config2.dayOffsetMinutes,
|
|
62672
|
+
...config2.dayOffsetFallbackFrom === null ? {} : { reason: config2.dayOffsetFallbackFrom }
|
|
62673
|
+
});
|
|
62454
62674
|
function dashboardDir() {
|
|
62455
62675
|
if (config2.staticDir !== null)
|
|
62456
62676
|
return config2.staticDir;
|
|
@@ -62467,7 +62687,8 @@ async function main() {
|
|
|
62467
62687
|
...config2.clusterMode && config2.databaseUrl !== null ? { url: config2.databaseUrl } : { path: config2.databasePath },
|
|
62468
62688
|
encryptionKey,
|
|
62469
62689
|
logger: logger2,
|
|
62470
|
-
nodeId
|
|
62690
|
+
nodeId,
|
|
62691
|
+
dayOffsetMinutes: config2.dayOffsetMinutes
|
|
62471
62692
|
});
|
|
62472
62693
|
await store.maintenance.heartbeat(now());
|
|
62473
62694
|
const swept = await store.usage.sweepPending(now());
|
|
@@ -62544,7 +62765,7 @@ async function main() {
|
|
|
62544
62765
|
coord,
|
|
62545
62766
|
nodeId,
|
|
62546
62767
|
mode: config2.clusterMode ? "cluster" : "single",
|
|
62547
|
-
...shared === null ? {} : { coordHealthy: shared.healthy },
|
|
62768
|
+
...shared === null ? {} : { coordHealthy: shared.healthy, coordFaults: shared.faults },
|
|
62548
62769
|
baseUrl: config2.baseUrl,
|
|
62549
62770
|
http,
|
|
62550
62771
|
now,
|