auto-model-router 0.32.0 → 0.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.omp-plugin/marketplace.json +2 -2
- package/README.md +39 -0
- package/docs/data-governance.md +42 -1
- package/package.json +1 -1
- package/src/cost/ledger-sql.ts +41 -3
- package/src/router/candidates.ts +9 -0
- package/src/router/types.ts +2 -0
- package/src/server/http.ts +7 -1
- package/src/server/turn.ts +3 -1
- package/src/upstream/anthropic.ts +5 -4
- package/src/upstream/compat.ts +13 -7
- package/src/upstream/ollama.ts +4 -3
- package/src/upstream/openrouter.ts +4 -3
- package/src/upstream/types.ts +7 -0
- package/src/util/schema.ts +148 -19
- package/src/wire/openai/request.ts +29 -0
- package/src/wire/types.ts +15 -0
- package/test/failover.test.ts +41 -0
- package/test/ledger-partitions.test.ts +321 -0
- package/test/upstream-keys.test.ts +240 -0
|
@@ -7,14 +7,14 @@
|
|
|
7
7
|
},
|
|
8
8
|
"metadata": {
|
|
9
9
|
"description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
|
|
10
|
-
"version": "0.
|
|
10
|
+
"version": "0.34.0",
|
|
11
11
|
"pluginRoot": "."
|
|
12
12
|
},
|
|
13
13
|
"plugins": [
|
|
14
14
|
{
|
|
15
15
|
"name": "auto-model-router",
|
|
16
16
|
"description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
|
|
17
|
-
"version": "0.
|
|
17
|
+
"version": "0.34.0",
|
|
18
18
|
"author": {
|
|
19
19
|
"name": "drewappling",
|
|
20
20
|
"email": "drewappling@gmail.com"
|
package/README.md
CHANGED
|
@@ -1192,6 +1192,16 @@ it lands on the other (`cache: keeping warm …`), and a replica that has served
|
|
|
1192
1192
|
nothing refuses with `402 budget_exceeded` once the shared spend is past its
|
|
1193
1193
|
cap — where the same cap against an empty store serves.
|
|
1194
1194
|
|
|
1195
|
+
On Postgres the `ledger` table is partitioned by UTC day
|
|
1196
|
+
(`PARTITION BY RANGE (created_at_ms)`), so `ledger.retentionDays` drops whole
|
|
1197
|
+
days as partitions instead of deleting rows — at the write rates a shared store
|
|
1198
|
+
is for, a bulk `DELETE` competes for I/O with the inserts it is making room
|
|
1199
|
+
for. Partitions are provisioned a few days ahead on every boot and created on
|
|
1200
|
+
demand if a turn arrives for a day nobody provisioned, so a write never fails
|
|
1201
|
+
for a missing partition. A Postgres ledger created before this is left exactly
|
|
1202
|
+
as it is (Postgres cannot convert a populated table in place) and logs how to
|
|
1203
|
+
convert; see [Data governance](docs/data-governance.md).
|
|
1204
|
+
|
|
1195
1205
|
A SQLite deployment is unchanged: the file is still migrated in place through
|
|
1196
1206
|
the nineteen shipped versions, and both halves live in the one file.
|
|
1197
1207
|
|
|
@@ -1426,6 +1436,35 @@ ignored rather than failing the turn. The decision trail records what the
|
|
|
1426
1436
|
policy changed (`policy: …`), and `GET /v1/router/catalog?policy=…` shows what a
|
|
1427
1437
|
policy admits, model by model, without routing a turn.
|
|
1428
1438
|
|
|
1439
|
+
## Per-turn upstream credentials
|
|
1440
|
+
|
|
1441
|
+
The same front door can also send the credentials one turn dispatches with, in
|
|
1442
|
+
an `X-Omp-Upstream-Keys` header carrying JSON that maps an upstream id to a
|
|
1443
|
+
credential:
|
|
1444
|
+
|
|
1445
|
+
```json
|
|
1446
|
+
{ "openrouter": "sk-or-v1-…", "azure-eu": "…" }
|
|
1447
|
+
```
|
|
1448
|
+
|
|
1449
|
+
The ids are the ones the catalog and `/health` use: `openrouter`, `ollama`, or
|
|
1450
|
+
a named entry's `id`. An upstream named here dispatches with that credential
|
|
1451
|
+
for the whole turn — every retry, same-tier failover and tier escalation
|
|
1452
|
+
included — instead of its configured `apiKey`; one not named keeps the
|
|
1453
|
+
configured key. An upstream mapped to `""` has **no** credential this turn and
|
|
1454
|
+
is excluded from selection rather than dispatched keyless, so a turn that
|
|
1455
|
+
carries nothing usable for a provider simply routes elsewhere.
|
|
1456
|
+
|
|
1457
|
+
Nothing is stored: the override lives on the parsed request and is read when a
|
|
1458
|
+
header is built, so the shared configuration is never written to and concurrent
|
|
1459
|
+
turns carrying different callers' keys cannot see each other's. The credential
|
|
1460
|
+
is a secret and is treated as one — it is never logged, never recorded in the
|
|
1461
|
+
ledger, and never repeated in an error or a decision reason. A malformed header
|
|
1462
|
+
is ignored like a malformed `X-Omp-Policy`, leaving the configured keys in
|
|
1463
|
+
force.
|
|
1464
|
+
|
|
1465
|
+
This is what lets one router serve callers who bring their own keys — the team
|
|
1466
|
+
edition's per-user credentials — without a process per credential set.
|
|
1467
|
+
|
|
1429
1468
|
## Data governance
|
|
1430
1469
|
|
|
1431
1470
|
Two things an operator with a compliance obligation needs from a router: that
|
package/docs/data-governance.md
CHANGED
|
@@ -166,7 +166,12 @@ one a library default should make for them. (Before v0.21.0 the default was
|
|
|
166
166
|
because the subquery needs the rows that are about to be deleted.
|
|
167
167
|
2. `ollama_meter_samples` past the cutoff. They only calibrate the ledger's own
|
|
168
168
|
Ollama estimate, so they age out with the rows they calibrate.
|
|
169
|
-
3. `ledger` rows past the cutoff.
|
|
169
|
+
3. `ledger` rows past the cutoff. On Postgres the ledger is partitioned by UTC
|
|
170
|
+
day, so every day the cutoff covers *completely* is dropped as a partition —
|
|
171
|
+
a metadata operation — and only the boundary day the cutoff falls inside is
|
|
172
|
+
deleted row by row. `deleted` still counts ROWS: each partition is counted
|
|
173
|
+
before it is dropped, so the number means turns, not tables. SQLite has no
|
|
174
|
+
declarative partitioning and deletes rows as it always has.
|
|
170
175
|
|
|
171
176
|
Then, when anything was deleted, `PRAGMA incremental_vacuum` hands freed pages
|
|
172
177
|
back to the filesystem and `PRAGMA wal_checkpoint(TRUNCATE)` folds the WAL so
|
|
@@ -182,6 +187,42 @@ behaviour and is fine.
|
|
|
182
187
|
honest answer to "how far back does this ledger go now", which is what the
|
|
183
188
|
question was actually about, and it is reported even when nothing was deleted.
|
|
184
189
|
|
|
190
|
+
### Partitions, on Postgres
|
|
191
|
+
|
|
192
|
+
A ledger row is ~2.6 kB, and a thousand-tenant deployment writes 33-165 GB a
|
|
193
|
+
day at 440-2200 writes a second. Deleting that competes for I/O with the
|
|
194
|
+
inserts it is trying to make room for, and leaves bloat autovacuum has to
|
|
195
|
+
chase. So on Postgres `ledger` is `PARTITION BY RANGE (created_at_ms)`, one
|
|
196
|
+
partition per UTC day, and retention becomes a `DROP TABLE` per day.
|
|
197
|
+
|
|
198
|
+
The bounds are the ms-epoch integers the column already holds, not a derived
|
|
199
|
+
timestamp, so every existing query (`created_at_ms >= ?`) prunes partitions on
|
|
200
|
+
its own: no read — export, spend, the caps, the decision trail — knows the
|
|
201
|
+
table is partitioned. The one visible consequence is the primary key, which a
|
|
202
|
+
partitioned table requires to include the partition key: it is
|
|
203
|
+
`(id, created_at_ms)` there. Ids are per-row UUIDs and a re-recorded entry
|
|
204
|
+
carries the same instant, so the `ON CONFLICT DO NOTHING` guard still collapses
|
|
205
|
+
a duplicate write.
|
|
206
|
+
|
|
207
|
+
`migrateStore` provisions yesterday through three days ahead on every boot. A
|
|
208
|
+
row for a day nobody provisioned is not an error either: the insert is retried
|
|
209
|
+
once after `ensureLedgerPartitions` creates that day (and the next few), so a
|
|
210
|
+
turn is never lost because a partition was late.
|
|
211
|
+
|
|
212
|
+
A Postgres ledger created before this shipped **stays exactly as it is**.
|
|
213
|
+
Postgres cannot convert a populated table to a partitioned one in place, and
|
|
214
|
+
copying a billing table at boot is not a failure mode a ledger can have — there
|
|
215
|
+
is no second copy of it. The router logs one line saying retention will keep
|
|
216
|
+
deleting rows there, with the conversion an operator can run deliberately, with
|
|
217
|
+
the router stopped:
|
|
218
|
+
|
|
219
|
+
```sql
|
|
220
|
+
ALTER TABLE ledger RENAME TO ledger_legacy; -- then start the router:
|
|
221
|
+
-- boot recreates it partitioned
|
|
222
|
+
INSERT INTO ledger SELECT * FROM ledger_legacy; -- verify the counts, then
|
|
223
|
+
DROP TABLE ledger_legacy;
|
|
224
|
+
```
|
|
225
|
+
|
|
185
226
|
### The schedule, and the route
|
|
186
227
|
|
|
187
228
|
`createRetentionRunner` (`src/cost/retention.ts`) owns the once-an-hour floor.
|
package/package.json
CHANGED
package/src/cost/ledger-sql.ts
CHANGED
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
import type { CatalogModel } from "../catalog/types.ts";
|
|
23
23
|
import type { RouterConfig } from "../config/types.ts";
|
|
24
24
|
import { consumePendingEstimate } from "../tokens/estimate.ts";
|
|
25
|
+
import { droppableLedgerPartitions, ensureLedgerPartitions } from "../util/schema.ts";
|
|
25
26
|
import { jsonParam, jsonValue, num, numOrNull, type SqlDb } from "../util/sql.ts";
|
|
26
27
|
import { foldBlendSamples, type BlendSample } from "./blended.ts";
|
|
27
28
|
import { computeCost } from "./forecast.ts";
|
|
@@ -290,7 +291,12 @@ export function createSqlLedger(db: SqlDb, cfg: RouterConfig, deps: LedgerDeps):
|
|
|
290
291
|
deps.findModel(entry.slug) ??
|
|
291
292
|
null;
|
|
292
293
|
const breakdown = model !== null ? computeCost(model, entry.usage) : null;
|
|
293
|
-
|
|
294
|
+
// No conflict target: on Postgres the ledger is partitioned by day, so
|
|
295
|
+
// its unique index has to include `created_at_ms` and `(id)` alone is
|
|
296
|
+
// not an arbiter. A re-recorded entry carries the same instant, which
|
|
297
|
+
// is the case this guard exists for.
|
|
298
|
+
const insert = async (): Promise<void> => {
|
|
299
|
+
await sql`
|
|
294
300
|
INSERT INTO ledger (
|
|
295
301
|
id, created_at_ms, conversation_key, session_id, turn, requested_model, harness_id, omp_session_id,
|
|
296
302
|
slug, served_slug, tier, classification_source, reasons, predicted_usd, reported_usd, usage,
|
|
@@ -309,7 +315,22 @@ export function createSqlLedger(db: SqlDb, cfg: RouterConfig, deps: LedgerDeps):
|
|
|
309
315
|
${entry.promptTokensSaved},
|
|
310
316
|
${entry.scope === undefined || entry.scope === "" ? null : entry.scope}, ${entry.redactions ?? null}
|
|
311
317
|
)
|
|
312
|
-
ON CONFLICT
|
|
318
|
+
ON CONFLICT DO NOTHING`;
|
|
319
|
+
};
|
|
320
|
+
try {
|
|
321
|
+
await insert();
|
|
322
|
+
} catch (err) {
|
|
323
|
+
// A day nobody provisioned: Postgres refuses the row with 23514
|
|
324
|
+
// ("no partition of relation ledger found for row"). The ledger has
|
|
325
|
+
// no CHECK constraints of its own, so on this statement that code
|
|
326
|
+
// can only mean partition routing. Create the day — and the next
|
|
327
|
+
// few, so this happens once rather than daily — and write again. A
|
|
328
|
+
// turn is never lost because a partition was late.
|
|
329
|
+
const code = err !== null && typeof err === "object" && "errno" in err ? String(err.errno) : "";
|
|
330
|
+
if (code !== "23514") throw err;
|
|
331
|
+
await ensureLedgerPartitions(db, entry.createdAtMs);
|
|
332
|
+
await insert();
|
|
333
|
+
}
|
|
313
334
|
// Always consume the pending estimate, even when the turn failed, so a
|
|
314
335
|
// dead turn's bytes can never pair with a later turn's tokens.
|
|
315
336
|
const pending = consumePendingEstimate(entry.conversationKey);
|
|
@@ -520,8 +541,25 @@ export function createSqlLedger(db: SqlDb, cfg: RouterConfig, deps: LedgerDeps):
|
|
|
520
541
|
// swept up too.
|
|
521
542
|
await sql`DELETE FROM feedback WHERE created_at_ms < ${cutoff} OR ledger_id IN (SELECT id FROM ledger WHERE created_at_ms < ${cutoff})`;
|
|
522
543
|
await sql`DELETE FROM ollama_meter_samples WHERE at_ms < ${cutoff}`;
|
|
544
|
+
// Whole days the cutoff covers go as metadata: a DROP of a day's
|
|
545
|
+
// partition is O(1) where deleting its rows is hours of I/O competing
|
|
546
|
+
// with the inserts it is making room for. Counted before the drop, so
|
|
547
|
+
// `deleted` still means rows, not partitions.
|
|
548
|
+
//
|
|
549
|
+
// ponytail: an exact COUNT(*) scans each doomed day once — a fraction of
|
|
550
|
+
// what deleting it costs, but not free. If that scan ever matters, read
|
|
551
|
+
// pg_class.reltuples instead and report the count as an estimate.
|
|
552
|
+
let dropped = 0;
|
|
553
|
+
for (const partition of await droppableLedgerPartitions(db, cutoff)) {
|
|
554
|
+
const rows = (await sql.unsafe(`SELECT COUNT(*) AS n FROM ${partition}`)) as { n: unknown }[];
|
|
555
|
+
dropped += num(rows[0]?.n);
|
|
556
|
+
await sql.unsafe(`DROP TABLE ${partition}`);
|
|
557
|
+
}
|
|
558
|
+
// What the drops could not cover: the boundary day the cutoff falls
|
|
559
|
+
// inside (a partial day, so row-wise), anything in a partition whose
|
|
560
|
+
// bounds could not be read, and every row on SQLite.
|
|
523
561
|
const deleted = (await sql`DELETE FROM ledger WHERE created_at_ms < ${cutoff} RETURNING id`) as { id: string }[];
|
|
524
|
-
return { deleted: deleted.length, oldestKeptMs: await oldest() };
|
|
562
|
+
return { deleted: dropped + deleted.length, oldestKeptMs: await oldest() };
|
|
525
563
|
},
|
|
526
564
|
|
|
527
565
|
async markWasted(id: string): Promise<void> {
|
package/src/router/candidates.ts
CHANGED
|
@@ -214,6 +214,15 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
|
|
|
214
214
|
continue;
|
|
215
215
|
}
|
|
216
216
|
|
|
217
|
+
// A per-turn credential map that names this model's upstream with an
|
|
218
|
+
// empty string says the turn HAS no key for it: dispatching would 401,
|
|
219
|
+
// so it is not a candidate this turn (and is a candidate again on the
|
|
220
|
+
// next turn that does carry one).
|
|
221
|
+
if (req.upstreamKeys?.[model.provider] === "") {
|
|
222
|
+
rejected.push({ slug, reason: "no_credential", detail: `upstream ${model.provider} has no credential on this turn` });
|
|
223
|
+
continue;
|
|
224
|
+
}
|
|
225
|
+
|
|
217
226
|
// Hard-coded denials, before any user configuration.
|
|
218
227
|
const builtIn = builtInDenial(model);
|
|
219
228
|
if (builtIn !== null) {
|
package/src/router/types.ts
CHANGED
|
@@ -164,6 +164,8 @@ export type RejectionReason =
|
|
|
164
164
|
| "free_tier_excluded"
|
|
165
165
|
| "reasoning_mandatory"
|
|
166
166
|
| "untrusted"
|
|
167
|
+
/** The turn's `X-Omp-Upstream-Keys` names this model's upstream with an empty credential: it cannot be dispatched to. */
|
|
168
|
+
| "no_credential"
|
|
167
169
|
/** Already failed on this turn; excluded so failover picks a different model. */
|
|
168
170
|
| "failed_this_turn";
|
|
169
171
|
|
package/src/server/http.ts
CHANGED
|
@@ -282,7 +282,7 @@ export function startServer(cfg: RouterConfig): StartedServer {
|
|
|
282
282
|
// A Postgres store has no bootstrap of its own to run synchronously, so the
|
|
283
283
|
// shape is created on the way up and every entry point waits for it once.
|
|
284
284
|
// Resolved already on SQLite, where `openDb` just did it.
|
|
285
|
-
const storeReady = postgres ? migrateStore(sqlDb) : Promise.resolve();
|
|
285
|
+
const storeReady = postgres ? migrateStore(sqlDb, log) : Promise.resolve();
|
|
286
286
|
// The ledger reads and writes through the engine-agnostic handle. `findModel`
|
|
287
287
|
// closes over the catalog built just below: a shared store has no catalog
|
|
288
288
|
// cache of its own to price a row from.
|
|
@@ -918,6 +918,12 @@ export function startServer(cfg: RouterConfig): StartedServer {
|
|
|
918
918
|
const snap = catalog.peek();
|
|
919
919
|
return json({
|
|
920
920
|
status: "ok",
|
|
921
|
+
// What a front door may rely on, by name rather than by version: a
|
|
922
|
+
// team edition that sends per-turn credentials to a router without
|
|
923
|
+
// `upstream-keys` would have them ignored, and its tenants served on
|
|
924
|
+
// the deployment's own credential — a silent cross-charge. A name it
|
|
925
|
+
// can check turns that into a refusal it can explain.
|
|
926
|
+
features: ["upstream-keys"],
|
|
921
927
|
apiKeyConfigured: cfg.openrouter.apiKey !== "",
|
|
922
928
|
// Which upstreams turns can actually be served from: OpenRouter needs
|
|
923
929
|
// its key; Ollama needs to be on and out of cooldown.
|
package/src/server/turn.ts
CHANGED
|
@@ -472,7 +472,9 @@ export async function runTurn(
|
|
|
472
472
|
let streamEnded = false;
|
|
473
473
|
|
|
474
474
|
try {
|
|
475
|
-
|
|
475
|
+
// Inside the attempt loop, so a retry, a same-tier failover and a tier
|
|
476
|
+
// escalation all dispatch with the same per-turn credentials.
|
|
477
|
+
dispatch = await upstream.dispatch({ body, sessionId: decision.sessionId, signal: attemptSignal, ...(req.upstreamKeys === undefined ? {} : { upstreamKeys: req.upstreamKeys }) });
|
|
476
478
|
} catch (err) {
|
|
477
479
|
streamError = err;
|
|
478
480
|
}
|
|
@@ -410,13 +410,13 @@ export function createAnthropicClient(cfg: RouterConfig, id: string, fetchImpl:
|
|
|
410
410
|
if (caller && timeout) return AbortSignal.any([caller, timeout]);
|
|
411
411
|
return caller ?? timeout;
|
|
412
412
|
}
|
|
413
|
-
async function post(e: UpstreamEntry, body: Record<string, unknown>, signal: AbortSignal | undefined): Promise<Response> {
|
|
413
|
+
async function post(e: UpstreamEntry, body: Record<string, unknown>, signal: AbortSignal | undefined, apiKey: string = e.apiKey): Promise<Response> {
|
|
414
414
|
const headers: Record<string, string> = { "content-type": "application/json", "anthropic-version": ANTHROPIC_VERSION, ...e.headers };
|
|
415
415
|
if (e.auth === "oauth-bearer") {
|
|
416
416
|
// A Claude Pro/Max subscription token: Bearer auth at the first-party API, with the OAuth beta. No per-token cost is reported.
|
|
417
|
-
headers["authorization"] = `Bearer ${
|
|
417
|
+
headers["authorization"] = `Bearer ${apiKey}`;
|
|
418
418
|
headers["anthropic-beta"] = e.headers["anthropic-beta"] ?? "oauth-2025-04-20,claude-code-20250219";
|
|
419
|
-
} else if (
|
|
419
|
+
} else if (apiKey !== "") headers["x-api-key"] = apiKey;
|
|
420
420
|
try {
|
|
421
421
|
return await fetchImpl(`${e.baseUrl.replace(/\/+$/, "")}/v1/messages`, { method: "POST", headers, body: JSON.stringify(body), signal: composeSignal(e, signal) });
|
|
422
422
|
} catch (err) {
|
|
@@ -444,7 +444,8 @@ export function createAnthropicClient(cfg: RouterConfig, id: string, fetchImpl:
|
|
|
444
444
|
async dispatch(opts: DispatchOptions): Promise<Dispatch> {
|
|
445
445
|
const e = entry();
|
|
446
446
|
const { rendered, servedSlug } = render(e, { ...opts.body, stream: true });
|
|
447
|
-
|
|
447
|
+
// A per-turn credential for this upstream wins for this dispatch only; the shared entry is never touched.
|
|
448
|
+
const res = await post(e, rendered, opts.signal, opts.upstreamKeys?.[id] ?? e.apiKey);
|
|
448
449
|
if (!res.ok) throw await httpError(res);
|
|
449
450
|
if (!res.body) throw new UpstreamError("upstream_error", res.status, "response had no body", true);
|
|
450
451
|
const translator = createAnthropicTranslator(servedSlug);
|
package/src/upstream/compat.ts
CHANGED
|
@@ -113,15 +113,20 @@ function transportError(id: string, err: unknown): UpstreamError {
|
|
|
113
113
|
return new UpstreamError("network", 0, err instanceof Error ? err.message : String(err), true);
|
|
114
114
|
}
|
|
115
115
|
|
|
116
|
-
/**
|
|
117
|
-
|
|
116
|
+
/**
|
|
117
|
+
* The chat-completions URL and auth for an entry: Azure names the deployment in the path and keys with `api-key`.
|
|
118
|
+
*
|
|
119
|
+
* `apiKey` overrides the entry's own credential for one dispatch (a per-turn
|
|
120
|
+
* key from the front door). The entry itself is never written to.
|
|
121
|
+
*/
|
|
122
|
+
export function compatEndpoint(entry: UpstreamEntry, modelId: string, apiKey: string = entry.apiKey): { url: string; headers: Record<string, string> } {
|
|
118
123
|
const base = entry.baseUrl.replace(/\/+$/, "");
|
|
119
124
|
const headers: Record<string, string> = { "content-type": "application/json", ...entry.headers };
|
|
120
125
|
if (entry.kind === "azure") {
|
|
121
|
-
if (
|
|
126
|
+
if (apiKey !== "") headers["api-key"] = apiKey;
|
|
122
127
|
return { url: `${base}/openai/deployments/${encodeURIComponent(modelId)}/chat/completions?api-version=${encodeURIComponent(entry.apiVersion)}`, headers };
|
|
123
128
|
}
|
|
124
|
-
if (
|
|
129
|
+
if (apiKey !== "") headers.authorization = `Bearer ${apiKey}`;
|
|
125
130
|
return { url: `${base}/chat/completions`, headers };
|
|
126
131
|
}
|
|
127
132
|
|
|
@@ -195,8 +200,8 @@ export function createCompatClient(cfg: RouterConfig, id: string, fetchImpl: Fet
|
|
|
195
200
|
return err;
|
|
196
201
|
}
|
|
197
202
|
|
|
198
|
-
async function post(e: UpstreamEntry, body: Record<string, unknown>, signal: AbortSignal | undefined): Promise<Response> {
|
|
199
|
-
const { url, headers } = compatEndpoint(e, typeof body.model === "string" ? body.model : "");
|
|
203
|
+
async function post(e: UpstreamEntry, body: Record<string, unknown>, signal: AbortSignal | undefined, apiKey: string = e.apiKey): Promise<Response> {
|
|
204
|
+
const { url, headers } = compatEndpoint(e, typeof body.model === "string" ? body.model : "", apiKey);
|
|
200
205
|
try {
|
|
201
206
|
return await fetchImpl(url, { method: "POST", headers, body: JSON.stringify(body), signal: composeSignal(e, signal) });
|
|
202
207
|
} catch (err) {
|
|
@@ -212,7 +217,8 @@ export function createCompatClient(cfg: RouterConfig, id: string, fetchImpl: Fet
|
|
|
212
217
|
|
|
213
218
|
async dispatch(opts: DispatchOptions): Promise<Dispatch> {
|
|
214
219
|
const e = entry();
|
|
215
|
-
|
|
220
|
+
// A per-turn credential for this upstream wins for this dispatch only; the shared entry is never touched.
|
|
221
|
+
const res = await post(e, toCompatBody(id, { ...opts.body, stream: true }), opts.signal, opts.upstreamKeys?.[id] ?? e.apiKey);
|
|
216
222
|
if (!res.ok) throw await httpError(res);
|
|
217
223
|
if (!res.body) throw new UpstreamError("upstream_error", res.status, "response had no body", true);
|
|
218
224
|
const parsed = parseSse(res.body, (msg, fields) => log.warn(msg, fields));
|
package/src/upstream/ollama.ts
CHANGED
|
@@ -146,9 +146,9 @@ export function createOllamaClient(cfg: RouterConfig, fetchImpl: FetchLike = fet
|
|
|
146
146
|
log.warn("ollama cloud unavailable; routing around it", { kind: err.kind, cooldownMs: ms, message: err.message });
|
|
147
147
|
};
|
|
148
148
|
|
|
149
|
-
function headers(extra: Record<string, string> = {}): Record<string, string> {
|
|
149
|
+
function headers(extra: Record<string, string> = {}, apiKey: string = o.apiKey): Record<string, string> {
|
|
150
150
|
const h: Record<string, string> = { "content-type": "application/json", ...extra };
|
|
151
|
-
if (
|
|
151
|
+
if (apiKey !== "") h.authorization = `Bearer ${apiKey}`;
|
|
152
152
|
return h;
|
|
153
153
|
}
|
|
154
154
|
|
|
@@ -195,7 +195,8 @@ export function createOllamaClient(cfg: RouterConfig, fetchImpl: FetchLike = fet
|
|
|
195
195
|
try {
|
|
196
196
|
res = await fetchImpl(`${baseUrl()}/chat/completions`, {
|
|
197
197
|
method: "POST",
|
|
198
|
-
|
|
198
|
+
// A per-turn credential wins for this dispatch only; cfg is never written to.
|
|
199
|
+
headers: headers({}, opts.upstreamKeys?.ollama ?? o.apiKey),
|
|
199
200
|
body: JSON.stringify(body),
|
|
200
201
|
signal: composeSignal(opts.signal),
|
|
201
202
|
});
|
|
@@ -114,14 +114,14 @@ export function createOpenRouterClient(cfg: RouterConfig): UpstreamClient {
|
|
|
114
114
|
return caller ?? timeout;
|
|
115
115
|
}
|
|
116
116
|
|
|
117
|
-
function headers(extra: Record<string, string
|
|
117
|
+
function headers(extra: Record<string, string>, apiKey: string = cfg.openrouter.apiKey): Record<string, string> {
|
|
118
118
|
const h: Record<string, string> = {
|
|
119
119
|
"content-type": "application/json",
|
|
120
120
|
"x-title": cfg.openrouter.title,
|
|
121
121
|
...extra,
|
|
122
122
|
};
|
|
123
123
|
// /models is public; an empty key must not produce a broken Bearer header.
|
|
124
|
-
if (
|
|
124
|
+
if (apiKey !== "") h.authorization = `Bearer ${apiKey}`;
|
|
125
125
|
if (cfg.openrouter.referer) h["http-referer"] = cfg.openrouter.referer;
|
|
126
126
|
return h;
|
|
127
127
|
}
|
|
@@ -139,7 +139,8 @@ export function createOpenRouterClient(cfg: RouterConfig): UpstreamClient {
|
|
|
139
139
|
try {
|
|
140
140
|
res = await fetch(`${baseUrl}/chat/completions`, {
|
|
141
141
|
method: "POST",
|
|
142
|
-
|
|
142
|
+
// A per-turn credential wins for this dispatch only; cfg is never written to.
|
|
143
|
+
headers: headers({ "x-session-id": opts.sessionId }, opts.upstreamKeys?.openrouter ?? cfg.openrouter.apiKey),
|
|
143
144
|
body: JSON.stringify(body),
|
|
144
145
|
signal,
|
|
145
146
|
});
|
package/src/upstream/types.ts
CHANGED
|
@@ -43,6 +43,13 @@ export interface DispatchOptions {
|
|
|
43
43
|
body: Record<string, unknown>;
|
|
44
44
|
/** Forwarded as the `x-session-id` header, mirroring body `session_id`. */
|
|
45
45
|
sessionId: string;
|
|
46
|
+
/**
|
|
47
|
+
* Per-turn credentials by upstream id (`NormRequest.upstreamKeys`). The
|
|
48
|
+
* client dispatching this body prefers its own entry over the configured
|
|
49
|
+
* `apiKey`, without ever writing to the shared config: concurrent turns
|
|
50
|
+
* carry different tenants' keys over the same `UpstreamEntry`.
|
|
51
|
+
*/
|
|
52
|
+
upstreamKeys?: Readonly<Record<string, string>>;
|
|
46
53
|
signal: AbortSignal;
|
|
47
54
|
}
|
|
48
55
|
|
package/src/util/schema.ts
CHANGED
|
@@ -8,19 +8,137 @@
|
|
|
8
8
|
* has no history to migrate, and on an already-migrated file every statement
|
|
9
9
|
* here is a no-op.
|
|
10
10
|
*
|
|
11
|
+
* On Postgres the ledger is additionally PARTITIONED BY RANGE over
|
|
12
|
+
* `created_at_ms`, one partition per UTC day. Retention is then a metadata
|
|
13
|
+
* DROP rather than a bulk DELETE, which matters at the sizes this store is
|
|
14
|
+
* built for: a ledger row is ~2.6 kB, and 10k-50k users write 33-165 GB a day
|
|
15
|
+
* at 440-2200 writes a second. Deleting that competes for I/O with the inserts
|
|
16
|
+
* it is trying to make room for, and leaves bloat autovacuum has to chase.
|
|
17
|
+
*
|
|
18
|
+
* Every read is unchanged: the bounds ARE ms-epoch integers, so a
|
|
19
|
+
* `created_at_ms >= x` filter prunes partitions on its own and no query needs
|
|
20
|
+
* to know the table is partitioned. SQLite has no declarative partitioning and
|
|
21
|
+
* keeps exactly the shape it always had.
|
|
22
|
+
*
|
|
11
23
|
* The two must agree, so the definitions below are transcribed from
|
|
12
24
|
* `util/sqlite.ts` with its incremental columns folded in, and
|
|
13
25
|
* `test/schema.test.ts` compares the two engine-by-engine rather than trusting
|
|
14
26
|
* that they were copied correctly.
|
|
15
27
|
*/
|
|
16
28
|
|
|
29
|
+
import type { Logger } from "./log.ts";
|
|
17
30
|
import type { SqlDb } from "./sql.ts";
|
|
18
31
|
|
|
32
|
+
const DAY_MS = 86_400_000;
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Days of ledger partitions provisioned ahead of today. A turn at 23:59:59
|
|
36
|
+
* must not depend on a partition created at 00:00:00, and a process that runs
|
|
37
|
+
* for days without rebooting still has runway.
|
|
38
|
+
*/
|
|
39
|
+
const LEDGER_PARTITION_AHEAD_DAYS = 3;
|
|
40
|
+
|
|
41
|
+
/** The start of the UTC day an ms-epoch instant falls in — a partition bound. */
|
|
42
|
+
export function ledgerDayStart(ms: number): number {
|
|
43
|
+
return Math.floor(ms / DAY_MS) * DAY_MS;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** The partition holding the UTC day that starts at `dayStartMs`, e.g. `ledger_p20260914`. */
|
|
47
|
+
export function ledgerPartitionName(dayStartMs: number): string {
|
|
48
|
+
return `ledger_p${new Date(dayStartMs).toISOString().slice(0, 10).replaceAll("-", "")}`;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* `IF NOT EXISTS` is not atomic on Postgres: two replicas booting against a
|
|
53
|
+
* fresh database both pass the existence check and the loser fails on the
|
|
54
|
+
* unique index over pg_type (23505), or on the table name itself (42P07 /
|
|
55
|
+
* 42710). Measured: one of two replicas started together died with
|
|
56
|
+
* "duplicate key value violates unique constraint pg_type_typname_nsp_index".
|
|
57
|
+
* The condition those errors report is the condition the statement asked to
|
|
58
|
+
* tolerate, so they are the success case arriving from the other replica.
|
|
59
|
+
*/
|
|
60
|
+
const RACED = new Set(["23505", "42P07", "42710"]);
|
|
61
|
+
|
|
62
|
+
async function createIfAbsent(db: SqlDb, statement: string): Promise<void> {
|
|
63
|
+
try {
|
|
64
|
+
await db.sql.unsafe(statement);
|
|
65
|
+
} catch (err) {
|
|
66
|
+
const code = err !== null && typeof err === "object" && "errno" in err ? String(err.errno) : "";
|
|
67
|
+
if (!RACED.has(code)) throw err;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* How this store holds the ledger.
|
|
73
|
+
*
|
|
74
|
+
* `plain` is SQLite, and also a Postgres ledger created before partitioning
|
|
75
|
+
* shipped: Postgres cannot convert a populated table to a partitioned one in
|
|
76
|
+
* place, so an existing deployment keeps the table it has (see `migrateStore`).
|
|
77
|
+
*/
|
|
78
|
+
export async function ledgerLayout(db: SqlDb): Promise<"partitioned" | "plain" | "absent"> {
|
|
79
|
+
if (db.dialect !== "postgres") return (await db.tableExists("ledger")) ? "plain" : "absent";
|
|
80
|
+
const rows = await db.query<{ relkind: string }>("SELECT relkind FROM pg_class WHERE oid = to_regclass('ledger')");
|
|
81
|
+
const kind = rows[0]?.relkind;
|
|
82
|
+
if (kind === "p") return "partitioned";
|
|
83
|
+
return kind === undefined ? "absent" : "plain";
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Creates the day partitions around `aroundMs` — yesterday through
|
|
88
|
+
* `aheadDays` — so the write path never meets a day nobody provisioned.
|
|
89
|
+
* Idempotent and a no-op on any store whose ledger is not partitioned.
|
|
90
|
+
*
|
|
91
|
+
* Yesterday is included because a row's `created_at_ms` is the instant the turn
|
|
92
|
+
* STARTED: a boot just after midnight, or a replica whose clock trails the one
|
|
93
|
+
* that provisioned, can still present the previous day.
|
|
94
|
+
*/
|
|
95
|
+
export async function ensureLedgerPartitions(db: SqlDb, aroundMs = Date.now(), aheadDays = LEDGER_PARTITION_AHEAD_DAYS): Promise<void> {
|
|
96
|
+
if ((await ledgerLayout(db)) !== "partitioned") return;
|
|
97
|
+
const today = ledgerDayStart(aroundMs);
|
|
98
|
+
for (let start = today - DAY_MS; start <= today + aheadDays * DAY_MS; start += DAY_MS) {
|
|
99
|
+
await createIfAbsent(
|
|
100
|
+
db,
|
|
101
|
+
`CREATE TABLE IF NOT EXISTS ${ledgerPartitionName(start)} PARTITION OF ledger FOR VALUES FROM (${start}) TO (${start + DAY_MS})`,
|
|
102
|
+
);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Ledger partitions whose whole range ends at or before `upToMs`, oldest
|
|
108
|
+
* first: the ones a retention cutoff covers completely.
|
|
109
|
+
*
|
|
110
|
+
* Bounds are read from the catalog rather than parsed out of the partition's
|
|
111
|
+
* name, so a partition an operator attached by hand is judged on what it
|
|
112
|
+
* actually holds, and one with an unreadable or DEFAULT bound is skipped
|
|
113
|
+
* rather than guessed at — the rows in it are then pruned by the row-wise
|
|
114
|
+
* DELETE, which is slower but never drops a day it did not verify.
|
|
115
|
+
*/
|
|
116
|
+
export async function droppableLedgerPartitions(db: SqlDb, upToMs: number): Promise<string[]> {
|
|
117
|
+
if ((await ledgerLayout(db)) !== "partitioned") return [];
|
|
118
|
+
const rows = await db.query<{ name: string; bound: string | null }>(
|
|
119
|
+
`SELECT c.relname AS name, pg_get_expr(c.relpartbound, c.oid) AS bound
|
|
120
|
+
FROM pg_class c JOIN pg_inherits i ON i.inhrelid = c.oid
|
|
121
|
+
WHERE i.inhparent = to_regclass('ledger')`,
|
|
122
|
+
);
|
|
123
|
+
const covered: { name: string; endMs: number }[] = [];
|
|
124
|
+
for (const row of rows) {
|
|
125
|
+
const upper = /TO \('?(-?\d+)'?\)/.exec(row.bound ?? "");
|
|
126
|
+
if (upper === null) continue;
|
|
127
|
+
const endMs = Number(upper[1]);
|
|
128
|
+
if (endMs <= upToMs) covered.push({ name: row.name, endMs });
|
|
129
|
+
}
|
|
130
|
+
covered.sort((a, b) => a.endMs - b.endMs);
|
|
131
|
+
return covered.map((p) => p.name);
|
|
132
|
+
}
|
|
133
|
+
|
|
19
134
|
/**
|
|
20
135
|
* Creates every table and index the router uses. Idempotent, so boot order
|
|
21
136
|
* never matters — the property the SQLite bootstrap has always had.
|
|
22
137
|
*/
|
|
23
|
-
export async function migrateStore(db: SqlDb): Promise<void> {
|
|
138
|
+
export async function migrateStore(db: SqlDb, log?: Logger): Promise<void> {
|
|
139
|
+
// Postgres partitions the ledger by day; SQLite has no declarative
|
|
140
|
+
// partitioning, so it keeps the single table it always had.
|
|
141
|
+
const partitioned = db.dialect === "postgres" && (await ledgerLayout(db)) !== "plain";
|
|
24
142
|
const json = db.type("json");
|
|
25
143
|
const float = db.type("float");
|
|
26
144
|
const big = db.type("bigint");
|
|
@@ -41,8 +159,14 @@ export async function migrateStore(db: SqlDb): Promise<void> {
|
|
|
41
159
|
|
|
42
160
|
// One row per dispatched upstream generation. The columns nineteen
|
|
43
161
|
// migrations added are declared here as they finally stand.
|
|
162
|
+
//
|
|
163
|
+
// Partitioned by day on Postgres, which forces the primary key to
|
|
164
|
+
// include the partition key — a unique index on a partitioned table has
|
|
165
|
+
// to. `id` alone stays unique in practice (it is a fresh UUID per entry),
|
|
166
|
+
// and a re-recorded entry carries the same `created_at_ms`, so the
|
|
167
|
+
// ON CONFLICT guard in `record` still collapses it.
|
|
44
168
|
`CREATE TABLE IF NOT EXISTS ledger (
|
|
45
|
-
id TEXT PRIMARY KEY,
|
|
169
|
+
id TEXT${partitioned ? "" : " PRIMARY KEY"},
|
|
46
170
|
created_at_ms ${big} NOT NULL,
|
|
47
171
|
conversation_key TEXT NOT NULL,
|
|
48
172
|
session_id TEXT NOT NULL,
|
|
@@ -77,8 +201,11 @@ export async function migrateStore(db: SqlDb): Promise<void> {
|
|
|
77
201
|
hold_arm INTEGER,
|
|
78
202
|
prompt_tokens_saved INTEGER,
|
|
79
203
|
scope TEXT,
|
|
80
|
-
redactions INTEGER
|
|
81
|
-
)`,
|
|
204
|
+
redactions INTEGER${partitioned ? ",\n\t\t\tPRIMARY KEY (id, created_at_ms)" : ""}
|
|
205
|
+
)${partitioned ? " PARTITION BY RANGE (created_at_ms)" : ""}`,
|
|
206
|
+
// The composite key above cannot serve a lookup by `id` alone, which is
|
|
207
|
+
// what the feedback join and `markWasted` do.
|
|
208
|
+
...(partitioned ? ["CREATE INDEX IF NOT EXISTS idx_ledger_id ON ledger (id)"] : []),
|
|
82
209
|
"CREATE INDEX IF NOT EXISTS idx_ledger_conversation ON ledger (conversation_key)",
|
|
83
210
|
"CREATE INDEX IF NOT EXISTS idx_ledger_created ON ledger (created_at_ms)",
|
|
84
211
|
"CREATE INDEX IF NOT EXISTS idx_ledger_slug ON ledger (slug)",
|
|
@@ -166,22 +293,24 @@ export async function migrateStore(db: SqlDb): Promise<void> {
|
|
|
166
293
|
)`,
|
|
167
294
|
];
|
|
168
295
|
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
for (const statement of statements) {
|
|
178
|
-
try {
|
|
179
|
-
await db.sql.unsafe(statement);
|
|
180
|
-
} catch (err) {
|
|
181
|
-
const code = (err as { errno?: unknown }).errno;
|
|
182
|
-
if (!RACED.has(String(code))) throw err;
|
|
183
|
-
}
|
|
296
|
+
for (const statement of statements) await createIfAbsent(db, statement);
|
|
297
|
+
|
|
298
|
+
if (db.dialect !== "postgres") return;
|
|
299
|
+
if (partitioned) {
|
|
300
|
+
// Ahead of need: the write path must never be the thing that discovers a
|
|
301
|
+
// day has no partition (it recovers, but a bill should not depend on that).
|
|
302
|
+
await ensureLedgerPartitions(db);
|
|
303
|
+
return;
|
|
184
304
|
}
|
|
305
|
+
// A Postgres ledger from before this shipped. Postgres cannot convert a
|
|
306
|
+
// populated table to a partitioned one in place, and copying a billing table
|
|
307
|
+
// at boot is the one failure mode the ledger must not have, so the existing
|
|
308
|
+
// table is left exactly as it is and retention keeps deleting rows. The
|
|
309
|
+
// conversion is an operator's decision, taken with the router stopped.
|
|
310
|
+
log?.warn("ledger is not partitioned by day, so retention will delete rows instead of dropping partitions; partitioning applies to new deployments", {
|
|
311
|
+
convert:
|
|
312
|
+
"stop every replica, then: ALTER TABLE ledger RENAME TO ledger_legacy; start the router (it recreates ledger partitioned); INSERT INTO ledger SELECT * FROM ledger_legacy; verify the counts match; DROP TABLE ledger_legacy",
|
|
313
|
+
});
|
|
185
314
|
}
|
|
186
315
|
|
|
187
316
|
/** Every table `migrateStore` creates, for tests and for teardown. */
|