auto-model-router 0.2.16 → 0.2.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.omp-plugin/marketplace.json +2 -2
- package/README.md +12 -6
- package/docs/routing-benchmark-findings.md +346 -0
- package/omp-extension/embed-logic.ts +21 -0
- package/omp-extension/router-embed.ts +32 -5
- package/package.json +1 -1
- package/src/cli/config-cmd.ts +39 -0
- package/src/config/defaults.ts +6 -0
- package/src/config/hot-reload.ts +163 -0
- package/src/config/load.ts +1 -1
- package/src/config/schema.ts +3 -0
- package/src/config/types.ts +36 -0
- package/src/context/agentdox.ts +6 -1
- package/src/context/bridge.ts +4 -2
- package/src/context/index.ts +1 -0
- package/src/cost/types.ts +6 -1
- package/src/router/candidates.ts +67 -1
- package/src/server/http.ts +33 -3
- package/src/wire/openai/sink.ts +15 -2
- package/test/context-bridge.test.ts +5 -3
- package/test/failover.test.ts +1 -1
- package/test/hot-reload.test.ts +131 -0
- package/test/http-resilience.test.ts +58 -0
- package/test/tier-plan.test.ts +97 -2
- package/test/turn.test.ts +1 -1
- package/test/wire-sink.test.ts +31 -0
- package/tools/replay.ts +82 -29
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hot reload for `config.yml`.
|
|
3
|
+
*
|
|
4
|
+
* The router is long-lived (an omp session embeds it), and the ranking knobs —
|
|
5
|
+
* tiers, filters, escalation, budgets, hysteresis — are exactly what a tuning
|
|
6
|
+
* session wants to change without restarting the harness. This module watches
|
|
7
|
+
* the config file, re-validates it through the same schema `loadConfig` uses,
|
|
8
|
+
* and mutates the SHARED config object in place.
|
|
9
|
+
*
|
|
10
|
+
* In-place mutation is the design: every consumer reads `cfg.tiers`,
|
|
11
|
+
* `cfg.filters`, `cfg.escalation` … at call time through the same object
|
|
12
|
+
* reference, so field assignment makes every per-turn read live with zero
|
|
13
|
+
* call-site changes. What is deliberately NOT reloaded is anything captured at
|
|
14
|
+
* construction — the listening socket (server.*), the OpenRouter client
|
|
15
|
+
* (openrouter.*), and the agentdox bridge (context.*). Those still require a
|
|
16
|
+
* restart; the watcher re-pinns them from the live object and reports skips.
|
|
17
|
+
*
|
|
18
|
+
* Safety properties:
|
|
19
|
+
* - Schema validation BEFORE any mutation; an invalid file leaves the running
|
|
20
|
+
* config untouched and logs the zod issues, exactly like loadConfig.
|
|
21
|
+
* - fs.watch fires several times per save; a trailing debounce collapses them.
|
|
22
|
+
* - A half-written or invalid file never throws into the watcher: the reload
|
|
23
|
+
* is skipped and the previous config keeps serving.
|
|
24
|
+
* - A deleted or emptied knob reverts to its DEFAULT, mirroring loadConfig's
|
|
25
|
+
* merge order (defaults <- file): the file is the source of truth, so
|
|
26
|
+
* disabling a feature by deleting its key works.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import { existsSync, readFileSync, watch, type FSWatcher } from "node:fs";
|
|
30
|
+
import { parse as parseYaml } from "yaml";
|
|
31
|
+
import { configInputSchema } from "./schema.ts";
|
|
32
|
+
import { DEFAULT_CONFIG } from "./defaults.ts";
|
|
33
|
+
import { deepMerge, resolveTilde } from "./load.ts";
|
|
34
|
+
import type { RouterConfig } from "./types.ts";
|
|
35
|
+
|
|
36
|
+
/** Milliseconds of quiet after the last fs event before a reload actually runs. */
|
|
37
|
+
const DEBOUNCE_MS = 250;
|
|
38
|
+
|
|
39
|
+
/** The validated file input, or why it could not be used. */
|
|
40
|
+
export type ConfigRead =
|
|
41
|
+
| { ok: true; cfg: RouterConfig }
|
|
42
|
+
| { ok: false; error: string };
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Re-reads and schema-validates the config file. Exported for tests: this is
|
|
46
|
+
* the exact gate a file must pass before it may touch the running config.
|
|
47
|
+
*
|
|
48
|
+
* The result is merged over DEFAULT_CONFIG — the file is the full source of
|
|
49
|
+
* truth, so a knob REMOVED from the file reverts to its default, matching what
|
|
50
|
+
* a restart would do. `server`/`openrouter`/`context` come back too, but the
|
|
51
|
+
* applier re-pinns those blocks from the live object, since they were captured
|
|
52
|
+
* by construction.
|
|
53
|
+
*/
|
|
54
|
+
export function readValidatedConfig(path: string): ConfigRead {
|
|
55
|
+
try {
|
|
56
|
+
if (!existsSync(path)) return { ok: false, error: "file missing" };
|
|
57
|
+
const raw: unknown = parseYaml(readFileSync(path, "utf8"));
|
|
58
|
+
if (raw === null || raw === undefined) return { ok: false, error: "file empty" };
|
|
59
|
+
const parsed = configInputSchema.safeParse(raw);
|
|
60
|
+
if (!parsed.success) {
|
|
61
|
+
const lines = parsed.error.issues
|
|
62
|
+
.slice(0, 5)
|
|
63
|
+
.map((issue) => ` - ${issue.path.join(".")}: ${issue.message}`)
|
|
64
|
+
.join("\n");
|
|
65
|
+
return { ok: false, error: `schema validation failed:\n${lines}` };
|
|
66
|
+
}
|
|
67
|
+
return { ok: true, cfg: deepMerge(DEFAULT_CONFIG, parsed.data) };
|
|
68
|
+
} catch (err) {
|
|
69
|
+
return { ok: false, error: err instanceof Error ? err.message : String(err) };
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** What changed in a reload, for logging. */
|
|
74
|
+
export interface ReloadSummary {
|
|
75
|
+
changed: string[];
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** A live config file watcher. */
|
|
79
|
+
export interface ConfigWatcher {
|
|
80
|
+
/** Stops watching. Idempotent. */
|
|
81
|
+
close(): void;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Options for watchConfig. */
|
|
85
|
+
export interface WatchConfigOptions {
|
|
86
|
+
/** Called after a successful in-place reload that changed something. */
|
|
87
|
+
onReload?: (summary: ReloadSummary) => void;
|
|
88
|
+
/** Called when a file could not be applied (invalid, unreadable). */
|
|
89
|
+
onError?: (message: string) => void;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Watches `path` and applies valid changes to `live` in place. `frozen` blocks
|
|
94
|
+
* (top-level names) are re-copied from `pinned` after every reload so file
|
|
95
|
+
* edits to construction-captured blocks cannot silently diverge.
|
|
96
|
+
*/
|
|
97
|
+
export function watchConfig(
|
|
98
|
+
path: string,
|
|
99
|
+
live: RouterConfig,
|
|
100
|
+
pinned: RouterConfig,
|
|
101
|
+
frozen: readonly (keyof RouterConfig)[],
|
|
102
|
+
opts: WatchConfigOptions = {},
|
|
103
|
+
): ConfigWatcher {
|
|
104
|
+
let closed = false;
|
|
105
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
106
|
+
let lastError = "";
|
|
107
|
+
|
|
108
|
+
const apply = (): void => {
|
|
109
|
+
if (closed) return;
|
|
110
|
+
const result = readValidatedConfig(path);
|
|
111
|
+
if (!result.ok) {
|
|
112
|
+
// A half-written file is normal (editors truncate-then-write): stay on
|
|
113
|
+
// the current config. Report each distinct error once.
|
|
114
|
+
if (result.error !== lastError) {
|
|
115
|
+
lastError = result.error;
|
|
116
|
+
opts.onError?.(result.error);
|
|
117
|
+
}
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
lastError = "";
|
|
121
|
+
|
|
122
|
+
const frozenSet = new Set(frozen);
|
|
123
|
+
const changed: string[] = [];
|
|
124
|
+
const next = result.cfg as unknown as Record<string, unknown>;
|
|
125
|
+
for (const key of Object.keys(next)) {
|
|
126
|
+
// Frozen blocks belong to construction: keep the pinned values.
|
|
127
|
+
const value = frozenSet.has(key as keyof RouterConfig)
|
|
128
|
+
? (pinned as unknown as Record<string, unknown>)[key]
|
|
129
|
+
: next[key];
|
|
130
|
+
const before = JSON.stringify((live as unknown as Record<string, unknown>)[key]);
|
|
131
|
+
const after = JSON.stringify(value);
|
|
132
|
+
if (before !== after) changed.push(key);
|
|
133
|
+
(live as unknown as Record<string, unknown>)[key] = value;
|
|
134
|
+
}
|
|
135
|
+
if (changed.length > 0) opts.onReload?.({ changed });
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
const schedule = (): void => {
|
|
139
|
+
clearTimeout(timer);
|
|
140
|
+
timer = setTimeout(() => {
|
|
141
|
+
timer = undefined;
|
|
142
|
+
apply();
|
|
143
|
+
}, DEBOUNCE_MS);
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
let watcher: FSWatcher | null = null;
|
|
147
|
+
try {
|
|
148
|
+
watcher = watch(resolveTilde(path), { persistent: false }, schedule);
|
|
149
|
+
} catch {
|
|
150
|
+
// Unwatchable file is not fatal: the router keeps its boot config.
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
return {
|
|
154
|
+
close() {
|
|
155
|
+
closed = true;
|
|
156
|
+
clearTimeout(timer);
|
|
157
|
+
timer = undefined;
|
|
158
|
+
watcher?.close();
|
|
159
|
+
watcher = null;
|
|
160
|
+
},
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
|
package/src/config/load.ts
CHANGED
|
@@ -36,7 +36,7 @@ function mergeValue(base: unknown, override: unknown): unknown {
|
|
|
36
36
|
return override;
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
function deepMerge(base: RouterConfig, override: unknown): RouterConfig {
|
|
39
|
+
export function deepMerge(base: RouterConfig, override: unknown): RouterConfig {
|
|
40
40
|
return mergeValue(base, override) as RouterConfig;
|
|
41
41
|
}
|
|
42
42
|
|
package/src/config/schema.ts
CHANGED
|
@@ -46,6 +46,8 @@ const tierConfig = z.strictObject({
|
|
|
46
46
|
maxInputPerMtok: z.number().nonnegative().optional(),
|
|
47
47
|
maxOutputPerMtok: z.number().nonnegative().optional(),
|
|
48
48
|
qualityExponent: z.number().nonnegative().optional(),
|
|
49
|
+
qualityNormalization: z.boolean().optional(),
|
|
50
|
+
capabilityFloorUsd: z.number().positive().optional(),
|
|
49
51
|
pin: z.array(z.string()).optional(),
|
|
50
52
|
});
|
|
51
53
|
|
|
@@ -141,6 +143,7 @@ const context = z.strictObject({
|
|
|
141
143
|
memoryLimit: z.number().int().positive().optional(),
|
|
142
144
|
docsLimit: z.number().int().nonnegative().optional(),
|
|
143
145
|
sessionLimit: z.number().int().nonnegative().optional(),
|
|
146
|
+
briefChars: z.number().int().nonnegative().optional(),
|
|
144
147
|
recordTurns: z.boolean().optional(),
|
|
145
148
|
maxQueue: z.number().int().positive().optional(),
|
|
146
149
|
});
|
package/src/config/types.ts
CHANGED
|
@@ -108,6 +108,32 @@ export interface TierConfig {
|
|
|
108
108
|
* above the floor (Pareto-style). Higher ⇒ pay for headroom above it.
|
|
109
109
|
*/
|
|
110
110
|
qualityExponent: number;
|
|
111
|
+
/**
|
|
112
|
+
* Rank on quality NORMALISED WITHIN the candidate set instead of on the raw
|
|
113
|
+
* 0-100 index. Default false (raw).
|
|
114
|
+
*
|
|
115
|
+
* Why it exists: raw scores occupy a narrow band (69-78 on the coding axis)
|
|
116
|
+
* while prices span ~250x ($0.02-$5.00/MTok), so `(quality/100)^exponent`
|
|
117
|
+
* over a forecast cost is a bounded numerator over an unbounded denominator
|
|
118
|
+
* — price wins unless the exponent is enormous (measured: ~140 to select a
|
|
119
|
+
* frontier model, where 0.715^140 is ~1e-20 and numerically fragile).
|
|
120
|
+
* Normalising maps the set's worst quality to 0 and its best to 1, so the
|
|
121
|
+
* exponent becomes a legible "how much do I pay for the best available
|
|
122
|
+
* model" dial at single digits instead of triple.
|
|
123
|
+
*/
|
|
124
|
+
qualityNormalization?: boolean;
|
|
125
|
+
/**
|
|
126
|
+
* Treat this tier as a CAPABILITY FLOOR rather than a cost ranking: pick the
|
|
127
|
+
* highest-quality candidate whose forecast turn cost is within this many USD,
|
|
128
|
+
* ignoring quality-per-dollar entirely. Unset ⇒ normal ranking.
|
|
129
|
+
*
|
|
130
|
+
* This is the top tier's real job. `hard` exists because the work needs a
|
|
131
|
+
* capable model, so "best quality under a spend cap" states the intent
|
|
132
|
+
* directly; ranking by quality/price cannot, since a bargain model always
|
|
133
|
+
* wins on the ratio however weak it is. Falls back to the ranked winner when
|
|
134
|
+
* no candidate fits the budget, so this can only ever upgrade a choice.
|
|
135
|
+
*/
|
|
136
|
+
capabilityFloorUsd?: number;
|
|
111
137
|
/** Slugs always allowed in this tier regardless of the quality floor. */
|
|
112
138
|
pin: string[];
|
|
113
139
|
}
|
|
@@ -379,6 +405,16 @@ export interface ContextConfig {
|
|
|
379
405
|
docsLimit: number;
|
|
380
406
|
/** Max recent session messages agentdox may select for the block. */
|
|
381
407
|
sessionLimit: number;
|
|
408
|
+
/**
|
|
409
|
+
* Character budget for the project brief inside the assembled block, 0 to
|
|
410
|
+
* omit. The brief is query-independent curated context (overview, style,
|
|
411
|
+
* gotchas, decision log) that renders FIRST, where the prompt cache holds it.
|
|
412
|
+
* It grows by one entry per recorded decision, so it takes an explicit
|
|
413
|
+
* budget; measured on two live scopes, static sections ~1.6k-8.6k chars and
|
|
414
|
+
* the decision log the rest. Keep this well inside `maxBlockChars` so the
|
|
415
|
+
* query-relevant tail always fits too.
|
|
416
|
+
*/
|
|
417
|
+
briefChars: number;
|
|
382
418
|
/** Write settled turns back to agentdox sessions, tagged with the served model. */
|
|
383
419
|
recordTurns: boolean;
|
|
384
420
|
/** Bound on queued write-backs; excess turns are dropped, never buffered unbounded. */
|
package/src/context/agentdox.ts
CHANGED
|
@@ -20,6 +20,8 @@ export interface AssembleLimits {
|
|
|
20
20
|
memoryLimit: number;
|
|
21
21
|
docsLimit: number;
|
|
22
22
|
sessionLimit: number;
|
|
23
|
+
/** Character budget for the project brief; 0 omits it (pre-brief servers ignore it). */
|
|
24
|
+
briefChars: number;
|
|
23
25
|
}
|
|
24
26
|
|
|
25
27
|
export interface AgentDoxClient {
|
|
@@ -84,13 +86,16 @@ export function createAgentDoxClient(opts: AgentDoxClientOptions): AgentDoxClien
|
|
|
84
86
|
return {
|
|
85
87
|
async assemble(scope, query, limits) {
|
|
86
88
|
// camelCase: the REST endpoint ignores snake_case limit keys entirely,
|
|
87
|
-
// which silently reads as "unbounded".
|
|
89
|
+
// which silently reads as "unbounded". briefChars is sent even when 0:
|
|
90
|
+
// an older server ignores the unknown key, and 0 is the documented
|
|
91
|
+
// "no brief" value there.
|
|
88
92
|
const res = await request("POST", "/context/assemble", {
|
|
89
93
|
scope,
|
|
90
94
|
query,
|
|
91
95
|
memoryLimit: limits.memoryLimit,
|
|
92
96
|
docsLimit: limits.docsLimit,
|
|
93
97
|
sessionLimit: limits.sessionLimit,
|
|
98
|
+
briefChars: limits.briefChars,
|
|
94
99
|
});
|
|
95
100
|
if (res !== null && res.status === 200) {
|
|
96
101
|
const prompt = promptOf(res.json);
|
package/src/context/bridge.ts
CHANGED
|
@@ -38,6 +38,8 @@ export interface BridgeOptions {
|
|
|
38
38
|
memoryLimit: number;
|
|
39
39
|
docsLimit: number;
|
|
40
40
|
sessionLimit: number;
|
|
41
|
+
/** Character budget for the project brief rendered first in the block; 0 omits it. */
|
|
42
|
+
briefChars: number;
|
|
41
43
|
/** Record settled turns back into agentdox sessions. */
|
|
42
44
|
recordTurns: boolean;
|
|
43
45
|
/** Bound on queued write-backs; excess is dropped rather than grown unbounded. */
|
|
@@ -81,7 +83,7 @@ function appendFragment(prior: string, next: string): string {
|
|
|
81
83
|
}
|
|
82
84
|
|
|
83
85
|
export function createContextBridge(opts: BridgeOptions): ContextBridge {
|
|
84
|
-
const { client, store, log, maxStalenessMs, maxBlockChars, memoryLimit, docsLimit, sessionLimit, recordTurns, maxQueue } = opts;
|
|
86
|
+
const { client, store, log, maxStalenessMs, maxBlockChars, memoryLimit, docsLimit, sessionLimit, briefChars, recordTurns, maxQueue } = opts;
|
|
85
87
|
|
|
86
88
|
// Serialized write-back queue. Session appends for one conversation must
|
|
87
89
|
// stay ordered, and agentdox is a local service — one worker is plenty.
|
|
@@ -119,7 +121,7 @@ export function createContextBridge(opts: BridgeOptions): ContextBridge {
|
|
|
119
121
|
return { ...pinned, fetchedAtMs: input.pinnedFetchedAtMs };
|
|
120
122
|
}
|
|
121
123
|
|
|
122
|
-
const raw = await client.assemble(input.scope, input.query, { memoryLimit, docsLimit, sessionLimit });
|
|
124
|
+
const raw = await client.assemble(input.scope, input.query, { memoryLimit, docsLimit, sessionLimit, briefChars });
|
|
123
125
|
if (raw === null) {
|
|
124
126
|
// agentdox unreachable or empty. Keep serving the pinned block if we
|
|
125
127
|
// have one: stale shared context beats none, and re-using it also
|
package/src/context/index.ts
CHANGED
|
@@ -30,6 +30,7 @@ export function createBridgeFromConfig(cfg: RouterConfig, db: Database): Context
|
|
|
30
30
|
memoryLimit: c.memoryLimit,
|
|
31
31
|
docsLimit: c.docsLimit,
|
|
32
32
|
sessionLimit: c.sessionLimit,
|
|
33
|
+
briefChars: c.briefChars,
|
|
33
34
|
recordTurns: c.recordTurns,
|
|
34
35
|
maxQueue: c.maxQueue,
|
|
35
36
|
});
|
package/src/cost/types.ts
CHANGED
|
@@ -124,7 +124,12 @@ export interface LedgerEntry {
|
|
|
124
124
|
/** Time to first content token, ms. */
|
|
125
125
|
ttftMs: number | null;
|
|
126
126
|
finishReason: string | null;
|
|
127
|
-
/**
|
|
127
|
+
/**
|
|
128
|
+
* Attempt superseded by a retry or escalation. NOT a cost figure: by design
|
|
129
|
+
* these rows never carry reported_usd, so "wasted spend" sums to $0.00.
|
|
130
|
+
* The meaningful waste measure is retry spend — rows with attempt > 0 that
|
|
131
|
+
* DID bill. Kept for compatibility; do not read it as money.
|
|
132
|
+
*/
|
|
128
133
|
wasted: boolean;
|
|
129
134
|
upstreamGenerationId: string | null;
|
|
130
135
|
error: string | null;
|
package/src/router/candidates.ts
CHANGED
|
@@ -134,6 +134,8 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
|
|
|
134
134
|
|
|
135
135
|
const candidates: Candidate[] = [];
|
|
136
136
|
const rejected: Rejection[] = [];
|
|
137
|
+
// Carries the trust/latency-adjusted cost into the second scoring pass.
|
|
138
|
+
const effectiveUsdBySlug = new Map<string, number>();
|
|
137
139
|
|
|
138
140
|
for (const model of snapshot.models) {
|
|
139
141
|
const slug = model.slug;
|
|
@@ -231,6 +233,20 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
|
|
|
231
233
|
continue;
|
|
232
234
|
}
|
|
233
235
|
|
|
236
|
+
// Every candidate is priced COLD, deliberately, and this has been measured
|
|
237
|
+
// rather than assumed. Two reasons:
|
|
238
|
+
// 1. `coldUsd` feeds the budget guard in select.ts, and a budget must
|
|
239
|
+
// survive a cache miss.
|
|
240
|
+
// 2. Discounting the warm slug here only ever LOWERS its effective cost,
|
|
241
|
+
// so it can only make the warm model win more often — and the warm
|
|
242
|
+
// model is already either the cheapest candidate or kept by the
|
|
243
|
+
// dedicated stay-vs-switch comparison in select.ts step 4, which does
|
|
244
|
+
// price staying at `cacheRead` against switching at cold+`cacheWrite`.
|
|
245
|
+
// So the ranking change has no headroom to alter an outcome.
|
|
246
|
+
// Verified with tools/replay.ts: scoring the warm candidate at hit rates
|
|
247
|
+
// 0.5 / 0.8 / 0.95 changed 0 of 897 decisions, and 0 of 702 on the subset
|
|
248
|
+
// whose conversations ran the expensive model. Cache economics belong in
|
|
249
|
+
// the switch decision, not in candidate scoring — do not "fix" this.
|
|
234
250
|
const fc = forecast(model, {
|
|
235
251
|
promptTokens: features.promptTokens,
|
|
236
252
|
completionTokens: expectedCompletionTokens,
|
|
@@ -249,7 +265,10 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
|
|
|
249
265
|
: null;
|
|
250
266
|
const latencyMult = latencyMultiplier(latency, filters, expectedCompletionTokens);
|
|
251
267
|
const effectiveUsd = (fc.expectedUsd / Math.max(trustScore, 0.5)) * latencyMult;
|
|
252
|
-
|
|
268
|
+
// Score is assigned in a SECOND PASS below: both qualityNormalization and
|
|
269
|
+
// capabilityFloorUsd are properties of the candidate SET, not of one
|
|
270
|
+
// model, so no per-model value can be computed here. Placeholder only.
|
|
271
|
+
const score = 0;
|
|
253
272
|
|
|
254
273
|
const reasons: string[] = [
|
|
255
274
|
quality === null
|
|
@@ -267,6 +286,30 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
|
|
|
267
286
|
}
|
|
268
287
|
if (pinned) reasons.push("pinned into tier");
|
|
269
288
|
candidates.push({ model, forecast: fc, qualityScore, trustScore, score, reasons });
|
|
289
|
+
effectiveUsdBySlug.set(slug, effectiveUsd);
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
// Second pass: both new tier modes need the whole set.
|
|
293
|
+
// - qualityNormalization rescales quality to the set's own [worst, best]
|
|
294
|
+
// range, so the exponent operates on a full 0-1 spread instead of the
|
|
295
|
+
// raw index's compressed 69-78 band.
|
|
296
|
+
// - capabilityFloorUsd ignores the ratio entirely and takes the highest
|
|
297
|
+
// quality candidate affordable within the cap.
|
|
298
|
+
const qualities = candidates.map((c) => c.qualityScore);
|
|
299
|
+
const qMin = qualities.length > 0 ? Math.min(...qualities) : 0;
|
|
300
|
+
const qMax = qualities.length > 0 ? Math.max(...qualities) : 0;
|
|
301
|
+
const qSpread = qMax - qMin;
|
|
302
|
+
const normalize = tierCfg.qualityNormalization === true && qSpread > 0;
|
|
303
|
+
for (const c of candidates) {
|
|
304
|
+
const effectiveUsd = effectiveUsdBySlug.get(c.model.slug) ?? c.forecast.expectedUsd;
|
|
305
|
+
// Normalised quality is unitless in [0,1]: the set's cheapest-quality
|
|
306
|
+
// model scores 0, its best scores 1. A single-model set has no spread,
|
|
307
|
+
// so it keeps the raw path (guarded by qSpread > 0).
|
|
308
|
+
const q = normalize ? (c.qualityScore - qMin) / qSpread : c.qualityScore / 100;
|
|
309
|
+
c.score = Math.pow(q, tierCfg.qualityExponent) / Math.max(effectiveUsd, 1e-9);
|
|
310
|
+
if (normalize) {
|
|
311
|
+
c.reasons.push(`quality normalised ${q.toFixed(3)} within set [${qMin}, ${qMax}]`);
|
|
312
|
+
}
|
|
270
313
|
}
|
|
271
314
|
|
|
272
315
|
candidates.sort((a, b) => {
|
|
@@ -285,5 +328,28 @@ export function buildCandidates(args: BuildCandidatesArgs): { candidates: Candid
|
|
|
285
328
|
if (b.model.slug === warmSlug) return 1;
|
|
286
329
|
return a.model.slug < b.model.slug ? -1 : 1;
|
|
287
330
|
});
|
|
331
|
+
|
|
332
|
+
// Capability-floor mode: the top tier's job is "best model the work needs",
|
|
333
|
+
// which quality-per-dollar cannot express — a bargain model always wins the
|
|
334
|
+
// ratio however weak it is. Promote the highest-quality candidate whose
|
|
335
|
+
// forecast turn cost fits the cap to the front. Strictly an upgrade: when
|
|
336
|
+
// nothing is affordable, or the ranked winner is already the best quality,
|
|
337
|
+
// the order is untouched.
|
|
338
|
+
const floorUsd = tierCfg.capabilityFloorUsd;
|
|
339
|
+
if (floorUsd !== undefined && candidates.length > 1) {
|
|
340
|
+
let best: Candidate | undefined;
|
|
341
|
+
for (const c of candidates) {
|
|
342
|
+
if (c.forecast.coldUsd > floorUsd) continue;
|
|
343
|
+
if (best === undefined || c.qualityScore > best.qualityScore) best = c;
|
|
344
|
+
}
|
|
345
|
+
if (best !== undefined && best !== candidates[0]) {
|
|
346
|
+
const idx = candidates.indexOf(best);
|
|
347
|
+
candidates.splice(idx, 1);
|
|
348
|
+
candidates.unshift(best);
|
|
349
|
+
best.reasons.push(
|
|
350
|
+
`capability floor: highest quality ${best.qualityScore} within $${floorUsd}/turn (cold $${best.forecast.coldUsd.toFixed(4)})`,
|
|
351
|
+
);
|
|
352
|
+
}
|
|
353
|
+
}
|
|
288
354
|
return { candidates, rejected };
|
|
289
355
|
}
|
package/src/server/http.ts
CHANGED
|
@@ -10,6 +10,8 @@ import { createConversationStore } from "../router/state.ts";
|
|
|
10
10
|
import { createOpenRouterClient } from "../upstream/openrouter.ts";
|
|
11
11
|
import { UpstreamError } from "../upstream/types.ts";
|
|
12
12
|
import { apiKeySource } from "../config/load.ts";
|
|
13
|
+
import { routerConfigPath } from "../cli/config-cmd.ts";
|
|
14
|
+
import { watchConfig } from "../config/hot-reload.ts";
|
|
13
15
|
import type { RouterConfig } from "../config/types.ts";
|
|
14
16
|
import { createLogger } from "../util/log.ts";
|
|
15
17
|
import { openDb } from "../util/sqlite.ts";
|
|
@@ -167,7 +169,7 @@ function isLoopbackHostHeader(hostHeader: string | null): boolean {
|
|
|
167
169
|
export function startServer(cfg: RouterConfig): StartedServer {
|
|
168
170
|
const log = createLogger(cfg.logLevel);
|
|
169
171
|
|
|
170
|
-
mkdirSync(dirname(cfg.ledger.path), { recursive: true });
|
|
172
|
+
if (cfg.ledger.path !== ":memory:") mkdirSync(dirname(cfg.ledger.path), { recursive: true });
|
|
171
173
|
const db = openDb(cfg.ledger.path);
|
|
172
174
|
const ledger = createLedger(db, cfg);
|
|
173
175
|
const upstream = createOpenRouterClient(cfg);
|
|
@@ -177,6 +179,27 @@ export function startServer(cfg: RouterConfig): StartedServer {
|
|
|
177
179
|
const context = createBridgeFromConfig(cfg, db);
|
|
178
180
|
const turnDeps = { config: cfg, router, upstream, ledger, conversations, catalog, context };
|
|
179
181
|
|
|
182
|
+
// Hot reload: ranking knobs (tiers, filters, escalation, budgets, …) take
|
|
183
|
+
// effect on the next turn without a restart, because every consumer reads
|
|
184
|
+
// the shared config object at call time. Construction-captured blocks
|
|
185
|
+
// (server socket, OpenRouter client, agentdox bridge) are pinned — editing
|
|
186
|
+
// those still requires a restart, and the watcher says so explicitly.
|
|
187
|
+
const pinned = { ...cfg };
|
|
188
|
+
const configWatcher = watchConfig(
|
|
189
|
+
routerConfigPath(),
|
|
190
|
+
cfg,
|
|
191
|
+
pinned,
|
|
192
|
+
["server", "openrouter", "context", "ledger"],
|
|
193
|
+
{
|
|
194
|
+
onReload: ({ changed }) => {
|
|
195
|
+
log.info("config reloaded", { changed: changed.join(", ") });
|
|
196
|
+
},
|
|
197
|
+
onError: (message) => {
|
|
198
|
+
log.warn("config reload rejected; keeping the running config", { error: message });
|
|
199
|
+
},
|
|
200
|
+
},
|
|
201
|
+
);
|
|
202
|
+
|
|
180
203
|
if (context.enabled) {
|
|
181
204
|
log.info("agentdox context bridge enabled", {
|
|
182
205
|
url: cfg.context.baseUrl,
|
|
@@ -258,7 +281,14 @@ export function startServer(cfg: RouterConfig): StartedServer {
|
|
|
258
281
|
runTurn(normReq, sink, turnDeps, req.signal)
|
|
259
282
|
.catch((err: unknown) => {
|
|
260
283
|
log.error("turn failed", { error: err instanceof Error ? err.message : String(err) });
|
|
261
|
-
|
|
284
|
+
// sink.error() is SYNCHRONOUS: it runs before Promise.resolve wraps
|
|
285
|
+
// anything, so a throw out of it escapes this .catch() handler and
|
|
286
|
+
// becomes an unhandled rejection that kills the process. Wrap it.
|
|
287
|
+
try {
|
|
288
|
+
Promise.resolve(sink.error(toWireError(err))).catch(() => {});
|
|
289
|
+
} catch {
|
|
290
|
+
// Stream already gone; the response cannot carry the error.
|
|
291
|
+
}
|
|
262
292
|
})
|
|
263
293
|
.finally(() => {
|
|
264
294
|
// Release the concurrency slot when the turn settles, not when the
|
|
@@ -348,8 +378,8 @@ export function startServer(cfg: RouterConfig): StartedServer {
|
|
|
348
378
|
return {
|
|
349
379
|
server,
|
|
350
380
|
stop: async () => {
|
|
381
|
+
configWatcher.close();
|
|
351
382
|
clearInterval(pruneTimer);
|
|
352
|
-
clearInterval(catalogRefreshTimer);
|
|
353
383
|
await server.stop(true);
|
|
354
384
|
// Drain queued agentdox write-backs before the DB closes under them.
|
|
355
385
|
context.close();
|
package/src/wire/openai/sink.ts
CHANGED
|
@@ -26,12 +26,25 @@ export function createStreamingSink(virtualModel: string): { sink: ResponseSink;
|
|
|
26
26
|
},
|
|
27
27
|
});
|
|
28
28
|
const send = (bytes: Uint8Array): void => {
|
|
29
|
-
if (
|
|
29
|
+
if (closed) return;
|
|
30
|
+
try {
|
|
31
|
+
controller?.enqueue(bytes);
|
|
32
|
+
} catch {
|
|
33
|
+
// The runtime can close the controller under us — a client cancelling
|
|
34
|
+
// the stream (reader.cancel()) is not exceptional and nothing above us
|
|
35
|
+
// observes it. Mark closed and drop the write; measured to throw
|
|
36
|
+
// ERR_INVALID_STATE synchronously on Bun 1.x otherwise.
|
|
37
|
+
closed = true;
|
|
38
|
+
}
|
|
30
39
|
};
|
|
31
40
|
const close = (): void => {
|
|
32
41
|
if (!closed) {
|
|
33
42
|
closed = true;
|
|
34
|
-
|
|
43
|
+
try {
|
|
44
|
+
controller?.close();
|
|
45
|
+
} catch {
|
|
46
|
+
// Already closed by the runtime; nothing left to do.
|
|
47
|
+
}
|
|
35
48
|
}
|
|
36
49
|
};
|
|
37
50
|
|
|
@@ -55,6 +55,7 @@ function mkBridge(client: AgentDoxClient, over: Partial<BridgeOpts> = {}) {
|
|
|
55
55
|
memoryLimit: 8,
|
|
56
56
|
docsLimit: 2,
|
|
57
57
|
sessionLimit: 6,
|
|
58
|
+
briefChars: 0,
|
|
58
59
|
recordTurns: true,
|
|
59
60
|
maxQueue: 64,
|
|
60
61
|
...over,
|
|
@@ -102,12 +103,12 @@ describe("context bridge refresh policy", () => {
|
|
|
102
103
|
// especially: docs are WHOLE documents and were left unbounded, and a single
|
|
103
104
|
// ashlands note-doc measured 41,921 chars — over the whole cap by itself.
|
|
104
105
|
// The REST endpoint also ignores snake_case limit keys, which silently reads
|
|
105
|
-
// as unbounded, so pin that all
|
|
106
|
+
// as unbounded, so pin that all four limits actually reach the client.
|
|
106
107
|
const client = mkClient();
|
|
107
|
-
const { bridge, db } = mkBridge(client, { memoryLimit: 5, docsLimit: 1, sessionLimit: 2 });
|
|
108
|
+
const { bridge, db } = mkBridge(client, { memoryLimit: 5, docsLimit: 1, sessionLimit: 2, briefChars: 9000 });
|
|
108
109
|
try {
|
|
109
110
|
await bridge.resolve(input());
|
|
110
|
-
expect(client.lastLimits).toEqual({ memoryLimit: 5, docsLimit: 1, sessionLimit: 2 });
|
|
111
|
+
expect(client.lastLimits).toEqual({ memoryLimit: 5, docsLimit: 1, sessionLimit: 2, briefChars: 9000 });
|
|
111
112
|
} finally {
|
|
112
113
|
db.close();
|
|
113
114
|
}
|
|
@@ -244,6 +245,7 @@ describe("context bridge refresh policy", () => {
|
|
|
244
245
|
memoryLimit: 8,
|
|
245
246
|
docsLimit: 2,
|
|
246
247
|
sessionLimit: 6,
|
|
248
|
+
briefChars: 0,
|
|
247
249
|
recordTurns: true,
|
|
248
250
|
maxQueue: 64,
|
|
249
251
|
};
|
package/test/failover.test.ts
CHANGED
|
@@ -69,7 +69,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
69
69
|
hysteresis: { holdTurns: 2, holdTurnsAfterEscalation: 4, switchMargin: 1.5, cacheWarmTtlMs: 600_000, maxDowngradePerTurn: 1 },
|
|
70
70
|
exploration: { enabled: false, rates: {}, stickyPolicy: "never", holdTurns: { enabled: false, values: [2, 3, 4] } },
|
|
71
71
|
cache: { injectBreakpoints: true, maxBreakpoints: 4, minPromptTokens: 1024, milestoneTokens: 20_000 },
|
|
72
|
-
context: { enabled: false, baseUrl: "", token: "", defaultScope: "", timeoutMs: 3_000, maxStalenessMs: 900_000, maxBlockChars: 24_000, memoryLimit: 8, docsLimit: 2, sessionLimit: 6, recordTurns: false, maxQueue: 64 },
|
|
72
|
+
context: { enabled: false, baseUrl: "", token: "", defaultScope: "", timeoutMs: 3_000, maxStalenessMs: 900_000, maxBlockChars: 24_000, memoryLimit: 8, docsLimit: 2, sessionLimit: 6, briefChars: 0, recordTurns: false, maxQueue: 64 },
|
|
73
73
|
compaction: { enabled: false, budgetTokens: 40_000, fitToWindow: true, protectRecentTurns: 4, maxToolResultBytes: 4_096, keepHeadBytes: 512, keepTailBytes: 512, elideSupersededReads: true, collapseDuplicateResults: true },
|
|
74
74
|
budget: { onExceeded: "downgrade" },
|
|
75
75
|
profiles: [],
|