auto-model-router 0.3.2 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.omp-plugin/marketplace.json +2 -2
- package/CLAUDE.md +18 -98
- package/README.md +1 -0
- package/docs/AGENTDOX-BRIDGE.md +8 -2
- package/package.json +1 -1
- package/src/cli/config-wizard.ts +1 -0
- package/src/config/defaults.ts +3 -0
- package/src/config/schema.ts +1 -0
- package/src/config/types.ts +13 -0
- package/src/context/bridge.ts +9 -1
- package/src/context/query.ts +58 -0
- package/src/context/types.ts +8 -1
- package/src/router/select.ts +43 -0
- package/src/router/state.ts +6 -2
- package/src/router/types.ts +9 -0
- package/src/server/turn.ts +7 -2
- package/src/util/sqlite.ts +8 -1
- package/test/context-bridge.test.ts +12 -0
- package/test/context-query.test.ts +67 -0
- package/test/failover.test.ts +2 -1
- package/test/select.test.ts +54 -0
- package/test/trust-attribution.test.ts +16 -2
- package/test/turn.test.ts +2 -1
- package/tools/agentdox-e2e.ts +1 -0
|
@@ -7,14 +7,14 @@
|
|
|
7
7
|
},
|
|
8
8
|
"metadata": {
|
|
9
9
|
"description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
|
|
10
|
-
"version": "0.3.
|
|
10
|
+
"version": "0.3.4",
|
|
11
11
|
"pluginRoot": "."
|
|
12
12
|
},
|
|
13
13
|
"plugins": [
|
|
14
14
|
{
|
|
15
15
|
"name": "auto-model-router",
|
|
16
16
|
"description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
|
|
17
|
-
"version": "0.3.
|
|
17
|
+
"version": "0.3.4",
|
|
18
18
|
"author": {
|
|
19
19
|
"name": "drewappling",
|
|
20
20
|
"email": "drewappling@gmail.com"
|
package/CLAUDE.md
CHANGED
|
@@ -11,108 +11,28 @@ It presents one keyless OpenAI-compatible provider and picks a concrete OpenRout
|
|
|
11
11
|
|
|
12
12
|
## agentdox — shared context/memory (**MANDATORY to keep updated**)
|
|
13
13
|
|
|
14
|
-
agentdox is this repo's memory + docs + live-conversation
|
|
15
|
-
**`omp-router`** — ALWAYS scope agentdox writes to it. The
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
the token grants every scope, so a wrong slug is **not** rejected — it silently writes into
|
|
19
|
-
another project. Getting `omp-router` right is on you, not on RBAC.
|
|
14
|
+
agentdox is this repo's memory + docs + live-conversation store. The project slug is
|
|
15
|
+
**`omp-router`** — ALWAYS scope agentdox writes to it. The bearer token (`AGENTDOX_TOKEN`)
|
|
16
|
+
is one global PAT that grants every scope, so a wrong slug is **not** rejected: it silently
|
|
17
|
+
files this project's data under another project. Getting `omp-router` right is on you.
|
|
20
18
|
|
|
21
|
-
**
|
|
19
|
+
**Full protocol: `.claude/skills/agentdox/SKILL.md`** (on connect, before asking the user,
|
|
20
|
+
batching writes to the end of the session, the REST fallback with request shapes, search
|
|
21
|
+
tips). Read it before using agentdox; do not improvise from this summary.
|
|
22
22
|
|
|
23
23
|
| What | Where |
|
|
24
24
|
| --- | --- |
|
|
25
|
-
| Token
|
|
26
|
-
| What `.mcp.json` reads | the `AGENTDOX_TOKEN` **environment variable
|
|
27
|
-
|
|
|
28
|
-
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
question. `docs_search` returns whole documents, which then get truncated, and the truncation
|
|
37
|
-
is rarely the relevant part.
|
|
38
|
-
- **If results look thin, run `index_stats {scope}` before concluding the store is empty.** It
|
|
39
|
-
reports how much of the scope is indexed and whether the embedding provider is reachable;
|
|
40
|
-
`embedded` far below `total`, or an unreachable provider, means you are getting keyword-only
|
|
41
|
-
results.
|
|
42
|
-
|
|
43
|
-
`.env.agentdox` is the durable record; the environment variable is what Claude Code actually
|
|
44
|
-
substitutes into `.mcp.json` at MCP-server startup. If agentdox MCP returns **401**, the
|
|
45
|
-
variable is missing from the environment — re-set it from `.env.agentdox` and restart Claude
|
|
46
|
-
Code (substitution happens once, at startup). Re-mint instructions are in `.env.agentdox`.
|
|
47
|
-
|
|
48
|
-
**Requirement: keeping agentdox current is part of completing any task, not optional.**
|
|
49
|
-
Do NOT close out a task while memory, docs, or conversation history for the area you touched
|
|
50
|
-
is stale or incomplete.
|
|
51
|
-
|
|
52
|
-
Concrete duties (all scoped to `omp-router`):
|
|
53
|
-
|
|
54
|
-
- **On connect:** run `project_ensure` with `slug: "omp-router"` before any memory/docs work.
|
|
55
|
-
- **On startup:** read `context_brief` to onboard on decisions, conventions, and gotchas
|
|
56
|
-
before rediscovering them.
|
|
57
|
-
- **Memory** (`memory_add` / `memory_update` / `memory_search`): record user-stated
|
|
58
|
-
preferences and corrections. When a fact changes, UPDATE the existing entry — never leave
|
|
59
|
-
contradictory facts. Keep entries compact and high-signal.
|
|
60
|
-
- **Docs** (`docs_write` / `docs_update`): keep architecture and decisions current as reality
|
|
61
|
-
changes; writing once is not enough.
|
|
62
|
-
- **Sessions** (`session_start` / `session_append`): append messages in real time, not as an
|
|
63
|
-
end-of-task summary.
|
|
64
|
-
- **Context** (`context_assemble`): consult it (with a query) before re-asking the user about
|
|
65
|
-
anything already captured.
|
|
66
|
-
- **Decisions** (`context_brief_record`): record decisions and conventions as they are made.
|
|
67
|
-
|
|
68
|
-
### How to actually call it (MCP tools vs REST)
|
|
69
|
-
|
|
70
|
-
The duties above name the **MCP tools** (`memory_add`, `docs_write`, `context_brief_record`, …),
|
|
71
|
-
served from this repo's `.mcp.json`.
|
|
72
|
-
|
|
73
|
-
**omp gets these tools too** — verified 2026-08-28. omp reads `.mcp.json` (repo root),
|
|
74
|
-
`.omp/mcp.json`, `.claude/mcp.json`, and `~/.omp/agent/mcp.json`, and it expands `${VAR}` in
|
|
75
|
-
headers. It mounts them **prefixed**: `agentdox_memory_add`, `agentdox_context_assemble`, …
|
|
76
|
-
(fully qualified `mcp__agentdox_*`). All 17 tools load.
|
|
77
|
-
|
|
78
|
-
The one prerequisite is `AGENTDOX_TOKEN` being present in the **launching shell's**
|
|
79
|
-
environment. It is persisted at Windows *User* scope, so only shells started afterwards
|
|
80
|
-
inherit it — an already-open terminal will show no agentdox tools until restarted.
|
|
81
|
-
|
|
82
|
-
**A harness genuinely without those tools (e.g. Hermes, or omp before the env var is
|
|
83
|
-
inherited) MUST use the REST API directly** — same live store, same RBAC. Don't skip recording
|
|
84
|
-
just because the MCP tools are absent.
|
|
85
|
-
|
|
86
|
-
REST basics: base `http://localhost:3003`, header `Authorization: Bearer <token>` where the token
|
|
87
|
-
is `AGENTDOX_TOKEN` from `.env.agentdox` (global; grants every scope). **memory uses
|
|
88
|
-
`category`, everything else uses `scope`; both are always `"omp-router"`.** MCP-tool → REST map:
|
|
89
|
-
|
|
90
|
-
| Duty / MCP tool | REST |
|
|
91
|
-
| --- | --- |
|
|
92
|
-
| `project_ensure` | `POST /projects` `{slug,name}` (idempotent; re-ensure returns the project) |
|
|
93
|
-
| `memory_search` | `GET /memory?category=omp-router&limit=N` · `GET /memory/search?q=…&category=omp-router` |
|
|
94
|
-
| `memory_add` | `POST /memory` `{content, category:"omp-router", importance:0..1}` |
|
|
95
|
-
| `memory_update` | `PATCH /memory/:id` `{content?, importance?, …}` (edit in place — never pile on dupes) |
|
|
96
|
-
| `docs_write` | `POST /docs` `{slug, title, content, scope:"omp-router", tags?}` |
|
|
97
|
-
| `docs_update` | `PATCH /docs/:id` `{title?, content?, tags?}` · list `GET /docs?scope=omp-router` |
|
|
98
|
-
| read `context_brief` | `GET /context/brief?scope=omp-router` (404 `no_brief` until seeded) |
|
|
99
|
-
| seed the brief | `PUT /context/brief` `{scope, overview?, repoLayout?, codeStyle?, buildTest?, assetConventions?, gotchas?}` |
|
|
100
|
-
| `context_brief_record` | `POST /context/brief/decision` `{scope, title, decision, rationale}` |
|
|
101
|
-
| `context_assemble` | `POST /context/assemble` `{scope, query}` · baseline `GET /context/snapshot?scope=` · `POST /context/refresh` `{scope}` |
|
|
102
|
-
| `session_start` / `session_append` | `POST /sessions` `{scope, title}` → `POST /sessions/:id/messages` `{role, content, refs?}` → `POST /sessions/:id/end` |
|
|
103
|
-
|
|
104
|
-
Cleanest call path (avoids Windows PowerShell mangling `$` in inline JSON): a throwaway
|
|
105
|
-
`bun` script that reads the token and `fetch`es —
|
|
106
|
-
|
|
107
|
-
```ts
|
|
108
|
-
const tok = /AGENTDOX_TOKEN=(.+)/.exec(await Bun.file(".env.agentdox").text())?.[1]?.trim() ?? "";
|
|
109
|
-
const H = { Authorization: `Bearer ${tok}`, "content-type": "application/json" };
|
|
110
|
-
await fetch("http://localhost:3003/memory", { method: "POST", headers: H,
|
|
111
|
-
body: JSON.stringify({ content: "…", category: "omp-router", importance: 0.9 }) });
|
|
112
|
-
```
|
|
113
|
-
|
|
114
|
-
Endpoints are defined in `E:/projects/agentdox/packages/server/src/index.ts`; the router's own
|
|
115
|
-
read/write client is `src/context/agentdox.ts` (assemble / createSession / append only).
|
|
25
|
+
| Token, URL, scope | `.env.agentdox` in this repo root (gitignored — never commit) |
|
|
26
|
+
| What `.mcp.json` reads | the `AGENTDOX_TOKEN` **environment variable** (Windows *User* scope; shells opened before it was set lack it) |
|
|
27
|
+
| Server | `http://localhost:3003` — Docker container `agentdox-server`; endpoints in `E:/projects/agentdox/packages/server/src/index.ts` |
|
|
28
|
+
| Admin token (re-mint) | `E:/projects/agentdox/deploy/.env` |
|
|
29
|
+
|
|
30
|
+
Two rules that cause silent mistakes: **memory calls take `category`, everything else takes
|
|
31
|
+
`scope`** (both always `"omp-router"`); and a **401 means the env var is missing** from the
|
|
32
|
+
launching shell — re-set it from `.env.agentdox` and restart the harness. omp mounts the
|
|
33
|
+
tools prefixed (`agentdox_memory_add`, …); a harness without them MUST use REST (the skill
|
|
34
|
+
has the map) rather than skip recording. Cleanest REST call path is a throwaway `bun`
|
|
35
|
+
script reading the token from `.env.agentdox`.
|
|
116
36
|
|
|
117
37
|
## This repo also *implements* an agentdox client
|
|
118
38
|
|
package/README.md
CHANGED
|
@@ -709,6 +709,7 @@ event has already arrived is treated as a completed turn, not an error.
|
|
|
709
709
|
| `holdTurnsAfterEscalation` | `4` | Hold longer after an escalation. |
|
|
710
710
|
| `switchMargin` | `1.3` | Switching must beat the warm-cache discount by this factor. Lower = switch away from a warm model more readily. |
|
|
711
711
|
| `switchHorizonTurns` | `1` | Turns the stay/switch comparison is amortised over: `H × stayWarm` vs `switchCold + (H − 1) × newWarm`. `1` is the one-turn comparison, which can keep a dear model warm indefinitely when the cheaper winner is itself dear cold; a small `H` lets a switch that pays for itself within a few turns go ahead. |
|
|
712
|
+
| `confirmUpgradesBelowConfidence` | `0.6` | A heuristic tier upgrade classified below this confidence waits one turn while the current model's cache is warm; a second consecutive upgrade classification confirms it. Escalations, explicit high reasoning and failing tool loops bypass the wait. `0` disables. Measured: 65 of 67 moderate→hard upgrades in a week bounced back within 3 turns, each paying a cold hard-tier read of a ~120k prompt. |
|
|
712
713
|
| `cacheWarmTtlMs` | `300000` (5 min) | How long a model's prompt cache is considered warm. |
|
|
713
714
|
| `maxDowngradePerTurn` | `1` | Max tiers a turn may drop in one step (avoids quality cliffs). |
|
|
714
715
|
| `breakHoldOnMechanical` | `false` | Let a tool-result continuation that classifies *below* the held tier escape the hold (still bounded by `maxDowngradePerTurn`). Worth enabling when the held tier is expensive. |
|
package/docs/AGENTDOX-BRIDGE.md
CHANGED
|
@@ -175,8 +175,14 @@ would fix that, at the cost of one credential reaching every project.
|
|
|
175
175
|
so router test turns feed back into the next block. The omp-router scope had accumulated 19
|
|
176
176
|
sessions of which 18 were noise (`hi`, `say hello`, `Reply with exactly the word: PONG`,
|
|
177
177
|
injection probes, `bridge e2e …`); they were deleted, and `tools/agentdox-e2e.ts` writes two
|
|
178
|
-
more every run.
|
|
179
|
-
|
|
178
|
+
more every run. **Done 2026-09-07:** recent sessions ride only on a conversation's FIRST
|
|
179
|
+
block (`ContextResolveInput.firstFetch`); refreshes ask for `sessionLimit: 0`, since by then
|
|
180
|
+
the section was this conversation's own turns (1.4-1.9k chars per block, changing every
|
|
181
|
+
refresh). In the same change the relevance query became `context/query.ts`
|
|
182
|
+
`relevanceQuery`: the last user message with real content after omp's wrapper elements
|
|
183
|
+
(`<system-reminder>`, `<recap>`, `<chat>`, image stubs) are stripped, capped at 400 chars —
|
|
184
|
+
live blocks had been queried with those wrappers verbatim, one header running to 2k chars.
|
|
185
|
+
Recorded user turns use the same cleaner.
|
|
180
186
|
- **`context.timeoutMs` is 3000ms** and failures degrade silently at `debug` level by design.
|
|
181
187
|
If agentdox is cold this can no-op invisibly. Consider logging the first failure at `warn`.
|
|
182
188
|
- **Four copies of this project exist** on this machine: this repo, the research checkout,
|
package/package.json
CHANGED
package/src/cli/config-wizard.ts
CHANGED
|
@@ -232,6 +232,7 @@ export const WIZARD_SECTIONS: readonly SectionSpec[] = [
|
|
|
232
232
|
{ path: "hysteresis.holdTurnsAfterEscalation", label: "Hold turns after escalation", kind: "number", min: 0 },
|
|
233
233
|
{ path: "hysteresis.switchMargin", label: "Switch margin", kind: "number", min: 0 },
|
|
234
234
|
{ path: "hysteresis.switchHorizonTurns", label: "Switch horizon", kind: "number", min: 1, hint: "turns amortised" },
|
|
235
|
+
{ path: "hysteresis.confirmUpgradesBelowConfidence", label: "Confirm upgrades below confidence", kind: "number", min: 0, max: 1, hint: "0=off; low-confidence tier-ups wait a turn" },
|
|
235
236
|
{ path: "hysteresis.cacheWarmTtlMs", label: "Cache-warm TTL", kind: "number", min: 0, hint: "ms" },
|
|
236
237
|
{ path: "hysteresis.maxDowngradePerTurn", label: "Max downgrade per turn", kind: "number", min: 0, hint: "tiers" },
|
|
237
238
|
{ path: "hysteresis.breakHoldOnMechanical", label: "Break hold on mechanical turns", kind: "boolean" },
|
package/src/config/defaults.ts
CHANGED
|
@@ -174,6 +174,9 @@ export const DEFAULT_CONFIG: RouterConfig = {
|
|
|
174
174
|
// 1 = the shipped one-turn comparison. Raise to amortise a switch over the
|
|
175
175
|
// turns that follow it; see HysteresisConfig.switchHorizonTurns.
|
|
176
176
|
switchHorizonTurns: 1,
|
|
177
|
+
// Low-confidence heuristic upgrades from a warm model wait one turn; see
|
|
178
|
+
// HysteresisConfig.confirmUpgradesBelowConfidence for the measurement.
|
|
179
|
+
confirmUpgradesBelowConfidence: 0.6,
|
|
177
180
|
// OpenRouter sticky sessions expire in 5-10 minutes.
|
|
178
181
|
cacheWarmTtlMs: 300_000,
|
|
179
182
|
maxDowngradePerTurn: 1,
|
package/src/config/schema.ts
CHANGED
|
@@ -136,6 +136,7 @@ const hysteresis = z.strictObject({
|
|
|
136
136
|
holdTurnsAfterEscalation: z.number().int().nonnegative().optional(),
|
|
137
137
|
switchMargin: z.number().positive().optional(),
|
|
138
138
|
switchHorizonTurns: z.number().int().positive().optional(),
|
|
139
|
+
confirmUpgradesBelowConfidence: z.number().min(0).max(1).optional(),
|
|
139
140
|
cacheWarmTtlMs: z.number().nonnegative().optional(),
|
|
140
141
|
maxDowngradePerTurn: z.number().int().nonnegative().optional(),
|
|
141
142
|
breakHoldOnMechanical: z.boolean().optional(),
|
package/src/config/types.ts
CHANGED
|
@@ -392,6 +392,19 @@ export interface HysteresisConfig {
|
|
|
392
392
|
* dispatches per user-visible turn, so single digits are conservative.
|
|
393
393
|
*/
|
|
394
394
|
switchHorizonTurns: number;
|
|
395
|
+
/**
|
|
396
|
+
* A heuristic tier UPGRADE whose classification confidence is below this
|
|
397
|
+
* waits one turn when the current model's cache is warm; a second
|
|
398
|
+
* consecutive upgrade classification confirms it. 0 disables. Escalations,
|
|
399
|
+
* explicit high reasoning and failing tool loops bypass the wait.
|
|
400
|
+
*
|
|
401
|
+
* Measured on 7 days of live traffic: 65 of 67 moderate→hard upgrades
|
|
402
|
+
* bounced back within 3 turns, 50 of them below 0.6 confidence, costing
|
|
403
|
+
* $17.90 in cold hard-tier prompt reads against $0.23 for staying warm.
|
|
404
|
+
* The stay/switch comparison never sees these because the warm cheap
|
|
405
|
+
* model is below the new tier's floor.
|
|
406
|
+
*/
|
|
407
|
+
confirmUpgradesBelowConfidence: number;
|
|
395
408
|
/** Assume a warm cache expires after this long. OpenRouter sticky sessions: 5-10 min. */
|
|
396
409
|
cacheWarmTtlMs: number;
|
|
397
410
|
/** Downgrade at most this many tiers per turn, so quality never falls off a cliff. */
|
package/src/context/bridge.ts
CHANGED
|
@@ -121,7 +121,15 @@ export function createContextBridge(opts: BridgeOptions): ContextBridge {
|
|
|
121
121
|
return { ...pinned, fetchedAtMs: input.pinnedFetchedAtMs };
|
|
122
122
|
}
|
|
123
123
|
|
|
124
|
-
|
|
124
|
+
// Recent sessions only on a conversation's first block: after that they
|
|
125
|
+
// are this conversation's own recorded turns, duplicating the prompt
|
|
126
|
+
// and changing on every refresh (measured 1.4-1.9k chars per block).
|
|
127
|
+
const raw = await client.assemble(input.scope, input.query, {
|
|
128
|
+
memoryLimit,
|
|
129
|
+
docsLimit,
|
|
130
|
+
sessionLimit: input.firstFetch ? sessionLimit : 0,
|
|
131
|
+
briefChars,
|
|
132
|
+
});
|
|
125
133
|
if (raw === null) {
|
|
126
134
|
// agentdox unreachable or empty. Keep serving the pinned block if we
|
|
127
135
|
// have one: stale shared context beats none, and re-using it also
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What to tell agentdox this turn is about.
|
|
3
|
+
*
|
|
4
|
+
* The relevance query biases which memories and docs the assembled block
|
|
5
|
+
* carries, and agentdox echoes it verbatim into the block's header. Taking
|
|
6
|
+
* the last user-role message literally sent omp's own machinery instead of
|
|
7
|
+
* the user's ask: `<system-reminder>` nudges, `<system-notice>` job results,
|
|
8
|
+
* `<recap>` prompts, "Attached image(s) from tool result:" continuations.
|
|
9
|
+
* Measured on the live ledger: the three most recent blocks for one scope
|
|
10
|
+
* were queried with `<chat>`, `<system-reminder>` and an image notice, one
|
|
11
|
+
* header alone ran to 2k chars, and every refresh re-ranked memory against
|
|
12
|
+
* noise — different bytes each time, at the head of the cached prefix.
|
|
13
|
+
*
|
|
14
|
+
* The query is the last user message with real content once wrapper blocks
|
|
15
|
+
* are stripped, falling back to the conversation's opening ask, capped so the
|
|
16
|
+
* header stays a line rather than a message.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import type { NormRequest } from "../wire/types.ts";
|
|
20
|
+
|
|
21
|
+
/** Longest query worth sending: enough to rank on, short enough to stay stable. */
|
|
22
|
+
export const MAX_QUERY_CHARS = 400;
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Harness wrapper tags whose whole element is machinery, not the user's ask.
|
|
26
|
+
* Stripped before judging whether a message has content.
|
|
27
|
+
*/
|
|
28
|
+
const WRAPPER_TAGS = ["system-reminder", "system-notice", "system-directive", "recap", "chat", "checkpoint-active-reminder", "interrupted-thinking"];
|
|
29
|
+
|
|
30
|
+
const WRAPPER_RE = new RegExp(`<(${WRAPPER_TAGS.join("|")})(\\s[^>]*)?>[\\s\\S]*?</\\1>`, "gi");
|
|
31
|
+
/** An unclosed wrapper at the start swallows the rest: treat the message as machinery. */
|
|
32
|
+
const OPEN_WRAPPER_RE = new RegExp(`^\\s*<(${WRAPPER_TAGS.join("|")})(\\s[^>]*)?>`, "i");
|
|
33
|
+
/** omp's tool-result image continuation and similar auto-generated stubs. */
|
|
34
|
+
const STUB_RE = /^(attached image\(s\)(\s.*)?|\[image\]|\(no output\)|continue[.!]?)$/i;
|
|
35
|
+
|
|
36
|
+
/** The user's own words in a message, or "" when it is all harness machinery. */
|
|
37
|
+
export function userContent(text: string): string {
|
|
38
|
+
let t = text.replace(WRAPPER_RE, " ");
|
|
39
|
+
if (OPEN_WRAPPER_RE.test(t)) return "";
|
|
40
|
+
t = t.replace(/\s+/g, " ").trim();
|
|
41
|
+
if (t === "" || STUB_RE.test(t)) return "";
|
|
42
|
+
return t;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function cap(text: string): string {
|
|
46
|
+
return text.length > MAX_QUERY_CHARS ? `${text.slice(0, MAX_QUERY_CHARS - 1)}…` : text;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** The relevance query for this turn: last real user message, else the opening ask, else "". */
|
|
50
|
+
export function relevanceQuery(req: Pick<NormRequest, "messages">): string {
|
|
51
|
+
for (let i = req.messages.length - 1; i >= 0; i--) {
|
|
52
|
+
const m = req.messages[i];
|
|
53
|
+
if (m === undefined || m.role !== "user") continue;
|
|
54
|
+
const content = userContent(m.text);
|
|
55
|
+
if (content !== "") return cap(content);
|
|
56
|
+
}
|
|
57
|
+
return "";
|
|
58
|
+
}
|
package/src/context/types.ts
CHANGED
|
@@ -38,8 +38,15 @@ export interface ContextResolveInput {
|
|
|
38
38
|
modelSwitching: boolean;
|
|
39
39
|
/** An escalation or failover retry — the prefix is cold either way. */
|
|
40
40
|
retrying: boolean;
|
|
41
|
-
/**
|
|
41
|
+
/** The user's ask this turn (see `context/query.ts`), used to bias agentdox relevance ranking. */
|
|
42
42
|
query: string;
|
|
43
|
+
/**
|
|
44
|
+
* This conversation has not been given a block before. Recent sessions
|
|
45
|
+
* are included only then: they carry a previous conversation forward, but
|
|
46
|
+
* on a refresh they would mostly be this conversation's own turns, which
|
|
47
|
+
* the prompt already holds.
|
|
48
|
+
*/
|
|
49
|
+
firstFetch: boolean;
|
|
43
50
|
}
|
|
44
51
|
|
|
45
52
|
/** One settled turn, recorded to agentdox with the model that served it. */
|
package/src/router/select.ts
CHANGED
|
@@ -102,6 +102,12 @@ function wideningOrder(tier: Tier, minTier: Tier, maxTier: Tier): Tier[] {
|
|
|
102
102
|
return out;
|
|
103
103
|
}
|
|
104
104
|
|
|
105
|
+
/** Evidence that an upgrade will stick: the conversation is failing or asked for deep reasoning. */
|
|
106
|
+
function hardSignal(f: Features): boolean {
|
|
107
|
+
const reasoning: string = f.requestedReasoning ?? "";
|
|
108
|
+
return f.lastToolFailed || f.circularToolCall || f.repeatedToolCall || reasoning === "high" || reasoning === "xhigh" || reasoning === "max";
|
|
109
|
+
}
|
|
110
|
+
|
|
105
111
|
export function select(args: SelectArgs): Decision {
|
|
106
112
|
const { req, features, classification, profile, state, snapshot, ledger, cfg, nowMs } = args;
|
|
107
113
|
const reasons: string[] = [];
|
|
@@ -161,6 +167,42 @@ export function select(args: SelectArgs): Decision {
|
|
|
161
167
|
// exploration (2c) and candidate building (3) so both agree on the term.
|
|
162
168
|
const cacheWarm = state.cacheWarmSlug !== null && nowMs - state.cacheWarmAtMs <= cfg.hysteresis.cacheWarmTtlMs;
|
|
163
169
|
|
|
170
|
+
// 2a. Cache-aware upgrade confirmation. A low-confidence heuristic upgrade
|
|
171
|
+
// from a warm model waits one turn; the next turn's classification
|
|
172
|
+
// confirms or forgets it. Measured on 7 days of live traffic: 65 of 67
|
|
173
|
+
// moderate→hard upgrades bounced back within 3 turns, 50 of them below
|
|
174
|
+
// 0.6 confidence, and each paid a cold hard-tier read of a ~120k prompt
|
|
175
|
+
// ($17.90 in total against $0.23 for staying warm). Step 4's stay/switch
|
|
176
|
+
// comparison never sees these — the warm cheap model is below the new
|
|
177
|
+
// tier's floor, so it is not a candidate there. Escalations, explicit
|
|
178
|
+
// high reasoning and failing tool loops bypass the wait: those are the
|
|
179
|
+
// upgrades that stick.
|
|
180
|
+
let upgradeDeferred: Tier | null = null;
|
|
181
|
+
const confirmBelow = cfg.hysteresis.confirmUpgradesBelowConfidence;
|
|
182
|
+
if (
|
|
183
|
+
confirmBelow > 0 &&
|
|
184
|
+
cls.source === "heuristic" &&
|
|
185
|
+
classification.confidence < confirmBelow &&
|
|
186
|
+
state.currentTier !== null &&
|
|
187
|
+
state.currentSlug !== null &&
|
|
188
|
+
tierIdx(effective) > tierIdx(clampTier(state.currentTier)) &&
|
|
189
|
+
cacheWarm &&
|
|
190
|
+
state.cacheWarmSlug === state.currentSlug &&
|
|
191
|
+
(args.excludeSlugs === undefined || args.excludeSlugs.length === 0) &&
|
|
192
|
+
!hardSignal(features)
|
|
193
|
+
) {
|
|
194
|
+
const held = clampTier(state.currentTier);
|
|
195
|
+
if (state.upgradeDeferredTier !== undefined && state.upgradeDeferredTier !== null) {
|
|
196
|
+
reasons.push(`upgrade ${held} → ${effective} confirmed: classified above ${held} on consecutive turns`);
|
|
197
|
+
} else {
|
|
198
|
+
reasons.push(
|
|
199
|
+
`upgrade ${held} → ${effective} deferred one turn: heuristic confidence ${classification.confidence.toFixed(2)} < ${confirmBelow} with ${state.currentSlug} warm`,
|
|
200
|
+
);
|
|
201
|
+
upgradeDeferred = effective;
|
|
202
|
+
effective = held;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
164
206
|
// 2b. Context compaction: shrink stale tool output before dispatch when the
|
|
165
207
|
// prompt exceeds the token budget (or would overflow the profile window).
|
|
166
208
|
// Deterministic and content-only (never removes a message), so downstream
|
|
@@ -531,5 +573,6 @@ export function select(args: SelectArgs): Decision {
|
|
|
531
573
|
reasons,
|
|
532
574
|
explored,
|
|
533
575
|
budgetDowngraded,
|
|
576
|
+
upgradeDeferred,
|
|
534
577
|
};
|
|
535
578
|
}
|
package/src/router/state.ts
CHANGED
|
@@ -32,6 +32,7 @@ interface Row {
|
|
|
32
32
|
context_fetched_at_ms: number;
|
|
33
33
|
compaction_plan: string | null;
|
|
34
34
|
compaction_plan_tokens: number;
|
|
35
|
+
upgrade_deferred_tier: string | null;
|
|
35
36
|
updated_at_ms: number;
|
|
36
37
|
}
|
|
37
38
|
|
|
@@ -53,6 +54,7 @@ function toState(row: Row): ConversationState {
|
|
|
53
54
|
contextFetchedAtMs: row.context_fetched_at_ms,
|
|
54
55
|
compactionPlan: row.compaction_plan === null ? null : (JSON.parse(row.compaction_plan) as CompactionEdit[]),
|
|
55
56
|
compactionPlanTokens: row.compaction_plan_tokens,
|
|
57
|
+
upgradeDeferredTier: row.upgrade_deferred_tier as Tier | null,
|
|
56
58
|
updatedAtMs: row.updated_at_ms,
|
|
57
59
|
};
|
|
58
60
|
}
|
|
@@ -71,10 +73,10 @@ export function createConversationStore(db: Database): ConversationStore {
|
|
|
71
73
|
INSERT INTO conversations (
|
|
72
74
|
key, session_id, turn, current_slug, current_tier, sticky_until_turn,
|
|
73
75
|
last_prompt_tokens, cache_warm_slug, cache_warm_at_ms,
|
|
74
|
-
context_version, context_fetched_at_ms, compaction_plan, compaction_plan_tokens, updated_at_ms
|
|
76
|
+
context_version, context_fetched_at_ms, compaction_plan, compaction_plan_tokens, upgrade_deferred_tier, updated_at_ms
|
|
75
77
|
) VALUES ($key, $sessionId, $turn, $currentSlug, $currentTier, $stickyUntilTurn,
|
|
76
78
|
$lastPromptTokens, $cacheWarmSlug, $cacheWarmAtMs,
|
|
77
|
-
$contextVersion, $contextFetchedAtMs, $compactionPlan, $compactionPlanTokens, $updatedAtMs)
|
|
79
|
+
$contextVersion, $contextFetchedAtMs, $compactionPlan, $compactionPlanTokens, $upgradeDeferredTier, $updatedAtMs)
|
|
78
80
|
ON CONFLICT(key) DO UPDATE SET
|
|
79
81
|
session_id = excluded.session_id,
|
|
80
82
|
turn = excluded.turn,
|
|
@@ -88,6 +90,7 @@ export function createConversationStore(db: Database): ConversationStore {
|
|
|
88
90
|
context_fetched_at_ms = excluded.context_fetched_at_ms,
|
|
89
91
|
compaction_plan = excluded.compaction_plan,
|
|
90
92
|
compaction_plan_tokens = excluded.compaction_plan_tokens,
|
|
93
|
+
upgrade_deferred_tier = excluded.upgrade_deferred_tier,
|
|
91
94
|
updated_at_ms = excluded.updated_at_ms
|
|
92
95
|
`);
|
|
93
96
|
// Read-modify-write in JS lost money: an aborted or failed dispatch is still
|
|
@@ -137,6 +140,7 @@ export function createConversationStore(db: Database): ConversationStore {
|
|
|
137
140
|
$lastPromptTokens: state.lastPromptTokens,
|
|
138
141
|
$compactionPlan: state.compactionPlan === null ? null : JSON.stringify(state.compactionPlan),
|
|
139
142
|
$compactionPlanTokens: state.compactionPlanTokens ?? 0,
|
|
143
|
+
$upgradeDeferredTier: state.upgradeDeferredTier ?? null,
|
|
140
144
|
$cacheWarmSlug: state.cacheWarmSlug,
|
|
141
145
|
$cacheWarmAtMs: state.cacheWarmAtMs,
|
|
142
146
|
$contextVersion: state.contextVersion,
|
package/src/router/types.ts
CHANGED
|
@@ -186,6 +186,13 @@ export interface ConversationState {
|
|
|
186
186
|
compactionPlanTokens?: number;
|
|
187
187
|
/** When that block was fetched, for the staleness TTL. */
|
|
188
188
|
contextFetchedAtMs: number;
|
|
189
|
+
/**
|
|
190
|
+
* The tier a low-confidence upgrade was deferred to on the previous turn
|
|
191
|
+
* (`hysteresis.confirmUpgradesBelowConfidence`), or null. One-turn memory:
|
|
192
|
+
* every turn overwrites it, so a second consecutive upgrade classification
|
|
193
|
+
* confirms the switch and anything else forgets it.
|
|
194
|
+
*/
|
|
195
|
+
upgradeDeferredTier?: Tier | null;
|
|
189
196
|
updatedAtMs: number;
|
|
190
197
|
}
|
|
191
198
|
|
|
@@ -270,6 +277,8 @@ export interface Decision {
|
|
|
270
277
|
explored: Exploration | null;
|
|
271
278
|
/** Budget guard forced a cheaper tier than the classifier asked for. */
|
|
272
279
|
budgetDowngraded: boolean;
|
|
280
|
+
/** A low-confidence upgrade to this tier was deferred one turn to keep the warm model. */
|
|
281
|
+
upgradeDeferred: Tier | null;
|
|
273
282
|
}
|
|
274
283
|
|
|
275
284
|
/** Why a guarded probe rejected an attempt. */
|
package/src/server/turn.ts
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
11
|
import type { CatalogSource } from "../catalog/types.ts";
|
|
12
|
+
import { relevanceQuery, userContent } from "../context/query.ts";
|
|
12
13
|
import type { ContextBridge } from "../context/types.ts";
|
|
13
14
|
import type { RouterConfig } from "../config/types.ts";
|
|
14
15
|
import { estimateUnreportedCache } from "../cost/cache-estimate.ts";
|
|
@@ -159,7 +160,8 @@ export async function runTurn(
|
|
|
159
160
|
pinnedFetchedAtMs: state.contextFetchedAtMs,
|
|
160
161
|
modelSwitching: state.currentSlug !== null && state.currentSlug !== decision.slug,
|
|
161
162
|
retrying: attempt > 0,
|
|
162
|
-
query:
|
|
163
|
+
query: relevanceQuery(req),
|
|
164
|
+
firstFetch: state.contextVersion === null,
|
|
163
165
|
});
|
|
164
166
|
if (pin !== null) {
|
|
165
167
|
contextBlock = pin.block;
|
|
@@ -565,6 +567,8 @@ export async function runTurn(
|
|
|
565
567
|
// shrunk so the prompt cache survives and the savings compound.
|
|
566
568
|
state.compactionPlan = decision.compactionPlan.length > 0 ? decision.compactionPlan : null;
|
|
567
569
|
state.compactionPlanTokens = decision.compactionPlanTokens;
|
|
570
|
+
// One-turn memory: a deferred upgrade is confirmed or forgotten next turn.
|
|
571
|
+
state.upgradeDeferredTier = decision.upgradeDeferred;
|
|
568
572
|
if (usage.cachedTokens > 0 || usage.cacheWriteTokens > 0) {
|
|
569
573
|
// Non-zero cache traffic is direct evidence the upstream cache exists.
|
|
570
574
|
state.cacheWarmSlug = servedSlug ?? decision.slug;
|
|
@@ -589,7 +593,8 @@ export async function runTurn(
|
|
|
589
593
|
// tool-less session is therefore not transcribed: silence beats garbage,
|
|
590
594
|
// because every junk record is re-injected into every later turn.
|
|
591
595
|
if (doxActive && req.tools.length > 0) {
|
|
592
|
-
|
|
596
|
+
// Record the user's words, not omp's wrappers (recap prompts, reminders).
|
|
597
|
+
const userText = userContent(lastUserText(req));
|
|
593
598
|
const turnEnded = finishReason !== "tool_calls";
|
|
594
599
|
log.debug("agentdox record turn", {
|
|
595
600
|
conversationKey: req.conversationKey.slice(0, 8),
|
package/src/util/sqlite.ts
CHANGED
|
@@ -18,7 +18,7 @@ import { mkdirSync } from "node:fs";
|
|
|
18
18
|
import { dirname } from "node:path";
|
|
19
19
|
|
|
20
20
|
/** Bump when a migration is added; guarded below so reopening never regresses it. */
|
|
21
|
-
const USER_VERSION =
|
|
21
|
+
const USER_VERSION = 17;
|
|
22
22
|
|
|
23
23
|
const MIGRATIONS = `
|
|
24
24
|
CREATE TABLE IF NOT EXISTS catalog_cache (
|
|
@@ -243,6 +243,12 @@ ALTER TABLE conversations ADD COLUMN compaction_plan_tokens INTEGER NOT NULL DEF
|
|
|
243
243
|
DELETE FROM token_calibration;
|
|
244
244
|
`;
|
|
245
245
|
|
|
246
|
+
// v17: one-turn memory of a deferred low-confidence upgrade
|
|
247
|
+
// (hysteresis.confirmUpgradesBelowConfidence).
|
|
248
|
+
const MIGRATE_V17 = `
|
|
249
|
+
ALTER TABLE conversations ADD COLUMN upgrade_deferred_tier TEXT;
|
|
250
|
+
`;
|
|
251
|
+
|
|
246
252
|
// v9: benchmark_cache holds the external benchmark feeds (Artificial Analysis,
|
|
247
253
|
// BenchLM) that backfill quality scores OpenRouter leaves unpublished. It is a
|
|
248
254
|
// whole new table, created idempotently by the MIGRATIONS block above, so there
|
|
@@ -288,6 +294,7 @@ export function openDb(path: string): Database {
|
|
|
288
294
|
if (!convCols.some((c) => c.name === "context_version")) db.exec(MIGRATE_V11);
|
|
289
295
|
if (!convCols.some((c) => c.name === "compaction_plan")) db.exec(MIGRATE_V13);
|
|
290
296
|
if (!convCols.some((c) => c.name === "compaction_plan_tokens")) db.exec(MIGRATE_V16);
|
|
297
|
+
if (!convCols.some((c) => c.name === "upgrade_deferred_tier")) db.exec(MIGRATE_V17);
|
|
291
298
|
db.exec(`PRAGMA user_version = ${USER_VERSION}`);
|
|
292
299
|
}
|
|
293
300
|
return db;
|
|
@@ -72,11 +72,23 @@ function input(over: Partial<ContextResolveInput> = {}): ContextResolveInput {
|
|
|
72
72
|
modelSwitching: false,
|
|
73
73
|
retrying: false,
|
|
74
74
|
query: "movement rules",
|
|
75
|
+
firstFetch: true,
|
|
75
76
|
...over,
|
|
76
77
|
};
|
|
77
78
|
}
|
|
78
79
|
|
|
79
80
|
describe("context bridge refresh policy", () => {
|
|
81
|
+
test("recent sessions ride only on a conversation's first block; refreshes ask for none", async () => {
|
|
82
|
+
const client = mkClient();
|
|
83
|
+
const { bridge } = mkBridge(client, { sessionLimit: 6 });
|
|
84
|
+
await bridge.resolve(input({ firstFetch: true }));
|
|
85
|
+
expect(client.lastLimits?.sessionLimit).toBe(6);
|
|
86
|
+
// A refresh (model switch) on a conversation that already had a block.
|
|
87
|
+
await bridge.resolve(input({ firstFetch: false, modelSwitching: true, pinnedVersion: "stale", pinnedFetchedAtMs: 1 }));
|
|
88
|
+
expect(client.lastLimits?.sessionLimit).toBe(0);
|
|
89
|
+
expect(client.assembleCalls).toBe(2);
|
|
90
|
+
});
|
|
91
|
+
|
|
80
92
|
test("fetches on the first turn, then pins without re-fetching", async () => {
|
|
81
93
|
const client = mkClient();
|
|
82
94
|
const { bridge, db } = mkBridge(client);
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import { MAX_QUERY_CHARS, relevanceQuery, userContent } from "../src/context/query.ts";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* The agentdox relevance query must be the user's ask, not omp's wrappers:
|
|
7
|
+
* live blocks were queried with `<chat>`, `<system-reminder>` and an image
|
|
8
|
+
* notice, re-ranking memory against noise on every refresh.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
const msg = (role: "user" | "assistant" | "system" | "tool", text: string) => ({ role, text }) as never;
|
|
12
|
+
|
|
13
|
+
describe("userContent", () => {
|
|
14
|
+
test("strips wrapper elements and keeps the user's words", () => {
|
|
15
|
+
expect(userContent("<system-reminder>\n5 todo items still open.\n</system-reminder>\nplease fix the water tests")).toBe("please fix the water tests");
|
|
16
|
+
expect(userContent("<recap>User stepped away; returning.</recap>")).toBe("");
|
|
17
|
+
expect(userContent("<system-notice>Background job bg_43 completed.</system-notice>")).toBe("");
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
test("an unclosed wrapper at the start is all machinery", () => {
|
|
21
|
+
expect(userContent("<chat>\nassistant said things\nuser said things")).toBe("");
|
|
22
|
+
expect(userContent("<system-reminder>Today: 2026-09-07")).toBe("");
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
test("auto-generated stubs are not content", () => {
|
|
26
|
+
expect(userContent("Attached image(s) from tool result:")).toBe("");
|
|
27
|
+
expect(userContent("(no output)")).toBe("");
|
|
28
|
+
expect(userContent("continue")).toBe("");
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test("ordinary text passes through with whitespace collapsed", () => {
|
|
32
|
+
expect(userContent(" why does the\n\nrouter pick glm? ")).toBe("why does the router pick glm?");
|
|
33
|
+
});
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
describe("relevanceQuery", () => {
|
|
37
|
+
test("takes the last user message with real content, skipping wrapper-only turns", () => {
|
|
38
|
+
const req = {
|
|
39
|
+
messages: [
|
|
40
|
+
msg("system", "You are omp."),
|
|
41
|
+
msg("user", "add a /router command with reports"),
|
|
42
|
+
msg("assistant", "done"),
|
|
43
|
+
msg("user", "<system-reminder>5 todo items still open.</system-reminder>"),
|
|
44
|
+
msg("tool", "Attached image(s) from tool result:"),
|
|
45
|
+
msg("user", "Attached image(s) from tool result:"),
|
|
46
|
+
],
|
|
47
|
+
};
|
|
48
|
+
expect(relevanceQuery(req)).toBe("add a /router command with reports");
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
test("prefers the newest real ask over the opening one", () => {
|
|
52
|
+
const req = { messages: [msg("user", "first ask"), msg("assistant", "ok"), msg("user", "<recap>x</recap> now fix the report window")] };
|
|
53
|
+
expect(relevanceQuery(req)).toBe("now fix the report window");
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
test("caps long asks so the block header stays a line", () => {
|
|
57
|
+
const long = "x".repeat(5_000);
|
|
58
|
+
const q = relevanceQuery({ messages: [msg("user", long)] });
|
|
59
|
+
expect(q.length).toBe(MAX_QUERY_CHARS);
|
|
60
|
+
expect(q.endsWith("…")).toBe(true);
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
test("nothing usable yields an empty query", () => {
|
|
64
|
+
expect(relevanceQuery({ messages: [msg("user", "<chat>\nlog"), msg("assistant", "hi")] })).toBe("");
|
|
65
|
+
expect(relevanceQuery({ messages: [] })).toBe("");
|
|
66
|
+
});
|
|
67
|
+
});
|
package/test/failover.test.ts
CHANGED
|
@@ -70,7 +70,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
70
70
|
escalateOnLengthStop: false,
|
|
71
71
|
...escalation,
|
|
72
72
|
},
|
|
73
|
-
hysteresis: { holdTurns: 2, holdTurnsAfterEscalation: 4, switchMargin: 1.5, cacheWarmTtlMs: 600_000, maxDowngradePerTurn: 1, breakHoldOnMechanical: false, switchHorizonTurns: 1 },
|
|
73
|
+
hysteresis: { holdTurns: 2, holdTurnsAfterEscalation: 4, switchMargin: 1.5, cacheWarmTtlMs: 600_000, maxDowngradePerTurn: 1, breakHoldOnMechanical: false, switchHorizonTurns: 1, confirmUpgradesBelowConfidence: 0.6 },
|
|
74
74
|
exploration: { enabled: false, rates: {}, stickyPolicy: "never", holdTurns: { enabled: false, values: [2, 3, 4] } },
|
|
75
75
|
cache: { injectBreakpoints: true, maxBreakpoints: 4, minPromptTokens: 1024, milestoneTokens: 20_000 },
|
|
76
76
|
context: { enabled: false, baseUrl: "", token: "", defaultScope: "", timeoutMs: 3_000, maxStalenessMs: 900_000, maxBlockChars: 24_000, memoryLimit: 8, docsLimit: 2, sessionLimit: 6, briefChars: 0, recordTurns: false, maxQueue: 64 },
|
|
@@ -160,6 +160,7 @@ function mkDecision(tier: Tier, slug: string, probe: Partial<ProbePlan> = {}): D
|
|
|
160
160
|
reasons: ["test decision"],
|
|
161
161
|
explored: null,
|
|
162
162
|
budgetDowngraded: false,
|
|
163
|
+
upgradeDeferred: null,
|
|
163
164
|
};
|
|
164
165
|
}
|
|
165
166
|
|
package/test/select.test.ts
CHANGED
|
@@ -996,3 +996,57 @@ describe("compaction.replanGrowthRatio (review 2026-09-05 §7)", () => {
|
|
|
996
996
|
});
|
|
997
997
|
});
|
|
998
998
|
|
|
999
|
+
|
|
1000
|
+
describe("hysteresis.confirmUpgradesBelowConfidence", () => {
|
|
1001
|
+
// A low-confidence heuristic upgrade from a warm model waits one turn.
|
|
1002
|
+
// Measured: 65 of 67 moderate→hard upgrades in a week bounced back within
|
|
1003
|
+
// 3 turns, each paying a cold hard-tier read of a ~120k prompt.
|
|
1004
|
+
const warmSlug = run({ tier: "moderate" }).slug;
|
|
1005
|
+
function upgrade(opts: { confidence?: number; source?: "heuristic" | "escalation"; st?: Partial<ConversationState>; cfg?: RouterConfig; lastToolFailed?: boolean }) {
|
|
1006
|
+
const cfg = opts.cfg ?? BASE;
|
|
1007
|
+
const req = request("now rework the whole scheduler");
|
|
1008
|
+
const base = extractFeatures(req, 120_000);
|
|
1009
|
+
const features = opts.lastToolFailed === true ? { ...base, lastToolFailed: true } : base;
|
|
1010
|
+
const heuristic = scoreHeuristic(features, cfg);
|
|
1011
|
+
return select({
|
|
1012
|
+
req,
|
|
1013
|
+
features,
|
|
1014
|
+
classification: { ...heuristic, tier: "hard", confidence: opts.confidence ?? 0.45, source: opts.source ?? "heuristic" },
|
|
1015
|
+
profile: PROFILE,
|
|
1016
|
+
state: state({ turn: 4, currentTier: "moderate", currentSlug: warmSlug, cacheWarmSlug: warmSlug, cacheWarmAtMs: Date.now(), lastPromptTokens: 110_000, ...opts.st }),
|
|
1017
|
+
snapshot: SNAPSHOT,
|
|
1018
|
+
ledger: null,
|
|
1019
|
+
cfg,
|
|
1020
|
+
nowMs: Date.now(),
|
|
1021
|
+
});
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
test("a low-confidence upgrade from a warm model is deferred to the held tier", () => {
|
|
1025
|
+
const d = upgrade({});
|
|
1026
|
+
expect(d.tier).toBe("moderate");
|
|
1027
|
+
expect(d.upgradeDeferred).toBe("hard");
|
|
1028
|
+
expect(d.reasons.some((r) => r.includes("upgrade moderate → hard deferred one turn"))).toBe(true);
|
|
1029
|
+
});
|
|
1030
|
+
|
|
1031
|
+
test("a second consecutive upgrade classification confirms it", () => {
|
|
1032
|
+
const d = upgrade({ st: { upgradeDeferredTier: "hard" } });
|
|
1033
|
+
expect(d.tier).toBe("hard");
|
|
1034
|
+
expect(d.upgradeDeferred).toBeNull();
|
|
1035
|
+
expect(d.reasons.some((r) => r.includes("upgrade moderate → hard confirmed"))).toBe(true);
|
|
1036
|
+
});
|
|
1037
|
+
|
|
1038
|
+
test("confident classifications, cold caches, escalations, failing tools and the off switch all upgrade at once", () => {
|
|
1039
|
+
expect(upgrade({ confidence: 0.9 }).tier).toBe("hard");
|
|
1040
|
+
expect(upgrade({ st: { cacheWarmAtMs: Date.now() - 3_600_000 } }).tier).toBe("hard");
|
|
1041
|
+
expect(upgrade({ source: "escalation" }).tier).toBe("hard");
|
|
1042
|
+
expect(upgrade({ lastToolFailed: true }).tier).toBe("hard");
|
|
1043
|
+
const off: RouterConfig = { ...BASE, hysteresis: { ...BASE.hysteresis, confirmUpgradesBelowConfidence: 0 } };
|
|
1044
|
+
expect(upgrade({ cfg: off }).tier).toBe("hard");
|
|
1045
|
+
for (const d of [upgrade({ confidence: 0.9 }), upgrade({ source: "escalation" })]) expect(d.upgradeDeferred).toBeNull();
|
|
1046
|
+
});
|
|
1047
|
+
|
|
1048
|
+
test("a downgrade or a same-tier turn is never deferred", () => {
|
|
1049
|
+
const d = run({ tier: "simple", st: state({ turn: 4, currentTier: "moderate", currentSlug: warmSlug, cacheWarmSlug: warmSlug, cacheWarmAtMs: Date.now() }) });
|
|
1050
|
+
expect(d.upgradeDeferred).toBeNull();
|
|
1051
|
+
});
|
|
1052
|
+
});
|
|
@@ -3,6 +3,7 @@ import { describe, expect, test } from "bun:test";
|
|
|
3
3
|
import { loadConfig } from "../src/config/load.ts";
|
|
4
4
|
import { createLedger, LATENCY_WINDOW_ROWS } from "../src/cost/ledger.ts";
|
|
5
5
|
import { EMPTY_USAGE, type LedgerEntry } from "../src/cost/types.ts";
|
|
6
|
+
import { createConversationStore } from "../src/router/state.ts";
|
|
6
7
|
import { openDb } from "../src/util/sqlite.ts";
|
|
7
8
|
|
|
8
9
|
const cfg = loadConfig({});
|
|
@@ -244,11 +245,24 @@ describe("v4 migration", () => {
|
|
|
244
245
|
}
|
|
245
246
|
});
|
|
246
247
|
|
|
247
|
-
test("
|
|
248
|
+
test("a deferred upgrade tier survives a save/load round trip", () => {
|
|
249
|
+
const db = openDb(":memory:");
|
|
250
|
+
const store = createConversationStore(db);
|
|
251
|
+
const st = store.load("conv-defer");
|
|
252
|
+
st.upgradeDeferredTier = "hard";
|
|
253
|
+
store.save(st);
|
|
254
|
+
expect(store.load("conv-defer").upgradeDeferredTier).toBe("hard");
|
|
255
|
+
st.upgradeDeferredTier = null;
|
|
256
|
+
store.save(st);
|
|
257
|
+
expect(store.load("conv-defer").upgradeDeferredTier).toBeNull();
|
|
258
|
+
db.close();
|
|
259
|
+
});
|
|
260
|
+
|
|
261
|
+
test("schema is at user_version 17", () => {
|
|
248
262
|
const db = openDb(":memory:");
|
|
249
263
|
try {
|
|
250
264
|
const row = db.query("PRAGMA user_version").get() as { user_version: number };
|
|
251
|
-
expect(row.user_version).toBe(
|
|
265
|
+
expect(row.user_version).toBe(17);
|
|
252
266
|
} finally {
|
|
253
267
|
db.close();
|
|
254
268
|
}
|
package/test/turn.test.ts
CHANGED
|
@@ -70,7 +70,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
70
70
|
escalateOnLengthStop: false,
|
|
71
71
|
...escalation,
|
|
72
72
|
},
|
|
73
|
-
hysteresis: { holdTurns: 2, holdTurnsAfterEscalation: 4, switchMargin: 1.5, cacheWarmTtlMs: 600_000, maxDowngradePerTurn: 1, breakHoldOnMechanical: false, switchHorizonTurns: 1 },
|
|
73
|
+
hysteresis: { holdTurns: 2, holdTurnsAfterEscalation: 4, switchMargin: 1.5, cacheWarmTtlMs: 600_000, maxDowngradePerTurn: 1, breakHoldOnMechanical: false, switchHorizonTurns: 1, confirmUpgradesBelowConfidence: 0.6 },
|
|
74
74
|
exploration: { enabled: false, rates: {}, stickyPolicy: "never", holdTurns: { enabled: false, values: [2, 3, 4] } },
|
|
75
75
|
cache: { injectBreakpoints: true, maxBreakpoints: 4, minPromptTokens: 1024, milestoneTokens: 20_000 },
|
|
76
76
|
context: { enabled: false, baseUrl: "", token: "", defaultScope: "", timeoutMs: 3_000, maxStalenessMs: 900_000, maxBlockChars: 24_000, memoryLimit: 8, docsLimit: 2, sessionLimit: 6, briefChars: 0, recordTurns: false, maxQueue: 64 },
|
|
@@ -160,6 +160,7 @@ function mkDecision(tier: Tier, slug: string, probe: Partial<ProbePlan> = {}): D
|
|
|
160
160
|
reasons: ["test decision"],
|
|
161
161
|
explored: null,
|
|
162
162
|
budgetDowngraded: false,
|
|
163
|
+
upgradeDeferred: null,
|
|
163
164
|
};
|
|
164
165
|
}
|
|
165
166
|
|