auto-model-router 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.omp-plugin/marketplace.json +2 -2
- package/README.md +20 -2
- package/omp-extension/report-logic.ts +5 -2
- package/package.json +1 -1
- package/src/cli/config-wizard.ts +1 -0
- package/src/config/defaults.ts +3 -0
- package/src/config/schema.ts +1 -0
- package/src/config/types.ts +6 -0
- package/src/cost/cache-estimate.ts +52 -0
- package/src/cost/report.ts +16 -5
- package/src/cost/types.ts +6 -0
- package/src/server/http.ts +3 -0
- package/src/server/turn.ts +17 -1
- package/src/upstream/ollama-usage.ts +65 -1
- package/test/cache-estimate.test.ts +48 -0
- package/test/failover.test.ts +1 -1
- package/test/ollama.test.ts +39 -5
- package/test/report-hub.test.ts +2 -0
- package/test/report-logic.test.ts +2 -1
- package/test/report.test.ts +16 -0
- package/test/turn.test.ts +50 -1
- package/tools/recompute-ollama-cache.ts +129 -0
|
@@ -7,14 +7,14 @@
|
|
|
7
7
|
},
|
|
8
8
|
"metadata": {
|
|
9
9
|
"description": "auto-model-router: a local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter",
|
|
10
|
-
"version": "0.3.
|
|
10
|
+
"version": "0.3.2",
|
|
11
11
|
"pluginRoot": "."
|
|
12
12
|
},
|
|
13
13
|
"plugins": [
|
|
14
14
|
{
|
|
15
15
|
"name": "auto-model-router",
|
|
16
16
|
"description": "Local cost/complexity-aware model router for Oh My Pi, backed by OpenRouter. Runs in-process, routes per turn by price and task complexity, with budget caps, mid-stream escalation, and cache-aware hysteresis.",
|
|
17
|
-
"version": "0.3.
|
|
17
|
+
"version": "0.3.2",
|
|
18
18
|
"author": {
|
|
19
19
|
"name": "drewappling",
|
|
20
20
|
"email": "drewappling@gmail.com"
|
package/README.md
CHANGED
|
@@ -508,8 +508,10 @@ What it shows, for the window:
|
|
|
508
508
|
Spend follows the ledger's rule — the provider's reported cost when it gave
|
|
509
509
|
one, else the usage-priced figure the router computed, else the forecast.
|
|
510
510
|
Speed uses only clean streamed rows (TTFT recorded, no error); tokens/s is
|
|
511
|
-
completion tokens over time after first token. Ollama Cloud
|
|
512
|
-
|
|
511
|
+
completion tokens over time after first token. Ollama Cloud caches prompt
|
|
512
|
+
prefixes and bills them at its cached rate but reports no count, so the
|
|
513
|
+
router estimates it (see [Ollama Cloud](#ollama-cloud)); cache rates that
|
|
514
|
+
include such rows are shown with a `~`.
|
|
513
515
|
|
|
514
516
|
## Configuring the router
|
|
515
517
|
|
|
@@ -621,6 +623,7 @@ an OpenRouter sibling). See [Ollama Cloud](#ollama-cloud) below.
|
|
|
621
623
|
| `usagePollMs` | `600000` (10 min) | How often plan usage is re-read. `0` disables it (static bias). Needs the API key; the daemon path without one keeps a static bias. |
|
|
622
624
|
| `quotaCooldownMs` | `900000` | Route around Ollama this long after a 402 (credits exhausted). |
|
|
623
625
|
| `rateLimitCooldownMs` | `60000` | Route around Ollama this long after a 429 (concurrency cap). |
|
|
626
|
+
| `planCreditsUsd` | `0` | Override for the plan's included monthly credits. `0` detects the plan from ollama.com (`POST /api/me`) and applies its published allowance (Pro $60, Max $300), so `/health` and `/router status` show ollama.com's reading as dollars next to the ledger's figure. Set it for a plan the router does not know. |
|
|
624
627
|
|
|
625
628
|
### `tiers` — per-tier economic envelope
|
|
626
629
|
|
|
@@ -809,6 +812,21 @@ What happens once it is on:
|
|
|
809
812
|
is what lets it serve `simple` and above. `ollama.twins` pins a match the
|
|
810
813
|
name normaliser cannot make; an unmatched model is unscored and serves only
|
|
811
814
|
`trivial`.
|
|
815
|
+
- **Cached prefixes are estimated, not reported.** ollama.com caches prompt
|
|
816
|
+
prefixes automatically and bills them at the published cached-input rate,
|
|
817
|
+
but neither its OpenAI-compatible usage nor the native API carries a cached
|
|
818
|
+
token count. Measured 2026-09-07: twelve identical 162k-token requests to
|
|
819
|
+
`glm-5.3-flash` moved the plan meter by $0.06 against $0.29 at the full
|
|
820
|
+
input rate, and repeats answered in ~1.5 s. Pricing every token fresh had
|
|
821
|
+
overstated a week of Ollama spend 3.7x ($23.01 booked, $6.24 metered). The
|
|
822
|
+
router now applies its own warm-cache rule to Ollama turns: when the same
|
|
823
|
+
model served the previous turn within `hysteresis.cacheWarmTtlMs`, the
|
|
824
|
+
previous prompt is taken as the cached prefix and priced at the cached
|
|
825
|
+
rate; a first turn, a switch, or a longer gap is priced cold. The ledger
|
|
826
|
+
flags these rows (`usage.cachedEstimated`) and reports show their cache
|
|
827
|
+
rate as `~N%`. `/router status` shows ollama.com's own dollar reading as
|
|
828
|
+
the cross-check: the plan is read from `POST /api/me` and its published
|
|
829
|
+
allowance applied (`planCreditsUsd` overrides it).
|
|
812
830
|
- **Same economics, same failover.** Candidates from both providers are ranked
|
|
813
831
|
together; `costBias` tilts the comparison while a plan's included credits
|
|
814
832
|
would otherwise go unused. **Credit-aware by default:** the router reads the
|
|
@@ -74,7 +74,8 @@ export interface HealthSnapshot {
|
|
|
74
74
|
available?: boolean;
|
|
75
75
|
cooldownUntilMs?: number | null;
|
|
76
76
|
lastTrip?: { kind?: string; atMs?: number; message?: string } | null;
|
|
77
|
-
usage?: { monthlyUsedFraction?: number | null; activityCostUsd?: number | null; fetchedAtMs?: number | null } | null;
|
|
77
|
+
usage?: { monthlyUsedFraction?: number | null; activityCostUsd?: number | null; plan?: string | null; fetchedAtMs?: number | null } | null;
|
|
78
|
+
meter?: { usedUsd?: number; creditsUsd?: number; plan?: string | null } | null;
|
|
78
79
|
costBias?: { configured?: number; effective?: number; biasUntilUsage?: number };
|
|
79
80
|
} | null;
|
|
80
81
|
catalog?: {
|
|
@@ -104,7 +105,9 @@ export function renderStatus(baseUrl: string, h: HealthSnapshot, nowMs = Date.no
|
|
|
104
105
|
} else {
|
|
105
106
|
const avail = o.available === true ? "available" : `COOLING DOWN${o.cooldownUntilMs ? ` until ${new Date(o.cooldownUntilMs).toLocaleTimeString()}` : ""}`;
|
|
106
107
|
const frac = o.usage?.monthlyUsedFraction;
|
|
107
|
-
const
|
|
108
|
+
const meter = o.meter !== undefined && o.meter !== null && o.meter.usedUsd !== undefined ? ` ($${o.meter.usedUsd.toFixed(2)} of $${o.meter.creditsUsd ?? "?"})` : "";
|
|
109
|
+
const planName = o.meter?.plan ?? o.usage?.plan ?? null;
|
|
110
|
+
const usage = frac === undefined || frac === null ? "plan usage unknown" : `${planName === null ? "plan" : `${planName} plan`} usage ${(frac * 100).toFixed(1)}%${meter}`;
|
|
108
111
|
const bias = o.costBias === undefined ? "" : ` · cost bias ×${o.costBias.effective ?? o.costBias.configured ?? 1} (until ${((o.costBias.biasUntilUsage ?? 1) * 100).toFixed(0)}%)`;
|
|
109
112
|
const trip = o.lastTrip !== undefined && o.lastTrip !== null ? ` · last trip ${o.lastTrip.kind ?? "?"}${o.lastTrip.atMs ? ` ${mins(nowMs - o.lastTrip.atMs)} ago` : ""}` : "";
|
|
110
113
|
out.push(`ollama cloud: ${o.models ?? 0} models · ${avail} · key ${o.apiKeySource ?? "?"} · ${usage}${bias}${trip}`);
|
package/package.json
CHANGED
package/src/cli/config-wizard.ts
CHANGED
|
@@ -143,6 +143,7 @@ export const WIZARD_SECTIONS: readonly SectionSpec[] = [
|
|
|
143
143
|
{ path: "ollama.usagePollMs", label: "Plan usage poll", kind: "number", min: 0, hint: "ms, 0=off" },
|
|
144
144
|
{ path: "ollama.quotaCooldownMs", label: "Quota (402) cooldown", kind: "number", min: 0, hint: "ms" },
|
|
145
145
|
{ path: "ollama.rateLimitCooldownMs", label: "Rate-limit (429) cooldown", kind: "number", min: 0, hint: "ms" },
|
|
146
|
+
{ path: "ollama.planCreditsUsd", label: "Plan credits per month $", kind: "number", min: 0, hint: "0=detect plan (Pro 60, Max 300)" },
|
|
146
147
|
],
|
|
147
148
|
},
|
|
148
149
|
{
|
package/src/config/defaults.ts
CHANGED
|
@@ -56,6 +56,9 @@ export const DEFAULT_CONFIG: RouterConfig = {
|
|
|
56
56
|
quotaCooldownMs: 15 * 60 * 1000,
|
|
57
57
|
// Concurrency caps clear as soon as an in-flight request finishes.
|
|
58
58
|
rateLimitCooldownMs: 60 * 1000,
|
|
59
|
+
// ollama.com reports usage as a share of the plan; the dollar figure on
|
|
60
|
+
// its dashboard is that share × the plan's credits. Unknown until set.
|
|
61
|
+
planCreditsUsd: 0,
|
|
59
62
|
},
|
|
60
63
|
benchmarks: {
|
|
61
64
|
// Keyless BenchLM alone fills real gaps, so this is on by default; the AA
|
package/src/config/schema.ts
CHANGED
|
@@ -53,6 +53,7 @@ const ollama = z.strictObject({
|
|
|
53
53
|
usagePollMs: z.number().nonnegative().optional(),
|
|
54
54
|
quotaCooldownMs: z.number().nonnegative().optional(),
|
|
55
55
|
rateLimitCooldownMs: z.number().nonnegative().optional(),
|
|
56
|
+
planCreditsUsd: z.number().nonnegative().optional(),
|
|
56
57
|
});
|
|
57
58
|
|
|
58
59
|
const benchmarks = z.strictObject({
|
package/src/config/types.ts
CHANGED
|
@@ -114,6 +114,12 @@ export interface OllamaConfig {
|
|
|
114
114
|
quotaCooldownMs: number;
|
|
115
115
|
/** How long to route around Ollama after a 429 (concurrency cap), ms. */
|
|
116
116
|
rateLimitCooldownMs: number;
|
|
117
|
+
/**
|
|
118
|
+
* Override for the plan's included monthly credits, USD. 0 (default) reads
|
|
119
|
+
* the plan from ollama.com (`POST /api/me`) and applies its published
|
|
120
|
+
* allowance (Pro 60, Max 300); set this for a plan the router does not know.
|
|
121
|
+
*/
|
|
122
|
+
planCreditsUsd: number;
|
|
117
123
|
}
|
|
118
124
|
|
|
119
125
|
/**
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cache accounting for upstreams that cache but do not say so.
|
|
3
|
+
*
|
|
4
|
+
* Ollama Cloud caches prompt prefixes automatically and bills them at its
|
|
5
|
+
* published cached-input rate, but neither its OpenAI-compatible usage nor
|
|
6
|
+
* the native API carries a cached-token count. Measured 2026-09-07: twelve
|
|
7
|
+
* identical 162k-token requests to glm-5.3-flash moved the plan meter by
|
|
8
|
+
* $0.06 against $0.29 at the full input rate, and repeats answered in ~1.5s.
|
|
9
|
+
* Pricing every token fresh overstated a week of Ollama spend 3.7x ($23.01
|
|
10
|
+
* booked against $6.24 metered).
|
|
11
|
+
*
|
|
12
|
+
* The estimate reuses the rule the router already applies when it decides
|
|
13
|
+
* whether staying on a model keeps a warm cache (select.ts step 4): the
|
|
14
|
+
* previous turn's prompt is the cached prefix when the same model served the
|
|
15
|
+
* previous turn within `cacheWarmTtlMs`. Tokens beyond that prefix are fresh.
|
|
16
|
+
* A conversation's first turn, a model switch, or an idle gap past the TTL
|
|
17
|
+
* count as fully cold — the same assumption the stay/switch comparison makes.
|
|
18
|
+
*
|
|
19
|
+
* Estimated counts are flagged (`cachedEstimated`) so reports can show them
|
|
20
|
+
* as estimates and never be mistaken for provider-reported figures.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import type { UsageCounts } from "./types.ts";
|
|
24
|
+
|
|
25
|
+
export interface CacheEstimateContext {
|
|
26
|
+
/** Model that served the previous turn of this conversation, if any. */
|
|
27
|
+
previousSlug: string | null;
|
|
28
|
+
/** Prompt tokens of that previous turn (0 when unknown). */
|
|
29
|
+
previousPromptTokens: number;
|
|
30
|
+
/** When the previous turn settled, ms epoch. */
|
|
31
|
+
previousAtMs: number;
|
|
32
|
+
/** Model that served this turn. */
|
|
33
|
+
servedSlug: string;
|
|
34
|
+
nowMs: number;
|
|
35
|
+
/** How long a warm prefix is assumed to survive; `hysteresis.cacheWarmTtlMs`. */
|
|
36
|
+
cacheWarmTtlMs: number;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Returns usage with `cachedTokens` filled in when the upstream reported no
|
|
41
|
+
* cache activity and the router's warm-cache rule says a prefix was warm.
|
|
42
|
+
* Reported cache counts (either field non-zero) are left untouched.
|
|
43
|
+
*/
|
|
44
|
+
export function estimateUnreportedCache(usage: UsageCounts, ctx: CacheEstimateContext): UsageCounts {
|
|
45
|
+
if (usage.cachedTokens > 0 || usage.cacheWriteTokens > 0) return usage;
|
|
46
|
+
if (usage.promptTokens <= 0) return usage;
|
|
47
|
+
if (ctx.previousSlug === null || ctx.previousSlug !== ctx.servedSlug) return usage;
|
|
48
|
+
if (ctx.previousPromptTokens <= 0) return usage;
|
|
49
|
+
if (ctx.cacheWarmTtlMs <= 0 || ctx.nowMs - ctx.previousAtMs > ctx.cacheWarmTtlMs) return usage;
|
|
50
|
+
const cachedTokens = Math.min(ctx.previousPromptTokens, usage.promptTokens);
|
|
51
|
+
return { ...usage, cachedTokens, cachedEstimated: true };
|
|
52
|
+
}
|
package/src/cost/report.ts
CHANGED
|
@@ -24,6 +24,8 @@ export interface ReportTotals {
|
|
|
24
24
|
aborted: number;
|
|
25
25
|
/** Turns that switched model mid-conversation. */
|
|
26
26
|
modelSwitches: number;
|
|
27
|
+
/** Any row in the window carries an estimated cache count. */
|
|
28
|
+
cacheEstimated: boolean;
|
|
27
29
|
}
|
|
28
30
|
|
|
29
31
|
export interface ReportRow {
|
|
@@ -33,6 +35,8 @@ export interface ReportRow {
|
|
|
33
35
|
/** Share of window spend, 0-1. */
|
|
34
36
|
share: number;
|
|
35
37
|
cacheHitRate: number;
|
|
38
|
+
/** Some rows carry router-estimated cache counts (Ollama); the rate is then an estimate. */
|
|
39
|
+
cacheEstimated: boolean;
|
|
36
40
|
avgPromptTokens: number;
|
|
37
41
|
/** Mean time to first token, ms, over streamed non-error rows; null without samples. */
|
|
38
42
|
avgTtftMs: number | null;
|
|
@@ -75,12 +79,14 @@ const CT = "json_extract(usage, '$.cachedTokens')";
|
|
|
75
79
|
const COMP = "json_extract(usage, '$.completionTokens')";
|
|
76
80
|
const PROVIDER = "CASE WHEN slug LIKE 'ollama/%' THEN 'ollama' ELSE 'openrouter' END";
|
|
77
81
|
const STREAMED = "ttft_ms IS NOT NULL AND ttft_ms > 0 AND error IS NULL";
|
|
82
|
+
const EST = "json_extract(usage, '$.cachedEstimated') = 1";
|
|
78
83
|
|
|
79
84
|
const ROW_SELECT = `
|
|
80
85
|
COUNT(*) AS dispatches,
|
|
81
86
|
COALESCE(SUM(${USD}), 0) AS spend,
|
|
82
87
|
COALESCE(SUM(${PT}), 0) AS prompt_tokens,
|
|
83
88
|
COALESCE(SUM(${CT}), 0) AS cached_tokens,
|
|
89
|
+
SUM(CASE WHEN ${EST} THEN 1 ELSE 0 END) AS estimated_rows,
|
|
84
90
|
COALESCE(AVG(${PT}), 0) AS avg_prompt_tokens,
|
|
85
91
|
AVG(CASE WHEN ${STREAMED} THEN ttft_ms END) AS ttft_ms,
|
|
86
92
|
SUM(CASE WHEN ${STREAMED} AND latency_ms > ttft_ms AND ${COMP} > 0 THEN ${COMP} END) AS ctok_sum,
|
|
@@ -94,6 +100,7 @@ interface RawRow {
|
|
|
94
100
|
spend: number;
|
|
95
101
|
prompt_tokens: number;
|
|
96
102
|
cached_tokens: number;
|
|
103
|
+
estimated_rows: number | null;
|
|
97
104
|
avg_prompt_tokens: number;
|
|
98
105
|
ttft_ms: number | null;
|
|
99
106
|
ctok_sum: number | null;
|
|
@@ -109,6 +116,7 @@ function toRow(r: RawRow, windowSpend: number): ReportRow {
|
|
|
109
116
|
spendUsd: r.spend,
|
|
110
117
|
share: windowSpend > 0 ? r.spend / windowSpend : 0,
|
|
111
118
|
cacheHitRate: r.prompt_tokens > 0 ? r.cached_tokens / r.prompt_tokens : 0,
|
|
119
|
+
cacheEstimated: (r.estimated_rows ?? 0) > 0,
|
|
112
120
|
avgPromptTokens: Math.round(r.avg_prompt_tokens),
|
|
113
121
|
avgTtftMs: r.ttft_ms === null ? null : Math.round(r.ttft_ms),
|
|
114
122
|
tokensPerSec: r.elapsed_ms !== null && r.elapsed_ms > 0 && r.ctok_sum !== null ? (r.ctok_sum * 1000) / r.elapsed_ms : null,
|
|
@@ -137,6 +145,7 @@ export function buildUsageReport(db: Database, opts: { windowDays: number; harne
|
|
|
137
145
|
COALESCE(SUM(${PT}), 0) AS prompt_tokens,
|
|
138
146
|
COALESCE(SUM(${CT}), 0) AS cached_tokens,
|
|
139
147
|
COALESCE(SUM(${COMP}), 0) AS completion_tokens,
|
|
148
|
+
SUM(CASE WHEN ${EST} THEN 1 ELSE 0 END) AS estimated_rows,
|
|
140
149
|
SUM(CASE WHEN escalation_signal IS NOT NULL THEN 1 ELSE 0 END) AS escalations,
|
|
141
150
|
SUM(CASE WHEN instr(reasons, 'failover:') > 0 THEN 1 ELSE 0 END) AS failovers,
|
|
142
151
|
SUM(CASE WHEN error IS NOT NULL THEN 1 ELSE 0 END) AS errors,
|
|
@@ -150,6 +159,7 @@ export function buildUsageReport(db: Database, opts: { windowDays: number; harne
|
|
|
150
159
|
prompt_tokens: number;
|
|
151
160
|
cached_tokens: number;
|
|
152
161
|
completion_tokens: number;
|
|
162
|
+
estimated_rows: number | null;
|
|
153
163
|
escalations: number | null;
|
|
154
164
|
failovers: number | null;
|
|
155
165
|
errors: number | null;
|
|
@@ -228,6 +238,7 @@ export function buildUsageReport(db: Database, opts: { windowDays: number; harne
|
|
|
228
238
|
errors: t.errors ?? 0,
|
|
229
239
|
aborted: t.aborted ?? 0,
|
|
230
240
|
modelSwitches: switches,
|
|
241
|
+
cacheEstimated: (t.estimated_rows ?? 0) > 0,
|
|
231
242
|
},
|
|
232
243
|
providers,
|
|
233
244
|
models,
|
|
@@ -241,7 +252,7 @@ export function buildUsageReport(db: Database, opts: { windowDays: number; harne
|
|
|
241
252
|
// ---------------------------------------------------------------------------
|
|
242
253
|
|
|
243
254
|
const usd = (v: number): string => (v >= 1 ? `$${v.toFixed(2)}` : `$${v.toFixed(4)}`);
|
|
244
|
-
const pct = (v: number): string => `${(v * 100).toFixed(0)}%`;
|
|
255
|
+
const pct = (v: number, estimated = false): string => `${estimated ? "~" : ""}${(v * 100).toFixed(0)}%`;
|
|
245
256
|
const num = (v: number): string => v.toLocaleString("en-US");
|
|
246
257
|
const ms = (v: number | null): string => (v === null ? "–" : v >= 1000 ? `${(v / 1000).toFixed(1)}s` : `${v}ms`);
|
|
247
258
|
const tps = (v: number | null): string => (v === null ? "–" : `${v.toFixed(0)} tok/s`);
|
|
@@ -281,7 +292,7 @@ export function reportView(r: UsageReport, opts: { maxModels?: number } = {}): R
|
|
|
281
292
|
const heading = `last ${r.windowDays}d${r.harnessId === "" ? "" : ` · harness ${r.harnessId}`} · ${new Date(r.generatedAtMs).toISOString().slice(0, 16).replace("T", " ")}Z`;
|
|
282
293
|
const summary = [
|
|
283
294
|
`spend ${usd(t.spendUsd)} over ${num(t.dispatches)} dispatches in ${num(t.conversations)} conversations · ${usd(t.dispatches > 0 ? t.spendUsd / t.dispatches : 0)}/dispatch`,
|
|
284
|
-
`prompt ${num(t.promptTokens)} tok (cache hit ${pct(t.cacheHitRate)}) · completion ${num(t.completionTokens)} tok · switches ${num(t.modelSwitches)} · escalations ${num(t.escalations)} · failovers ${num(t.failovers)} · errors ${num(t.errors)} (${num(t.aborted)} aborted)`,
|
|
295
|
+
`prompt ${num(t.promptTokens)} tok (cache hit ${pct(t.cacheHitRate, t.cacheEstimated)}) · completion ${num(t.completionTokens)} tok · switches ${num(t.modelSwitches)} · escalations ${num(t.escalations)} · failovers ${num(t.failovers)} · errors ${num(t.errors)} (${num(t.aborted)} aborted)`,
|
|
285
296
|
];
|
|
286
297
|
const tables: ReportTable[] = [];
|
|
287
298
|
if (r.providers.length > 0) {
|
|
@@ -289,7 +300,7 @@ export function reportView(r: UsageReport, opts: { maxModels?: number } = {}): R
|
|
|
289
300
|
id: "providers",
|
|
290
301
|
title: "providers",
|
|
291
302
|
headers: ["provider", "dispatches", "spend", "share", "cache", "ttft", "speed", "esc", "err"],
|
|
292
|
-
rows: r.providers.map((p) => [p.key, num(p.dispatches), usd(p.spendUsd), pct(p.share), pct(p.cacheHitRate), ms(p.avgTtftMs), tps(p.tokensPerSec), num(p.escalations), num(p.errors)]),
|
|
303
|
+
rows: r.providers.map((p) => [p.key, num(p.dispatches), usd(p.spendUsd), pct(p.share), pct(p.cacheHitRate, p.cacheEstimated), ms(p.avgTtftMs), tps(p.tokensPerSec), num(p.escalations), num(p.errors)]),
|
|
293
304
|
});
|
|
294
305
|
}
|
|
295
306
|
if (r.models.length > 0) {
|
|
@@ -302,7 +313,7 @@ export function reportView(r: UsageReport, opts: { maxModels?: number } = {}): R
|
|
|
302
313
|
num(m.dispatches),
|
|
303
314
|
usd(m.spendUsd),
|
|
304
315
|
pct(m.share),
|
|
305
|
-
pct(m.cacheHitRate),
|
|
316
|
+
pct(m.cacheHitRate, m.cacheEstimated),
|
|
306
317
|
ms(m.avgTtftMs),
|
|
307
318
|
tps(m.tokensPerSec),
|
|
308
319
|
Object.entries(m.tiers)
|
|
@@ -317,7 +328,7 @@ export function reportView(r: UsageReport, opts: { maxModels?: number } = {}): R
|
|
|
317
328
|
id: "tiers",
|
|
318
329
|
title: "tiers",
|
|
319
330
|
headers: ["tier", "dispatches", "spend", "share", "cache", "avg prompt", "esc"],
|
|
320
|
-
rows: r.tiers.map((x) => [x.key, num(x.dispatches), usd(x.spendUsd), pct(x.share), pct(x.cacheHitRate), num(x.avgPromptTokens), num(x.escalations)]),
|
|
331
|
+
rows: r.tiers.map((x) => [x.key, num(x.dispatches), usd(x.spendUsd), pct(x.share), pct(x.cacheHitRate, x.cacheEstimated), num(x.avgPromptTokens), num(x.escalations)]),
|
|
321
332
|
});
|
|
322
333
|
}
|
|
323
334
|
if (r.days.length > 1) {
|
package/src/cost/types.ts
CHANGED
|
@@ -26,6 +26,12 @@ export interface UsageCounts {
|
|
|
26
26
|
reasoningTokens: number;
|
|
27
27
|
/** Images in the prompt, for per-image surcharges. */
|
|
28
28
|
images: number;
|
|
29
|
+
/**
|
|
30
|
+
* `cachedTokens` was estimated by the router (see `cache-estimate.ts`)
|
|
31
|
+
* because the upstream caches without reporting it (Ollama Cloud). Absent
|
|
32
|
+
* or false ⇒ the count came from the provider.
|
|
33
|
+
*/
|
|
34
|
+
cachedEstimated?: boolean;
|
|
29
35
|
}
|
|
30
36
|
|
|
31
37
|
export const EMPTY_USAGE: UsageCounts = {
|
package/src/server/http.ts
CHANGED
|
@@ -10,6 +10,7 @@ import { createRouter } from "../router/index.ts";
|
|
|
10
10
|
import { createConversationStore } from "../router/state.ts";
|
|
11
11
|
import { UpstreamError } from "../upstream/types.ts";
|
|
12
12
|
import { apiKeySource, ollamaKeySource } from "../config/load.ts";
|
|
13
|
+
import { ollamaMeter } from "../upstream/ollama-usage.ts";
|
|
13
14
|
import { routerConfigPath } from "../cli/config-cmd.ts";
|
|
14
15
|
import { watchConfig } from "../config/hot-reload.ts";
|
|
15
16
|
import type { RouterConfig } from "../config/types.ts";
|
|
@@ -410,6 +411,8 @@ export function startServer(cfg: RouterConfig): StartedServer {
|
|
|
410
411
|
// Plan usage as ollama.com reports it (share of included monthly
|
|
411
412
|
// credits) and the cost multiplier currently in force.
|
|
412
413
|
usage: ollamaUsage.peek(),
|
|
414
|
+
// The dashboard's dollar figure: plan share × included credits, when known.
|
|
415
|
+
meter: ollamaMeter(ollamaUsage.peek(), cfg.ollama.planCreditsUsd),
|
|
413
416
|
costBias: { configured: cfg.ollama.costBias, effective: catalog.ollamaBias?.() ?? cfg.ollama.costBias, biasUntilUsage: cfg.ollama.biasUntilUsage },
|
|
414
417
|
},
|
|
415
418
|
catalog: snap === null
|
package/src/server/turn.ts
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
import type { CatalogSource } from "../catalog/types.ts";
|
|
12
12
|
import type { ContextBridge } from "../context/types.ts";
|
|
13
13
|
import type { RouterConfig } from "../config/types.ts";
|
|
14
|
+
import { estimateUnreportedCache } from "../cost/cache-estimate.ts";
|
|
14
15
|
import { computeCost } from "../cost/forecast.ts";
|
|
15
16
|
import { EMPTY_USAGE, type Ledger, type UsageCounts } from "../cost/types.ts";
|
|
16
17
|
import { createProbe, type Probe } from "../router/escalate.ts";
|
|
@@ -462,7 +463,22 @@ export async function runTurn(
|
|
|
462
463
|
// in for a turn that produced 26.
|
|
463
464
|
if (reportedUsd === null && usage.promptTokens > 0) {
|
|
464
465
|
const served = deps.catalog.find(servedSlug ?? decision.slug);
|
|
465
|
-
if (served !== undefined)
|
|
466
|
+
if (served !== undefined) {
|
|
467
|
+
// Ollama caches prompt prefixes and bills them at its cached rate
|
|
468
|
+
// without reporting a count; estimate it with the router's own
|
|
469
|
+
// warm-cache rule so the ledger stops booking every token fresh.
|
|
470
|
+
if (served.provider === "ollama") {
|
|
471
|
+
usage = estimateUnreportedCache(usage, {
|
|
472
|
+
previousSlug: state.currentSlug,
|
|
473
|
+
previousPromptTokens: state.lastPromptTokens,
|
|
474
|
+
previousAtMs: state.updatedAtMs,
|
|
475
|
+
servedSlug: served.slug,
|
|
476
|
+
nowMs: Date.now(),
|
|
477
|
+
cacheWarmTtlMs: config.hysteresis.cacheWarmTtlMs,
|
|
478
|
+
});
|
|
479
|
+
}
|
|
480
|
+
reportedUsd = computeCost(served, usage).total;
|
|
481
|
+
}
|
|
466
482
|
}
|
|
467
483
|
|
|
468
484
|
if (sinkDied) {
|
|
@@ -35,9 +35,26 @@ export interface OllamaUsage {
|
|
|
35
35
|
activityCostUsd: number | null;
|
|
36
36
|
/** Requests this billing month, summed over models. */
|
|
37
37
|
requestsThisMonth: number;
|
|
38
|
+
/** Subscription name from `POST /api/me` (`pro`, `max`, …), or null when unknown. */
|
|
39
|
+
plan: string | null;
|
|
38
40
|
fetchedAtMs: number;
|
|
39
41
|
}
|
|
40
42
|
|
|
43
|
+
/**
|
|
44
|
+
* Included monthly credits per plan, USD, from ollama.com/pricing (2026-09-07):
|
|
45
|
+
* Pro $20/mo carries $60 of usage, Max $100/mo carries $300. The dashboard's
|
|
46
|
+
* dollar figure is `limits.monthly.usage` × this. A plan not listed here
|
|
47
|
+
* (free, team, an unseen tier) yields no dollar reading rather than a guess.
|
|
48
|
+
*/
|
|
49
|
+
export const PLAN_CREDITS_USD: Readonly<Record<string, number>> = { pro: 60, max: 300 };
|
|
50
|
+
|
|
51
|
+
/** The plan named by an `/api/me` payload, lower-cased, or null. */
|
|
52
|
+
export function parseOllamaPlan(json: unknown): string | null {
|
|
53
|
+
const root = asRec(json);
|
|
54
|
+
const plan = root?.Plan ?? root?.plan;
|
|
55
|
+
return typeof plan === "string" && plan.trim() !== "" ? plan.trim().toLowerCase() : null;
|
|
56
|
+
}
|
|
57
|
+
|
|
41
58
|
function asRec(v: unknown): Record<string, unknown> | null {
|
|
42
59
|
return typeof v === "object" && v !== null && !Array.isArray(v) ? (v as Record<string, unknown>) : null;
|
|
43
60
|
}
|
|
@@ -72,6 +89,7 @@ export function parseOllamaUsage(json: unknown, nowMs = Date.now()): OllamaUsage
|
|
|
72
89
|
monthlyUsageRaw: usageRaw,
|
|
73
90
|
activityCostUsd: Number.isFinite(cost) ? cost : null,
|
|
74
91
|
requestsThisMonth: requests,
|
|
92
|
+
plan: null,
|
|
75
93
|
fetchedAtMs: nowMs,
|
|
76
94
|
};
|
|
77
95
|
}
|
|
@@ -98,8 +116,35 @@ export function createOllamaUsageSource(
|
|
|
98
116
|
let checkedAtMs = 0;
|
|
99
117
|
let inflight: Promise<OllamaUsage | null> | null = null;
|
|
100
118
|
let warned = false;
|
|
119
|
+
let plan: string | null = null;
|
|
120
|
+
let planWarned = false;
|
|
121
|
+
|
|
122
|
+
/** Best-effort: a missing plan only costs the dollar reading, never the bias. */
|
|
123
|
+
async function refreshPlan(): Promise<void> {
|
|
124
|
+
try {
|
|
125
|
+
const res = await fetchImpl(`${root}/api/me`, {
|
|
126
|
+
method: "POST",
|
|
127
|
+
headers: { authorization: `Bearer ${opts.apiKey}` },
|
|
128
|
+
signal: AbortSignal.timeout(opts.timeoutMs),
|
|
129
|
+
});
|
|
130
|
+
if (res.ok) {
|
|
131
|
+
const parsed = parseOllamaPlan(await res.json());
|
|
132
|
+
if (parsed !== null) plan = parsed;
|
|
133
|
+
} else if (!planWarned) {
|
|
134
|
+
planWarned = true;
|
|
135
|
+
opts.log.warn("ollama account endpoint unavailable; plan stays unknown", { status: res.status });
|
|
136
|
+
}
|
|
137
|
+
} catch (err) {
|
|
138
|
+
if (!planWarned) {
|
|
139
|
+
planWarned = true;
|
|
140
|
+
opts.log.warn("ollama account fetch failed; plan stays unknown", { error: err instanceof Error ? err.message : String(err) });
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
101
144
|
|
|
102
145
|
async function refresh(): Promise<OllamaUsage | null> {
|
|
146
|
+
// The plan changes rarely, but the call is one small request per poll.
|
|
147
|
+
await refreshPlan();
|
|
103
148
|
try {
|
|
104
149
|
const res = await fetchImpl(`${root}/api/usage`, {
|
|
105
150
|
headers: { authorization: `Bearer ${opts.apiKey}` },
|
|
@@ -108,7 +153,7 @@ export function createOllamaUsageSource(
|
|
|
108
153
|
if (res.ok) {
|
|
109
154
|
const parsed = parseOllamaUsage(await res.json());
|
|
110
155
|
if (parsed !== null) {
|
|
111
|
-
current = parsed;
|
|
156
|
+
current = { ...parsed, plan };
|
|
112
157
|
warned = false;
|
|
113
158
|
} else if (!warned) {
|
|
114
159
|
warned = true;
|
|
@@ -155,3 +200,22 @@ export function effectiveOllamaBias(costBias: number, biasUntilUsage: number, us
|
|
|
155
200
|
if (used === null) return costBias;
|
|
156
201
|
return used >= biasUntilUsage ? 1 : costBias;
|
|
157
202
|
}
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* Included credits for this account: the configured override when set, else
|
|
206
|
+
* the detected plan's published allowance, else null.
|
|
207
|
+
*/
|
|
208
|
+
export function ollamaPlanCredits(usage: OllamaUsage | null, overrideUsd: number): number | null {
|
|
209
|
+
if (overrideUsd > 0) return overrideUsd;
|
|
210
|
+
const plan = usage?.plan ?? null;
|
|
211
|
+
if (plan === null) return null;
|
|
212
|
+
return PLAN_CREDITS_USD[plan] ?? null;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/** The dashboard's dollar reading: plan share × included credits, when both are known. */
|
|
216
|
+
export function ollamaMeter(usage: OllamaUsage | null, overrideUsd: number): { usedUsd: number; creditsUsd: number; plan: string | null } | null {
|
|
217
|
+
if (usage === null || usage.monthlyUsedFraction === null) return null;
|
|
218
|
+
const credits = ollamaPlanCredits(usage, overrideUsd);
|
|
219
|
+
if (credits === null) return null;
|
|
220
|
+
return { usedUsd: Math.round(usage.monthlyUsedFraction * credits * 100) / 100, creditsUsd: credits, plan: usage.plan };
|
|
221
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import { estimateUnreportedCache } from "../src/cost/cache-estimate.ts";
|
|
4
|
+
import { EMPTY_USAGE, type UsageCounts } from "../src/cost/types.ts";
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Ollama Cloud caches prompt prefixes and bills them at its cached rate but
|
|
8
|
+
* reports no count (measured 2026-09-07: 12 identical 162k-token requests
|
|
9
|
+
* moved the plan meter $0.06 against $0.29 at the full rate). The estimate
|
|
10
|
+
* follows the router's warm-cache rule: same model as the previous turn
|
|
11
|
+
* within the TTL ⇒ the previous prompt is the cached prefix.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
const base = (over: Partial<UsageCounts> = {}): UsageCounts => ({ ...EMPTY_USAGE, promptTokens: 120_000, completionTokens: 300, ...over });
|
|
15
|
+
const ctx = { previousSlug: "ollama/glm", previousPromptTokens: 100_000, previousAtMs: 1_000_000, servedSlug: "ollama/glm", nowMs: 1_060_000, cacheWarmTtlMs: 300_000 };
|
|
16
|
+
|
|
17
|
+
describe("estimateUnreportedCache", () => {
|
|
18
|
+
test("same model within the TTL: the previous prompt is the cached prefix, flagged as estimated", () => {
|
|
19
|
+
const out = estimateUnreportedCache(base(), ctx);
|
|
20
|
+
expect(out.cachedTokens).toBe(100_000);
|
|
21
|
+
expect(out.cachedEstimated).toBe(true);
|
|
22
|
+
expect(out.promptTokens).toBe(120_000);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
test("a shorter prompt than the previous one caps the cached count at the prompt", () => {
|
|
26
|
+
expect(estimateUnreportedCache(base({ promptTokens: 40_000 }), ctx).cachedTokens).toBe(40_000);
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
test("first turn, model switch, or idle past the TTL count as cold", () => {
|
|
30
|
+
expect(estimateUnreportedCache(base(), { ...ctx, previousSlug: null }).cachedTokens).toBe(0);
|
|
31
|
+
expect(estimateUnreportedCache(base(), { ...ctx, previousSlug: "ollama/other" }).cachedTokens).toBe(0);
|
|
32
|
+
expect(estimateUnreportedCache(base(), { ...ctx, nowMs: ctx.previousAtMs + 300_001 }).cachedTokens).toBe(0);
|
|
33
|
+
expect(estimateUnreportedCache(base(), { ...ctx, previousPromptTokens: 0 }).cachedTokens).toBe(0);
|
|
34
|
+
expect(estimateUnreportedCache(base(), { ...ctx, cacheWarmTtlMs: 0 }).cachedTokens).toBe(0);
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
test("provider-reported cache counts are never overwritten", () => {
|
|
38
|
+
const reported = base({ cachedTokens: 5_000 });
|
|
39
|
+
expect(estimateUnreportedCache(reported, ctx)).toBe(reported);
|
|
40
|
+
const written = base({ cacheWriteTokens: 5_000 });
|
|
41
|
+
expect(estimateUnreportedCache(written, ctx)).toBe(written);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
test("no prompt tokens: nothing to estimate", () => {
|
|
45
|
+
const empty = base({ promptTokens: 0 });
|
|
46
|
+
expect(estimateUnreportedCache(empty, ctx)).toBe(empty);
|
|
47
|
+
});
|
|
48
|
+
});
|
package/test/failover.test.ts
CHANGED
|
@@ -31,7 +31,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
31
31
|
return {
|
|
32
32
|
server: { host: "127.0.0.1", port: 8787, maxConcurrentTurns: 24 },
|
|
33
33
|
openrouter: { baseUrl: "https://openrouter.ai/api/v1", apiKey: "", title: "test", timeoutMs: 30_000, catalogTtlMs: 3_600_000, catalogRefreshMs: 0 },
|
|
34
|
-
ollama: { enabled: false, baseUrl: "http://127.0.0.1:11434/v1", apiKey: "", timeoutMs: 30_000, catalogTtlMs: 300_000, includeLocal: false, prices: {}, twins: {}, costBias: 1, biasUntilUsage: 0.9, usagePollMs: 0, quotaCooldownMs: 0, rateLimitCooldownMs: 0 },
|
|
34
|
+
ollama: { enabled: false, baseUrl: "http://127.0.0.1:11434/v1", apiKey: "", timeoutMs: 30_000, catalogTtlMs: 300_000, includeLocal: false, prices: {}, twins: {}, costBias: 1, biasUntilUsage: 0.9, usagePollMs: 0, quotaCooldownMs: 0, rateLimitCooldownMs: 0, planCreditsUsd: 0 },
|
|
35
35
|
benchmarks: { enabled: false, artificialAnalysisApiKey: "", benchlm: true, refreshMs: 86_400_000, timeoutMs: 30_000, useLocalScores: false },
|
|
36
36
|
tiers: {
|
|
37
37
|
trivial: { minQuality: 0, maxInputPerMtok: 0.3, qualityExponent: 0, pin: [] },
|
package/test/ollama.test.ts
CHANGED
|
@@ -22,7 +22,7 @@ import { buildCandidates } from "../src/router/candidates.ts";
|
|
|
22
22
|
import { extractFeatures } from "../src/router/features.ts";
|
|
23
23
|
import { createMultiUpstream } from "../src/upstream/multi.ts";
|
|
24
24
|
import { classifyOllamaStatus, createOllamaClient, toOllamaBody } from "../src/upstream/ollama.ts";
|
|
25
|
-
import { createOllamaUsageSource, effectiveOllamaBias, NO_USAGE, parseOllamaUsage, usageFraction } from "../src/upstream/ollama-usage.ts";
|
|
25
|
+
import { createOllamaUsageSource, effectiveOllamaBias, NO_USAGE, ollamaMeter, parseOllamaPlan, parseOllamaUsage, usageFraction } from "../src/upstream/ollama-usage.ts";
|
|
26
26
|
import type { Dispatch, DispatchOptions, UpstreamClient } from "../src/upstream/types.ts";
|
|
27
27
|
import { createLogger } from "../src/util/log.ts";
|
|
28
28
|
import { parseChatRequest } from "../src/wire/openai/request.ts";
|
|
@@ -314,7 +314,7 @@ describe("ollama client", () => {
|
|
|
314
314
|
|
|
315
315
|
test("a 429 with a zero cooldown does not trip the breaker", async () => {
|
|
316
316
|
const fetchImpl = (async (): Promise<Response> => new Response("slow down", { status: 429 }));
|
|
317
|
-
const client = createOllamaClient(cfgWith({ rateLimitCooldownMs: 0 }), fetchImpl);
|
|
317
|
+
const client = createOllamaClient(cfgWith({ rateLimitCooldownMs: 0, planCreditsUsd: 0 }), fetchImpl);
|
|
318
318
|
await client.dispatch({ body: { model: "ollama/x", messages: [] }, sessionId: "s", signal: new AbortController().signal }).catch(() => {});
|
|
319
319
|
expect(client.available()).toBe(true);
|
|
320
320
|
});
|
|
@@ -444,7 +444,7 @@ describe("ollama plan usage (credit-aware bias)", () => {
|
|
|
444
444
|
});
|
|
445
445
|
|
|
446
446
|
test("the bias holds under the threshold, switches to list price above it, and stays on when usage is unknown", () => {
|
|
447
|
-
const at = (f: number | null) => (f === null ? null : { monthlyUsedFraction: f, monthlyUsageRaw: f, activityCostUsd: null, requestsThisMonth: 0, fetchedAtMs: 0 });
|
|
447
|
+
const at = (f: number | null) => (f === null ? null : { monthlyUsedFraction: f, monthlyUsageRaw: f, activityCostUsd: null, requestsThisMonth: 0, plan: null, fetchedAtMs: 0 });
|
|
448
448
|
expect(effectiveOllamaBias(0.1, 0.9, at(0.5))).toBe(0.1);
|
|
449
449
|
expect(effectiveOllamaBias(0.1, 0.9, at(0.9))).toBe(1);
|
|
450
450
|
expect(effectiveOllamaBias(0.1, 0.9, at(1))).toBe(1);
|
|
@@ -457,15 +457,21 @@ describe("ollama plan usage (credit-aware bias)", () => {
|
|
|
457
457
|
let calls = 0;
|
|
458
458
|
let fail = false;
|
|
459
459
|
const fetchImpl = async (url: string, init?: RequestInit): Promise<Response> => {
|
|
460
|
+
expect((init?.headers as Record<string, string>).authorization).toBe("Bearer k");
|
|
461
|
+
// The plan rides along on every poll: POST /api/me (GET answers 405).
|
|
462
|
+
if (url === "https://ollama.com/api/me") {
|
|
463
|
+
expect(init?.method).toBe("POST");
|
|
464
|
+
return Response.json({ ID: "x", Email: "e", Plan: "Pro" });
|
|
465
|
+
}
|
|
460
466
|
calls++;
|
|
461
467
|
expect(url).toBe("https://ollama.com/api/usage");
|
|
462
|
-
expect((init?.headers as Record<string, string>).authorization).toBe("Bearer k");
|
|
463
468
|
if (fail) return new Response("down", { status: 503 });
|
|
464
469
|
return Response.json({ ...PAYLOAD, limits: { monthly: { usage: 42, models: [] } } });
|
|
465
470
|
};
|
|
466
471
|
const src = createOllamaUsageSource({ apiKey: "k", pollMs: 20, timeoutMs: 1000, log, fetchImpl });
|
|
467
472
|
expect(src.peek()).toBeNull();
|
|
468
473
|
expect((await src.get())?.monthlyUsedFraction).toBeCloseTo(0.42, 6);
|
|
474
|
+
expect(src.peek()?.plan).toBe("pro");
|
|
469
475
|
await src.get();
|
|
470
476
|
expect(calls).toBe(1); // within the interval
|
|
471
477
|
fail = true;
|
|
@@ -482,7 +488,7 @@ describe("ollama plan usage (credit-aware bias)", () => {
|
|
|
482
488
|
const source = { get: async () => ollamaModels, peek: () => ollamaModels, invalidate: () => {} };
|
|
483
489
|
const breaker = { available: () => true, cooldownUntilMs: () => null, lastTrip: () => null };
|
|
484
490
|
let used = 0.2;
|
|
485
|
-
const usage = { get: async () => ({ monthlyUsedFraction: used, monthlyUsageRaw: used * 100, activityCostUsd: null, requestsThisMonth: 0, fetchedAtMs: 0 }), peek: () => ({ monthlyUsedFraction: used, monthlyUsageRaw: used * 100, activityCostUsd: null, requestsThisMonth: 0, fetchedAtMs: 0 }) };
|
|
491
|
+
const usage = { get: async () => ({ monthlyUsedFraction: used, monthlyUsageRaw: used * 100, activityCostUsd: null, requestsThisMonth: 0, plan: null, fetchedAtMs: 0 }), peek: () => ({ monthlyUsedFraction: used, monthlyUsageRaw: used * 100, activityCostUsd: null, requestsThisMonth: 0, plan: null, fetchedAtMs: 0 }) };
|
|
486
492
|
const catalog = createCompositeCatalog(openrouter, source, breaker, { costBias: 0.1, biasUntilUsage: 0.9, usage });
|
|
487
493
|
|
|
488
494
|
const a = await catalog.get();
|
|
@@ -504,3 +510,31 @@ describe("ollama plan usage (credit-aware bias)", () => {
|
|
|
504
510
|
});
|
|
505
511
|
});
|
|
506
512
|
|
|
513
|
+
|
|
514
|
+
describe("ollamaMeter", () => {
|
|
515
|
+
const usage = (plan: string | null, frac: number | null = 0.104) => ({ monthlyUsedFraction: frac, monthlyUsageRaw: frac, activityCostUsd: 0, requestsThisMonth: 1250, plan, fetchedAtMs: 1 });
|
|
516
|
+
|
|
517
|
+
test("a detected plan applies its published allowance: 10.4% of Pro's $60 is the $6.24 ollama.com shows", () => {
|
|
518
|
+
expect(ollamaMeter(usage("pro"), 0)).toEqual({ usedUsd: 6.24, creditsUsd: 60, plan: "pro" });
|
|
519
|
+
expect(ollamaMeter(usage("max"), 0)).toEqual({ usedUsd: 31.2, creditsUsd: 300, plan: "max" });
|
|
520
|
+
});
|
|
521
|
+
|
|
522
|
+
test("a configured override wins over the detected plan; an unknown plan without one yields no meter", () => {
|
|
523
|
+
expect(ollamaMeter(usage("pro"), 100)).toEqual({ usedUsd: 10.4, creditsUsd: 100, plan: "pro" });
|
|
524
|
+
expect(ollamaMeter(usage("team"), 0)).toBeNull();
|
|
525
|
+
expect(ollamaMeter(usage("team"), 500)?.creditsUsd).toBe(500);
|
|
526
|
+
expect(ollamaMeter(usage(null), 0)).toBeNull();
|
|
527
|
+
});
|
|
528
|
+
|
|
529
|
+
test("no usage reading yields no meter", () => {
|
|
530
|
+
expect(ollamaMeter(null, 60)).toBeNull();
|
|
531
|
+
expect(ollamaMeter(usage("pro", null), 60)).toBeNull();
|
|
532
|
+
});
|
|
533
|
+
|
|
534
|
+
test("parseOllamaPlan reads the account payload case-insensitively", () => {
|
|
535
|
+
expect(parseOllamaPlan({ ID: "x", Plan: "Pro" })).toBe("pro");
|
|
536
|
+
expect(parseOllamaPlan({ plan: "max" })).toBe("max");
|
|
537
|
+
expect(parseOllamaPlan({ Plan: "" })).toBeNull();
|
|
538
|
+
expect(parseOllamaPlan("nope")).toBeNull();
|
|
539
|
+
});
|
|
540
|
+
});
|
package/test/report-hub.test.ts
CHANGED
|
@@ -51,6 +51,7 @@ function report(over: Partial<UsageReport> = {}): UsageReport {
|
|
|
51
51
|
spendUsd: spend,
|
|
52
52
|
share: 0.5,
|
|
53
53
|
cacheHitRate: 0.8,
|
|
54
|
+
cacheEstimated: false,
|
|
54
55
|
avgPromptTokens: 1000,
|
|
55
56
|
avgTtftMs: 900,
|
|
56
57
|
tokensPerSec: 120,
|
|
@@ -74,6 +75,7 @@ function report(over: Partial<UsageReport> = {}): UsageReport {
|
|
|
74
75
|
errors: 0,
|
|
75
76
|
aborted: 0,
|
|
76
77
|
modelSwitches: 1,
|
|
78
|
+
cacheEstimated: false,
|
|
77
79
|
},
|
|
78
80
|
providers: [row("openrouter", 2), row("ollama", 1)],
|
|
79
81
|
models: [
|
|
@@ -65,6 +65,7 @@ describe("renderStatus", () => {
|
|
|
65
65
|
apiKeySource: "omp",
|
|
66
66
|
lastTrip: { kind: "quota", atMs: now - 120_000, message: "402" },
|
|
67
67
|
usage: { monthlyUsedFraction: 0.42, activityCostUsd: 3.1, fetchedAtMs: now },
|
|
68
|
+
meter: { usedUsd: 25.2, creditsUsd: 60, plan: "pro" },
|
|
68
69
|
costBias: { configured: 0.1, effective: 0.1, biasUntilUsage: 0.9 },
|
|
69
70
|
},
|
|
70
71
|
catalog: { models: 240, ageMs: 5 * 60_000, keyScoped: true, shrink: { fromModels: 300, toModels: 120, atMs: now } },
|
|
@@ -75,7 +76,7 @@ describe("renderStatus", () => {
|
|
|
75
76
|
expect(text).toContain("refreshed 5m ago");
|
|
76
77
|
expect(text).toContain("SHRANK 300 -> 120");
|
|
77
78
|
expect(text).toContain("COOLING DOWN");
|
|
78
|
-
expect(text).toContain("plan usage 42%");
|
|
79
|
+
expect(text).toContain("pro plan usage 42.0% ($25.20 of $60)");
|
|
79
80
|
expect(text).toContain("cost bias ×0.1 (until 90%)");
|
|
80
81
|
expect(text).toContain("last trip quota 2m ago");
|
|
81
82
|
expect(text).toContain("scope omp-router");
|
package/test/report.test.ts
CHANGED
|
@@ -171,6 +171,7 @@ describe("buildUsageReport", () => {
|
|
|
171
171
|
errors: 0,
|
|
172
172
|
aborted: 0,
|
|
173
173
|
modelSwitches: 0,
|
|
174
|
+
cacheEstimated: false,
|
|
174
175
|
});
|
|
175
176
|
expect(r.providers).toEqual([]);
|
|
176
177
|
expect(r.models).toEqual([]);
|
|
@@ -189,6 +190,21 @@ describe("buildUsageReport", () => {
|
|
|
189
190
|
});
|
|
190
191
|
|
|
191
192
|
describe("renderUsageReport", () => {
|
|
193
|
+
test("router-estimated cache counts render as an estimate", async () => {
|
|
194
|
+
const { db, ledger } = seeded();
|
|
195
|
+
ledger.record(entry({ slug: "ollama/glm", servedSlug: "ollama/glm", usage: { promptTokens: 1000, cachedTokens: 900, cacheWriteTokens: 0, completionTokens: 10, reasoningTokens: 0, images: 0, cachedEstimated: true } }));
|
|
196
|
+
ledger.record(entry({ usage: { promptTokens: 1000, cachedTokens: 500, cacheWriteTokens: 0, completionTokens: 10, reasoningTokens: 0, images: 0 } }));
|
|
197
|
+
const r = buildUsageReport(db, { windowDays: 7, nowMs: NOW });
|
|
198
|
+
expect(r.totals.cacheEstimated).toBe(true);
|
|
199
|
+
expect(r.providers.find((p) => p.key === "ollama")!.cacheEstimated).toBe(true);
|
|
200
|
+
expect(r.providers.find((p) => p.key === "openrouter")!.cacheEstimated).toBe(false);
|
|
201
|
+
const text = renderUsageReport(r);
|
|
202
|
+
expect(text).toContain("(cache hit ~70%)");
|
|
203
|
+
expect(text).toMatch(/ollama\s+1\s+\S+\s+\S+\s+~90%/);
|
|
204
|
+
expect(text).toMatch(/openrouter\s+1\s+\S+\s+\S+\s+50%/);
|
|
205
|
+
db.close();
|
|
206
|
+
});
|
|
207
|
+
|
|
192
208
|
test("renders every section as plain fixed-width text", () => {
|
|
193
209
|
const { db, ledger } = seeded();
|
|
194
210
|
ledger.record(entry({ reportedUsd: 1.25, createdAtMs: NOW - HOUR }));
|
package/test/turn.test.ts
CHANGED
|
@@ -31,7 +31,7 @@ function mkConfig(escalation: Partial<EscalationConfig> = {}): RouterConfig {
|
|
|
31
31
|
return {
|
|
32
32
|
server: { host: "127.0.0.1", port: 8787, maxConcurrentTurns: 24 },
|
|
33
33
|
openrouter: { baseUrl: "https://openrouter.ai/api/v1", apiKey: "", title: "test", timeoutMs: 30_000, catalogTtlMs: 3_600_000, catalogRefreshMs: 0 },
|
|
34
|
-
ollama: { enabled: false, baseUrl: "http://127.0.0.1:11434/v1", apiKey: "", timeoutMs: 30_000, catalogTtlMs: 300_000, includeLocal: false, prices: {}, twins: {}, costBias: 1, biasUntilUsage: 0.9, usagePollMs: 0, quotaCooldownMs: 0, rateLimitCooldownMs: 0 },
|
|
34
|
+
ollama: { enabled: false, baseUrl: "http://127.0.0.1:11434/v1", apiKey: "", timeoutMs: 30_000, catalogTtlMs: 300_000, includeLocal: false, prices: {}, twins: {}, costBias: 1, biasUntilUsage: 0.9, usagePollMs: 0, quotaCooldownMs: 0, rateLimitCooldownMs: 0, planCreditsUsd: 0 },
|
|
35
35
|
benchmarks: { enabled: false, artificialAnalysisApiKey: "", benchlm: true, refreshMs: 86_400_000, timeoutMs: 30_000, useLocalScores: false },
|
|
36
36
|
tiers: {
|
|
37
37
|
trivial: { minQuality: 0, maxInputPerMtok: 0.3, qualityExponent: 0, pin: [] },
|
|
@@ -846,4 +846,53 @@ describe("latency measurement covers the work the router actually does", () => {
|
|
|
846
846
|
expect(finishes[0]!.reportedUsd).toBeCloseTo(0.65, 6);
|
|
847
847
|
});
|
|
848
848
|
|
|
849
|
+
test("an Ollama-served turn after another on the same model prices the previous prompt as cached", async () => {
|
|
850
|
+
// ollama.com caches prefixes and bills them at the cached rate without
|
|
851
|
+
// reporting a count; the router estimates it with its warm-cache rule.
|
|
852
|
+
const ollamaModel: CatalogModel = {
|
|
853
|
+
slug: "ollama/glm-5.3-flash",
|
|
854
|
+
provider: "ollama",
|
|
855
|
+
canonicalSlug: "ollama/glm-5.3-flash",
|
|
856
|
+
name: "glm",
|
|
857
|
+
contextLength: 1_000_000,
|
|
858
|
+
supportsTools: true,
|
|
859
|
+
supportsReasoning: true,
|
|
860
|
+
reasoningMandatory: false,
|
|
861
|
+
supportsToolChoice: false,
|
|
862
|
+
inputModalities: ["text"],
|
|
863
|
+
price: { prompt: 0.15 / 1e6, cacheRead: 0.03 / 1e6, completion: 0.5 / 1e6 },
|
|
864
|
+
priceTiers: [],
|
|
865
|
+
quality: {},
|
|
866
|
+
tokenizer: "Other",
|
|
867
|
+
isFree: false,
|
|
868
|
+
createdAtMs: 0,
|
|
869
|
+
author: "ollama",
|
|
870
|
+
};
|
|
871
|
+
const priced = { ...catalog, find: (slug: string) => (slug === ollamaModel.slug ? ollamaModel : undefined) };
|
|
872
|
+
const turn = (prompt: number) => ({
|
|
873
|
+
kind: "chunks" as const,
|
|
874
|
+
chunks: [startChunk(ollamaModel.slug), textChunk("ok"), finishChunk("stop"), usageChunk({ promptTokens: prompt, completionTokens: 0 }, null)],
|
|
875
|
+
});
|
|
876
|
+
const { router } = mkRouter([mkDecision("simple", ollamaModel.slug)]);
|
|
877
|
+
const { upstream } = mkUpstream([turn(100_000), turn(120_000)]);
|
|
878
|
+
const { ledger, entries } = mkLedger();
|
|
879
|
+
const { store, map } = mkConversations();
|
|
880
|
+
const deps = { config: mkConfig(), router, upstream, ledger, conversations: store, catalog: priced, context: createDisabledBridge() };
|
|
881
|
+
|
|
882
|
+
await runTurn(mkReq(), mkSink().sink, deps, new AbortController().signal);
|
|
883
|
+
// First turn: nothing to be cached yet, full input rate.
|
|
884
|
+
expect(entries[0]!.usage.cachedTokens).toBe(0);
|
|
885
|
+
expect(entries[0]!.usage.cachedEstimated).toBeUndefined();
|
|
886
|
+
expect(entries[0]!.reportedUsd).toBeCloseTo(0.015, 6);
|
|
887
|
+
|
|
888
|
+
await runTurn(mkReq(), mkSink().sink, deps, new AbortController().signal);
|
|
889
|
+
// Second turn on the same model: the 100k previous prompt is the cached
|
|
890
|
+
// prefix at $0.03/M, the 20k of growth is fresh at $0.15/M.
|
|
891
|
+
expect(entries[1]!.usage.cachedTokens).toBe(100_000);
|
|
892
|
+
expect(entries[1]!.usage.cachedEstimated).toBe(true);
|
|
893
|
+
expect(entries[1]!.reportedUsd).toBeCloseTo(0.003 + 0.003, 6);
|
|
894
|
+
// The estimate is evidence enough to keep the Ollama model warm for the stay/switch comparison.
|
|
895
|
+
expect(map.get("conv-test")!.cacheWarmSlug).toBe(ollamaModel.slug);
|
|
896
|
+
});
|
|
897
|
+
|
|
849
898
|
});
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
/**
|
|
3
|
+
* One-off backfill: re-price historical Ollama ledger rows with the router's
|
|
4
|
+
* cache estimate (`src/cost/cache-estimate.ts`).
|
|
5
|
+
*
|
|
6
|
+
* Rows recorded before the estimate existed booked every prompt token at the
|
|
7
|
+
* full input rate, overstating Ollama spend ~3.7x against ollama.com's meter.
|
|
8
|
+
* This walks each conversation's Ollama rows in order, applies the same rule
|
|
9
|
+
* the live path applies (same model as the previous kept row within
|
|
10
|
+
* `hysteresis.cacheWarmTtlMs` ⇒ the previous prompt is the cached prefix),
|
|
11
|
+
* and rewrites `usage`, `cost_breakdown` and `reported_usd` for rows whose
|
|
12
|
+
* cost the router itself computed (Ollama never reports a cost). Rows that
|
|
13
|
+
* already carry a cache count are left alone, so it is safe to re-run.
|
|
14
|
+
*
|
|
15
|
+
* Usage:
|
|
16
|
+
* bun tools/recompute-ollama-cache.ts # dry run: prints the delta
|
|
17
|
+
* bun tools/recompute-ollama-cache.ts --apply # backs up router.db, then writes
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { copyFileSync, mkdirSync } from "node:fs";
|
|
21
|
+
import { dirname, join } from "node:path";
|
|
22
|
+
import { Database } from "bun:sqlite";
|
|
23
|
+
import { ollamaRateFor } from "../src/catalog/ollama-prices.ts";
|
|
24
|
+
import { loadConfig } from "../src/config/load.ts";
|
|
25
|
+
import { estimateUnreportedCache } from "../src/cost/cache-estimate.ts";
|
|
26
|
+
import type { UsageCounts } from "../src/cost/types.ts";
|
|
27
|
+
|
|
28
|
+
const apply = process.argv.includes("--apply");
|
|
29
|
+
const cfg = loadConfig({});
|
|
30
|
+
const ttl = cfg.hysteresis.cacheWarmTtlMs;
|
|
31
|
+
|
|
32
|
+
interface Row {
|
|
33
|
+
id: string;
|
|
34
|
+
ck: string;
|
|
35
|
+
t: number;
|
|
36
|
+
latency: number;
|
|
37
|
+
slug: string;
|
|
38
|
+
usage: string;
|
|
39
|
+
reported: number | null;
|
|
40
|
+
wasted: number;
|
|
41
|
+
error: string | null;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const db = new Database(cfg.ledger.path);
|
|
45
|
+
db.exec("PRAGMA busy_timeout = 5000");
|
|
46
|
+
const rows = db
|
|
47
|
+
.query(
|
|
48
|
+
`SELECT id, conversation_key ck, created_at_ms t, latency_ms latency, COALESCE(served_slug, slug) slug, usage, reported_usd reported, wasted, error
|
|
49
|
+
FROM ledger WHERE COALESCE(served_slug, slug) LIKE 'ollama/%' ORDER BY conversation_key, created_at_ms`,
|
|
50
|
+
)
|
|
51
|
+
.all() as Row[];
|
|
52
|
+
|
|
53
|
+
let before = 0;
|
|
54
|
+
let after = 0;
|
|
55
|
+
let changed = 0;
|
|
56
|
+
const updates: { id: string; usage: string; breakdown: string; usd: number }[] = [];
|
|
57
|
+
let prev: Row | undefined;
|
|
58
|
+
let prevPrompt = 0;
|
|
59
|
+
for (const r of rows) {
|
|
60
|
+
const usage = JSON.parse(r.usage) as UsageCounts;
|
|
61
|
+
const sameConv = prev !== undefined && prev.ck === r.ck;
|
|
62
|
+
const rate = ollamaRateFor(r.slug.slice("ollama/".length), cfg.ollama.prices);
|
|
63
|
+
before += r.reported ?? 0;
|
|
64
|
+
if (rate === null || r.reported === null || r.error !== null) {
|
|
65
|
+
after += r.reported ?? 0;
|
|
66
|
+
if (r.error === null) {
|
|
67
|
+
prev = r;
|
|
68
|
+
prevPrompt = usage.promptTokens;
|
|
69
|
+
}
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
const est = estimateUnreportedCache(usage, {
|
|
73
|
+
previousSlug: sameConv ? (prev?.slug ?? null) : null,
|
|
74
|
+
previousPromptTokens: sameConv ? prevPrompt : 0,
|
|
75
|
+
previousAtMs: sameConv ? (prev?.t ?? 0) : 0,
|
|
76
|
+
servedSlug: r.slug,
|
|
77
|
+
nowMs: r.t,
|
|
78
|
+
cacheWarmTtlMs: ttl,
|
|
79
|
+
});
|
|
80
|
+
const input = rate.rate.input / 1e6;
|
|
81
|
+
const cached = (rate.rate.cachedInput ?? rate.rate.input) / 1e6;
|
|
82
|
+
const output = rate.rate.output / 1e6;
|
|
83
|
+
const fresh = Math.max(0, est.promptTokens - est.cachedTokens);
|
|
84
|
+
const breakdown = {
|
|
85
|
+
freshPrompt: fresh * input,
|
|
86
|
+
cacheRead: est.cachedTokens * cached,
|
|
87
|
+
cacheWrite: 0,
|
|
88
|
+
completion: est.completionTokens * output,
|
|
89
|
+
reasoning: 0,
|
|
90
|
+
images: 0,
|
|
91
|
+
request: 0,
|
|
92
|
+
total: 0,
|
|
93
|
+
tierAtPromptTokens: 0,
|
|
94
|
+
};
|
|
95
|
+
breakdown.total = breakdown.freshPrompt + breakdown.cacheRead + breakdown.completion;
|
|
96
|
+
after += breakdown.total;
|
|
97
|
+
if (est.cachedTokens !== usage.cachedTokens) {
|
|
98
|
+
changed++;
|
|
99
|
+
updates.push({ id: r.id, usage: JSON.stringify(est), breakdown: JSON.stringify(breakdown), usd: breakdown.total });
|
|
100
|
+
}
|
|
101
|
+
// The next row's "previous" is this row as dispatched, whether or not it was wasted:
|
|
102
|
+
// a wasted probe still warmed the prefix.
|
|
103
|
+
prev = r;
|
|
104
|
+
prevPrompt = est.promptTokens;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
console.log(`ollama rows: ${rows.length}, re-priced: ${changed}`);
|
|
108
|
+
console.log(`ledger Ollama spend: $${before.toFixed(2)} → $${after.toFixed(2)}`);
|
|
109
|
+
if (!apply) {
|
|
110
|
+
console.log("dry run; pass --apply to write (router.db is backed up first)");
|
|
111
|
+
db.close();
|
|
112
|
+
process.exit(0);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
const backupDir = join(dirname(cfg.ledger.path), "backups");
|
|
116
|
+
mkdirSync(backupDir, { recursive: true });
|
|
117
|
+
const stamp = new Date().toISOString().replace(/[-:]/g, "").replace(/\..+/, "");
|
|
118
|
+
const backup = join(backupDir, `router-pre-ollama-cache-${stamp}.db`);
|
|
119
|
+
db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
|
|
120
|
+
copyFileSync(cfg.ledger.path, backup);
|
|
121
|
+
console.log(`backup: ${backup}`);
|
|
122
|
+
|
|
123
|
+
const stmt = db.prepare("UPDATE ledger SET usage = $usage, cost_breakdown = $breakdown, reported_usd = $usd WHERE id = $id");
|
|
124
|
+
const tx = db.transaction((list: typeof updates) => {
|
|
125
|
+
for (const u of list) stmt.run({ $usage: u.usage, $breakdown: u.breakdown, $usd: u.usd, $id: u.id });
|
|
126
|
+
});
|
|
127
|
+
tx(updates);
|
|
128
|
+
console.log(`updated ${updates.length} rows`);
|
|
129
|
+
db.close();
|