@rizal72/oc-models-pro 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -19
- package/aa.ts +129 -0
- package/bench.ts +5 -3
- package/multipliers.ts +2 -1
- package/package.json +4 -3
- package/tps.ts +212 -0
- package/tui.js +93 -29
package/README.md
CHANGED
|
@@ -1,10 +1,7 @@
|
|
|
1
1
|
# @rizal72/oc-models-pro
|
|
2
2
|
|
|
3
|
-
A model picker
|
|
4
|
-
|
|
5
|
-
It lists every model with its input and output price, context window, intelligence index, `tok/s`
|
|
6
|
-
when a source has it, the Go credit multiplier, and the effective price per request; it sorts by
|
|
7
|
-
what a request actually costs and switches the model on Enter.
|
|
3
|
+
A model picker that shows what a request costs: price in and out, context, intelligence, `tok/s` and
|
|
4
|
+
the Go credit, sorted by what a request actually costs. Enter switches.
|
|
8
5
|
|
|
9
6
|
## Install
|
|
10
7
|
|
|
@@ -20,7 +17,7 @@ Or by hand, one line in `~/.config/opencode/opencode.jsonc`:
|
|
|
20
17
|
|
|
21
18
|
## Use
|
|
22
19
|
|
|
23
|
-
`/models-pro`, or `Ctrl+X` then `p
|
|
20
|
+
`/models-pro`, or `Ctrl+X` then `p`, inside a session.
|
|
24
21
|
|
|
25
22
|
| key | does |
|
|
26
23
|
|---|---|
|
|
@@ -33,21 +30,29 @@ Or by hand, one line in `~/.config/opencode/opencode.jsonc`:
|
|
|
33
30
|
| `ctrl+u` | clear the search |
|
|
34
31
|
| `esc` | clear the search, else close |
|
|
35
32
|
|
|
36
|
-
##
|
|
33
|
+
## Where the numbers come from
|
|
34
|
+
|
|
35
|
+
| column | source |
|
|
36
|
+
|---|---|
|
|
37
|
+
| price, ctx | the host's own catalogue |
|
|
38
|
+
| int | catalog.stencil.so, cached 6 h |
|
|
39
|
+
| tok/s | this machine's own sessions: last 7 days, or all history when that window is thin |
|
|
40
|
+
| ~tok/s | Artificial Analysis, with a free key |
|
|
41
|
+
| cred | the Go plan page, cached 1 day, `opencode-go` only |
|
|
37
42
|
|
|
38
|
-
|
|
39
|
-
- the intelligence index from `catalog.stencil.so`, cached six hours;
|
|
40
|
-
- the Go credit from OpenCode's Go plan page, cached one day — `opencode-go` models only;
|
|
41
|
-
- favourites and recent models from the app's own `~/.local/state/opencode/model.json`, so they are
|
|
42
|
-
the same ones the native picker keeps.
|
|
43
|
+
Favourites and recents are the native picker's (`~/.local/state/opencode/model.json`), so both agree.
|
|
43
44
|
|
|
44
|
-
|
|
45
|
+
## Optional: tok/s for models you have not run
|
|
45
46
|
|
|
46
|
-
|
|
47
|
+
`tok/s` is measured from your own sessions, so a model you never used has no number. A free key from
|
|
48
|
+
<https://artificialanalysis.ai/> (1,000 requests a day) fills those cells with a `~`:
|
|
47
49
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
50
|
+
```sh
|
|
51
|
+
# macOS and Linux, zsh or bash
|
|
52
|
+
echo 'export ARTIFICIAL_ANALYSIS_API_KEY="your-key"' >> ~/.zshrc && source ~/.zshrc
|
|
53
|
+
```
|
|
52
54
|
|
|
53
|
-
|
|
55
|
+
```powershell
|
|
56
|
+
# Windows
|
|
57
|
+
setx ARTIFICIAL_ANALYSIS_API_KEY "your-key"
|
|
58
|
+
```
|
package/aa.ts
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
// oc-models-pro/aa.ts — tokens per second for the models this machine has never run, from
|
|
2
|
+
// Artificial Analysis' free API. Attribution: https://artificialanalysis.ai/
|
|
3
|
+
//
|
|
4
|
+
// Their number is the same measurement as ours — "the average number of tokens received per second,
|
|
5
|
+
// after the first token is received" — so filling an empty cell with it does not mix two families,
|
|
6
|
+
// the way `int` and speed already do. The panel marks those values with a leading `~`, because they
|
|
7
|
+
// were measured on their infrastructure and not on this person's provider.
|
|
8
|
+
//
|
|
9
|
+
// No key, no calls: without ARTIFICIAL_ANALYSIS_API_KEY this module returns an empty map and nothing
|
|
10
|
+
// leaves the machine. With a key the response is cached six hours, which their terms ask for.
|
|
11
|
+
//
|
|
12
|
+
// Runnable standalone for its self-check: node plugins/oc-models-pro/aa.ts --self-check
|
|
13
|
+
|
|
14
|
+
import { mkdir, readFile, rename, writeFile } from "node:fs/promises"
|
|
15
|
+
import { homedir } from "node:os"
|
|
16
|
+
import { dirname, join } from "node:path"
|
|
17
|
+
import { benchKey } from "./bench.ts"
|
|
18
|
+
|
|
19
|
+
export const AA = "https://artificialanalysis.ai/api/v2/data/llms/models"
|
|
20
|
+
|
|
21
|
+
const CACHE_DIR = process.env.XDG_CACHE_HOME || join(homedir(), ".cache")
|
|
22
|
+
const CACHE = join(CACHE_DIR, "oc-models-pro", "aa.json")
|
|
23
|
+
const GRACE_MS = 6 * 60 * 60 * 1000
|
|
24
|
+
/** Optional decoration must never hold the panel: a hanging endpoint gives up after this. */
|
|
25
|
+
const FETCH_MS = 5_000
|
|
26
|
+
|
|
27
|
+
interface Cached {
|
|
28
|
+
fetched: number
|
|
29
|
+
index: Record<string, number>
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function apiKey(): string | undefined {
|
|
33
|
+
const key = process.env.ARTIFICIAL_ANALYSIS_API_KEY
|
|
34
|
+
return key && key.trim() ? key.trim() : undefined
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** Name first, slug second — the same meeting point `bench.ts` uses, so one model has one key. */
|
|
38
|
+
export function buildIndex(payload: any): Map<string, number> {
|
|
39
|
+
const index = new Map<string, number>()
|
|
40
|
+
for (const entry of payload?.data ?? []) {
|
|
41
|
+
const tps = entry?.median_output_tokens_per_second
|
|
42
|
+
if (typeof tps !== "number" || !(tps > 0)) continue
|
|
43
|
+
for (const candidate of [entry.name, entry.slug]) {
|
|
44
|
+
if (typeof candidate !== "string") continue
|
|
45
|
+
const key = benchKey(candidate)
|
|
46
|
+
// A key met twice keeps the faster figure: label stripping can fuse a variant into its base
|
|
47
|
+
// model, and the slower of the two would understate what the faster one shows.
|
|
48
|
+
if (key) index.set(key, Math.max(index.get(key) ?? 0, tps))
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return index
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** A cache is a file on disk: a value that is not a positive number is not a measurement. */
|
|
55
|
+
function fromCache(previous: Cached): Map<string, number> {
|
|
56
|
+
const index = new Map<string, number>()
|
|
57
|
+
for (const [key, value] of Object.entries(previous.index ?? {})) {
|
|
58
|
+
if (typeof value === "number" && Number.isFinite(value) && value > 0) index.set(key, value)
|
|
59
|
+
}
|
|
60
|
+
return index
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async function cached(): Promise<Cached | undefined> {
|
|
64
|
+
try {
|
|
65
|
+
return JSON.parse(await readFile(CACHE, "utf8")) as Cached
|
|
66
|
+
} catch {
|
|
67
|
+
return undefined
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Empty without a key: this is the whole cost of the column being optional. */
|
|
72
|
+
export async function aaIndex(): Promise<Map<string, number>> {
|
|
73
|
+
const key = apiKey()
|
|
74
|
+
if (!key) return new Map()
|
|
75
|
+
const previous = await cached()
|
|
76
|
+
if (previous && Date.now() - previous.fetched < GRACE_MS) return fromCache(previous)
|
|
77
|
+
try {
|
|
78
|
+
const response = await fetch(AA, {
|
|
79
|
+
headers: { "x-api-key": key, "user-agent": "oc-models-pro" },
|
|
80
|
+
signal: AbortSignal.timeout(FETCH_MS),
|
|
81
|
+
})
|
|
82
|
+
if (!response.ok) throw new Error("HTTP " + response.status)
|
|
83
|
+
const built = buildIndex(await response.json())
|
|
84
|
+
if (built.size === 0) throw new Error("response shape changed")
|
|
85
|
+
await mkdir(dirname(CACHE), { recursive: true })
|
|
86
|
+
await writeFile(CACHE + ".tmp", JSON.stringify({ fetched: Date.now(), index: Object.fromEntries(built) }))
|
|
87
|
+
await rename(CACHE + ".tmp", CACHE)
|
|
88
|
+
return built
|
|
89
|
+
} catch (error) {
|
|
90
|
+
console.error(
|
|
91
|
+
`[oc-models-pro] artificial analysis not refreshed: ${error instanceof Error ? error.message : String(error)}`,
|
|
92
|
+
)
|
|
93
|
+
return previous ? fromCache(previous) : new Map()
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// Self-check: node plugins/oc-models-pro/aa.ts --self-check
|
|
98
|
+
if (process.argv[2] === "--self-check") {
|
|
99
|
+
const run = async () => {
|
|
100
|
+
// The mapping is the part that can be wrong without anyone noticing, so it is tested against
|
|
101
|
+
// their documented response shape — no key, no spend, no network.
|
|
102
|
+
const canned = {
|
|
103
|
+
data: [
|
|
104
|
+
{ name: "o3-mini", slug: "o3-mini", median_output_tokens_per_second: 153.831 },
|
|
105
|
+
{ name: "Claude Fable 5", slug: "claude-fable-5", median_output_tokens_per_second: 69.6 },
|
|
106
|
+
{ name: "Foo", slug: "foo", median_output_tokens_per_second: 40 },
|
|
107
|
+
{ name: "Foo Latest", slug: "foo-latest", median_output_tokens_per_second: 90 },
|
|
108
|
+
{ name: "no speed", slug: "no-speed" },
|
|
109
|
+
{ name: "zero", slug: "zero", median_output_tokens_per_second: 0 },
|
|
110
|
+
],
|
|
111
|
+
}
|
|
112
|
+
const index = buildIndex(canned)
|
|
113
|
+
if (index.get(benchKey("o3-mini")) !== 153.831) throw new Error("self-check: o3-mini not mapped")
|
|
114
|
+
if (index.get("claude-fable-5") !== 69.6) throw new Error("self-check: slug not mapped")
|
|
115
|
+
if (index.has("no-speed") || index.has("zero")) throw new Error("self-check: an entry without a speed was mapped")
|
|
116
|
+
if (index.get(benchKey("Foo")) !== 90) throw new Error("self-check: two entries on one key did not keep the faster")
|
|
117
|
+
if (!apiKey()) {
|
|
118
|
+
console.log("self-check ok: mapping holds; no ARTIFICIAL_ANALYSIS_API_KEY, live path not exercised")
|
|
119
|
+
return
|
|
120
|
+
}
|
|
121
|
+
const live = await aaIndex()
|
|
122
|
+
if (live.size === 0) throw new Error("self-check: a key is set but nothing came back")
|
|
123
|
+
console.log("self-check ok: mapping holds; live " + live.size + " models keyed")
|
|
124
|
+
}
|
|
125
|
+
run().catch((error) => {
|
|
126
|
+
console.error(error)
|
|
127
|
+
process.exit(1)
|
|
128
|
+
})
|
|
129
|
+
}
|
package/bench.ts
CHANGED
|
@@ -7,8 +7,9 @@
|
|
|
7
7
|
// stripped of the labels a catalogue attaches (`latest`, `contributor`, `free`, `preview`) and of
|
|
8
8
|
// snapshot stamps (`1017`, `0731`); the model id is the fallback when the name finds nothing.
|
|
9
9
|
//
|
|
10
|
-
// `int` is on
|
|
11
|
-
// speed column
|
|
10
|
+
// `int` is on most entries; `tps` was on none of them when this was last measured (2026-10-19), so the
|
|
11
|
+
// speed column is fed by `tps.ts` and `aa.ts` instead — this file's `tps` stays only as the last resort.
|
|
12
|
+
// A figure that is not there is never guessed.
|
|
12
13
|
//
|
|
13
14
|
// Runnable standalone for its self-check: node plugins/oc-models-pro/bench.ts --self-check
|
|
14
15
|
|
|
@@ -18,7 +19,8 @@ import { dirname, join } from "node:path"
|
|
|
18
19
|
|
|
19
20
|
export const CATALOG = "https://catalog.stencil.so/models.json"
|
|
20
21
|
|
|
21
|
-
const
|
|
22
|
+
const CACHE_DIR = process.env.XDG_CACHE_HOME || join(homedir(), ".cache")
|
|
23
|
+
const CACHE = join(CACHE_DIR, "oc-models-pro", "bench.json")
|
|
22
24
|
const GRACE_MS = 6 * 60 * 60 * 1000
|
|
23
25
|
|
|
24
26
|
export interface Bench {
|
package/multipliers.ts
CHANGED
|
@@ -54,7 +54,8 @@ export interface Credits {
|
|
|
54
54
|
peak?: number
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
-
const
|
|
57
|
+
const CACHE_DIR = process.env.XDG_CACHE_HOME || join(homedir(), ".cache")
|
|
58
|
+
const CACHE = join(CACHE_DIR, "oc-models-pro", "go-limits.json")
|
|
58
59
|
|
|
59
60
|
function text(html: string): string {
|
|
60
61
|
return html
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rizal72/oc-models-pro",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"type": "module",
|
|
5
|
-
"description": "OpenCode v2 model picker with prices, context, intelligence, Go credit and the cost of a request",
|
|
5
|
+
"description": "OpenCode v2 model picker with prices, context, intelligence, tok/s, Go credit and the cost of a request",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
8
8
|
"url": "git+https://github.com/rizal72/opencode-plugins.git"
|
|
@@ -17,7 +17,8 @@
|
|
|
17
17
|
"opencode",
|
|
18
18
|
"opencode-plugin",
|
|
19
19
|
"models",
|
|
20
|
-
"pricing"
|
|
20
|
+
"pricing",
|
|
21
|
+
"throughput"
|
|
21
22
|
],
|
|
22
23
|
"publishConfig": {
|
|
23
24
|
"access": "public"
|
package/tps.ts
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
// oc-models-pro/tps.ts — tokens per second, measured on this machine.
|
|
2
|
+
//
|
|
3
|
+
// Not an estimate from a catalogue: it is what this provider actually did, for the models this
|
|
4
|
+
// person actually ran. The number is the one the industry publishes, so it can be read next to
|
|
5
|
+
// anything online: (output + reasoning tokens) over the window from the first streamed token to the
|
|
6
|
+
// last, time-to-first-token excluded — Artificial Analysis' "output speed", and `completion_tokens`
|
|
7
|
+
// as OpenAI, Anthropic and OpenRouter bill it, where reasoning is part of the output.
|
|
8
|
+
//
|
|
9
|
+
// The window is the last seven days when it holds at least three turns, and all of history when it
|
|
10
|
+
// does not: a number from a provider's good week is worth less than the provider's behaviour now,
|
|
11
|
+
// but an empty cell for a model used once last month is worth less still.
|
|
12
|
+
//
|
|
13
|
+
// Runnable standalone for its self-check: node plugins/oc-models-pro/tps.ts --self-check
|
|
14
|
+
|
|
15
|
+
import { execFile } from "node:child_process"
|
|
16
|
+
import { existsSync } from "node:fs"
|
|
17
|
+
import { homedir } from "node:os"
|
|
18
|
+
import { join } from "node:path"
|
|
19
|
+
|
|
20
|
+
export interface Speed {
|
|
21
|
+
/** Median tokens per second for this provider/model. */
|
|
22
|
+
tps: number
|
|
23
|
+
/** How many turns the median came from. */
|
|
24
|
+
samples: number
|
|
25
|
+
/** True when the seven-day window was too thin and the whole history was used. */
|
|
26
|
+
allTime: boolean
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
interface Turn {
|
|
30
|
+
key: string
|
|
31
|
+
sid: string
|
|
32
|
+
tps: number
|
|
33
|
+
at: number
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** A window thinner than this is noise, not a measurement. */
|
|
37
|
+
const RECENT_DAYS = 7
|
|
38
|
+
const MIN_SAMPLES = 3
|
|
39
|
+
/** A turn shorter than this cannot separate decode time from the rounding of the clock. */
|
|
40
|
+
const MIN_SPAN_MS = 200
|
|
41
|
+
const MIN_TOKENS = 20
|
|
42
|
+
|
|
43
|
+
export function databasePath(): string {
|
|
44
|
+
const data = process.env.XDG_DATA_HOME || join(homedir(), ".local", "share")
|
|
45
|
+
return process.env.OPENCODE_DB || join(data, "opencode", "opencode.db")
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** The host's own SQLite. Absent outside Bun, where the CLI carries the self-check. */
|
|
49
|
+
async function sqlite(): Promise<typeof import("bun:sqlite") | undefined> {
|
|
50
|
+
try {
|
|
51
|
+
return await import("bun:sqlite")
|
|
52
|
+
} catch {
|
|
53
|
+
return undefined
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async function read(database: string): Promise<Array<{ data: string; at: number; sid: string }>> {
|
|
58
|
+
if (!existsSync(database)) return []
|
|
59
|
+
const bun = await sqlite()
|
|
60
|
+
if (bun) {
|
|
61
|
+
try {
|
|
62
|
+
const db = new bun.Database(database, { readonly: true })
|
|
63
|
+
try {
|
|
64
|
+
return db
|
|
65
|
+
.query("select data, time_created as at, session_id as sid from session_message where type = 'assistant'")
|
|
66
|
+
.all() as Array<{ data: string; at: number; sid: string }>
|
|
67
|
+
} finally {
|
|
68
|
+
db.close()
|
|
69
|
+
}
|
|
70
|
+
} catch {
|
|
71
|
+
// A read-only open of a live WAL database is the one thing that differs per host.
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return new Promise((resolve, reject) => {
|
|
75
|
+
execFile(
|
|
76
|
+
"sqlite3",
|
|
77
|
+
[
|
|
78
|
+
"file:" + database + "?mode=ro",
|
|
79
|
+
"-json",
|
|
80
|
+
"select data, time_created as at, session_id as sid from session_message where type = 'assistant'",
|
|
81
|
+
],
|
|
82
|
+
{ maxBuffer: 256 * 1024 * 1024 },
|
|
83
|
+
(error, stdout) => (error ? reject(error) : resolve(JSON.parse(stdout || "[]"))),
|
|
84
|
+
)
|
|
85
|
+
})
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** One assistant turn as a rate, or nothing when the clock or the token counts cannot carry one. */
|
|
89
|
+
export function turn(entry: any, sid = ""): Turn | undefined {
|
|
90
|
+
if (!entry || !entry.tokens || !entry.time || !Array.isArray(entry.content)) return undefined
|
|
91
|
+
const provider = entry.model?.providerID
|
|
92
|
+
const model = entry.model?.id
|
|
93
|
+
if (typeof provider !== "string" || typeof model !== "string") return undefined
|
|
94
|
+
// The window opens at the first content part that carries a timestamp — a part without one (a file
|
|
95
|
+
// reference, say) would start the clock late and overstate the rate.
|
|
96
|
+
const first = entry.content.find((part: any) => typeof part?.time?.created === "number")?.time?.created
|
|
97
|
+
// The last streamed token; `completed` only when the host recorded nothing (it carries the same
|
|
98
|
+
// instant, and post-stream bookkeeping would drag the rate down).
|
|
99
|
+
const last = entry.time.streamed ?? entry.time.completed
|
|
100
|
+
if (typeof first !== "number" || typeof last !== "number") return undefined
|
|
101
|
+
const tokens = (entry.tokens.output ?? 0) + (entry.tokens.reasoning ?? 0)
|
|
102
|
+
const span = last - first
|
|
103
|
+
if (!(span >= MIN_SPAN_MS) || tokens < MIN_TOKENS) return undefined
|
|
104
|
+
return { key: provider + "/" + model, sid, tps: tokens / (span / 1000), at: entry.time.created ?? 0 }
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export function median(values: number[]): number {
|
|
108
|
+
const sorted = [...values].sort((a, b) => a - b)
|
|
109
|
+
const middle = sorted.length >> 1
|
|
110
|
+
return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle]
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** The rates of every model, keyed `providerID/modelID` as the host writes them. */
|
|
114
|
+
export async function speedIndex(database = databasePath()): Promise<{
|
|
115
|
+
speeds: Map<string, Speed>
|
|
116
|
+
turns: number
|
|
117
|
+
rows: number
|
|
118
|
+
}> {
|
|
119
|
+
let rows: Array<{ data: string; at: number; sid: string }>
|
|
120
|
+
try {
|
|
121
|
+
rows = await read(database)
|
|
122
|
+
} catch {
|
|
123
|
+
return { speeds: new Map(), turns: 0, rows: 0 }
|
|
124
|
+
}
|
|
125
|
+
const turns: Turn[] = []
|
|
126
|
+
for (const row of rows) {
|
|
127
|
+
let entry: any
|
|
128
|
+
try {
|
|
129
|
+
entry = JSON.parse(row.data)
|
|
130
|
+
} catch {
|
|
131
|
+
continue
|
|
132
|
+
}
|
|
133
|
+
const measured = turn(entry, row.sid)
|
|
134
|
+
if (measured) turns.push({ ...measured, at: measured.at || row.at })
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
const since = Date.now() - RECENT_DAYS * 86_400_000
|
|
138
|
+
const speeds = new Map<string, Speed>()
|
|
139
|
+
for (const key of new Set(turns.map((t) => t.key))) {
|
|
140
|
+
const all = turns.filter((t) => t.key === key)
|
|
141
|
+
const recent = all.filter((t) => t.at >= since)
|
|
142
|
+
const use = recent.length >= MIN_SAMPLES ? recent : all
|
|
143
|
+
if (use.length === 0) continue
|
|
144
|
+
// One voice per session, then the median of those voices: without it a single long-running
|
|
145
|
+
// session elects the number on its own, and this person's sessions run for days.
|
|
146
|
+
const perSession = new Map<string, number[]>()
|
|
147
|
+
for (const t of use) {
|
|
148
|
+
const list = perSession.get(t.sid) ?? []
|
|
149
|
+
list.push(t.tps)
|
|
150
|
+
perSession.set(t.sid, list)
|
|
151
|
+
}
|
|
152
|
+
speeds.set(key, {
|
|
153
|
+
tps: median([...perSession.values()].map((list) => median(list))),
|
|
154
|
+
samples: use.length,
|
|
155
|
+
allTime: recent.length < MIN_SAMPLES,
|
|
156
|
+
})
|
|
157
|
+
}
|
|
158
|
+
return { speeds, turns: turns.length, rows: rows.length }
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// Self-check: node plugins/oc-models-pro/tps.ts --self-check
|
|
162
|
+
if (process.argv[2] === "--self-check") {
|
|
163
|
+
const run = async () => {
|
|
164
|
+
const started = Date.now()
|
|
165
|
+
const { speeds, turns, rows } = await speedIndex()
|
|
166
|
+
const ms = Date.now() - started
|
|
167
|
+
|
|
168
|
+
// The thresholds and the key, on turns made up here: real data cannot exercise a floor it never hits.
|
|
169
|
+
const base = {
|
|
170
|
+
model: { providerID: "p", id: "m" },
|
|
171
|
+
time: { created: 0, streamed: 1000 },
|
|
172
|
+
tokens: { output: 30, reasoning: 20 },
|
|
173
|
+
content: [{ type: "text", time: { created: 200 } }],
|
|
174
|
+
}
|
|
175
|
+
const good = turn(base, "ses_x")
|
|
176
|
+
if (!good || good.key !== "p/m" || good.sid !== "ses_x") throw new Error("self-check: a plain turn did not parse")
|
|
177
|
+
if (Math.abs(good.tps - 62.5) > 0.001) throw new Error("self-check: 50 tokens over 800 ms is not 62.5 t/s: " + good.tps)
|
|
178
|
+
const short = { ...base, time: { created: 0, streamed: 300 } }
|
|
179
|
+
if (turn(short)) throw new Error("self-check: a 100 ms window must be dropped")
|
|
180
|
+
const few = { ...base, tokens: { output: 5, reasoning: 0 } }
|
|
181
|
+
if (turn(few)) throw new Error("self-check: five tokens must be dropped")
|
|
182
|
+
if (turn({ time: {}, tokens: {}, content: [] })) throw new Error("self-check: a turn without a model must be dropped")
|
|
183
|
+
if (median([3, 1, 2]) !== 2 || median([1, 4]) !== 2.5) throw new Error("self-check: median is wrong")
|
|
184
|
+
|
|
185
|
+
if (rows === 0) {
|
|
186
|
+
console.log("self-check ok: no assistant messages on this machine, nothing to measure")
|
|
187
|
+
return
|
|
188
|
+
}
|
|
189
|
+
// Rows but no measurement is not an empty machine: it is a shape that moved, and it must fail.
|
|
190
|
+
if (turns === 0) throw new Error("self-check: " + rows + " assistant messages and none measurable, the message shape changed")
|
|
191
|
+
if (speeds.size === 0) throw new Error("self-check: " + turns + " turns but no model measured")
|
|
192
|
+
for (const [key, speed] of speeds) {
|
|
193
|
+
if (!Number.isFinite(speed.tps) || speed.tps <= 0) throw new Error("self-check: " + key + " not finite: " + speed.tps)
|
|
194
|
+
if (speed.samples < 1) throw new Error("self-check: " + key + " has no samples")
|
|
195
|
+
}
|
|
196
|
+
const best = [...speeds.entries()].sort((a, b) => b[1].samples - a[1].samples).slice(0, 3)
|
|
197
|
+
console.log(
|
|
198
|
+
"self-check ok: " +
|
|
199
|
+
turns +
|
|
200
|
+
" turns, " +
|
|
201
|
+
speeds.size +
|
|
202
|
+
" models, " +
|
|
203
|
+
ms +
|
|
204
|
+
" ms -> " +
|
|
205
|
+
best.map(([k, s]) => k + " " + s.tps.toFixed(1) + " t/s (" + s.samples + (s.allTime ? " all-time" : " 7d") + ")").join(", "),
|
|
206
|
+
)
|
|
207
|
+
}
|
|
208
|
+
run().catch((error) => {
|
|
209
|
+
console.error(error)
|
|
210
|
+
process.exit(1)
|
|
211
|
+
})
|
|
212
|
+
}
|
package/tui.js
CHANGED
|
@@ -21,7 +21,9 @@ import { For, Show, createEffect, createMemo, createSignal, onCleanup } from "so
|
|
|
21
21
|
import { appendFileSync, readFileSync, renameSync, watch, writeFileSync } from "node:fs";
|
|
22
22
|
import { homedir, tmpdir } from "node:os";
|
|
23
23
|
import { join } from "node:path";
|
|
24
|
-
import { benchFor, benchIndex } from "./bench.ts";
|
|
24
|
+
import { benchFor, benchIndex, benchKey } from "./bench.ts";
|
|
25
|
+
import { aaIndex } from "./aa.ts";
|
|
26
|
+
import { speedIndex } from "./tps.ts";
|
|
25
27
|
import { credits, goPlan, key } from "./multipliers.ts";
|
|
26
28
|
import { LEGEND_MIN, fit, hintLines, rowWidth } from "./fit.ts";
|
|
27
29
|
import { tagsFor } from "./tags.ts";
|
|
@@ -138,7 +140,7 @@ const SORTS = [{
|
|
|
138
140
|
}, {
|
|
139
141
|
label: "speed",
|
|
140
142
|
short: "speed↓",
|
|
141
|
-
value: row => -(row.tps ?? -1)
|
|
143
|
+
value: row => -(row.measured ?? row.tps ?? -1)
|
|
142
144
|
}, {
|
|
143
145
|
label: "recently used",
|
|
144
146
|
short: "recent",
|
|
@@ -197,11 +199,36 @@ function free(row) {
|
|
|
197
199
|
function headerRow(f) {
|
|
198
200
|
return [" ", " ", "model".padEnd(f.name), cell("in", 6), cell("out", 6), ...(f.ctx ? [cell("ctx", 5)] : []), ...(f.int ? [cell("int", 5)] : []), ...(f.tkps ? [cell("tok/s", 6)] : []), cell("cred", 4), ...(f.note ? [cell("note", 6)] : [])].join(" ");
|
|
199
201
|
}
|
|
200
|
-
|
|
202
|
+
|
|
203
|
+
/** The speed cell on its own, so the renderer can colour it apart from the rest of the row. */
|
|
204
|
+
function speedCell(row, f) {
|
|
205
|
+
if (!f.tkps) return "";
|
|
206
|
+
const text = row.measured !== undefined ? String(Math.round(row.measured)) : row.tps !== undefined ? "~" + Math.round(row.tps) : "";
|
|
207
|
+
return cell(text, 6);
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/** A row in three pieces: everything before the speed, the speed, everything after. */
|
|
211
|
+
function rowParts(row, mine, favourite, f) {
|
|
201
212
|
const notes = [row.peak ? "peak*" + row.peak : "", row.promo ? "+" + row.promo + "x" : ""].filter(Boolean);
|
|
202
|
-
|
|
213
|
+
const head = [(mine ? "●" : " ") + (favourite ? "*" : " "), padTag(row.tag), label(row.name, f.name), price(row.input, 6), price(row.output, 6), ...(f.ctx ? [cell(tokens(row.context), 5)] : []), ...(f.int ? [cell(row.int !== undefined ? row.int.toFixed(1) : "", 5)] : [])];
|
|
214
|
+
const tail = [
|
|
203
215
|
// Any credit other than 1 changes the effective price, so any of them is shown.
|
|
204
|
-
cell(row.credit !== 1 ? "×" + row.credit : "", 4), ...(f.note ? [cell(notes.join(" "), 6)] : [])]
|
|
216
|
+
cell(row.credit !== 1 ? "×" + row.credit : "", 4), ...(f.note ? [cell(notes.join(" "), 6)] : [])];
|
|
217
|
+
const speed = speedCell(row, f);
|
|
218
|
+
if (!speed) return {
|
|
219
|
+
before: head.concat(tail).join(" "),
|
|
220
|
+
speed: "",
|
|
221
|
+
after: ""
|
|
222
|
+
};
|
|
223
|
+
return {
|
|
224
|
+
before: head.join(" ") + " ",
|
|
225
|
+
speed,
|
|
226
|
+
after: tail.length ? " " + tail.join(" ") : ""
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
function modelRow(row, mine, favourite, f) {
|
|
230
|
+
const parts = rowParts(row, mine, favourite, f);
|
|
231
|
+
return parts.before + parts.speed + parts.after;
|
|
205
232
|
}
|
|
206
233
|
const TRACE = join(tmpdir(), "oc-models-pro.log");
|
|
207
234
|
|
|
@@ -254,6 +281,10 @@ export default Plugin.define({
|
|
|
254
281
|
prefs = readPrefs();
|
|
255
282
|
let creditIndex = new Map();
|
|
256
283
|
let bench = new Map();
|
|
284
|
+
/** Measured on this machine, keyed `providerID/modelID`: it wins over any catalogue figure. */
|
|
285
|
+
let measured = new Map();
|
|
286
|
+
/** From Artificial Analysis, for the models this machine never ran. */
|
|
287
|
+
let estimated = new Map();
|
|
257
288
|
/** False when the Go page could not be read and no cache was there: the credits are all 1. */
|
|
258
289
|
let creditsKnown = false;
|
|
259
290
|
// Signals, not plain variables: the header shows them, and a plain variable left the title
|
|
@@ -296,6 +327,10 @@ export default Plugin.define({
|
|
|
296
327
|
const cost = (model.cost ?? []).find(entry => !entry.tier) ?? model.cost?.[0];
|
|
297
328
|
const limit = model.limit ?? {};
|
|
298
329
|
const score = benchFor(bench, model.name, model.modelID);
|
|
330
|
+
// Three sources, one order: what this machine measured, then Artificial Analysis, then
|
|
331
|
+
// the catalogue. A measured figure is never displaced by an estimate.
|
|
332
|
+
const mine = measured.get(model.providerID + "/" + model.modelID);
|
|
333
|
+
const theirs = mine === undefined ? estimated.get(benchKey(model.name)) ?? estimated.get(benchKey(model.modelID)) ?? score?.tps : undefined;
|
|
299
334
|
result.push({
|
|
300
335
|
name: model.name,
|
|
301
336
|
providerID: model.providerID,
|
|
@@ -316,7 +351,8 @@ export default Plugin.define({
|
|
|
316
351
|
promo: isGo(model.providerID) ? creditIndex.get(key(model.modelID))?.promo : undefined,
|
|
317
352
|
peak: isGo(model.providerID) ? creditIndex.get(key(model.modelID))?.peak : undefined,
|
|
318
353
|
int: score?.int,
|
|
319
|
-
tps:
|
|
354
|
+
tps: theirs,
|
|
355
|
+
measured: mine,
|
|
320
356
|
enabled: model.enabled !== false
|
|
321
357
|
});
|
|
322
358
|
}
|
|
@@ -340,7 +376,21 @@ export default Plugin.define({
|
|
|
340
376
|
// One fetch a day at most; a failure only costs the credit column and the effective ordering.
|
|
341
377
|
const plan = await goPlan();
|
|
342
378
|
creditIndex = plan ? credits(plan) : new Map();
|
|
343
|
-
|
|
379
|
+
// The three reads run together: the panel waits for the slowest, not for their sum. Measured
|
|
380
|
+
// comes first in the ordering below — the only figure that is this provider's own behaviour.
|
|
381
|
+
const started = Date.now();
|
|
382
|
+
const [benchmarks, speeds, estimates] = await Promise.all([benchIndex(), speedIndex(), aaIndex()]);
|
|
383
|
+
bench = benchmarks;
|
|
384
|
+
measured = new Map([...speeds.speeds].map(([key, speed]) => [key, speed.tps]));
|
|
385
|
+
estimated = estimates;
|
|
386
|
+
// How many of the models on offer actually come out with a figure: the one number that says
|
|
387
|
+
// whether a thin column is a thin source or a broken key.
|
|
388
|
+
const offer = listModels();
|
|
389
|
+
const withSpeed = offer.filter(model => {
|
|
390
|
+
if (measured.has(model.providerID + "/" + model.modelID)) return true;
|
|
391
|
+
return estimated.get(benchKey(model.name)) !== undefined || estimated.get(benchKey(model.modelID)) !== undefined;
|
|
392
|
+
}).length;
|
|
393
|
+
trace("open: tps=" + speeds.turns + " turns, " + measured.size + " measured, " + estimated.size + " from AA, " + withSpeed + "/" + offer.length + " with a speed, " + (Date.now() - started) + " ms");
|
|
344
394
|
creditsKnown = creditIndex.size > 0;
|
|
345
395
|
trace("open: models=" + listModels().length + " bench=" + bench.size + " plan=" + (plan ? "$" + plan.price + " " + plan.limits.length + " rows" : "none") + " credits=" + creditIndex.size + " prefs=" + prefs.favorite.size + "/" + prefs.recent.size);
|
|
346
396
|
if (!ui.panel.open("oc-models-pro")) {
|
|
@@ -899,29 +949,43 @@ export default Plugin.define({
|
|
|
899
949
|
return scope() !== "favorites" && prefs.favorite.has(key);
|
|
900
950
|
};
|
|
901
951
|
return (() => {
|
|
902
|
-
var _el$19 = _$createElement("box")
|
|
903
|
-
_el$20 = _$createElement("text");
|
|
904
|
-
_$insertNode(_el$19, _el$20);
|
|
952
|
+
var _el$19 = _$createElement("box");
|
|
905
953
|
_$setProp(_el$19, "id", "mp-" + key);
|
|
906
954
|
_$setProp(_el$19, "height", 1);
|
|
955
|
+
_$setProp(_el$19, "flexDirection", "row");
|
|
907
956
|
_$setProp(_el$19, "onMouseOver", () => hover(key));
|
|
908
957
|
_$setProp(_el$19, "onMouseMove", () => hover(key));
|
|
909
958
|
_$setProp(_el$19, "onMouseDown", () => {
|
|
910
959
|
setSelected(key);
|
|
911
960
|
choose(row);
|
|
912
961
|
});
|
|
913
|
-
_$
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
962
|
+
_$insert(_el$19, () => {
|
|
963
|
+
const parts = rowParts(row, mine(), favourite(), fitted());
|
|
964
|
+
const fg = active() ? ON_SELECTED : mine() ? ACCENT : BASE;
|
|
965
|
+
// Dim marks the borrowed figure; the tilde says it too, for a reader who
|
|
966
|
+
// cannot see the colour. Measured values keep the row's own colour.
|
|
967
|
+
const speedFg = active() ? ON_SELECTED : row.tps !== undefined ? DIM : fg;
|
|
968
|
+
return [(() => {
|
|
969
|
+
var _el$20 = _$createElement("text");
|
|
970
|
+
_$setProp(_el$20, "fg", fg);
|
|
971
|
+
_$setProp(_el$20, "wrapMode", "none");
|
|
972
|
+
_$insert(_el$20, () => parts.before);
|
|
973
|
+
return _el$20;
|
|
974
|
+
})(), _$memo(() => _$memo(() => !!parts.speed)() ? (() => {
|
|
975
|
+
var _el$21 = _$createElement("text");
|
|
976
|
+
_$setProp(_el$21, "fg", speedFg);
|
|
977
|
+
_$setProp(_el$21, "wrapMode", "none");
|
|
978
|
+
_$insert(_el$21, () => parts.speed);
|
|
979
|
+
return _el$21;
|
|
980
|
+
})() : null), _$memo(() => _$memo(() => !!parts.after)() ? (() => {
|
|
981
|
+
var _el$22 = _$createElement("text");
|
|
982
|
+
_$setProp(_el$22, "fg", fg);
|
|
983
|
+
_$setProp(_el$22, "wrapMode", "none");
|
|
984
|
+
_$insert(_el$22, () => parts.after);
|
|
985
|
+
return _el$22;
|
|
986
|
+
})() : null)];
|
|
924
987
|
});
|
|
988
|
+
_$effect(_$p => _$setProp(_el$19, "backgroundColor", active() ? SELECTED : undefined, _$p));
|
|
925
989
|
return _el$19;
|
|
926
990
|
})();
|
|
927
991
|
}
|
|
@@ -934,14 +998,14 @@ export default Plugin.define({
|
|
|
934
998
|
return hints();
|
|
935
999
|
},
|
|
936
1000
|
children: line => (() => {
|
|
937
|
-
var _el$
|
|
938
|
-
_el$
|
|
939
|
-
_$insertNode(_el$
|
|
940
|
-
_$setProp(_el$
|
|
941
|
-
_$setProp(_el$
|
|
942
|
-
_$setProp(_el$
|
|
943
|
-
_$insert(_el$
|
|
944
|
-
return _el$
|
|
1001
|
+
var _el$23 = _$createElement("box"),
|
|
1002
|
+
_el$24 = _$createElement("text");
|
|
1003
|
+
_$insertNode(_el$23, _el$24);
|
|
1004
|
+
_$setProp(_el$23, "height", 1);
|
|
1005
|
+
_$setProp(_el$24, "fg", DIM);
|
|
1006
|
+
_$setProp(_el$24, "wrapMode", "none");
|
|
1007
|
+
_$insert(_el$24, line);
|
|
1008
|
+
return _el$23;
|
|
945
1009
|
})()
|
|
946
1010
|
}));
|
|
947
1011
|
_$effect(_p$ => {
|