openzoo 0.49.18 → 0.50.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/worktree.mjs DELETED
@@ -1,424 +0,0 @@
1
- // Desktop grokui SPAWN isolation — Claude Code worktrees + UltraCode
2
- // spawn_agent_worktree. Not ported to iOS/Android/Seeker/PSG1 (no local FS).
3
- //
4
- // Git goes through dugite's bundled binary, never PATH `git`.
5
- import { execFileSync } from 'node:child_process';
6
- import {
7
- appendFileSync, cpSync, existsSync, mkdirSync, readdirSync,
8
- readFileSync, rmSync,
9
- } from 'node:fs';
10
- import { homedir } from 'node:os';
11
- import path from 'node:path';
12
- import { setupEnvironment } from 'dugite';
13
-
14
- const FETCH_MS = 8000;
15
- const GIT_MS = 15000;
16
-
17
- export function agentSlug(name) {
18
- const pr = parsePrRef(name);
19
- if (pr) return `pr-${pr.n}`;
20
- const s = String(name || 'agent').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 64);
21
- return s || 'agent';
22
- }
23
-
24
- /** #123, GitHub PR URL, GitLab MR URL, or a generic /pull/N. */
25
- export function parsePrRef(text) {
26
- const s = String(text || '').trim();
27
- if (!s) return null;
28
- const hash = /^#(\d+)\b/.exec(s);
29
- if (hash) return { n: Number(hash[1]) };
30
- const gh = /github\.com\/[^/\s]+\/[^/\s]+\/pull\/(\d+)/i.exec(s);
31
- if (gh) return { n: Number(gh[1]), host: 'github.com' };
32
- const gl = /gitlab\.com\/\S+\/-\/merge_requests\/(\d+)/i.exec(s);
33
- if (gl) return { n: Number(gl[1]), host: 'gitlab.com' };
34
- const pull = /\/pull\/(\d+)\b/i.exec(s);
35
- if (pull) return { n: Number(pull[1]) };
36
- return null;
37
- }
38
-
39
- export function extractPrFromSpawn(name, spec) {
40
- const fromName = parsePrRef(name);
41
- if (fromName) return fromName;
42
- const body = String(spec || '').trim();
43
- if (!body) return null;
44
- const first = body.split(/\s+/)[0];
45
- return parsePrRef(first) || parsePrRef(body.split('|')[0].trim()) || parsePrRef(body);
46
- }
47
-
48
- /** github.com → pull/N/head; gitlab.com → merge-requests/N/head; else pull first. */
49
- export function fetchSpecsForOrigin(originUrl, n) {
50
- const url = String(originUrl || '');
51
- if (/github\.com/i.test(url)) return [`pull/${n}/head`];
52
- if (/gitlab\.com/i.test(url)) return [`merge-requests/${n}/head`];
53
- return [`pull/${n}/head`, `merge-requests/${n}/head`];
54
- }
55
-
56
- /** Absolute dugite binary, or '' if the embed is missing. Never PATH `git`. */
57
- export function bundledGitPath() {
58
- try {
59
- const { gitLocation } = setupEnvironment({});
60
- return gitLocation && existsSync(gitLocation) ? gitLocation : '';
61
- } catch {
62
- return '';
63
- }
64
- }
65
-
66
- function resolveGitRunner() {
67
- let gitLocation = '';
68
- let env = {};
69
- try {
70
- const setup = setupEnvironment({
71
- GIT_TERMINAL_PROMPT: '0',
72
- GIT_CONFIG_NOSYSTEM: '1',
73
- });
74
- gitLocation = setup.gitLocation;
75
- env = setup.env;
76
- } catch { /* setup failed */ }
77
- if (!gitLocation || !existsSync(gitLocation)) {
78
- throw new Error('dugite embedded git is missing — SPAWN will not call PATH git');
79
- }
80
- return { bin: gitLocation, env, bundled: true };
81
- }
82
-
83
- let lastGitBinary = '';
84
- export function gitBinary() { return lastGitBinary; }
85
-
86
- /** Dugite bundled git only. Never exec `git` from PATH. */
87
- export function git(args, cwd, { allowFail = false, timeout = GIT_MS } = {}) {
88
- let bin, env;
89
- try {
90
- ({ bin, env } = resolveGitRunner());
91
- } catch (e) {
92
- if (allowFail) return '';
93
- throw e;
94
- }
95
- lastGitBinary = bin;
96
- try {
97
- return execFileSync(bin, ['-c', 'safe.directory=*', ...args], {
98
- cwd,
99
- env,
100
- encoding: 'utf8',
101
- timeout,
102
- maxBuffer: 8 * 1024 * 1024,
103
- stdio: ['ignore', 'pipe', 'pipe'],
104
- }).trim();
105
- } catch (e) {
106
- if (allowFail) return '';
107
- const err = new Error((e.stderr || e.message || '').toString().trim() || 'git failed');
108
- err.code = e.status;
109
- throw err;
110
- }
111
- }
112
-
113
- export function githubRepoFromOrigin(originUrl) {
114
- const m = /github\.com[:/]([^/]+)\/([^/.]+)/i.exec(String(originUrl || ''));
115
- return m ? { owner: m[1], repo: m[2] } : null;
116
- }
117
-
118
- export function githubPullApiUrl(originUrl, n) {
119
- const r = githubRepoFromOrigin(originUrl);
120
- return r ? `https://api.github.com/repos/${r.owner}/${r.repo}/pulls/${n}` : null;
121
- }
122
-
123
- export function gitlabProjectFromOrigin(originUrl) {
124
- const m = /gitlab\.com[:/](.+?)(?:\.git)?$/i.exec(String(originUrl || '').replace(/\/+$/, ''));
125
- return m ? m[1].replace(/\.git$/i, '') : null;
126
- }
127
-
128
- export function gitlabMrApiUrl(originUrl, n) {
129
- const project = gitlabProjectFromOrigin(originUrl);
130
- return project
131
- ? `https://gitlab.com/api/v4/projects/${encodeURIComponent(project)}/merge_requests/${n}`
132
- : null;
133
- }
134
-
135
- function httpGetJsonSync(url) {
136
- const src = `
137
- const r = await fetch(${JSON.stringify(url)}, {
138
- headers: { 'user-agent': 'openzoo-grokui', accept: 'application/json' },
139
- });
140
- if (!r.ok) process.exit(1);
141
- process.stdout.write(await r.text());
142
- `;
143
- const body = execFileSync(process.execPath, ['--input-type=module', '-e', src], {
144
- encoding: 'utf8',
145
- timeout: FETCH_MS,
146
- maxBuffer: 2 * 1024 * 1024,
147
- stdio: ['ignore', 'pipe', 'pipe'],
148
- });
149
- return JSON.parse(body);
150
- }
151
-
152
- /** GitHub/GitLab API → commit SHA when `git fetch pull/N/head` cannot. */
153
- export function fetchPrHeadShaViaApi(originUrl, n) {
154
- const gh = githubPullApiUrl(originUrl, n);
155
- if (gh) {
156
- try {
157
- const j = httpGetJsonSync(gh);
158
- if (j?.head?.sha) return String(j.head.sha);
159
- } catch { /* */ }
160
- }
161
- const gl = gitlabMrApiUrl(originUrl, n);
162
- if (gl) {
163
- try {
164
- const j = httpGetJsonSync(gl);
165
- if (j?.sha) return String(j.sha);
166
- } catch { /* */ }
167
- }
168
- return '';
169
- }
170
-
171
- export function isolatedWorktreesHome(home = homedir()) {
172
- return path.join(home, '.openzoo', 'grokui-worktrees');
173
- }
174
-
175
- export function isGitRepo(dir) {
176
- if (!dir || !existsSync(dir)) return false;
177
- return git(['rev-parse', '--is-inside-work-tree'], dir, { allowFail: true }) === 'true';
178
- }
179
-
180
- export function mainRepoRoot(dir) {
181
- if (!isGitRepo(dir)) return null;
182
- let common = git(['rev-parse', '--path-format=absolute', '--git-common-dir'], dir, { allowFail: true });
183
- if (!common) {
184
- const rel = git(['rev-parse', '--git-common-dir'], dir, { allowFail: true });
185
- common = rel ? path.resolve(dir, rel) : '';
186
- }
187
- if (!common) return null;
188
- if (common.endsWith('.git')) return path.dirname(common);
189
- return common;
190
- }
191
-
192
- export function originUrl(repo) {
193
- return git(['remote', 'get-url', 'origin'], repo, { allowFail: true });
194
- }
195
-
196
- export function freshBaseRef(repo) {
197
- const sym = git(['symbolic-ref', '--quiet', 'refs/remotes/origin/HEAD'], repo, { allowFail: true });
198
- if (sym) return sym.replace(/^refs\/remotes\//, '');
199
- for (const cand of ['origin/main', 'origin/master', 'main', 'master']) {
200
- if (git(['rev-parse', '--verify', cand], repo, { allowFail: true })) return cand;
201
- }
202
- return 'HEAD';
203
- }
204
-
205
- function ensureExclude(repo) {
206
- const common = git(['rev-parse', '--path-format=absolute', '--git-common-dir'], repo, { allowFail: true })
207
- || path.join(repo, '.git');
208
- const exclude = path.join(common, 'info', 'exclude');
209
- try { mkdirSync(path.dirname(exclude), { recursive: true }); } catch { /* */ }
210
- let cur = '';
211
- try { cur = readFileSync(exclude, 'utf8'); } catch { /* */ }
212
- if (!cur.includes('.openzoo/worktrees')) {
213
- appendFileSync(exclude, (cur.endsWith('\n') || !cur ? '' : '\n') + '.openzoo/worktrees/\n');
214
- }
215
- }
216
-
217
- function worktreeListed(repo, dest) {
218
- const list = git(['worktree', 'list', '--porcelain'], repo, { allowFail: true });
219
- const want = path.resolve(dest);
220
- for (const block of list.split('\n\n')) {
221
- const line = block.split('\n').find((l) => l.startsWith('worktree '));
222
- if (line && path.resolve(line.slice(9).trim()) === want) return true;
223
- }
224
- return false;
225
- }
226
-
227
- function simpleGlobMatch(pattern, rel) {
228
- const p = String(pattern || '').replace(/\\/g, '/').replace(/\/+$/, '');
229
- const r = String(rel || '').replace(/\\/g, '/');
230
- if (!p) return false;
231
- if (!p.includes('*') && !p.includes('?')) {
232
- return r === p || r.endsWith('/' + p) || path.basename(r) === p;
233
- }
234
- const re = new RegExp('^' + p.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*').replace(/\?/g, '.') + '$');
235
- return re.test(r) || re.test(path.basename(r));
236
- }
237
-
238
- /** Claude .worktreeinclude — copy matching gitignored files. Optional, never blocks. */
239
- export function copyWorktreeIncludes(repo, dest) {
240
- const spec = path.join(repo, '.worktreeinclude');
241
- if (!existsSync(spec) || !existsSync(dest)) return 0;
242
- const patterns = readFileSync(spec, 'utf8').split(/\r?\n/)
243
- .map((l) => l.trim()).filter((l) => l && !l.startsWith('#'));
244
- if (!patterns.length) return 0;
245
- const ignored = git(['ls-files', '-oi', '--exclude-standard', '-z'], repo, { allowFail: true })
246
- .split('\0').filter(Boolean);
247
- let n = 0;
248
- for (const rel of ignored) {
249
- if (!patterns.some((p) => simpleGlobMatch(p, rel))) continue;
250
- const from = path.join(repo, rel);
251
- const to = path.join(dest, rel);
252
- try {
253
- if (!existsSync(from)) continue;
254
- mkdirSync(path.dirname(to), { recursive: true });
255
- cpSync(from, to);
256
- n += 1;
257
- } catch { /* optional */ }
258
- }
259
- return n;
260
- }
261
-
262
- export function worktreePath(repo, slug) {
263
- return path.join(repo, '.openzoo', 'worktrees', slug);
264
- }
265
-
266
- export function worktreeBranch(slug) {
267
- return `worktree-${slug}`;
268
- }
269
-
270
- function addWorktree(repo, slug, { pr = null } = {}) {
271
- const dest = worktreePath(repo, slug);
272
- const branch = worktreeBranch(slug);
273
- mkdirSync(path.dirname(dest), { recursive: true });
274
- ensureExclude(repo);
275
-
276
- if (existsSync(dest) && worktreeListed(repo, dest)) {
277
- return {
278
- path: dest, branch, repo, reused: true,
279
- fetchRef: pr?.n ? fetchSpecsForOrigin(originUrl(repo), pr.n)[0] : '',
280
- };
281
- }
282
- if (existsSync(dest)) {
283
- try { rmSync(dest, { recursive: true, force: true }); } catch { /* */ }
284
- }
285
-
286
- let base = freshBaseRef(repo);
287
- let fetchRef = '';
288
- if (pr?.n) {
289
- const origin = originUrl(repo);
290
- const specs = fetchSpecsForOrigin(origin, pr.n);
291
- for (const spec of specs) {
292
- try {
293
- git(['fetch', '--no-tags', 'origin', spec], repo, { timeout: FETCH_MS });
294
- if (git(['rev-parse', '--verify', 'FETCH_HEAD'], repo, { allowFail: true })) {
295
- fetchRef = spec;
296
- base = 'FETCH_HEAD';
297
- break;
298
- }
299
- } catch { /* next spec */ }
300
- }
301
- if (!fetchRef && origin) {
302
- const sha = fetchPrHeadShaViaApi(origin, pr.n);
303
- if (sha) {
304
- try {
305
- git(['fetch', '--no-tags', 'origin', sha], repo, { timeout: FETCH_MS });
306
- if (git(['rev-parse', '--verify', sha], repo, { allowFail: true })) {
307
- fetchRef = specs[0];
308
- base = sha;
309
- }
310
- } catch { /* isolated fallback */ }
311
- }
312
- }
313
- }
314
-
315
- const hasBranch = Boolean(git(['rev-parse', '--verify', `refs/heads/${branch}`], repo, { allowFail: true }));
316
- try {
317
- if (hasBranch) git(['worktree', 'add', dest, branch], repo);
318
- else git(['worktree', 'add', '-b', branch, dest, base], repo);
319
- } catch (e) {
320
- if (!existsSync(dest)) throw e;
321
- }
322
- try { copyWorktreeIncludes(repo, dest); } catch { /* optional */ }
323
- return {
324
- path: dest, branch, repo, reused: hasBranch, baseRef: base,
325
- ...(fetchRef ? { fetchRef } : {}),
326
- ...(pr?.n ? { pr: pr.n } : {}),
327
- };
328
- }
329
-
330
- function isolatedChildDir(parentDir, slug) {
331
- let dest = path.join(isolatedWorktreesHome(), slug);
332
- const parent = path.resolve(parentDir);
333
- if (dest === parent || dest.startsWith(parent + path.sep)) {
334
- dest = path.join(homedir(), '.openzoo', 'grokui-worktrees-out', slug);
335
- }
336
- if (dest === parent) throw new Error('refusing to isolate a child into the parent cwd');
337
- mkdirSync(dest, { recursive: true });
338
- return { path: dest, branch: null, kind: 'isolated' };
339
- }
340
-
341
- /**
342
- * Isolated cwd for a SPAWNed grokui thread (same Node process; shell cwd = t.dir).
343
- * Git parent → <repo>/.openzoo/worktrees/<slug> on worktree-<slug>.
344
- * Non-git → ~/.openzoo/grokui-worktrees/<slug>, never the parent dir.
345
- */
346
- export function prepareChildDir(parent, name, spec) {
347
- const parentDir = path.resolve(parent?.dir || process.env.OZ_WORKSPACE_DIR
348
- || path.join(homedir(), '.openzoo', 'grokui-workspace'));
349
- const pr = extractPrFromSpawn(name, spec);
350
- const slug = pr && parsePrRef(name) ? `pr-${pr.n}` : agentSlug(name);
351
- const repo = mainRepoRoot(parentDir);
352
- if (repo) {
353
- try {
354
- const ws = addWorktree(repo, slug, { pr });
355
- return {
356
- path: ws.path,
357
- branch: ws.branch,
358
- parentDir,
359
- kind: 'worktree',
360
- repo,
361
- fetchRef: ws.fetchRef || '',
362
- baseRef: ws.baseRef,
363
- };
364
- } catch { /* fall back to isolated, still not parent cwd */ }
365
- }
366
- return { ...isolatedChildDir(parentDir, slug), parentDir };
367
- }
368
-
369
- export function worktreeHasWork(wt) {
370
- if (!wt?.path || !existsSync(wt.path)) return false;
371
- if (wt.kind === 'isolated' || !wt.branch) {
372
- try { return readdirSync(wt.path).length > 0; } catch { return false; }
373
- }
374
- if (git(['status', '--porcelain'], wt.path, { allowFail: true })) return true;
375
- const base = wt.baseRef && wt.baseRef !== 'FETCH_HEAD' && wt.baseRef !== 'HEAD'
376
- ? wt.baseRef
377
- : (wt.repo ? freshBaseRef(wt.repo) : 'HEAD');
378
- if (Number(git(['rev-list', '--count', `${base}..HEAD`], wt.path, { allowFail: true })) > 0) return true;
379
- const unpushed = git(['rev-list', '--count', '@{upstream}..HEAD'], wt.path, { allowFail: true });
380
- return Boolean(unpushed && Number(unpushed) > 0);
381
- }
382
-
383
- export function lockWorktree(wtOrThread) {
384
- const wt = wtOrThread?.worktree || wtOrThread;
385
- if (!wt?.path || !wt.branch || !existsSync(wt.path)) return false;
386
- git(['worktree', 'lock', '--reason', 'openzoo spawn', wt.path], wt.repo || wt.path, { allowFail: true });
387
- wt.locked = true;
388
- return true;
389
- }
390
-
391
- export function unlockWorktree(wtOrThread) {
392
- const wt = wtOrThread?.worktree || wtOrThread;
393
- if (!wt?.path || !wt.branch) return false;
394
- git(['worktree', 'unlock', wt.path], wt.repo || wt.path, { allowFail: true });
395
- wt.locked = false;
396
- return true;
397
- }
398
-
399
- /** Clean worktree → remove + delete the branch we created. Dirty → keep. */
400
- export function finishChildDir(wtOrThread) {
401
- const thread = wtOrThread && wtOrThread.worktree ? wtOrThread : null;
402
- const wt = thread?.worktree || wtOrThread;
403
- if (!wt?.path) return { skipped: true };
404
- unlockWorktree(wt);
405
- if (worktreeHasWork(wt)) return { kept: true, path: wt.path, branch: wt.branch };
406
- if (wt.kind === 'worktree' && wt.repo && wt.branch) {
407
- git(['worktree', 'remove', '--force', wt.path], wt.repo, { allowFail: true });
408
- git(['branch', '-D', wt.branch], wt.repo, { allowFail: true });
409
- }
410
- try { if (existsSync(wt.path)) rmSync(wt.path, { recursive: true, force: true }); } catch { /* */ }
411
- if (thread) delete thread.worktree;
412
- return { removed: true, path: wt.path, branch: wt.branch };
413
- }
414
-
415
- export function isWorktreeLocked(repo, dest) {
416
- const list = git(['worktree', 'list', '--porcelain'], repo, { allowFail: true });
417
- const want = path.resolve(dest);
418
- let current = '';
419
- for (const line of list.split('\n')) {
420
- if (line.startsWith('worktree ')) current = path.resolve(line.slice(9).trim());
421
- if ((line === 'locked' || line.startsWith('locked ')) && current === want) return true;
422
- }
423
- return false;
424
- }
@@ -1,132 +0,0 @@
1
- # modelroute — current state
2
-
3
- _Snapshot: 2026-08-20. Read `HANDOFF.md` for how to integrate; this file is what is true, what is
4
- guessed, and what is unfinished._
5
-
6
- ---
7
-
8
- ## Status at a glance
9
-
10
- | part | state |
11
- |---|---|
12
- | Router runtime (`route()`) | **working**, 22 contract tests + module selftest green |
13
- | Catalogue (414 gateway-callable models) | **working**, rebuilt from the gateway, 11 KB |
14
- | Task classifier (10 classes) | **working**, 0.642 held-out / 0.772 5-fold, 40 KB |
15
- | End-to-end CLI (`ask.py`) | **working** — routes, calls openzoo, records outcome |
16
- | `P_success` measured on real models | **in progress** — hard-suite run ~20% done |
17
- | Base-suite outcomes | **discarded** (saturated, low-n) — kept at `outcomes.base.json.bak` |
18
- | Vision / creative / advice classes | **never measured** — no objective grader, prior only |
19
-
20
- The router works today on priors. It works *better* the moment the hard-suite run lands and every
21
- time you record a real outcome.
22
-
23
- ---
24
-
25
- ## What is measured vs. what is authored
26
-
27
- This distinction is the whole point of the design. Do not blur it.
28
-
29
- ### MEASURED (evidence)
30
-
31
- - **Classifier accuracy.** 0.642 on a register-shifted held-out split, against a bag-of-words
32
- baseline at 0.333 and majority at 0.100. Ablated: ridge head +0.12 over perceptron, hashed IDF
33
- +0.09, anchor gazetteer +0.075. Caveat: anchors were written *after* inspecting held-out errors,
34
- so 0.642 is optimistically biased. The 5-fold number (0.772) is clean.
35
- - **Routing regret.** 5.0% of held-out requests route below the bar their true class demands, at a
36
- median 1.12× the oracle route's cost. The top-2 hedge causes that: hedge off gives 13.3% under-bar
37
- at 1.00× cost. Ability bought with money, as the objective asks.
38
- - **Model success rates.** In progress. See below.
39
-
40
- ### AUTHORED (policy, not evidence — argue with these)
41
-
42
- - **`_BAR`** — the required `P_success` per (class, difficulty). The single most important knob.
43
- - **`_OUT_TOKENS`** — expected completion length per class. Cost is linear in it.
44
- - **`_CLASS_CATEGORIES`** — which OpenRouter leaderboards count as evidence for which task class.
45
- `vision` uses `technology` as an explicit *proxy* (there is no vision leaderboard); `bulk` maps to
46
- nothing on purpose, so its objective degenerates to "cheapest feasible", which is correct for it.
47
- - **`P_TOP/P_BOT/P_UNKNOWN`** — how a leaderboard rank becomes a probability, and what an unranked
48
- model is assumed to be worth (0.45, deliberately below every bar except bulk/chat).
49
- - **`BIND_ABOVE_TOKENS` (60k), `BIND_SLICE_TOKENS` (8k)** — when to bind instead of buying context.
50
- - **`difficulty()`** — a 3-signal heuristic (rigour cues, long input, low classifier margin). There
51
- are no difficulty labels in the corpus, so a trained head here would be a guess in a lab coat.
52
-
53
- ---
54
-
55
- ## The measurement run
56
-
57
- **Hard suite** (`suite_hard.py`): 64 tasks, every answer key **computed by the generator** and
58
- verified before spending — a reference answer must pass the real grader and a wrong answer must fail
59
- it. 5 classes: reasoning (24), agentic (12), longctx (10), bulk (10), code (8).
60
-
61
- It discriminates, which the base suite did not:
62
-
63
- | class | pass rate across models so far |
64
- |---|---|
65
- | agentic | 0.96 |
66
- | longctx | 0.88 |
67
- | code | 0.79 |
68
- | reasoning | 0.67 |
69
- | bulk | 0.54 |
70
-
71
- Model spread so far runs 0.60 → 0.95 (vs. the base suite, where 30 of 124 models tied at a perfect
72
- 1.00 and it could not rank anything above the floor).
73
-
74
- **Run state:** ~1,200 of 6,336 calls, 20 of 99 models, ~$5 spent, ~4 calls/s, ETA well under an hour.
75
- Restart with the same command and `--resume` — it skips completed calls and *retries* errored ones.
76
-
77
- ### Why 99 models, not 414
78
-
79
- 409 models is not 409 pieces of information. `gpt-5.6-luna/sol/terra` scored identically; so did the
80
- `deepseek-v4-flash` and `gemini-3.x-flash` families. A fixed budget is models × tasks, and redundant
81
- variants buy nothing while every per-class estimate stays at n≈4 — a 95% CI **0.5 wide**, wider than
82
- the entire range the router cares about. `--per-provider 2` takes each provider's flagship (by
83
- leaderboard placement, then capability) plus its cheapest option. Same money, 10× the depth.
84
-
85
- ### Statistical power, plainly
86
-
87
- | tasks per model | 95% CI width at p≈0.9 |
88
- |---|---|
89
- | 5 | 0.59 |
90
- | 10 | 0.39 |
91
- | 30 | 0.22 |
92
- | 64 (hard suite) | ~0.18 overall, but only ~n=10–24 *per class* |
93
-
94
- **Per-class estimates are still thin.** Overall model quality is now solidly measured; a
95
- per-(class, model) number at n=10 still leans on the prior. If you need per-class confidence, add
96
- tasks to that class in `suite_hard.py` — the generators make that cheap and key-safe.
97
-
98
- ---
99
-
100
- ## Known weaknesses (ranked by how much they should bother you)
101
-
102
- 1. **Three classes are never measured.** `creative`, `advice` and `vision` have no objective grader
103
- (`vision` also needs image fixtures). They run on the leaderboard prior and always will until
104
- someone decides what "good" means for them. An LLM judge was rejected: it is a second opinion, not
105
- a measurement, and it would poison the same table the benchmark fills with real results.
106
- 2. **`code` is the worst classifier class** (0.25 on the register-shifted held-out split). Requests
107
- like "the linter hates everything I write" share no vocabulary with the training split. The
108
- anchor gazetteer helps; more real examples would help more.
109
- 3. **The corpus is one person's idea of what requests look like** — 400 authored sentences. Your
110
- actual traffic is the real distribution. Drop a JSONL at
111
- `lecore_data/modelroute/eval_real.jsonl` (`{"text":…, "label":…}`) and `train_router.py` will
112
- report against it automatically.
113
- 4. **21% of gateway calls return 429.** It is a per-account limit, not a concurrency problem —
114
- dropping from 48 to 16 workers made it *worse*. The shared throttle handles it; it costs
115
- wall-clock, not money or correctness.
116
- 5. **Real prompts are fragments.** In a live agent session most turns are "fix it" / "did this
117
- work?", which carry no routable task alone. Pass the conversation as `context=`; the router will
118
- otherwise hedge (correctly) and route as if the task were hard, which costs money.
119
-
120
- ---
121
-
122
- ## What I would build next, in order
123
-
124
- 1. **Wire outcome recording into the openzoo call path.** Everything else exists. This is the step
125
- that makes `P_success` yours instead of borrowed.
126
- 2. **Grow `suite_hard.py` per class** until per-class n ≥ 30, using the generators (keys stay
127
- correct by construction).
128
- 3. **Decide what "good" means for creative/advice**, or accept them as prior-only forever and say so
129
- in the UI.
130
- 4. **Thompson sampling for exploration.** Right now the cheapest clearing model always wins, so the
131
- alternatives never get observed and the table can only improve where you already route. A small
132
- exploration budget would fix that; the Beta posteriors needed for it are already there.
@@ -1,110 +0,0 @@
1
- # The auto-router, for Moose
2
-
3
- *How `openzoo/auto` picks a model — a VSA classifier trained closed-form, 40KB,
4
- no torch, no embeddings, no pretrained anything. Your math, our objective.*
5
-
6
- ## The objective (this is the part that isn't a classifier)
7
-
8
- Routing is not "what topic is this" and not "what's the best model":
9
-
10
- ```
11
- pick argmin cost(m, task) subject to P_success(m | task) >= bar(task)
12
- m ∈ feasible(task)
13
- ```
14
-
15
- If nothing clears the bar, say so and return the highest-P option **labelled as
16
- such** — never silently pretend the cheapest one was fine. Four terms, four
17
- sources:
18
-
19
- - **feasible()** — hard metadata facts only (image input, tool calling, strict
20
- JSON, context length). A model without an image modality can't read a
21
- screenshot at any price. Never guessed.
22
- - **cost()** — `price_in × est. prompt tokens + price_out × est. completion
23
- tokens`, divided by the cheapest feasible candidate's, so ranking is
24
- scale-free.
25
- - **bar()** — an authored policy knob per (class, difficulty). Stated, not
26
- hidden, not measured.
27
- - **P_success()** — the interesting one, below.
28
-
29
- **A trap we deliberately walked around:** price is tempting as a capability
30
- proxy (expensive ≈ stronger). It is NOT used that way, because cost is the
31
- thing being minimised — letting price raise P_success would make the objective
32
- partly cancel itself and quietly re-rank toward expensive models for no
33
- measured reason. Price appears in cost(), nowhere else.
34
-
35
- ## The classifier — random indexing, two gradient-free heads
36
-
37
- Text → hypervector by **Kanerva random indexing**: each token deterministically
38
- hashes (sha256, never Python's salted `hash()` — deterministic under any
39
- `PYTHONHASHSEED`) to k sparse ±1 positions in d=4096. A request is the bundle
40
- of its token atoms over **three separately-normalised channels** — words, word
41
- bigrams, char 4-grams — so a misspelling degrades a vector instead of erasing a
42
- word. Channel atoms are weighted by a hashed IDF table (16KB).
43
-
44
- Two heads produce the **same artifact shape** (K×d matrix, K=10 classes:
45
- code / reasoning / longctx / vision / bulk / creative / translate / agentic /
46
- chat / advice):
47
-
48
- - **ridge (shipped)** — closed-form one-vs-rest least squares in the dual:
49
- `W = Xᵀ(XXᵀ + λI)⁻¹Y`. One 535×535 solve. No epochs, no learning rate, no
50
- shuffling, no lucky seed.
51
- - **adapthd (kept)** — perceptron prototypes; the reference the ridge head is
52
- checked against.
53
-
54
- Shipped artifact: **40KB int8 matrix + 16KB IDF**. Inference is a dot product.
55
- `router.json` carries `{classes, q_b64 (int8), q_shape [10,4096], scale, idf}`.
56
-
57
- ## Measured, with ablations (train_router.py --ablate)
58
-
59
- 10 classes, 400 authored requests + anchors, held-out split written in a
60
- deliberately different register:
61
-
62
- | configuration | held-out | 5-fold |
63
- |---|---|---|
64
- | ridge + IDF + anchors | **0.642** | **0.772** |
65
- | bag-of-words nearest centroid | 0.333 | — |
66
- | majority class | 0.100 | — |
67
-
68
- Each of the three choices (ridge, IDF, anchors) was ablated separately and each
69
- earns its place.
70
-
71
- **But class accuracy is not the headline.** `routing_regret.py` measures the
72
- classifier in dollars and failures instead: **5.0% of held-out requests route
73
- below the bar their true class would have demanded, at a median 1.12× the
74
- oracle route's cost.** A misclassification that lands on an adequate cheaper
75
- model is not an error that matters.
76
-
77
- ## P_success — a prior that becomes a measurement
78
-
79
- No API reports "model m completes class c with probability p". What exists is
80
- OpenRouter's per-category leaderboards: top-20 **by usage** — revealed
81
- preference, not benchmark (popularity tracks price, marketing, defaults too,
82
- and it's labelled as such).
83
-
84
- So: prior = leaderboard rank in the categories the class maps to, shrunk toward
85
- a metadata floor for the ~87% of the 414-model catalogue on no leaderboard at
86
- all. Then `record_outcome(class, model, ok)` turns it into a **Beta posterior**
87
- whose prior weight is the leaderboard number and whose data is our own
88
- completions. Day one it answers `evidence: "prior"`; after real traffic it
89
- answers `evidence: "measured(n=…)"`. As of today `outcomes.json` holds **391
90
- (class, model) pairs of live measurements**.
91
-
92
- The classifier's output is never the answer — it sets the bar and picks which
93
- leaderboards to trust. The catalogue is a lookup table, not a softmax, so a
94
- model added tomorrow is routable today with zero retraining.
95
-
96
- ## Serving
97
-
98
- Gateway (`x402-tokens/src/auto.ts`): `openzoo/auto` → `route(text)` → shortlist
99
- → **first-2-of-5 race** over the shortlist (RACE_X=2, RACE_Y=5, min score 6);
100
- tool-bearing bodies race a tool-capable pool, never a bare flash. If the router
101
- artifacts are missing it degrades to a static cheap pool — routing is an
102
- enhancement, never a dependency. Named models are never rerouted.
103
-
104
- Files: `holographic_modelroute.py` (the whole thing, heavily commented),
105
- `router.json` / `outcomes.json` / `catalog.json` (artifacts),
106
- `tools/modelroute/train_router.py` (training + ablations),
107
- `tools/modelroute/routing_regret.py` (the dollars-and-failures metric).
108
-
109
- *Live result on X right now: @openzoobot answers tweet-sized questions at
110
- $0.00001–0.007 via this router, receipts printed per reply.*