@zenera/cli 1.1.32 → 1.1.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +586 -734
- package/dist/catalog.d.ts +8 -3
- package/dist/catalog.js +49 -9
- package/dist/commands/meta.js +18 -7
- package/dist/commands/models.js +82 -21
- package/dist/meta/host.d.ts +3 -1
- package/dist/meta/host.js +3 -1
- package/dist/meta/model.d.ts +8 -0
- package/dist/meta/model.js +19 -1
- package/dist/meta/wire.d.ts +2 -0
- package/dist/meta/wire.js +4 -0
- package/dist/rank.d.ts +24 -0
- package/dist/rank.js +153 -0
- package/package.json +3 -3
- package/templates/editor/.github/skills/zen-cli/references/models.md +7 -0
package/dist/catalog.d.ts
CHANGED
|
@@ -142,8 +142,13 @@ export declare function matches(row: CatalogEntry, query: string, filters?: Filt
|
|
|
142
142
|
export declare const catalogProviders: () => readonly Provider[];
|
|
143
143
|
/** Resold on Vertex rather than made by Google. */
|
|
144
144
|
export declare const isPartner: (e: CatalogEntry) => boolean;
|
|
145
|
-
/** Provider, then Google's own models, then each partner's together. */
|
|
145
|
+
/** Provider, then Google's own models, then each partner's together; best first within each. */
|
|
146
146
|
export declare function byGroup(a: CatalogEntry, b: CatalogEntry): number;
|
|
147
|
-
/** The heading a row is listed under
|
|
148
|
-
export declare function groupOf(e: CatalogEntry): string
|
|
147
|
+
/** The heading a row is listed under: its provider, split by maker where the provider resells. */
|
|
148
|
+
export declare function groupOf(e: CatalogEntry): string;
|
|
149
|
+
/**
|
|
150
|
+
* At most `limit` rows, shared evenly between providers and kept in order, so a
|
|
151
|
+
* long first listing does not push the others off a truncated screen.
|
|
152
|
+
*/
|
|
153
|
+
export declare function sampleByProvider(rows: readonly CatalogEntry[], limit: number): CatalogEntry[];
|
|
149
154
|
//# sourceMappingURL=catalog.d.ts.map
|
package/dist/catalog.js
CHANGED
|
@@ -6,9 +6,9 @@
|
|
|
6
6
|
//
|
|
7
7
|
// Note what is *not* here: any judgement about which model is good. The lists
|
|
8
8
|
// below are ordered by how likely a call is to succeed and how little it costs
|
|
9
|
-
// to find out, not by quality — a recovery path, not a recommender. The
|
|
10
|
-
//
|
|
11
|
-
//
|
|
9
|
+
// to find out, not by quality — a recovery path, not a recommender. The one
|
|
10
|
+
// judgement the CLI does make lives in `rank.ts`, read off the ids a listing
|
|
11
|
+
// returns, so that no table here has to be argued about every quarter.
|
|
12
12
|
//
|
|
13
13
|
// Nor is there a single hard-coded table of every model. Vendors ship models
|
|
14
14
|
// faster than a release cycle, so the shipped table is a *fallback* and the
|
|
@@ -20,6 +20,7 @@ import { GATEWAY_KEY_HEADER, GATEWAY_NAME, GATEWAY_UPSTREAMS, gatewayOf, ModelRe
|
|
|
20
20
|
import { Cache, cacheKey } from "./cache.js";
|
|
21
21
|
import { PROVIDERS } from "./keys.js";
|
|
22
22
|
import { classify } from "./liveness.js";
|
|
23
|
+
import { compareModels } from "./rank.js";
|
|
23
24
|
/** A day. Model lists change on the scale of weeks; a stale row costs a retry. */
|
|
24
25
|
export const CATALOG_TTL_MS = 24 * 60 * 60 * 1000;
|
|
25
26
|
export const CATALOG_KIND = 'catalog';
|
|
@@ -419,7 +420,13 @@ async function fetchGatewayListing(at) {
|
|
|
419
420
|
return by;
|
|
420
421
|
}
|
|
421
422
|
function publisherOf(id) {
|
|
422
|
-
|
|
423
|
+
if (/^claude-/.test(id)) {
|
|
424
|
+
return 'anthropic';
|
|
425
|
+
}
|
|
426
|
+
if (/^(mistral|codestral)-/.test(id)) {
|
|
427
|
+
return 'mistral';
|
|
428
|
+
}
|
|
429
|
+
return id.includes('/') ? id.split('/')[0] : 'google';
|
|
423
430
|
}
|
|
424
431
|
/** What the gateway serves under one provider's name, enriched like a vendor listing. */
|
|
425
432
|
export async function fetchGatewayCatalog(provider, at) {
|
|
@@ -669,20 +676,53 @@ export const catalogProviders = () => PROVIDERS;
|
|
|
669
676
|
// ---------------------------------------------------------------------------
|
|
670
677
|
/** Resold on Vertex rather than made by Google. */
|
|
671
678
|
export const isPartner = (e) => e.publisher !== undefined && e.publisher !== 'google';
|
|
672
|
-
/**
|
|
679
|
+
/** The order listings are shown in: the vendors in rank order, then the resellers. */
|
|
680
|
+
const GROUP_ORDER = ['anthropic', 'openai', 'google', 'vertex', 'openrouter'];
|
|
681
|
+
/** Provider, then Google's own models, then each partner's together; best first within each. */
|
|
673
682
|
export function byGroup(a, b) {
|
|
674
|
-
return (a.provider.
|
|
683
|
+
return (GROUP_ORDER.indexOf(a.provider) - GROUP_ORDER.indexOf(b.provider) ||
|
|
675
684
|
Number(isPartner(a)) - Number(isPartner(b)) ||
|
|
676
685
|
(a.publisher ?? '').localeCompare(b.publisher ?? '') ||
|
|
677
|
-
a
|
|
686
|
+
compareModels(a, b));
|
|
678
687
|
}
|
|
679
|
-
/** The heading a row is listed under
|
|
688
|
+
/** The heading a row is listed under: its provider, split by maker where the provider resells. */
|
|
680
689
|
export function groupOf(e) {
|
|
681
690
|
if (!e.publisher) {
|
|
682
|
-
return
|
|
691
|
+
return e.provider;
|
|
683
692
|
}
|
|
684
693
|
return isPartner(e)
|
|
685
694
|
? `${e.provider} · partner · ${e.publisher}`
|
|
686
695
|
: `${e.provider} · Google (Gemini, Gemma, Imagen, Veo)`;
|
|
687
696
|
}
|
|
697
|
+
/**
|
|
698
|
+
* At most `limit` rows, shared evenly between providers and kept in order, so a
|
|
699
|
+
* long first listing does not push the others off a truncated screen.
|
|
700
|
+
*/
|
|
701
|
+
export function sampleByProvider(rows, limit) {
|
|
702
|
+
if (rows.length <= limit) {
|
|
703
|
+
return [...rows];
|
|
704
|
+
}
|
|
705
|
+
const counts = new Map();
|
|
706
|
+
for (const row of rows) {
|
|
707
|
+
const c = counts.get(row.provider) ?? { total: 0, take: 0 };
|
|
708
|
+
c.total++;
|
|
709
|
+
counts.set(row.provider, c);
|
|
710
|
+
}
|
|
711
|
+
for (let left = limit, gave = true; left > 0 && gave;) {
|
|
712
|
+
gave = false;
|
|
713
|
+
for (const c of counts.values()) {
|
|
714
|
+
if (left > 0 && c.take < c.total) {
|
|
715
|
+
c.take++;
|
|
716
|
+
left--;
|
|
717
|
+
gave = true;
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
const seen = new Map();
|
|
722
|
+
return rows.filter((row) => {
|
|
723
|
+
const n = (seen.get(row.provider) ?? 0) + 1;
|
|
724
|
+
seen.set(row.provider, n);
|
|
725
|
+
return n <= counts.get(row.provider).take;
|
|
726
|
+
});
|
|
727
|
+
}
|
|
688
728
|
//# sourceMappingURL=catalog.js.map
|
package/dist/commands/meta.js
CHANGED
|
@@ -11,7 +11,7 @@ import { copilotEventLog } from "../eventlog.js";
|
|
|
11
11
|
import { ensureHome } from "../home.js";
|
|
12
12
|
import { assertOwner, isProvider, KeyStore, PROVIDERS } from "../keys.js";
|
|
13
13
|
import { probeChat, probeModel } from "../liveness.js";
|
|
14
|
-
import { answerBox, chooseModel, DATASET_DETAILS, DATASET_SUMMARY, DATASET_USAGE, datasetProjectFlag, DEFAULT_RESUMES,
|
|
14
|
+
import { answerBox, chooseModel, DATASET_DETAILS, DATASET_SUMMARY, DATASET_USAGE, datasetProjectFlag, DEFAULT_RESUMES, editorFiles, fallbackRef, FINETUNE_DETAILS, FINETUNE_SUMMARY, FINETUNE_USAGE, finetuneProjectFlag, lastSession, launch, listPrompts, loadPrompt, locate, masked, metaCompatible, misspelledProvider, MODEL_ENV, openLog, ORDER, PROMPT_DIR, providersWarning, readMeta, recordSession, RESUME_PROMPT, resumeDelayMs, runDataset, runFinetune, sameModel, sessionModel, sessionTotals, SOURCE_LABELS, startVertexRelay, tailSpans, Tally, terminalSink, transient, wire, wireGateway, writeMeta, } from "../meta/index.js";
|
|
15
15
|
import { splitRef } from "../modelref.js";
|
|
16
16
|
import { duration } from "../narrate.js";
|
|
17
17
|
import * as Projects from "../projects.js";
|
|
@@ -169,10 +169,13 @@ export const meta = {
|
|
|
169
169
|
'The model is looked for in this order: --model, ZENERA_META_MODEL in the',
|
|
170
170
|
"shell, ZENERA_META_MODEL in the project's .env, `zen meta model`, then",
|
|
171
171
|
'agents.yaml. With none of them set it uses the best model it knows of for',
|
|
172
|
-
'a provider you hold a key for, and says which on stderr.',
|
|
172
|
+
'a provider you hold a key for, and says which on stderr. Through a gateway',
|
|
173
|
+
'it asks what the gateway serves and takes the best of that, as ranked by',
|
|
174
|
+
'`zen models ls --chat --best`.',
|
|
173
175
|
'',
|
|
174
176
|
'Setting one asks the provider a one-word question first, so a model that',
|
|
175
|
-
|
|
177
|
+
"will not answer is never stored. A model whose API rejects copilot's tools",
|
|
178
|
+
'(gpt-4o and older) is refused before that. --force skips both.',
|
|
176
179
|
'',
|
|
177
180
|
'`zen meta model --pick` browses every model your keys reach: / filters,',
|
|
178
181
|
't asks one a question, p takes it.',
|
|
@@ -557,14 +560,14 @@ async function go(ctx, values, project, prompt, promptName) {
|
|
|
557
560
|
// After materialize: a keyring in bifrost mode is what sets the url.
|
|
558
561
|
const gateway = gatewayOf();
|
|
559
562
|
const only = values.provider ? asProvider(values.provider) : undefined;
|
|
563
|
+
// The fallback can cost a gateway listing, so it is only looked up when nothing else answers.
|
|
560
564
|
const chosen = chooseModel({
|
|
561
565
|
flag: values.model,
|
|
562
566
|
shell,
|
|
563
567
|
env: process.env[MODEL_ENV],
|
|
564
568
|
stored: (await readMeta()).model,
|
|
565
569
|
project: projectModel(project.dir),
|
|
566
|
-
|
|
567
|
-
});
|
|
570
|
+
}) ?? chooseModel({ fallback: await fallbackRef(store, only, gateway !== undefined) });
|
|
568
571
|
if (!chosen) {
|
|
569
572
|
if (only) {
|
|
570
573
|
throw credentialError(`no ${only} key`, `add one: zen key add ${only}`);
|
|
@@ -623,7 +626,10 @@ async function go(ctx, values, project, prompt, promptName) {
|
|
|
623
626
|
warn(competing);
|
|
624
627
|
}
|
|
625
628
|
if (chosen.from === 'default') {
|
|
626
|
-
|
|
629
|
+
const what = gateway
|
|
630
|
+
? 'the best model the gateway serves'
|
|
631
|
+
: `the best ${owner} model zen knows of`;
|
|
632
|
+
note(dim(`no model set — using ${chosen.ref}, ${what}`));
|
|
627
633
|
note(dim('set another: zen meta model <provider>/<id>'));
|
|
628
634
|
}
|
|
629
635
|
note(cyan(`${owner} · ${wiring.model} · ${project.name}`));
|
|
@@ -922,13 +928,15 @@ async function model(ctx, values, args) {
|
|
|
922
928
|
if (ref) {
|
|
923
929
|
return set(ctx, values, project, ref);
|
|
924
930
|
}
|
|
931
|
+
const store = await KeyStore.open();
|
|
932
|
+
store.materialize();
|
|
925
933
|
const sources = {
|
|
926
934
|
flag: undefined,
|
|
927
935
|
shell,
|
|
928
936
|
env: process.env[MODEL_ENV],
|
|
929
937
|
stored: (await readMeta()).model,
|
|
930
938
|
project: project ? projectModel(project.dir) : undefined,
|
|
931
|
-
fallback:
|
|
939
|
+
fallback: await fallbackRef(store, undefined, gatewayOf() !== undefined),
|
|
932
940
|
};
|
|
933
941
|
const chosen = chooseModel(sources);
|
|
934
942
|
const at = {
|
|
@@ -1000,6 +1008,9 @@ async function vet(ctx, ref) {
|
|
|
1000
1008
|
}
|
|
1001
1009
|
const { provider, id } = splitRef(ref);
|
|
1002
1010
|
const owner = provider ?? 'openai';
|
|
1011
|
+
if (!metaCompatible(owner, id)) {
|
|
1012
|
+
throw usageError(`${owner}/${id} cannot run the meta agent: its API rejects copilot's tools`, 'pick a reasoning model, e.g. from: zen models ls --chat --best');
|
|
1013
|
+
}
|
|
1003
1014
|
const store = await KeyStore.open();
|
|
1004
1015
|
store.materialize();
|
|
1005
1016
|
if (!gatewayOf() && !store.active(owner)) {
|
package/dist/commands/models.js
CHANGED
|
@@ -13,11 +13,12 @@
|
|
|
13
13
|
// ---------------------------------------------------------------------------
|
|
14
14
|
import { createEmbedder, GATEWAY_NAME, gatewayOf, ModelRegistry } from '@zenera/neo';
|
|
15
15
|
import { parse } from "../args.js";
|
|
16
|
-
import { byGroup, groupOf, isPartner, loadCatalog, loadCatalogs, matches, routerBalance, } from "../catalog.js";
|
|
16
|
+
import { byGroup, groupOf, isPartner, loadCatalog, loadCatalogs, matches, routerBalance, sampleByProvider, } from "../catalog.js";
|
|
17
17
|
import { ensureHome } from "../home.js";
|
|
18
18
|
import { envNames, form, isProvider, KeyStore, PROVIDERS } from "../keys.js";
|
|
19
19
|
import { probeChat, probeModel } from "../liveness.js";
|
|
20
|
-
import {
|
|
20
|
+
import { compareModels } from "../rank.js";
|
|
21
|
+
import { ago, bold, CliError, credentialError, cut, cyan, dim, EXIT, green, isInteractive, json, note, plain, progress, red, table, usageError, write, writeAll, yellow, } from "../term.js";
|
|
21
22
|
const USAGE = 'zen models <providers|ls|search|show|test|browse> [ref] [options]';
|
|
22
23
|
/**
|
|
23
24
|
* The keyring is materialised here, and only here.
|
|
@@ -142,18 +143,54 @@ function row(entry) {
|
|
|
142
143
|
dim(entry.name ?? ''),
|
|
143
144
|
];
|
|
144
145
|
}
|
|
145
|
-
/**
|
|
146
|
-
function
|
|
147
|
-
const
|
|
146
|
+
/** Drops every column no row has anything in, so a blank one costs no gap. */
|
|
147
|
+
function filled(rows) {
|
|
148
|
+
const keep = (rows[0] ?? []).map((_, c) => rows.some((r) => plain(r[c] ?? '') !== ''));
|
|
149
|
+
return rows.map((r) => r.filter((_, c) => keep[c]));
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* One section per provider (and per maker, where a provider resells), each
|
|
153
|
+
* headed by how much of it is on screen and how fresh it is. The maker is in
|
|
154
|
+
* the heading, so it is not repeated as a column. Columns line up across
|
|
155
|
+
* sections, so the eye can run down them past a heading.
|
|
156
|
+
*/
|
|
157
|
+
function sections(shown, all, cats) {
|
|
158
|
+
const totals = new Map();
|
|
159
|
+
for (const e of all) {
|
|
160
|
+
totals.set(groupOf(e), (totals.get(groupOf(e)) ?? 0) + 1);
|
|
161
|
+
}
|
|
162
|
+
const counts = new Map();
|
|
163
|
+
for (const e of shown) {
|
|
164
|
+
counts.set(groupOf(e), (counts.get(groupOf(e)) ?? 0) + 1);
|
|
165
|
+
}
|
|
166
|
+
const fresh = new Map(cats.map((c) => [c.provider, c]));
|
|
167
|
+
// An empty cell stays unstyled: style codes around nothing defeat the table's trimEnd.
|
|
168
|
+
const faint = (s) => (s ? dim(s) : '');
|
|
169
|
+
const lines = table(filled(shown.map((e) => [
|
|
170
|
+
cyan(e.ref),
|
|
171
|
+
faint(roleMark(e)),
|
|
172
|
+
faint(tokens(e.contextLength)),
|
|
173
|
+
faint(e.pricing?.free ? 'free' : ''),
|
|
174
|
+
// A gateway names a model by its id, which the ref already says.
|
|
175
|
+
faint(e.name && e.name !== e.id ? e.name : ''),
|
|
176
|
+
])));
|
|
148
177
|
const out = [];
|
|
178
|
+
const headed = new Set();
|
|
179
|
+
// A wrapped row reads as two models; a piped listing keeps every character.
|
|
180
|
+
const fit = (line) => process.stdout.isTTY && process.stdout.columns ? cut(line, process.stdout.columns) : line;
|
|
149
181
|
let last;
|
|
150
|
-
|
|
182
|
+
shown.forEach((e, i) => {
|
|
151
183
|
const group = groupOf(e);
|
|
152
|
-
if (group
|
|
153
|
-
|
|
184
|
+
if (group !== last) {
|
|
185
|
+
const total = totals.get(group);
|
|
186
|
+
const on = counts.get(group);
|
|
187
|
+
const size = on < total ? `${on} of ${total}` : String(total);
|
|
188
|
+
const cat = headed.has(e.provider) ? undefined : fresh.get(e.provider);
|
|
189
|
+
headed.add(e.provider);
|
|
190
|
+
out.push(...(out.length > 0 ? [''] : []), `${bold(group)} ${dim(size)}${cat ? ` ${freshness(cat)}` : ''}`);
|
|
154
191
|
}
|
|
155
192
|
last = group;
|
|
156
|
-
out.push(lines[i]);
|
|
193
|
+
out.push(fit(` ${lines[i]}`));
|
|
157
194
|
});
|
|
158
195
|
return out;
|
|
159
196
|
}
|
|
@@ -162,10 +199,19 @@ function grouped(entries) {
|
|
|
162
199
|
* on stderr, because it is narration about the answer rather than the answer.
|
|
163
200
|
*/
|
|
164
201
|
function explain(cats) {
|
|
202
|
+
// Through a gateway every provider shares one request, so one failure is said once.
|
|
203
|
+
const said = new Set();
|
|
165
204
|
for (const cat of cats) {
|
|
166
205
|
if (cat.problem) {
|
|
167
|
-
|
|
168
|
-
|
|
206
|
+
const who = cat.via ? cat.via.name : cat.provider;
|
|
207
|
+
const detail = cat.problem.detail ?? 'no reason given';
|
|
208
|
+
if (said.has(`${who}\0${detail}`)) {
|
|
209
|
+
continue;
|
|
210
|
+
}
|
|
211
|
+
said.add(`${who}\0${detail}`);
|
|
212
|
+
// A gateway has no built-in list to fall back on.
|
|
213
|
+
const shown = cat.via ? 'nothing listed' : `showing the ${cat.origin} list`;
|
|
214
|
+
note(`${yellow(who)} could not be listed: ${detail} ` + dim(`(${shown})`));
|
|
169
215
|
if (cat.problem.fix) {
|
|
170
216
|
note(dim(` ${cat.problem.fix}`));
|
|
171
217
|
}
|
|
@@ -225,7 +271,8 @@ const ls = async (ctx, args) => {
|
|
|
225
271
|
refresh: { type: 'boolean' },
|
|
226
272
|
limit: { type: 'string' },
|
|
227
273
|
all: { type: 'boolean' },
|
|
228
|
-
|
|
274
|
+
best: { type: 'boolean' },
|
|
275
|
+
}, 'zen models ls [provider] [--chat] [--embeddings] [--best] [--refresh] [--limit N] [--all]');
|
|
229
276
|
const where = await credentials();
|
|
230
277
|
const targets = scope(where, positionals[0]);
|
|
231
278
|
const roles = rolesFrom(values);
|
|
@@ -237,7 +284,7 @@ const ls = async (ctx, args) => {
|
|
|
237
284
|
const all = cats
|
|
238
285
|
.flatMap((c) => c.entries)
|
|
239
286
|
.filter((e) => matches(e, '', { roles }))
|
|
240
|
-
.sort(byGroup);
|
|
287
|
+
.sort(values.best ? compareModels : byGroup);
|
|
241
288
|
if (ctx.json) {
|
|
242
289
|
json({
|
|
243
290
|
models: all,
|
|
@@ -253,14 +300,20 @@ const ls = async (ctx, args) => {
|
|
|
253
300
|
announce(ctx, where);
|
|
254
301
|
explain(cats);
|
|
255
302
|
if (all.length === 0) {
|
|
256
|
-
|
|
303
|
+
if (!cats.every((c) => c.problem)) {
|
|
304
|
+
note(dim('nothing matched — try: zen models ls --refresh'));
|
|
305
|
+
}
|
|
257
306
|
return;
|
|
258
307
|
}
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
308
|
+
// A ranking keeps its top; a grouped listing shows some of every provider.
|
|
309
|
+
const shown = values.best ? all.slice(0, limit) : sampleByProvider(all, limit);
|
|
310
|
+
writeAll(values.best ? table(shown.map(row)) : sections(shown, all, cats));
|
|
311
|
+
if (all.length > shown.length) {
|
|
312
|
+
const one = targets.length === 1 ? targets[0] : '<provider>';
|
|
313
|
+
note('');
|
|
314
|
+
note(dim(`showing ${shown.length} of ${all.length} — zen models ls ${one} --all, or zen models search <words>`));
|
|
262
315
|
}
|
|
263
|
-
if (!where.gateway &&
|
|
316
|
+
if (!where.gateway && shown.some(isPartner)) {
|
|
264
317
|
note(dim('partner models are billed through Vertex and must be enabled in Model Garden first: https://console.cloud.google.com/vertex-ai/model-garden'));
|
|
265
318
|
}
|
|
266
319
|
};
|
|
@@ -277,7 +330,8 @@ const search = async (ctx, args) => {
|
|
|
277
330
|
'min-context': { type: 'string' },
|
|
278
331
|
limit: { type: 'string' },
|
|
279
332
|
refresh: { type: 'boolean' },
|
|
280
|
-
|
|
333
|
+
best: { type: 'boolean' },
|
|
334
|
+
}, 'zen models search <query> [--provider p] [--embeddings] [--tools] [--vision] [--free] [--min-context N] [--best] [--limit N]');
|
|
281
335
|
const query = positionals.join(' ').trim();
|
|
282
336
|
const where = await credentials();
|
|
283
337
|
const targets = scope(where, values.provider);
|
|
@@ -298,8 +352,10 @@ const search = async (ctx, args) => {
|
|
|
298
352
|
.filter((e) => matches(e, query, filters))
|
|
299
353
|
// Cheapest first when price is known: search is usually the step before
|
|
300
354
|
// picking one, and the free ones are what most people are looking for.
|
|
301
|
-
.sort(
|
|
302
|
-
|
|
355
|
+
.sort(values.best
|
|
356
|
+
? compareModels
|
|
357
|
+
: (a, b) => Number(b.pricing?.free ?? false) - Number(a.pricing?.free ?? false) ||
|
|
358
|
+
byGroup(a, b));
|
|
303
359
|
if (ctx.json) {
|
|
304
360
|
json({ query, matched: hits.length, models: hits.slice(0, limit) });
|
|
305
361
|
return;
|
|
@@ -622,6 +678,11 @@ export const models = {
|
|
|
622
678
|
' zen models test <ref> … Ask it one real question.',
|
|
623
679
|
' zen models browse [provider] Walk providers and models with the',
|
|
624
680
|
' arrow keys; t asks one, p prints its ref.',
|
|
681
|
+
'',
|
|
682
|
+
'Listings are grouped by provider, best model first in each. `ls --best`',
|
|
683
|
+
'and `search --best` drop the groups for one list, best first: Anthropic,',
|
|
684
|
+
'OpenAI, Google, then the rest; newer generations and higher tiers ahead;',
|
|
685
|
+
'mini, nano, lite and haiku last. A heuristic read off the ids, not a benchmark.',
|
|
625
686
|
],
|
|
626
687
|
run: async (ctx) => {
|
|
627
688
|
const [name, ...rest] = ctx.args;
|
package/dist/meta/host.d.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
export { invokedAs, parse } from '../args.ts';
|
|
2
|
+
export { loadCatalogs } from '../catalog.ts';
|
|
2
3
|
export type { EventLine } from '../eventlog.ts';
|
|
3
4
|
export { paths, readJson, writeJson } from '../home.ts';
|
|
4
|
-
export { envOf, type KeyEntry, type KeyStore, type Provider } from '../keys.ts';
|
|
5
|
+
export { envOf, PROVIDERS, type KeyEntry, type KeyStore, type Provider } from '../keys.ts';
|
|
5
6
|
export { PROVIDER_NAMES, splitRef } from '../modelref.ts';
|
|
6
7
|
export { duration } from '../narrate.ts';
|
|
8
|
+
export { rankModels } from '../rank.ts';
|
|
7
9
|
export { copyTree, SHARED_RULES, TEMPLATES } from '../scaffold.ts';
|
|
8
10
|
export { bold, CliError, confirm, credentialError, cut, cyan, dim, EXIT, green, invalidError, json, note, pad, plain, red, styled, table, usageError, write, writeAll, yellow, } from '../term.ts';
|
|
9
11
|
export { safe } from '../trace.ts';
|
package/dist/meta/host.js
CHANGED
|
@@ -6,10 +6,12 @@
|
|
|
6
6
|
// a dependency. `test/meta-boundary.test.ts` holds the line.
|
|
7
7
|
// ---------------------------------------------------------------------------
|
|
8
8
|
export { invokedAs, parse } from "../args.js";
|
|
9
|
+
export { loadCatalogs } from "../catalog.js";
|
|
9
10
|
export { paths, readJson, writeJson } from "../home.js";
|
|
10
|
-
export { envOf } from "../keys.js";
|
|
11
|
+
export { envOf, PROVIDERS } from "../keys.js";
|
|
11
12
|
export { PROVIDER_NAMES, splitRef } from "../modelref.js";
|
|
12
13
|
export { duration } from "../narrate.js";
|
|
14
|
+
export { rankModels } from "../rank.js";
|
|
13
15
|
export { copyTree, SHARED_RULES, TEMPLATES } from "../scaffold.js";
|
|
14
16
|
export { bold, CliError, confirm, credentialError, cut, cyan, dim, EXIT, green, invalidError, json, note, pad, plain, red, styled, table, usageError, write, writeAll, yellow, } from "../term.js";
|
|
15
17
|
export { safe } from "../trace.js";
|
package/dist/meta/model.d.ts
CHANGED
|
@@ -36,6 +36,14 @@ export declare const RECOMMENDED: Partial<Record<Provider, readonly string[]>>;
|
|
|
36
36
|
* `only` is `--provider`: the recommendation for that provider, if it is held.
|
|
37
37
|
*/
|
|
38
38
|
export declare function defaultRef(store: KeyStore, only?: Provider, gateway?: boolean): string | undefined;
|
|
39
|
+
/**
|
|
40
|
+
* The best chat model a gateway actually serves, by `rank.ts`: a vendor
|
|
41
|
+
* recommendation it does not carry would fail on the first call. The store must
|
|
42
|
+
* already be materialized, since that is what puts the gateway in the environment.
|
|
43
|
+
*/
|
|
44
|
+
export declare function gatewayDefaultRef(only?: Provider): Promise<string | undefined>;
|
|
45
|
+
/** The last resort: through a gateway, what it serves; otherwise what a held key recommends. */
|
|
46
|
+
export declare function fallbackRef(store: KeyStore, only?: Provider, gateway?: boolean): Promise<string | undefined>;
|
|
39
47
|
export interface MetaFile {
|
|
40
48
|
version: 1;
|
|
41
49
|
model?: string;
|
package/dist/meta/model.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { paths, PROVIDER_NAMES, readJson, writeJson, } from "./host.js";
|
|
1
|
+
import { loadCatalogs, paths, PROVIDER_NAMES, PROVIDERS, rankModels, readJson, writeJson, } from "./host.js";
|
|
2
|
+
import { metaCompatible } from "./wire.js";
|
|
2
3
|
// ---------------------------------------------------------------------------
|
|
3
4
|
// The model
|
|
4
5
|
// ---------------------------------------------------------------------------
|
|
@@ -76,6 +77,23 @@ export function defaultRef(store, only, gateway = false) {
|
|
|
76
77
|
}
|
|
77
78
|
return undefined;
|
|
78
79
|
}
|
|
80
|
+
/**
|
|
81
|
+
* The best chat model a gateway actually serves, by `rank.ts`: a vendor
|
|
82
|
+
* recommendation it does not carry would fail on the first call. The store must
|
|
83
|
+
* already be materialized, since that is what puts the gateway in the environment.
|
|
84
|
+
*/
|
|
85
|
+
export async function gatewayDefaultRef(only) {
|
|
86
|
+
const cats = await loadCatalogs(only ? [only] : PROVIDERS);
|
|
87
|
+
const usable = cats
|
|
88
|
+
.flatMap((c) => c.entries)
|
|
89
|
+
.filter((e) => e.roles.includes('chat') && metaCompatible(e.provider, e.id));
|
|
90
|
+
const best = rankModels(usable)[0];
|
|
91
|
+
return best ? `${best.provider}/${best.id}` : undefined;
|
|
92
|
+
}
|
|
93
|
+
/** The last resort: through a gateway, what it serves; otherwise what a held key recommends. */
|
|
94
|
+
export async function fallbackRef(store, only, gateway = false) {
|
|
95
|
+
return ((gateway ? await gatewayDefaultRef(only) : undefined) ?? defaultRef(store, only, gateway));
|
|
96
|
+
}
|
|
79
97
|
const EMPTY = { version: 1 };
|
|
80
98
|
export const readMeta = () => readJson(paths.meta(), EMPTY);
|
|
81
99
|
export function writeMeta(file) {
|
package/dist/meta/wire.d.ts
CHANGED
|
@@ -8,6 +8,8 @@ import { type KeyEntry, type KeyStore, type Provider } from './host.ts';
|
|
|
8
8
|
* listed rather than newer ones, so each new generation works without a release.
|
|
9
9
|
*/
|
|
10
10
|
export declare function wireApi(provider: Provider, id: string): string | undefined;
|
|
11
|
+
/** Whether copilot's tools survive the model: OpenAI's completions-only models refuse them. */
|
|
12
|
+
export declare function metaCompatible(provider: Provider, id: string): boolean;
|
|
11
13
|
export interface Wiring {
|
|
12
14
|
provider: Provider;
|
|
13
15
|
/** what the model is called on the wire */
|
package/dist/meta/wire.js
CHANGED
|
@@ -27,6 +27,10 @@ export function wireApi(provider, id) {
|
|
|
27
27
|
? 'responses'
|
|
28
28
|
: undefined;
|
|
29
29
|
}
|
|
30
|
+
/** Whether copilot's tools survive the model: OpenAI's completions-only models refuse them. */
|
|
31
|
+
export function metaCompatible(provider, id) {
|
|
32
|
+
return provider !== 'openai' || wireApi(provider, id) !== undefined;
|
|
33
|
+
}
|
|
30
34
|
/** Where the relay's url goes once it is listening. */
|
|
31
35
|
export const RELAY_URL = 'http://127.0.0.1:<relay>';
|
|
32
36
|
const vertexClaude = (id) => /^(anthropic\/)?claude-/.test(id);
|
package/dist/rank.d.ts
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
export type Maker = 'anthropic' | 'openai' | 'google' | 'other';
|
|
2
|
+
export type Tier = 'top' | 'mid' | 'small';
|
|
3
|
+
export interface ModelTraits {
|
|
4
|
+
maker: Maker;
|
|
5
|
+
/** `[5, 6]` for `gpt-5.6-sol`; empty when the id carries none */
|
|
6
|
+
version: number[];
|
|
7
|
+
tier: Tier;
|
|
8
|
+
/** built for one job: codex, search, audio, image… */
|
|
9
|
+
special: boolean;
|
|
10
|
+
preview: boolean;
|
|
11
|
+
/** a dated snapshot, as `YYYYMMDD` or the vendor's shorter `MMDD` */
|
|
12
|
+
date?: string;
|
|
13
|
+
}
|
|
14
|
+
/** What a listing row needs to be ranked: the provider routing it and its id. */
|
|
15
|
+
export interface Ranked {
|
|
16
|
+
provider: string;
|
|
17
|
+
id: string;
|
|
18
|
+
}
|
|
19
|
+
export declare function parseModel(id: string): ModelTraits;
|
|
20
|
+
/** Best first. Usable as a sort comparator on any listing row. */
|
|
21
|
+
export declare function compareModels(a: Ranked, b: Ranked): number;
|
|
22
|
+
/** A sorted copy, best first. */
|
|
23
|
+
export declare function rankModels<T extends Ranked>(rows: readonly T[]): T[];
|
|
24
|
+
//# sourceMappingURL=rank.d.ts.map
|
package/dist/rank.js
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
// ---------------------------------------------------------------------------
|
|
2
|
+
// Ranking models
|
|
3
|
+
//
|
|
4
|
+
// A heuristic read off the id alone, not a benchmark. Vendors name models by
|
|
5
|
+
// generation and tier, so both can be parsed and compared without a table
|
|
6
|
+
// that goes stale with every release: `gpt-6-sol` outranks `gpt-5.6-sol` the
|
|
7
|
+
// day it is listed.
|
|
8
|
+
//
|
|
9
|
+
// Best first: general models before specialised ones and small tiers last;
|
|
10
|
+
// then Anthropic, OpenAI, Google, everyone else; then the newer generation;
|
|
11
|
+
// then the higher tier; then the rest of the version.
|
|
12
|
+
// ---------------------------------------------------------------------------
|
|
13
|
+
const MAKERS = ['anthropic', 'openai', 'google', 'other'];
|
|
14
|
+
const TIERS = ['top', 'mid', 'small'];
|
|
15
|
+
const SMALL = new Set(['haiku', 'luna', 'mini', 'nano', 'lite', 'small', 'tiny']);
|
|
16
|
+
const TOP = new Set(['opus', 'sol', 'pro', 'ultra', 'argon', 'max']);
|
|
17
|
+
const PREVIEW = new Set(['preview', 'exp', 'experimental', 'beta', 'alpha']);
|
|
18
|
+
// Not `instruct`: open-weight chat models carry it (`qwen3-…-instruct`) as their general release.
|
|
19
|
+
const SPECIAL = /codex|search|audio|realtime|image|tts|transcribe|computer-use|embedding|research|(^|-)live(-|$)/;
|
|
20
|
+
function makerOf(name) {
|
|
21
|
+
if (name.startsWith('claude')) {
|
|
22
|
+
return 'anthropic';
|
|
23
|
+
}
|
|
24
|
+
// Open weights under OpenAI's name, served by others.
|
|
25
|
+
if (name.startsWith('gpt-oss')) {
|
|
26
|
+
return 'other';
|
|
27
|
+
}
|
|
28
|
+
if (/^(gpt|chatgpt|o\d)/.test(name)) {
|
|
29
|
+
return 'openai';
|
|
30
|
+
}
|
|
31
|
+
return name.startsWith('gemini') ? 'google' : 'other';
|
|
32
|
+
}
|
|
33
|
+
const parts = (text) => text.split('.').map(Number);
|
|
34
|
+
export function parseModel(id) {
|
|
35
|
+
// Routed ids carry their maker (`anthropic/claude-opus-5`) and OpenRouter a variant (`:free`).
|
|
36
|
+
const routed = (id.toLowerCase().split('/').pop() ?? '').replace(/:.*$/, '');
|
|
37
|
+
// Vertex dates Claude with an at sign: `claude-opus-4-5@20251101`.
|
|
38
|
+
const [name = '', at] = routed.split('@');
|
|
39
|
+
const maker = makerOf(name);
|
|
40
|
+
const words = name.split('-');
|
|
41
|
+
const traits = {
|
|
42
|
+
maker,
|
|
43
|
+
version: [],
|
|
44
|
+
tier: 'mid',
|
|
45
|
+
special: SPECIAL.test(name),
|
|
46
|
+
preview: false,
|
|
47
|
+
...(at && /^\d{8}$/.test(at) ? { date: at } : {}),
|
|
48
|
+
};
|
|
49
|
+
let ended = false;
|
|
50
|
+
for (let i = 0; i < words.length; i++) {
|
|
51
|
+
const word = words[i];
|
|
52
|
+
if (/^\d{8}$/.test(word)) {
|
|
53
|
+
traits.date = word;
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
if (/^20\d\d$/.test(word) &&
|
|
57
|
+
/^\d\d$/.test(words[i + 1] ?? '') &&
|
|
58
|
+
/^\d\d$/.test(words[i + 2] ?? '')) {
|
|
59
|
+
traits.date = word + words[i + 1] + words[i + 2];
|
|
60
|
+
i += 2;
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
// `gpt-4-0613`, `gpt-3.5-turbo-0125`
|
|
64
|
+
if (/^\d{4}$/.test(word) && traits.version.length > 0) {
|
|
65
|
+
traits.date = word;
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
if (PREVIEW.has(word)) {
|
|
69
|
+
traits.preview = true;
|
|
70
|
+
}
|
|
71
|
+
if (SMALL.has(word)) {
|
|
72
|
+
traits.tier = 'small';
|
|
73
|
+
}
|
|
74
|
+
else if (TOP.has(word) && traits.tier !== 'small') {
|
|
75
|
+
traits.tier = 'top';
|
|
76
|
+
}
|
|
77
|
+
if (ended) {
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
// OpenAI's o-series came between GPT-4 and GPT-5.
|
|
81
|
+
const series = i === 0 && maker === 'openai' ? /^o(\d+)$/.exec(word) : null;
|
|
82
|
+
if (series) {
|
|
83
|
+
traits.version = [4, Number(series[1])];
|
|
84
|
+
ended = true;
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
// `qwen3`, `llama4`: the generation glued to the family name.
|
|
88
|
+
const glued = i === 0 ? /^[a-z]+(\d+(?:\.\d+)*)$/.exec(word) : null;
|
|
89
|
+
if (glued) {
|
|
90
|
+
traits.version = parts(glued[1]);
|
|
91
|
+
ended = true;
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
94
|
+
// `5.6`, `4o`, `v3`, or the `4`, `5` of `opus-4-5`; sizes like `120b` are not versions.
|
|
95
|
+
const number = /^v?(\d{1,2}(?:\.\d+)*)o?$/.exec(word);
|
|
96
|
+
if (number) {
|
|
97
|
+
const next = parts(number[1]);
|
|
98
|
+
traits.version = traits.version.length > 0 ? [...traits.version, next[0]] : next;
|
|
99
|
+
if (word.endsWith('o')) {
|
|
100
|
+
ended = true;
|
|
101
|
+
}
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
if (traits.version.length > 0) {
|
|
105
|
+
ended = true;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return traits;
|
|
109
|
+
}
|
|
110
|
+
/** The vendor's own endpoint first, then the resellers. */
|
|
111
|
+
function routeRank(provider, maker) {
|
|
112
|
+
if (provider === maker) {
|
|
113
|
+
return 0;
|
|
114
|
+
}
|
|
115
|
+
return provider === 'vertex' ? 1 : provider === 'openrouter' ? 2 : 3;
|
|
116
|
+
}
|
|
117
|
+
const bucket = (t) => (t.tier === 'small' ? 2 : t.special ? 1 : 0);
|
|
118
|
+
function compareTraits(a, b) {
|
|
119
|
+
const majorA = a.version[0] ?? -1;
|
|
120
|
+
const majorB = b.version[0] ?? -1;
|
|
121
|
+
return (bucket(a) - bucket(b) ||
|
|
122
|
+
MAKERS.indexOf(a.maker) - MAKERS.indexOf(b.maker) ||
|
|
123
|
+
majorB - majorA ||
|
|
124
|
+
TIERS.indexOf(a.tier) - TIERS.indexOf(b.tier) ||
|
|
125
|
+
compareRest(b.version, a.version) ||
|
|
126
|
+
Number(a.preview) - Number(b.preview) ||
|
|
127
|
+
// The alias that follows the newest snapshot, then the newest snapshot.
|
|
128
|
+
Number(a.date !== undefined) - Number(b.date !== undefined) ||
|
|
129
|
+
(b.date ?? '').localeCompare(a.date ?? ''));
|
|
130
|
+
}
|
|
131
|
+
function compareRest(a, b) {
|
|
132
|
+
for (let i = 1; i < Math.max(a.length, b.length); i++) {
|
|
133
|
+
const diff = (a[i] ?? 0) - (b[i] ?? 0);
|
|
134
|
+
if (diff !== 0) {
|
|
135
|
+
return diff;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
return 0;
|
|
139
|
+
}
|
|
140
|
+
/** Best first. Usable as a sort comparator on any listing row. */
|
|
141
|
+
export function compareModels(a, b) {
|
|
142
|
+
const ta = parseModel(a.id);
|
|
143
|
+
const tb = parseModel(b.id);
|
|
144
|
+
return (compareTraits(ta, tb) ||
|
|
145
|
+
routeRank(a.provider, ta.maker) - routeRank(b.provider, tb.maker) ||
|
|
146
|
+
a.id.localeCompare(b.id) ||
|
|
147
|
+
a.provider.localeCompare(b.provider));
|
|
148
|
+
}
|
|
149
|
+
/** A sorted copy, best first. */
|
|
150
|
+
export function rankModels(rows) {
|
|
151
|
+
return [...rows].sort(compareModels);
|
|
152
|
+
}
|
|
153
|
+
//# sourceMappingURL=rank.js.map
|