@zenera/cli 1.1.3 → 1.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +160 -40
- package/dist/audit.d.ts +5 -2
- package/dist/audit.js +7 -2
- package/dist/catalog.d.ts +111 -0
- package/dist/catalog.js +439 -0
- package/dist/commands/check.js +39 -11
- package/dist/commands/index.d.ts +2 -2
- package/dist/commands/index.js +3 -2
- package/dist/commands/key.js +18 -0
- package/dist/commands/models.d.ts +0 -6
- package/dist/commands/models.js +546 -101
- package/dist/home.d.ts +2 -0
- package/dist/home.js +2 -0
- package/dist/keys.d.ts +9 -1
- package/dist/lib.d.ts +1 -0
- package/dist/lib.js +1 -0
- package/dist/liveness.d.ts +32 -0
- package/dist/liveness.js +194 -5
- package/dist/validate.d.ts +17 -1
- package/dist/validate.js +97 -7
- package/package.json +2 -2
- package/templates/editor/.github/copilot-instructions.md +30 -3
- package/templates/editor/.github/skills/zen-cli/SKILL.md +6 -3
- package/templates/editor/.github/skills/zen-cli/references/check.md +15 -19
- package/templates/editor/.github/skills/zen-cli/references/keys.md +5 -0
- package/templates/editor/.github/skills/zen-cli/references/models.md +108 -0
- package/templates/project/sandbox/Dockerfile +1 -1
package/dist/home.d.ts
CHANGED
|
@@ -5,6 +5,8 @@ export declare const paths: {
|
|
|
5
5
|
keys: () => string;
|
|
6
6
|
keyDir: () => string;
|
|
7
7
|
faker: () => string;
|
|
8
|
+
/** cached model listings, one file per provider — public data, not secrets */
|
|
9
|
+
catalog: () => string;
|
|
8
10
|
};
|
|
9
11
|
/** Creates a directory owner-only, and leaves an existing one's mode alone. */
|
|
10
12
|
export declare function ensureDir(dir: string, mode?: number): string;
|
package/dist/home.js
CHANGED
|
@@ -26,6 +26,8 @@ export const paths = {
|
|
|
26
26
|
keys: () => join(home(), 'keys.json'),
|
|
27
27
|
keyDir: () => join(home(), 'keys'),
|
|
28
28
|
faker: () => join(home(), 'faker'),
|
|
29
|
+
/** cached model listings, one file per provider — public data, not secrets */
|
|
30
|
+
catalog: () => join(home(), 'catalog'),
|
|
29
31
|
};
|
|
30
32
|
/** Creates a directory owner-only, and leaves an existing one's mode alone. */
|
|
31
33
|
export function ensureDir(dir, mode = DIR_MODE) {
|
package/dist/keys.d.ts
CHANGED
|
@@ -63,12 +63,20 @@ export declare function envOf(entry: Pick<KeyEntry, 'provider' | 'holds' | 'env'
|
|
|
63
63
|
export declare function isProvider(name: string): name is Provider;
|
|
64
64
|
export declare function isOwner(name: string): name is KeyOwner;
|
|
65
65
|
export declare function assertOwner(name: string): KeyOwner;
|
|
66
|
-
|
|
66
|
+
/**
|
|
67
|
+
* `blocked` is the credential being fine and the *account* not: an api switched
|
|
68
|
+
* off in the project, a spent balance, a model this key was never granted. It
|
|
69
|
+
* is split out of `dead` because the two want opposite actions — rotating a key
|
|
70
|
+
* that authenticated perfectly is the wrong afternoon.
|
|
71
|
+
*/
|
|
72
|
+
export type Liveness = 'live' | 'dead' | 'blocked' | 'unknown';
|
|
67
73
|
export interface KeyCheck {
|
|
68
74
|
state: Liveness;
|
|
69
75
|
at: string;
|
|
70
76
|
/** the provider's own words when it said no, or ours when we could not ask */
|
|
71
77
|
detail?: string;
|
|
78
|
+
/** what to actually do about it, when the refusal implies something specific */
|
|
79
|
+
fix?: string;
|
|
72
80
|
}
|
|
73
81
|
export interface KeyEntry {
|
|
74
82
|
provider: KeyOwner;
|
package/dist/lib.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
export { extract, invokedAs, one, parse, split, type Parsed, type Split } from './args.ts';
|
|
2
2
|
export { printBanner, type BannerText } from './banner.ts';
|
|
3
|
+
export { CATALOG_TTL_MS, CURATED, fetchCatalog, loadCatalog, loadCatalogs, matches, PREFERRED, type Catalog, type CatalogEntry, type CatalogOptions, type Filters, type Role, } from './catalog.ts';
|
|
3
4
|
export type { Command, Context } from './command.ts';
|
|
4
5
|
export { assertPrivate, ensureDir, ensureHome, home, paths, readJson, writeJson } from './home.ts';
|
|
5
6
|
export { assertNotEmpty, assertOwner, assertUsable, describe, envNames, envOf, form, isOwner, isProvider, keyId, KeyStore, mask, OWNERS, parseRef, PROVIDERS, SERVICES, SHAPES, type CredentialForm, type KeyCheck, type KeyEntry, type KeyOwner, type Liveness, type Provider, type Service, } from './keys.ts';
|
package/dist/lib.js
CHANGED
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
// ---------------------------------------------------------------------------
|
|
24
24
|
export { extract, invokedAs, one, parse, split } from "./args.js";
|
|
25
25
|
export { printBanner } from "./banner.js";
|
|
26
|
+
export { CATALOG_TTL_MS, CURATED, fetchCatalog, loadCatalog, loadCatalogs, matches, PREFERRED, } from "./catalog.js";
|
|
26
27
|
export { assertPrivate, ensureDir, ensureHome, home, paths, readJson, writeJson } from "./home.js";
|
|
27
28
|
export { assertNotEmpty, assertOwner, assertUsable, describe, envNames, envOf, form, isOwner, isProvider, keyId, KeyStore, mask, OWNERS, parseRef, PROVIDERS, SERVICES, SHAPES, } from "./keys.js";
|
|
28
29
|
export { probe, probeAll } from "./liveness.js";
|
package/dist/liveness.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
import { type Embedder, type Model } from '@zenera/neo';
|
|
1
2
|
import { type KeyCheck, type KeyEntry, type KeyStore } from './keys.ts';
|
|
3
|
+
export declare function classify(err: unknown): KeyCheck;
|
|
2
4
|
/**
|
|
3
5
|
* The cheapest authenticated call each SDK has. Nothing here reads a model or
|
|
4
6
|
* spends a token: the question is only whether the credential is accepted.
|
|
@@ -30,4 +32,34 @@ export declare function probe(store: KeyStore, entry: KeyEntry): Promise<KeyChec
|
|
|
30
32
|
* un-joined again.
|
|
31
33
|
*/
|
|
32
34
|
export declare function probeAll(store: KeyStore, entries: readonly KeyEntry[], onProbe?: (entry: KeyEntry, done: number, total: number) => void): Promise<[KeyEntry, KeyCheck][]>;
|
|
35
|
+
/** What the project calls a model, and the thing that call resolved to. */
|
|
36
|
+
export type ModelTarget = {
|
|
37
|
+
ref: string;
|
|
38
|
+
kind: 'model';
|
|
39
|
+
model: Model;
|
|
40
|
+
} | {
|
|
41
|
+
ref: string;
|
|
42
|
+
kind: 'embedding';
|
|
43
|
+
embedder: Embedder;
|
|
44
|
+
};
|
|
45
|
+
export interface ModelProbe {
|
|
46
|
+
/** the reference as the config writes it — an alias, or a full ref */
|
|
47
|
+
ref: string;
|
|
48
|
+
/** the id that goes on the wire */
|
|
49
|
+
id: string;
|
|
50
|
+
kind: 'model' | 'embedding';
|
|
51
|
+
check: KeyCheck;
|
|
52
|
+
/** how long the round trip took, for the one that is merely slow */
|
|
53
|
+
ms: number;
|
|
54
|
+
/** embeddings only: the width the model actually returned */
|
|
55
|
+
dimensions?: number;
|
|
56
|
+
}
|
|
57
|
+
export declare function probeModel(target: ModelTarget): Promise<ModelProbe>;
|
|
58
|
+
/**
|
|
59
|
+
* One round trip per model, run together — they are independent questions, and
|
|
60
|
+
* a project with four models should not take four deadlines to answer. Nothing
|
|
61
|
+
* here touches `process.env`, so unlike the credential probes there is no case
|
|
62
|
+
* that has to go alone.
|
|
63
|
+
*/
|
|
64
|
+
export declare function probeModels(targets: readonly ModelTarget[], onProbe?: (target: ModelTarget, done: number, total: number) => void): Promise<ModelProbe[]>;
|
|
33
65
|
//# sourceMappingURL=liveness.d.ts.map
|
package/dist/liveness.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { EXA_BASE_URL, ModelRegistry } from '@zenera/neo';
|
|
1
|
+
import { EXA_BASE_URL, ModelRegistry, text, } from '@zenera/neo';
|
|
2
2
|
import { envOf, SHAPES, } from "./keys.js";
|
|
3
3
|
// ---------------------------------------------------------------------------
|
|
4
4
|
// Liveness
|
|
@@ -7,6 +7,11 @@ import { envOf, SHAPES, } from "./keys.js";
|
|
|
7
7
|
// the credential and said no — versus *unknown* — we could not ask. Collapsing
|
|
8
8
|
// them into one red mark is the classic way to send someone hunting for the
|
|
9
9
|
// wrong bug: rotating a perfectly good key because the office wifi was down.
|
|
10
|
+
//
|
|
11
|
+
// *blocked* is the third: the credential authenticated and the account then
|
|
12
|
+
// refused. A disabled api, an empty balance, a model this key was never granted
|
|
13
|
+
// — all arrive as a 403 alongside genuine rejections, and all of them are made
|
|
14
|
+
// worse by rotating the key.
|
|
10
15
|
// ---------------------------------------------------------------------------
|
|
11
16
|
/** Words a provider uses when the credential itself is the problem. */
|
|
12
17
|
const REJECTED = [
|
|
@@ -33,7 +38,38 @@ const UNREACHED = [
|
|
|
33
38
|
'timeout',
|
|
34
39
|
'socket hang up',
|
|
35
40
|
];
|
|
36
|
-
|
|
41
|
+
/** Words for an api the account has switched off, or never switched on. */
|
|
42
|
+
const DISABLED = [
|
|
43
|
+
'service_disabled',
|
|
44
|
+
'accessnotconfigured',
|
|
45
|
+
'has not been used in project',
|
|
46
|
+
'is disabled',
|
|
47
|
+
'api is not enabled',
|
|
48
|
+
];
|
|
49
|
+
/** Words for an account that authenticated and then declined to serve. */
|
|
50
|
+
const UNFUNDED = [
|
|
51
|
+
'insufficient_quota',
|
|
52
|
+
'billing',
|
|
53
|
+
'credit balance is too low',
|
|
54
|
+
'exceeded your current quota',
|
|
55
|
+
'quota exceeded',
|
|
56
|
+
'resource_exhausted',
|
|
57
|
+
'payment required',
|
|
58
|
+
];
|
|
59
|
+
/**
|
|
60
|
+
* Google names both the api and the project in its refusal, and buries them in
|
|
61
|
+
* a console url. Digging them back out turns a paragraph into the one command
|
|
62
|
+
* that fixes it.
|
|
63
|
+
*/
|
|
64
|
+
function enablement(message) {
|
|
65
|
+
const service = /apis\/api\/([a-z0-9.-]+\.googleapis\.com)/i.exec(message)?.[1];
|
|
66
|
+
if (!service) {
|
|
67
|
+
return undefined;
|
|
68
|
+
}
|
|
69
|
+
const project = /[?&]project=([a-z0-9-]+)/i.exec(message)?.[1];
|
|
70
|
+
return `gcloud services enable ${service}${project ? ` --project ${project}` : ''}`;
|
|
71
|
+
}
|
|
72
|
+
export function classify(err) {
|
|
37
73
|
const at = new Date().toISOString();
|
|
38
74
|
// OpenAI, Anthropic and the GenAI SDK all say `status`; OpenRouter's says
|
|
39
75
|
// `statusCode`. Reading only the first would classify a revoked key as
|
|
@@ -42,6 +78,26 @@ function classify(err) {
|
|
|
42
78
|
const status = e?.status ?? e?.statusCode;
|
|
43
79
|
const message = err instanceof Error ? err.message : String(err);
|
|
44
80
|
const haystack = `${status ?? ''} ${message}`.toLowerCase();
|
|
81
|
+
// Ahead of the 401/403 arm, because both of these arrive as a 403 and both
|
|
82
|
+
// are about the account rather than the key.
|
|
83
|
+
if (DISABLED.some((needle) => haystack.includes(needle))) {
|
|
84
|
+
return {
|
|
85
|
+
state: 'blocked',
|
|
86
|
+
at,
|
|
87
|
+
detail: firstLine(message),
|
|
88
|
+
fix: enablement(message) ?? 'enable the api for this project in the vendor console',
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
// A 429 is this minute's rate limit, not an empty account, and the two
|
|
92
|
+
// share vocabulary — so a status that says "slow down" wins.
|
|
93
|
+
if (status !== 429 && UNFUNDED.some((needle) => haystack.includes(needle))) {
|
|
94
|
+
return {
|
|
95
|
+
state: 'blocked',
|
|
96
|
+
at,
|
|
97
|
+
detail: firstLine(message),
|
|
98
|
+
fix: 'add credit or raise the quota in the vendor console',
|
|
99
|
+
};
|
|
100
|
+
}
|
|
45
101
|
if (status === 401 || status === 403) {
|
|
46
102
|
return { state: 'dead', at, detail: `${status} ${firstLine(message)}` };
|
|
47
103
|
}
|
|
@@ -58,7 +114,29 @@ function classify(err) {
|
|
|
58
114
|
}
|
|
59
115
|
return { state: 'unknown', at, detail: firstLine(message) };
|
|
60
116
|
}
|
|
61
|
-
|
|
117
|
+
/**
|
|
118
|
+
* The one sentence worth showing.
|
|
119
|
+
*
|
|
120
|
+
* Google's SDK throws with the whole JSON error body as the message, and a
|
|
121
|
+
* table cell holding `{"error":{"code":403,"message":"…` is a cell nobody
|
|
122
|
+
* reads. The sentence inside it is the part a person or an agent acts on, so
|
|
123
|
+
* that is what comes out when there is one.
|
|
124
|
+
*/
|
|
125
|
+
const firstLine = (s) => {
|
|
126
|
+
const start = s.indexOf('{');
|
|
127
|
+
if (start !== -1) {
|
|
128
|
+
try {
|
|
129
|
+
const body = JSON.parse(s.slice(start));
|
|
130
|
+
if (typeof body.error?.message === 'string') {
|
|
131
|
+
return body.error.message.split('\n')[0].slice(0, 200);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
catch {
|
|
135
|
+
// Not JSON, or truncated JSON. The raw line is still better than nothing.
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
return s.split('\n')[0].slice(0, 200);
|
|
139
|
+
};
|
|
62
140
|
// ---------------------------------------------------------------------------
|
|
63
141
|
// Deadline
|
|
64
142
|
//
|
|
@@ -69,15 +147,20 @@ const firstLine = (s) => s.split('\n')[0].slice(0, 160);
|
|
|
69
147
|
// been answered by now, the honest answer is *unknown*.
|
|
70
148
|
// ---------------------------------------------------------------------------
|
|
71
149
|
const DEADLINE_MS = 15_000;
|
|
150
|
+
/**
|
|
151
|
+
* A model is asked to think, not merely to authenticate, so it is given longer
|
|
152
|
+
* — a reasoning model can spend half a minute on one word and still be working.
|
|
153
|
+
*/
|
|
154
|
+
const MODEL_DEADLINE_MS = 90_000;
|
|
72
155
|
class Deadline extends Error {
|
|
73
156
|
}
|
|
74
|
-
async function within(work) {
|
|
157
|
+
async function within(work, ms = DEADLINE_MS) {
|
|
75
158
|
let timer;
|
|
76
159
|
try {
|
|
77
160
|
return await Promise.race([
|
|
78
161
|
work,
|
|
79
162
|
new Promise((_, reject) => {
|
|
80
|
-
timer = setTimeout(() => reject(new Deadline()),
|
|
163
|
+
timer = setTimeout(() => reject(new Deadline()), ms);
|
|
81
164
|
// The abandoned request must not keep the process alive.
|
|
82
165
|
timer.unref?.();
|
|
83
166
|
}),
|
|
@@ -269,4 +352,110 @@ export async function probeAll(store, entries, onProbe) {
|
|
|
269
352
|
// the network, and a list that reshuffles itself between runs is unreadable.
|
|
270
353
|
return entries.map((entry) => [entry, results.get(entry)]);
|
|
271
354
|
}
|
|
355
|
+
// ---------------------------------------------------------------------------
|
|
356
|
+
// Models
|
|
357
|
+
//
|
|
358
|
+
// A credential that authenticates says nothing about the model id it is spent
|
|
359
|
+
// on: `gemini-3.5-flash` with an OpenAI key, a deprecated snapshot, a model the
|
|
360
|
+
// account was never granted — all of them pass every check this file otherwise
|
|
361
|
+
// performs and then fail on the first turn of a real run. The only thing that
|
|
362
|
+
// answers the question is asking the model itself, so this asks it: one word
|
|
363
|
+
// in, one word out, per distinct model the project would use.
|
|
364
|
+
// ---------------------------------------------------------------------------
|
|
365
|
+
/** Words a vendor uses when the credential was fine and the model id was not. */
|
|
366
|
+
const UNSERVED = [
|
|
367
|
+
'model_not_found',
|
|
368
|
+
'does not exist',
|
|
369
|
+
'not found',
|
|
370
|
+
'unknown model',
|
|
371
|
+
'invalid model',
|
|
372
|
+
'no endpoints found',
|
|
373
|
+
'is not supported',
|
|
374
|
+
'not supported',
|
|
375
|
+
'no access',
|
|
376
|
+
];
|
|
377
|
+
const targetId = (target) => target.kind === 'model' ? target.model.id : target.embedder.id;
|
|
378
|
+
/**
|
|
379
|
+
* Refused by the provider, unreachable, or served.
|
|
380
|
+
*
|
|
381
|
+
* A model this account cannot use is `blocked`, not `dead`: the credential was
|
|
382
|
+
* accepted and the id was the thing refused, so the fix is another model rather
|
|
383
|
+
* than another key. `dead` is left to mean the credential itself was rejected.
|
|
384
|
+
*/
|
|
385
|
+
function classifyModel(err, target) {
|
|
386
|
+
const at = new Date().toISOString();
|
|
387
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
388
|
+
const e = err;
|
|
389
|
+
const haystack = `${e?.status ?? e?.statusCode ?? ''} ${message}`.toLowerCase();
|
|
390
|
+
if (err instanceof Deadline || e?.name === 'AbortError' || e?.name === 'TimeoutError') {
|
|
391
|
+
return { state: 'unknown', at, detail: `no answer in ${MODEL_DEADLINE_MS / 1000}s` };
|
|
392
|
+
}
|
|
393
|
+
if (UNSERVED.some((needle) => haystack.includes(needle))) {
|
|
394
|
+
return { state: 'blocked', at, detail: firstLine(message), fix: instead(target) };
|
|
395
|
+
}
|
|
396
|
+
const check = classify(err);
|
|
397
|
+
return check.state === 'blocked' && !check.fix ? { ...check, fix: instead(target) } : check;
|
|
398
|
+
}
|
|
399
|
+
/** The command that finds something this account can actually use. */
|
|
400
|
+
function instead(target) {
|
|
401
|
+
const provider = target.ref.includes(':') ? target.ref.split(':')[0] : undefined;
|
|
402
|
+
return provider
|
|
403
|
+
? `zen models ls ${provider}`
|
|
404
|
+
: `zen models pick --${target.kind === 'embedding' ? 'embedding' : 'chat'}`;
|
|
405
|
+
}
|
|
406
|
+
/**
|
|
407
|
+
* The smallest real call the model can be asked for. It costs a handful of
|
|
408
|
+
* tokens, which is the point: anything cheaper than a completion does not
|
|
409
|
+
* exercise the thing that breaks.
|
|
410
|
+
*
|
|
411
|
+
* An embedding answers with its width, which is worth carrying back: a model
|
|
412
|
+
* that serves the wrong number of dimensions is not interchangeable with the
|
|
413
|
+
* one an index was built on.
|
|
414
|
+
*/
|
|
415
|
+
async function askModel(target, signal) {
|
|
416
|
+
if (target.kind === 'embedding') {
|
|
417
|
+
const res = await target.embedder.embed({ input: ['ping'], signal });
|
|
418
|
+
return res.dimensions;
|
|
419
|
+
}
|
|
420
|
+
await target.model.generate({
|
|
421
|
+
system: 'Reply with the single word: ok',
|
|
422
|
+
messages: [{ role: 'user', content: [text('ping')] }],
|
|
423
|
+
tools: [],
|
|
424
|
+
signal,
|
|
425
|
+
});
|
|
426
|
+
return undefined;
|
|
427
|
+
}
|
|
428
|
+
export async function probeModel(target) {
|
|
429
|
+
const started = Date.now();
|
|
430
|
+
const common = { ref: target.ref, id: targetId(target), kind: target.kind };
|
|
431
|
+
try {
|
|
432
|
+
// The signal cancels the request; the deadline a little behind it is
|
|
433
|
+
// the answer for an SDK that decides to ignore the signal.
|
|
434
|
+
const work = askModel(target, AbortSignal.timeout(MODEL_DEADLINE_MS));
|
|
435
|
+
const dimensions = await within(work, MODEL_DEADLINE_MS + 5_000);
|
|
436
|
+
return {
|
|
437
|
+
...common,
|
|
438
|
+
check: { state: 'live', at: new Date().toISOString() },
|
|
439
|
+
ms: Date.now() - started,
|
|
440
|
+
...(dimensions === undefined ? {} : { dimensions }),
|
|
441
|
+
};
|
|
442
|
+
}
|
|
443
|
+
catch (err) {
|
|
444
|
+
return { ...common, check: classifyModel(err, target), ms: Date.now() - started };
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
/**
|
|
448
|
+
* One round trip per model, run together — they are independent questions, and
|
|
449
|
+
* a project with four models should not take four deadlines to answer. Nothing
|
|
450
|
+
* here touches `process.env`, so unlike the credential probes there is no case
|
|
451
|
+
* that has to go alone.
|
|
452
|
+
*/
|
|
453
|
+
export async function probeModels(targets, onProbe) {
|
|
454
|
+
let done = 0;
|
|
455
|
+
return await Promise.all(targets.map(async (target) => {
|
|
456
|
+
const result = await probeModel(target);
|
|
457
|
+
onProbe?.(target, ++done, targets.length);
|
|
458
|
+
return result;
|
|
459
|
+
}));
|
|
460
|
+
}
|
|
272
461
|
//# sourceMappingURL=liveness.js.map
|
package/dist/validate.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type AnyTool, type ProjectConfig, type Runner } from '@zenera/neo';
|
|
2
2
|
import { type DeclaredRole } from './audit.ts';
|
|
3
|
-
import { type KeyStore } from './keys.ts';
|
|
3
|
+
import { type KeyStore, type Liveness } from './keys.ts';
|
|
4
4
|
/**
|
|
5
5
|
* `error` — the project will not load, or will not run.
|
|
6
6
|
* `warning` — it loads, and something about it is probably not what was meant.
|
|
@@ -82,6 +82,13 @@ export interface ModelReport {
|
|
|
82
82
|
env?: string;
|
|
83
83
|
credential: 'present' | 'missing' | 'rejected' | 'unknown';
|
|
84
84
|
detail?: string;
|
|
85
|
+
/** the provider's own answer, when the model was actually asked */
|
|
86
|
+
check?: {
|
|
87
|
+
state: Liveness;
|
|
88
|
+
detail?: string;
|
|
89
|
+
fix?: string;
|
|
90
|
+
ms: number;
|
|
91
|
+
};
|
|
85
92
|
/** agents that would use it */
|
|
86
93
|
usedBy: string[];
|
|
87
94
|
}
|
|
@@ -156,6 +163,15 @@ export interface ValidateOptions {
|
|
|
156
163
|
/** called with each slow step, so the caller can narrate one */
|
|
157
164
|
onProgress?: (what: string) => void;
|
|
158
165
|
};
|
|
166
|
+
/**
|
|
167
|
+
* Whether to ask each model that has a credential to answer once. Costs a
|
|
168
|
+
* few tokens apiece and needs the network, so like the sandbox it is named
|
|
169
|
+
* separately. Without `keys` there is nothing to ask with and it is skipped.
|
|
170
|
+
*/
|
|
171
|
+
models?: {
|
|
172
|
+
enabled: boolean;
|
|
173
|
+
onProgress?: (what: string) => void;
|
|
174
|
+
};
|
|
159
175
|
}
|
|
160
176
|
export declare function validateProject(opts: ValidateOptions): Promise<Report>;
|
|
161
177
|
export declare function availableTools(root: string, config: ProjectConfig): AnyTool<unknown>[];
|
package/dist/validate.js
CHANGED
|
@@ -5,6 +5,7 @@ import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:pat
|
|
|
5
5
|
import { auditModels, credentialFor } from "./audit.js";
|
|
6
6
|
import { resolveBuild } from "./image.js";
|
|
7
7
|
import { SHAPES, envNames, form } from "./keys.js";
|
|
8
|
+
import { probeModels } from "./liveness.js";
|
|
8
9
|
import { BuildError, ensurePodmanReady } from "./podman.js";
|
|
9
10
|
// Mirrors the loader's own constants (`packages/neo/src/project/load.ts`).
|
|
10
11
|
// Duplicated rather than exported, because a check that agreed with the loader
|
|
@@ -198,18 +199,80 @@ export async function validateProject(opts) {
|
|
|
198
199
|
// -----------------------------------------------------------------------
|
|
199
200
|
// Models and credentials
|
|
200
201
|
// -----------------------------------------------------------------------
|
|
201
|
-
|
|
202
|
+
// Nothing to ask with is the same as not asking: the missing credential is
|
|
203
|
+
// already a finding, and a call that cannot be made teaches nothing twice.
|
|
204
|
+
const asking = opts.models?.enabled && opts.keys ? opts.models : undefined;
|
|
205
|
+
const resolved = checkModels(root, config, opts.keys, add, Boolean(asking));
|
|
202
206
|
providers = resolved.providers;
|
|
203
207
|
models.push(...resolved.models);
|
|
204
208
|
checkServices(agents, available, opts.keys, add);
|
|
205
|
-
// Last, because
|
|
206
|
-
// reading of the files can tell you is already on the
|
|
207
|
-
// interrupted check is still a useful one.
|
|
209
|
+
// Last, and in this order, because these are the two steps that leave the
|
|
210
|
+
// machine: everything a reading of the files can tell you is already on the
|
|
211
|
+
// report by now, so an interrupted check is still a useful one.
|
|
212
|
+
if (asking && resolved.targets.length) {
|
|
213
|
+
await askModels(resolved.targets, asking, add);
|
|
214
|
+
}
|
|
208
215
|
if (opts.sandbox?.enabled && sandboxSummary(config, agents).used) {
|
|
209
216
|
probed = await probeSandbox(config, build, opts.sandbox, add);
|
|
210
217
|
}
|
|
211
218
|
return done();
|
|
212
219
|
}
|
|
220
|
+
/**
|
|
221
|
+
* One round trip per model, and a verdict per model.
|
|
222
|
+
*
|
|
223
|
+
* A refusal is an error: the provider looked at this reference and said no, and
|
|
224
|
+
* every run of this project will meet the same answer. Silence is a warning —
|
|
225
|
+
* it is the network's problem, not the project's, and a report that failed
|
|
226
|
+
* because a train went into a tunnel would teach the wrong lesson.
|
|
227
|
+
*
|
|
228
|
+
* A *blocked* model is an error too, but a different one: the credential was
|
|
229
|
+
* accepted and the account then refused, so the fix is the vendor's own —
|
|
230
|
+
* enabling an api, adding credit, choosing a model this account is granted.
|
|
231
|
+
* Telling someone to check their spelling there costs them an afternoon.
|
|
232
|
+
*/
|
|
233
|
+
async function askModels(targets, opts, add) {
|
|
234
|
+
opts.onProgress?.(`asking ${targets.length} model${targets.length === 1 ? '' : 's'}`);
|
|
235
|
+
const probes = await probeModels(targets.map(([, target]) => target), (target, done, total) => opts.onProgress?.(`asked ${target.ref} … ${done}/${total}`));
|
|
236
|
+
probes.forEach((probe, i) => {
|
|
237
|
+
const [report] = targets[i];
|
|
238
|
+
report.check = {
|
|
239
|
+
state: probe.check.state,
|
|
240
|
+
...(probe.check.detail ? { detail: probe.check.detail } : {}),
|
|
241
|
+
...(probe.check.fix ? { fix: probe.check.fix } : {}),
|
|
242
|
+
ms: probe.ms,
|
|
243
|
+
};
|
|
244
|
+
if (probe.check.state === 'live') {
|
|
245
|
+
return;
|
|
246
|
+
}
|
|
247
|
+
const where = `${report.role} "${report.name}"${report.provider ? ` (${report.provider})` : ''}`;
|
|
248
|
+
if (probe.check.state === 'blocked') {
|
|
249
|
+
const pick = report.role === 'embedding' ? '--embedding' : '--chat';
|
|
250
|
+
add({
|
|
251
|
+
severity: 'error',
|
|
252
|
+
code: `${report.role}.blocked`,
|
|
253
|
+
where,
|
|
254
|
+
message: `the credential was accepted and then refused: ${probe.check.detail ?? 'no reason given'}`,
|
|
255
|
+
fix: `${probe.check.fix ?? 'resolve it in the vendor console'}, or find one that works: zen models pick ${pick}`,
|
|
256
|
+
});
|
|
257
|
+
return;
|
|
258
|
+
}
|
|
259
|
+
add(probe.check.state === 'dead'
|
|
260
|
+
? {
|
|
261
|
+
severity: 'error',
|
|
262
|
+
code: `${report.role}.refused`,
|
|
263
|
+
where,
|
|
264
|
+
message: `the provider refused it: ${probe.check.detail ?? 'no reason given'}`,
|
|
265
|
+
fix: `check that ${probe.id} is spelt right, still served, and granted to this account`,
|
|
266
|
+
}
|
|
267
|
+
: {
|
|
268
|
+
severity: 'warning',
|
|
269
|
+
code: `${report.role}.unreachable`,
|
|
270
|
+
where,
|
|
271
|
+
message: `it could not be asked: ${probe.check.detail ?? 'no answer'}`,
|
|
272
|
+
fix: 'try again, or pass --no-models to skip this',
|
|
273
|
+
});
|
|
274
|
+
});
|
|
275
|
+
}
|
|
213
276
|
/**
|
|
214
277
|
* A tool that needs a key of its own is invisible to the model audit, which
|
|
215
278
|
* walks `models:` and finds nothing to say about `web_search`. The project is
|
|
@@ -1039,7 +1102,7 @@ function siblings(skillFile) {
|
|
|
1039
1102
|
// ---------------------------------------------------------------------------
|
|
1040
1103
|
// Models
|
|
1041
1104
|
// ---------------------------------------------------------------------------
|
|
1042
|
-
function checkModels(root, config, keys, add) {
|
|
1105
|
+
function checkModels(root, config, keys, add, probe = false) {
|
|
1043
1106
|
let registry;
|
|
1044
1107
|
try {
|
|
1045
1108
|
registry = projectRegistry(config);
|
|
@@ -1052,7 +1115,7 @@ function checkModels(root, config, keys, add) {
|
|
|
1052
1115
|
message: err instanceof Error ? err.message : String(err),
|
|
1053
1116
|
fix: 'see the `providers:` section of docs/agents-yaml.md',
|
|
1054
1117
|
});
|
|
1055
|
-
return { providers: [], models: [] };
|
|
1118
|
+
return { providers: [], models: [], targets: [] };
|
|
1056
1119
|
}
|
|
1057
1120
|
// Declared under an alias first, so an alias keeps its own name in the
|
|
1058
1121
|
// report and a `model:` that names one collapses onto it. Mirrors the
|
|
@@ -1103,6 +1166,7 @@ function checkModels(root, config, keys, add) {
|
|
|
1103
1166
|
}
|
|
1104
1167
|
}
|
|
1105
1168
|
const models = [];
|
|
1169
|
+
const targets = [];
|
|
1106
1170
|
const describe = (role, name, ref, usedBy) => {
|
|
1107
1171
|
const report = {
|
|
1108
1172
|
name,
|
|
@@ -1153,6 +1217,32 @@ function checkModels(root, config, keys, add) {
|
|
|
1153
1217
|
});
|
|
1154
1218
|
}
|
|
1155
1219
|
models.push(report);
|
|
1220
|
+
// Only what there is something to ask with. Constructing the client is
|
|
1221
|
+
// what reads the credential, so a model with none cannot be built, let
|
|
1222
|
+
// alone asked — and the missing key is already a finding of its own.
|
|
1223
|
+
if (probe && report.credential === 'present') {
|
|
1224
|
+
try {
|
|
1225
|
+
targets.push([
|
|
1226
|
+
report,
|
|
1227
|
+
role === 'model'
|
|
1228
|
+
? { ref: name, kind: 'model', model: registry.model(ref) }
|
|
1229
|
+
: {
|
|
1230
|
+
ref: name,
|
|
1231
|
+
kind: 'embedding',
|
|
1232
|
+
embedder: registry.embedder(ref),
|
|
1233
|
+
},
|
|
1234
|
+
]);
|
|
1235
|
+
}
|
|
1236
|
+
catch (err) {
|
|
1237
|
+
add({
|
|
1238
|
+
severity: 'error',
|
|
1239
|
+
code: `${role}.unusable`,
|
|
1240
|
+
where: whereFor(config, role, name, usedBy),
|
|
1241
|
+
message: err instanceof Error ? err.message : String(err),
|
|
1242
|
+
fix: 'the provider is declared but cannot be built — see `providers:`',
|
|
1243
|
+
});
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1156
1246
|
};
|
|
1157
1247
|
for (const [name, { ref, usedBy }] of declared) {
|
|
1158
1248
|
describe('model', name, ref, usedBy);
|
|
@@ -1165,7 +1255,7 @@ function checkModels(root, config, keys, add) {
|
|
|
1165
1255
|
if (config.embedding && !config.embeddings?.[config.embedding]) {
|
|
1166
1256
|
describe('embedding', config.embedding, config.embedding, []);
|
|
1167
1257
|
}
|
|
1168
|
-
return { providers: registry.names(), models };
|
|
1258
|
+
return { providers: registry.names(), models, targets };
|
|
1169
1259
|
}
|
|
1170
1260
|
/** The config key a bad reference was written under. */
|
|
1171
1261
|
function whereFor(config, role, name, usedBy) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zenera/cli",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.4",
|
|
4
4
|
"description": "Command-line front end for @zenera/neo: agentic projects you can run, share and commit.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"agents",
|
|
@@ -51,7 +51,7 @@
|
|
|
51
51
|
"@inkjs/ui": "^2.0.0",
|
|
52
52
|
"ink": "^7.1.1",
|
|
53
53
|
"react": "^19.2.8",
|
|
54
|
-
"@zenera/neo": "^1.1.
|
|
54
|
+
"@zenera/neo": "^1.1.4",
|
|
55
55
|
"@anthropic-ai/sdk": "^0.120.0",
|
|
56
56
|
"@google/genai": "^2.18.0",
|
|
57
57
|
"@openrouter/sdk": "^1.2.80",
|
|
@@ -382,8 +382,8 @@ Google's `embedContent` takes one document per request for every
|
|
|
382
382
|
hide that. `maxBatch` therefore defaults to `1`; raise it only for a
|
|
383
383
|
`text-embedding-*` model, which accepts more.
|
|
384
384
|
|
|
385
|
-
`zen check`
|
|
386
|
-
|
|
385
|
+
`zen check` reports every declared embedding beside the models, with the
|
|
386
|
+
credential each one would need.
|
|
387
387
|
|
|
388
388
|
### 3.2 `INSTRUCTIONS.md`
|
|
389
389
|
|
|
@@ -1540,8 +1540,34 @@ There is no compiler for prose. Substitutes, in order of value:
|
|
|
1540
1540
|
credential resolves, not that the route serves images or honours `tools`; on a
|
|
1541
1541
|
gateway that gap is a request-time 404 (§7.5).
|
|
1542
1542
|
|
|
1543
|
+
### When a model or embedder will not answer
|
|
1544
|
+
|
|
1545
|
+
`zen check` names the model and the verdict. Act on the verdict, not on the
|
|
1546
|
+
first thing that looks like a credential problem:
|
|
1547
|
+
|
|
1548
|
+
```sh
|
|
1549
|
+
zen models test <ref> # one real call, one verdict
|
|
1550
|
+
```
|
|
1551
|
+
|
|
1552
|
+
- **refused** — the credential was rejected. `zen key check <provider>`.
|
|
1553
|
+
- **blocked** — the credential was _accepted_ and the account said no: an API
|
|
1554
|
+
switched off, an empty balance, a model this key was never granted. **Do not
|
|
1555
|
+
rotate the key** — a new one is refused identically. The fix is printed under
|
|
1556
|
+
the verdict; for a disabled Google API it is the exact `gcloud services
|
|
1557
|
+
enable …` line.
|
|
1558
|
+
- **no answer** — the network. Try again.
|
|
1559
|
+
|
|
1560
|
+
Either run the printed fix, or take a model that works and put the ref it prints
|
|
1561
|
+
into `agents.yaml`:
|
|
1562
|
+
|
|
1563
|
+
```sh
|
|
1564
|
+
zen models pick --embedding # or --chat; the ref goes to stdout, alone
|
|
1565
|
+
```
|
|
1566
|
+
|
|
1567
|
+
Then re-run `zen check`.
|
|
1568
|
+
|
|
1543
1569
|
CLI (`zen --help` for the authoritative list): `zen init`, `zen run`, `zen check`,
|
|
1544
|
-
`zen inspect`, `zen
|
|
1570
|
+
`zen inspect`, `zen key`, `zen models`, `zen list`. **stdout is the answer, stderr
|
|
1545
1571
|
is the narration**; every command takes `--json`. Exit codes: `0` ok, `1` failed,
|
|
1546
1572
|
`2` usage, `3` invalid project, `4` no usable credential.
|
|
1547
1573
|
|
|
@@ -1631,6 +1657,7 @@ Before finishing any change here:
|
|
|
1631
1657
|
| Answers from stale knowledge of the world | Grant `web_search` + `web_read`, and say when — §3.8 |
|
|
1632
1658
|
| Cites a page it only saw the excerpt of | A prompt line: `web_read` before quoting — §3.8 |
|
|
1633
1659
|
| Every web call refuses | No Exa key: `zen key add exa` — `zen check` warns — §3.8 |
|
|
1660
|
+
| A model or embedder refuses every call | `zen models test <ref>` — if `blocked`, `zen models pick` — §8 |
|
|
1634
1661
|
| Answers instead of routing | Router prompt prohibition; check `handoffs:` |
|
|
1635
1662
|
| Routes to the wrong specialist | The target agents' `description:` fields |
|
|
1636
1663
|
| Loses a detail after a handoff | Say it in the handoff; check the collapse policy |
|
|
@@ -21,8 +21,8 @@ zen list [--sessions] [--prune]
|
|
|
21
21
|
zen run [project] [prompt] [options]
|
|
22
22
|
zen open [project] [--editor <cmd>] [--wait]
|
|
23
23
|
zen key <ls|add|use|check|rm|show|env> [ref] [options]
|
|
24
|
-
zen models
|
|
25
|
-
zen check [name|dir] [--project <name|dir>] [--no-sandbox] [--strict] [--quiet]
|
|
24
|
+
zen models <providers|ls|search|show|test|pick> [ref] [options]
|
|
25
|
+
zen check [name|dir] [--project <name|dir>] [--no-sandbox] [--no-models] [--strict] [--quiet]
|
|
26
26
|
zen inspect [run] [--session <id>] [--open] [--rebuild] [--serve [port]]
|
|
27
27
|
zen sandbox [status|up|pull|clean|disk] [options]
|
|
28
28
|
zen version
|
|
@@ -42,8 +42,9 @@ guess a flag, and do not read them all.
|
|
|
42
42
|
| Global flags, `--json`, exit codes, environment, where files live | [frame.md](./references/frame.md) |
|
|
43
43
|
| Creating, finding or opening a project | [projects.md](./references/projects.md) |
|
|
44
44
|
| Running: the TUI, one-shot answers, sessions, workspaces, overrides | [run.md](./references/run.md) |
|
|
45
|
-
| Validating: `zen check`,
|
|
45
|
+
| Validating: `zen check`, what it can and cannot see | [check.md](./references/check.md) |
|
|
46
46
|
| API keys, providers, the keyring, "no credential" errors | [keys.md](./references/keys.md) |
|
|
47
|
+
| Which models exist, whether one works, recovering from a refusal | [models.md](./references/models.md) |
|
|
47
48
|
| The container shell commands run in, images, `persist` | [sandbox.md](./references/sandbox.md) |
|
|
48
49
|
| Run reports, trajectories, what a session directory holds | [inspect.md](./references/inspect.md) |
|
|
49
50
|
| `zen faker` — a mock API from an OpenAPI/Swagger document | [faker.md](./references/faker.md) |
|
|
@@ -58,6 +59,8 @@ zen run the TUI
|
|
|
58
59
|
zen run "what changed?" one answer, this directory as the workspace
|
|
59
60
|
zen inspect --open what the model was actually given
|
|
60
61
|
zen key ls --check which credentials still work
|
|
62
|
+
zen models test <ref> whether one model actually answers
|
|
63
|
+
zen models pick --embedding the first embedder that does, on stdout
|
|
61
64
|
```
|
|
62
65
|
|
|
63
66
|
Run `zen check` after any edit to `agents.yaml`, a prompt or a skill: it reads
|