faberwright 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +62 -48
- package/dist/agent.js +11 -0
- package/dist/index.js +32 -5
- package/dist/llm.js +268 -6
- package/dist/models.js +111 -1
- package/dist/onboard.js +126 -16
- package/dist/pricing.js +20 -2
- package/dist/prompt.js +86 -26
- package/dist/usage.js +127 -82
- package/package.json +1 -1
package/dist/llm.js
CHANGED
|
@@ -1,6 +1,14 @@
|
|
|
1
1
|
import { signRequest, discoverAwsCredentials } from "./sigv4.js";
|
|
2
|
+
import { priceFor } from "./pricing.js";
|
|
2
3
|
import { FatalError, TransientAPIError, withRetries, CancelledError } from "./errors.js";
|
|
3
4
|
const RETRYABLE = new Set([408, 409, 429, 500, 502, 503, 504, 529]);
|
|
5
|
+
/**
|
|
6
|
+
* Why the last model listing came back empty. "Couldn't reach the provider"
|
|
7
|
+
* is not a useful thing to tell someone whose key demonstrably works — the
|
|
8
|
+
* status code or error usually says exactly what went wrong.
|
|
9
|
+
*/
|
|
10
|
+
let lastListError;
|
|
11
|
+
export function lastModelListError() { return lastListError; }
|
|
4
12
|
/** AWS credentials discovered once per process for SigV4 routes. */
|
|
5
13
|
let awsCredsPromise;
|
|
6
14
|
export class LLMClient {
|
|
@@ -30,7 +38,34 @@ export class LLMClient {
|
|
|
30
38
|
* minus display_name. Returns [] on any failure — this is a convenience,
|
|
31
39
|
* never a blocker, so an offline or restricted key just falls back.
|
|
32
40
|
*/
|
|
41
|
+
/**
|
|
42
|
+
* Check whether the provider accepts this credential.
|
|
43
|
+
* "rejected" means the key is definitively wrong (401/403) — that is a
|
|
44
|
+
* failed setup. "unreachable" covers being offline or an endpoint without a
|
|
45
|
+
* models route, which says nothing about the key and must not block setup.
|
|
46
|
+
*/
|
|
47
|
+
async verifyKey() {
|
|
48
|
+
const anthropic = this.config.provider === "anthropic";
|
|
49
|
+
const url = anthropic ? `${this.config.baseUrl}/v1/models?limit=1`
|
|
50
|
+
: `${this.config.baseUrl}/models`;
|
|
51
|
+
const headers = anthropic
|
|
52
|
+
? { "x-api-key": this.config.apiKey, "anthropic-version": "2023-06-01" }
|
|
53
|
+
: { authorization: `Bearer ${this.config.apiKey}` };
|
|
54
|
+
try {
|
|
55
|
+
const ctl = new AbortController();
|
|
56
|
+
const timer = setTimeout(() => ctl.abort(), 8000);
|
|
57
|
+
const res = await fetch(url, { headers, signal: ctl.signal });
|
|
58
|
+
clearTimeout(timer);
|
|
59
|
+
if (res.status === 401 || res.status === 403)
|
|
60
|
+
return "rejected";
|
|
61
|
+
return res.ok ? "ok" : "unreachable";
|
|
62
|
+
}
|
|
63
|
+
catch {
|
|
64
|
+
return "unreachable";
|
|
65
|
+
}
|
|
66
|
+
}
|
|
33
67
|
async listModels(signal) {
|
|
68
|
+
lastListError = undefined;
|
|
34
69
|
const anthropic = this.config.provider === "anthropic";
|
|
35
70
|
const url = anthropic
|
|
36
71
|
? `${this.config.baseUrl}/v1/models?limit=100`
|
|
@@ -44,14 +79,25 @@ export class LLMClient {
|
|
|
44
79
|
signal?.addEventListener("abort", () => ctl.abort(), { once: true });
|
|
45
80
|
const res = await fetch(url, { headers, signal: ctl.signal });
|
|
46
81
|
clearTimeout(timer);
|
|
47
|
-
if (!res.ok)
|
|
82
|
+
if (!res.ok) {
|
|
83
|
+
lastListError = `HTTP ${res.status} from ${url}`;
|
|
48
84
|
return [];
|
|
85
|
+
}
|
|
49
86
|
const body = await res.json();
|
|
50
87
|
return (body.data ?? [])
|
|
51
88
|
.filter((m) => typeof m.id === "string")
|
|
52
|
-
.map((m) => ({
|
|
89
|
+
.map((m) => ({
|
|
90
|
+
id: m.id,
|
|
91
|
+
name: m.display_name,
|
|
92
|
+
// OpenAI sends a unix timestamp; Anthropic an ISO date. Either way
|
|
93
|
+
// this is the release date the picker sorts on.
|
|
94
|
+
created: typeof m.created === "number" ? m.created
|
|
95
|
+
: m.created_at ? Math.floor(Date.parse(m.created_at) / 1000) || undefined
|
|
96
|
+
: undefined,
|
|
97
|
+
}));
|
|
53
98
|
}
|
|
54
|
-
catch {
|
|
99
|
+
catch (e) {
|
|
100
|
+
lastListError = e instanceof Error ? e.message : String(e);
|
|
55
101
|
return []; // offline, no permission, or an endpoint without the route
|
|
56
102
|
}
|
|
57
103
|
}
|
|
@@ -84,9 +130,14 @@ export class LLMClient {
|
|
|
84
130
|
/** Switch model at runtime (/model). Cache prefixes are per-model. */
|
|
85
131
|
setModel(id) { this.config = { ...this.config, model: id }; }
|
|
86
132
|
complete(system, messages, tools, onText, signal, modelOverride) {
|
|
133
|
+
// Which OpenAI API a model speaks isn't a user setting — the dataset
|
|
134
|
+
// records it per model, so route on that rather than asking anyone.
|
|
135
|
+
const model = modelOverride ?? this.config.model;
|
|
87
136
|
return withRetries(() => this.config.provider === "anthropic"
|
|
88
137
|
? this.anthropicStream(system, messages, tools, onText, signal, modelOverride)
|
|
89
|
-
:
|
|
138
|
+
: usesResponsesApi(model)
|
|
139
|
+
? this.responsesStream(system, messages, tools, onText, signal, modelOverride)
|
|
140
|
+
: this.openaiWithResponsesFallback(system, messages, tools, onText, signal, modelOverride), { maxAttempts: this.config.retryMaxAttempts, signal });
|
|
90
141
|
}
|
|
91
142
|
/** Internal summarization: routed to the cheap weak model when configured. */
|
|
92
143
|
async summarize(text, instruction) {
|
|
@@ -252,6 +303,145 @@ export class LLMClient {
|
|
|
252
303
|
rawContent.push({ type: "tool_use", id: c.id, name: c.name, input: c.input });
|
|
253
304
|
return { text, toolCalls, rawContent, stopReason: toolCalls.length ? "tool_use" : "end_turn", usage };
|
|
254
305
|
}
|
|
306
|
+
/**
|
|
307
|
+
* Chat completions, retrying on the Responses API if the provider says the
|
|
308
|
+
* model belongs there.
|
|
309
|
+
*
|
|
310
|
+
* Routing normally comes from the dataset, but a model can be missing from
|
|
311
|
+
* it, or the local copy can predate the fields we read. Rather than failing
|
|
312
|
+
* with a 404 the user can do nothing about, take the provider at its word
|
|
313
|
+
* and retry on the endpoint it named.
|
|
314
|
+
*/
|
|
315
|
+
async openaiWithResponsesFallback(system, messages, tools, onText, signal, modelOverride) {
|
|
316
|
+
try {
|
|
317
|
+
return await this.openaiStream(system, messages, tools, onText, signal, modelOverride);
|
|
318
|
+
}
|
|
319
|
+
catch (e) {
|
|
320
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
321
|
+
if (!/responses/i.test(msg) || !/endpoint|not supported/i.test(msg))
|
|
322
|
+
throw e;
|
|
323
|
+
return this.responsesStream(system, messages, tools, onText, signal, modelOverride);
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
// ------------------------------------------------------------- responses
|
|
327
|
+
/**
|
|
328
|
+
* OpenAI's Responses API — the only way to reach the codex models, which are
|
|
329
|
+
* the coding-tuned ones a coding agent actually wants.
|
|
330
|
+
*
|
|
331
|
+
* Three things differ from chat completions, and each is a place to get it
|
|
332
|
+
* wrong: the request carries `instructions` and `input` rather than a
|
|
333
|
+
* `messages` array; the stream is a sequence of named events keyed by a
|
|
334
|
+
* `type` field rather than `choices[].delta`; and usage arrives as
|
|
335
|
+
* input_tokens/output_tokens, which the cost ledger reads directly, so a
|
|
336
|
+
* mismatch here silently under-reports spend rather than failing loudly.
|
|
337
|
+
*/
|
|
338
|
+
async responsesStream(system, messages, tools, onText, signal, modelOverride) {
|
|
339
|
+
const input = [];
|
|
340
|
+
for (const m of messages)
|
|
341
|
+
input.push(...toResponsesInput(m));
|
|
342
|
+
const body = {
|
|
343
|
+
model: modelOverride ?? this.config.model,
|
|
344
|
+
instructions: system,
|
|
345
|
+
input,
|
|
346
|
+
stream: true,
|
|
347
|
+
max_output_tokens: this.config.maxTokens,
|
|
348
|
+
};
|
|
349
|
+
if (tools.length) {
|
|
350
|
+
// Flatter than chat completions: no nested `function` object.
|
|
351
|
+
body.tools = tools.map((t) => ({
|
|
352
|
+
type: "function",
|
|
353
|
+
name: t.name,
|
|
354
|
+
description: t.description,
|
|
355
|
+
parameters: t.input_schema,
|
|
356
|
+
}));
|
|
357
|
+
}
|
|
358
|
+
const res = await this.post(`${this.config.baseUrl}/responses`, {
|
|
359
|
+
Authorization: `Bearer ${this.config.apiKey}`,
|
|
360
|
+
"content-type": "application/json",
|
|
361
|
+
}, body, signal);
|
|
362
|
+
let text = "";
|
|
363
|
+
const usage = { input: 0, cacheRead: 0, cacheWrite: 0, output: 0 };
|
|
364
|
+
// Tool calls arrive as items announced first, then filled in by argument
|
|
365
|
+
// deltas addressed to the item's id.
|
|
366
|
+
const calls = new Map();
|
|
367
|
+
for await (const evt of sseEvents(res, signal)) {
|
|
368
|
+
const e = evt;
|
|
369
|
+
switch (e.type) {
|
|
370
|
+
case "response.output_text.delta": {
|
|
371
|
+
const d = typeof e.delta === "string" ? e.delta : "";
|
|
372
|
+
if (d) {
|
|
373
|
+
text += d;
|
|
374
|
+
onText?.(d);
|
|
375
|
+
}
|
|
376
|
+
break;
|
|
377
|
+
}
|
|
378
|
+
case "response.output_item.added": {
|
|
379
|
+
const item = e.item;
|
|
380
|
+
if (item?.type === "function_call") {
|
|
381
|
+
calls.set(String(item.id ?? item.call_id), {
|
|
382
|
+
id: String(item.call_id ?? item.id ?? ""),
|
|
383
|
+
name: String(item.name ?? ""),
|
|
384
|
+
args: typeof item.arguments === "string" ? item.arguments : "",
|
|
385
|
+
});
|
|
386
|
+
}
|
|
387
|
+
break;
|
|
388
|
+
}
|
|
389
|
+
case "response.function_call_arguments.delta": {
|
|
390
|
+
const key = String(e.item_id ?? "");
|
|
391
|
+
const cur = calls.get(key) ?? { id: key, name: "", args: "" };
|
|
392
|
+
cur.args += typeof e.delta === "string" ? e.delta : "";
|
|
393
|
+
calls.set(key, cur);
|
|
394
|
+
break;
|
|
395
|
+
}
|
|
396
|
+
case "response.function_call_arguments.done": {
|
|
397
|
+
// Some responses send the complete arguments here instead of deltas.
|
|
398
|
+
const key = String(e.item_id ?? "");
|
|
399
|
+
const cur = calls.get(key);
|
|
400
|
+
if (cur && !cur.args && typeof e.arguments === "string")
|
|
401
|
+
cur.args = e.arguments;
|
|
402
|
+
break;
|
|
403
|
+
}
|
|
404
|
+
case "response.completed":
|
|
405
|
+
case "response.incomplete": {
|
|
406
|
+
const u = e.response?.usage;
|
|
407
|
+
if (u) {
|
|
408
|
+
const cached = u.input_tokens_details?.cached_tokens ?? 0;
|
|
409
|
+
usage.input = (u.input_tokens ?? 0) - cached;
|
|
410
|
+
usage.cacheRead = cached;
|
|
411
|
+
usage.output = u.output_tokens ?? 0;
|
|
412
|
+
}
|
|
413
|
+
break;
|
|
414
|
+
}
|
|
415
|
+
case "error":
|
|
416
|
+
case "response.failed": {
|
|
417
|
+
const msg = e.message ?? e.response?.error?.message ?? "stream failed";
|
|
418
|
+
throw new FatalError(`Responses API error: ${String(msg)}`);
|
|
419
|
+
}
|
|
420
|
+
default: break; // ignore lifecycle events we don't need
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
const toolCalls = [...calls.values()]
|
|
424
|
+
.filter((c) => c.name)
|
|
425
|
+
.map((c) => {
|
|
426
|
+
let parsed = {};
|
|
427
|
+
try {
|
|
428
|
+
parsed = c.args ? JSON.parse(c.args) : {};
|
|
429
|
+
}
|
|
430
|
+
catch { /* leave empty */ }
|
|
431
|
+
return { id: c.id, name: c.name, input: parsed };
|
|
432
|
+
});
|
|
433
|
+
const rawContent = [];
|
|
434
|
+
if (text)
|
|
435
|
+
rawContent.push({ type: "text", text });
|
|
436
|
+
for (const c of toolCalls) {
|
|
437
|
+
rawContent.push({ type: "tool_use", id: c.id, name: c.name, input: c.input });
|
|
438
|
+
}
|
|
439
|
+
return {
|
|
440
|
+
text, toolCalls, rawContent,
|
|
441
|
+
stopReason: toolCalls.length ? "tool_use" : "end_turn",
|
|
442
|
+
usage,
|
|
443
|
+
};
|
|
444
|
+
}
|
|
255
445
|
// ------------------------------------------------------------------ http
|
|
256
446
|
async post(url, headers, body, signal) {
|
|
257
447
|
let res;
|
|
@@ -270,8 +460,28 @@ export class LLMClient {
|
|
|
270
460
|
if (res.status === 401 || res.status === 403) {
|
|
271
461
|
throw new FatalError(`Authentication failed (HTTP ${res.status}). Check your API key.`);
|
|
272
462
|
}
|
|
273
|
-
if (!res.ok)
|
|
274
|
-
|
|
463
|
+
if (!res.ok) {
|
|
464
|
+
const body = (await res.text()).slice(0, 500);
|
|
465
|
+
// Record what the provider just told us. Neither OpenAI's catalogue nor
|
|
466
|
+
// its /v1/models response flags retired models or says which endpoint a
|
|
467
|
+
// model needs, so the failing request is the only reliable signal — and
|
|
468
|
+
// remembering it keeps the model out of every future menu.
|
|
469
|
+
const { classifyModelError, markUnusable } = await import("./models.js");
|
|
470
|
+
const why = classifyModelError(body);
|
|
471
|
+
if (why)
|
|
472
|
+
markUnusable(this.config.model, why);
|
|
473
|
+
// Some newer OpenAI models are only served by the Responses API, which
|
|
474
|
+
// Faber doesn't speak yet. The raw 404 doesn't say what to do about it.
|
|
475
|
+
if (/v1\/responses endpoint/i.test(body)) {
|
|
476
|
+
// Not fatal: the caller retries this request on the Responses API.
|
|
477
|
+
throw new FatalError(`${this.config.model} is served by the responses endpoint, not chat completions.`);
|
|
478
|
+
}
|
|
479
|
+
if (/has been deprecated/i.test(body)) {
|
|
480
|
+
throw new FatalError(`${this.config.model} has been deprecated by the provider.\n` +
|
|
481
|
+
` It won't be offered again. Pick another with /model.`);
|
|
482
|
+
}
|
|
483
|
+
throw new FatalError(`API error HTTP ${res.status}: ${body}`);
|
|
484
|
+
}
|
|
275
485
|
return res;
|
|
276
486
|
}
|
|
277
487
|
}
|
|
@@ -310,6 +520,58 @@ async function* sseEvents(res, signal) {
|
|
|
310
520
|
reader.releaseLock();
|
|
311
521
|
}
|
|
312
522
|
}
|
|
523
|
+
/**
|
|
524
|
+
* Convert a message to Responses API input items.
|
|
525
|
+
*
|
|
526
|
+
* The Responses API doesn't take a `messages` array of chat turns. It takes a
|
|
527
|
+
* flat list of items where a tool call and its result are siblings rather than
|
|
528
|
+
* nested inside an assistant turn — so one Faber message can expand into
|
|
529
|
+
* several items. Tool calls are joined to their results by `call_id`.
|
|
530
|
+
*/
|
|
531
|
+
/**
|
|
532
|
+
* Does this model require the Responses API? Answered by the pricing dataset's
|
|
533
|
+
* `mode` and `supported_endpoints`, refreshed on every launch, so a model
|
|
534
|
+
* released tomorrow routes correctly without a Faber update.
|
|
535
|
+
*/
|
|
536
|
+
export function usesResponsesApi(modelId) {
|
|
537
|
+
const p = priceFor(modelId);
|
|
538
|
+
if (p?.mode === "responses")
|
|
539
|
+
return true;
|
|
540
|
+
if (p?.endpoints?.length) {
|
|
541
|
+
return p.endpoints.includes("/v1/responses")
|
|
542
|
+
&& !p.endpoints.some((e) => e.includes("chat/completions"));
|
|
543
|
+
}
|
|
544
|
+
return false;
|
|
545
|
+
}
|
|
546
|
+
function toResponsesInput(m) {
|
|
547
|
+
const out = [];
|
|
548
|
+
const texts = [];
|
|
549
|
+
for (const b of m.content) {
|
|
550
|
+
if (b.type === "text")
|
|
551
|
+
texts.push(b.text);
|
|
552
|
+
else if (b.type === "tool_use") {
|
|
553
|
+
out.push({
|
|
554
|
+
type: "function_call",
|
|
555
|
+
call_id: b.id,
|
|
556
|
+
name: b.name,
|
|
557
|
+
arguments: JSON.stringify(b.input),
|
|
558
|
+
});
|
|
559
|
+
}
|
|
560
|
+
else if (b.type === "tool_result") {
|
|
561
|
+
out.push({
|
|
562
|
+
type: "function_call_output",
|
|
563
|
+
call_id: b.tool_use_id,
|
|
564
|
+
output: typeof b.content === "string" ? b.content : JSON.stringify(b.content),
|
|
565
|
+
});
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
if (texts.length) {
|
|
569
|
+
// input_text for what we send, output_text for what the model said
|
|
570
|
+
const kind = m.role === "assistant" ? "output_text" : "input_text";
|
|
571
|
+
out.unshift({ role: m.role, content: [{ type: kind, text: texts.join("\n") }] });
|
|
572
|
+
}
|
|
573
|
+
return out;
|
|
574
|
+
}
|
|
313
575
|
function toOpenAI(m) {
|
|
314
576
|
const out = [];
|
|
315
577
|
const texts = [];
|
package/dist/models.js
CHANGED
|
@@ -14,6 +14,34 @@ import * as fs from "node:fs";
|
|
|
14
14
|
import * as os from "node:os";
|
|
15
15
|
import * as path from "node:path";
|
|
16
16
|
import { modelsForRoute } from "./routes.js";
|
|
17
|
+
import { priceFor } from "./pricing.js";
|
|
18
|
+
/**
|
|
19
|
+
* Order the model list the way someone choosing one would want it.
|
|
20
|
+
*
|
|
21
|
+
* On OpenAI the coding-tuned models matter most to a coding agent, so those
|
|
22
|
+
* come first as a group; everything else follows. Within each group, newest
|
|
23
|
+
* first, since a release date is the only ranking the provider actually gives
|
|
24
|
+
* us. Anthropic has no such split, so it's purely newest first.
|
|
25
|
+
*/
|
|
26
|
+
export function isCodexModel(id) {
|
|
27
|
+
return /codex/i.test(id);
|
|
28
|
+
}
|
|
29
|
+
export function sortModels(models, wire) {
|
|
30
|
+
const byDate = (a, b) => {
|
|
31
|
+
if (a.created && b.created)
|
|
32
|
+
return b.created - a.created;
|
|
33
|
+
if (a.created)
|
|
34
|
+
return -1; // dated entries above undated ones
|
|
35
|
+
if (b.created)
|
|
36
|
+
return 1;
|
|
37
|
+
return a.id.localeCompare(b.id);
|
|
38
|
+
};
|
|
39
|
+
if (wire !== "openai")
|
|
40
|
+
return [...models].sort(byDate);
|
|
41
|
+
const codex = models.filter((m) => isCodexModel(m.id)).sort(byDate);
|
|
42
|
+
const rest = models.filter((m) => !isCodexModel(m.id)).sort(byDate);
|
|
43
|
+
return [...codex, ...rest];
|
|
44
|
+
}
|
|
17
45
|
const TTL_MS = 24 * 60 * 60 * 1000; // a day: new models are rare, staleness is cheap
|
|
18
46
|
function cacheFile(routeId) {
|
|
19
47
|
return path.join(os.homedir(), ".faber", "cache", `models-${routeId}.json`);
|
|
@@ -43,6 +71,43 @@ export function clearCache(routeId) {
|
|
|
43
71
|
}
|
|
44
72
|
catch { /* already gone */ }
|
|
45
73
|
}
|
|
74
|
+
/**
|
|
75
|
+
* Can this model actually run an agent loop?
|
|
76
|
+
*
|
|
77
|
+
* Provider catalogues list everything the key can call: embeddings, speech,
|
|
78
|
+
* transcription, moderation, image generation. None of them accept a chat
|
|
79
|
+
* request, so offering them as a choice is offering a guaranteed failure.
|
|
80
|
+
* Excluding by capability keyword is deliberately conservative — anything
|
|
81
|
+
* unrecognised is kept, since a new chat model must never be filtered out.
|
|
82
|
+
*/
|
|
83
|
+
const NOT_CHAT = /(^|[-_/])(embedding|embed|tts|whisper|moderation|dall-e|dalle|image|audio|transcribe|realtime|speech|rerank|search-preview|codex-mini-latest)([-_/]|$)/i;
|
|
84
|
+
/** Retired completion-only families that predate the chat API. */
|
|
85
|
+
const LEGACY = /^(davinci|babbage|curie|ada|text-davinci|code-davinci)/i;
|
|
86
|
+
export function isChatModel(id) {
|
|
87
|
+
// The dataset states each model's mode and endpoints, so use that rather
|
|
88
|
+
// than reading the name. Guessing from an id misclassifies every new naming
|
|
89
|
+
// scheme; this is the provider's own answer, refreshed on every launch.
|
|
90
|
+
const p = priceFor(id);
|
|
91
|
+
// "responses" counts as a chat model now that Faber speaks that API too.
|
|
92
|
+
if (p?.mode)
|
|
93
|
+
return p.mode === "chat" || p.mode === "completion" || p.mode === "responses";
|
|
94
|
+
if (p?.endpoints?.length) {
|
|
95
|
+
return p.endpoints.some((e) => e.includes("chat/completions") || e === "/v1/responses");
|
|
96
|
+
}
|
|
97
|
+
// Only fall back to name-based rules when the dataset has never seen it,
|
|
98
|
+
// which is the case for a brand new model or a local one.
|
|
99
|
+
return !NOT_CHAT.test(id) && !LEGACY.test(id);
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Does this model need an API Faber doesn't speak? The dataset records it —
|
|
103
|
+
* gpt-5.3-codex reports mode "responses" and endpoint /v1/responses — so
|
|
104
|
+
* these can be labelled honestly instead of failing at request time.
|
|
105
|
+
*/
|
|
106
|
+
export function requiresUnsupportedApi(_id) {
|
|
107
|
+
// Responses-only models are supported now, so nothing is excluded on this
|
|
108
|
+
// basis. Kept as the hook for the next API Faber doesn't yet speak.
|
|
109
|
+
return undefined;
|
|
110
|
+
}
|
|
46
111
|
/**
|
|
47
112
|
* Build the /model menu: built-in aliases first (stable names people type),
|
|
48
113
|
* then anything discovered that an alias doesn't already cover.
|
|
@@ -57,9 +122,16 @@ export function buildPicker(route, discovered, currentId) {
|
|
|
57
122
|
live: false,
|
|
58
123
|
}));
|
|
59
124
|
const covered = new Set(aliases.map((a) => a.id));
|
|
60
|
-
|
|
125
|
+
const unusable = readUnusable();
|
|
126
|
+
for (const m of sortModels(discovered, route.wire)) {
|
|
61
127
|
if (covered.has(m.id))
|
|
62
128
|
continue;
|
|
129
|
+
if (!isChatModel(m.id))
|
|
130
|
+
continue; // embeddings, speech, moderation…
|
|
131
|
+
// Only genuinely dead models are hidden. An endpoint mismatch is handled
|
|
132
|
+
// by retrying on the right endpoint, so it must not remove the model.
|
|
133
|
+
if (unusable[m.id] === "deprecated")
|
|
134
|
+
continue;
|
|
63
135
|
entries.push({ value: m.id, label: m.id, blurb: m.name ?? "", live: true });
|
|
64
136
|
}
|
|
65
137
|
// if the model in use is neither an alias nor discovered, keep it visible
|
|
@@ -68,3 +140,41 @@ export function buildPicker(route, discovered, currentId) {
|
|
|
68
140
|
}
|
|
69
141
|
return entries;
|
|
70
142
|
}
|
|
143
|
+
function unusableFile() {
|
|
144
|
+
return path.join(os.homedir(), ".faber", "cache", "unusable-models.json");
|
|
145
|
+
}
|
|
146
|
+
export function readUnusable() {
|
|
147
|
+
try {
|
|
148
|
+
const raw = JSON.parse(fs.readFileSync(unusableFile(), "utf8"));
|
|
149
|
+
return raw && typeof raw === "object" ? raw : {};
|
|
150
|
+
}
|
|
151
|
+
catch {
|
|
152
|
+
return {};
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
export function markUnusable(modelId, why) {
|
|
156
|
+
try {
|
|
157
|
+
const f = unusableFile();
|
|
158
|
+
fs.mkdirSync(path.dirname(f), { recursive: true });
|
|
159
|
+
fs.writeFileSync(f, JSON.stringify({ ...readUnusable(), [modelId]: why }, null, 2));
|
|
160
|
+
}
|
|
161
|
+
catch { /* best effort */ }
|
|
162
|
+
}
|
|
163
|
+
export function clearUnusable() {
|
|
164
|
+
try {
|
|
165
|
+
fs.unlinkSync(unusableFile());
|
|
166
|
+
}
|
|
167
|
+
catch { /* already gone */ }
|
|
168
|
+
}
|
|
169
|
+
/** Classify a provider error, so a failure teaches us something. */
|
|
170
|
+
export function classifyModelError(message) {
|
|
171
|
+
const m = message.toLowerCase();
|
|
172
|
+
if (m.includes("deprecat") || m.includes("model_not_found") || m.includes("has been retired")) {
|
|
173
|
+
return "deprecated";
|
|
174
|
+
}
|
|
175
|
+
if (m.includes("/v1/responses") || m.includes("responses endpoint")
|
|
176
|
+
|| m.includes("not supported in the v1/chat/completions")) {
|
|
177
|
+
return "wrong-endpoint";
|
|
178
|
+
}
|
|
179
|
+
return undefined;
|
|
180
|
+
}
|
package/dist/onboard.js
CHANGED
|
@@ -3,7 +3,7 @@ import { select, readSecret } from "./prompt.js";
|
|
|
3
3
|
import { vendors, modelsForRoute, baseUrlFor, getRoute, DEFAULT_REGION, } from "./routes.js";
|
|
4
4
|
import * as fs from "node:fs";
|
|
5
5
|
import { loadSettings, saveSettings, settingsPath } from "./settings.js";
|
|
6
|
-
import { credentialSource, saveCredential, maskCredential, credentialsPath, looksLikeKey, } from "./credentials.js";
|
|
6
|
+
import { credentialSource, saveCredential, maskCredential, credentialsPath, looksLikeKey, deleteCredential, } from "./credentials.js";
|
|
7
7
|
/** First run = no settings file yet. */
|
|
8
8
|
/** Where a credential comes from, so setup can point people at the right page. */
|
|
9
9
|
const KEY_SOURCE = {
|
|
@@ -124,7 +124,32 @@ export async function ensureCredential(rl, route, profile) {
|
|
|
124
124
|
return {};
|
|
125
125
|
const found = credentialSource(route.keyEnv);
|
|
126
126
|
if (found?.from === "store") {
|
|
127
|
-
|
|
127
|
+
// Running setup again usually means something needs changing — often the
|
|
128
|
+
// key itself, because it expired or was wrong. Silently reusing the saved
|
|
129
|
+
// one makes that impossible, so offer the same choice as for an
|
|
130
|
+
// environment key.
|
|
131
|
+
console.log();
|
|
132
|
+
console.log(`${route.keyEnv} is already saved on this computer (${maskCredential(found.value)}).`);
|
|
133
|
+
const choice = await select(rl, "Use it?", [
|
|
134
|
+
`Keep using it ${pc.dim("no change")}`,
|
|
135
|
+
`Replace it ${pc.dim("paste a new key")}`,
|
|
136
|
+
`Remove it ${pc.dim("delete the saved key and stop")}`,
|
|
137
|
+
]);
|
|
138
|
+
if (choice === 1) {
|
|
139
|
+
const key = await readKey(rl, route.keyEnv, ` ${route.keyEnv} (hidden): `);
|
|
140
|
+
if (key) {
|
|
141
|
+
saveCredential(route.keyEnv, key);
|
|
142
|
+
console.log(pc.dim(` replaced (${maskCredential(key)})`));
|
|
143
|
+
}
|
|
144
|
+
else {
|
|
145
|
+
console.log(pc.dim(" nothing entered — keeping the saved key"));
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
else if (choice === 2) {
|
|
149
|
+
deleteCredential(route.keyEnv);
|
|
150
|
+
console.log(pc.dim(` removed. Run faber again to set one up.`));
|
|
151
|
+
return { switchedTo: "quit" };
|
|
152
|
+
}
|
|
128
153
|
return {};
|
|
129
154
|
}
|
|
130
155
|
if (found?.from === "environment") {
|
|
@@ -182,6 +207,52 @@ export async function ensureCredential(rl, route, profile) {
|
|
|
182
207
|
// starts over from the beginning rather than resuming into a broken state.
|
|
183
208
|
return { switchedTo: "quit" };
|
|
184
209
|
}
|
|
210
|
+
/** Does the provider accept this credential? Never blocks on being offline. */
|
|
211
|
+
async function verifyCredential(route, profile) {
|
|
212
|
+
if (!route.keyEnv)
|
|
213
|
+
return "ok";
|
|
214
|
+
try {
|
|
215
|
+
const { LLMClient } = await import("./llm.js");
|
|
216
|
+
const { resolveCredential } = await import("./credentials.js");
|
|
217
|
+
const key = resolveCredential(route.keyEnv);
|
|
218
|
+
const baseUrl = profile.baseUrl ?? baseUrlFor(route, profile.region);
|
|
219
|
+
if (!key || !baseUrl)
|
|
220
|
+
return "unreachable";
|
|
221
|
+
process.stdout.write(pc.dim(" checking your key… "));
|
|
222
|
+
const llm = new LLMClient({
|
|
223
|
+
provider: route.wire, baseUrl, apiKey: key, model: "x", route: route.id,
|
|
224
|
+
});
|
|
225
|
+
const verdict = await llm.verifyKey();
|
|
226
|
+
process.stdout.write("\r\x1b[2K");
|
|
227
|
+
return verdict;
|
|
228
|
+
}
|
|
229
|
+
catch {
|
|
230
|
+
process.stdout.write("\r\x1b[2K");
|
|
231
|
+
return "unreachable";
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
/** Ask the provider which models this key can use. Empty on any failure. */
|
|
235
|
+
async function discoverModels(route, profile) {
|
|
236
|
+
try {
|
|
237
|
+
const { LLMClient } = await import("./llm.js");
|
|
238
|
+
const { resolveCredential } = await import("./credentials.js");
|
|
239
|
+
const key = route.keyEnv ? resolveCredential(route.keyEnv) : undefined;
|
|
240
|
+
const baseUrl = profile.baseUrl ?? baseUrlFor(route, profile.region);
|
|
241
|
+
if (!baseUrl)
|
|
242
|
+
return [];
|
|
243
|
+
process.stdout.write(pc.dim(" checking which models your key can use… "));
|
|
244
|
+
const llm = new LLMClient({
|
|
245
|
+
provider: route.wire, baseUrl, apiKey: key, model: "x", route: route.id,
|
|
246
|
+
});
|
|
247
|
+
const models = await llm.listModels();
|
|
248
|
+
process.stdout.write("\r\x1b[2K");
|
|
249
|
+
return models;
|
|
250
|
+
}
|
|
251
|
+
catch {
|
|
252
|
+
process.stdout.write("\r\x1b[2K");
|
|
253
|
+
return [];
|
|
254
|
+
}
|
|
255
|
+
}
|
|
185
256
|
async function finish(rl, route, base) {
|
|
186
257
|
const profile = { ...base, route: route.id };
|
|
187
258
|
const activeRoute = route;
|
|
@@ -203,25 +274,64 @@ async function finish(rl, route, base) {
|
|
|
203
274
|
return { profile, route: activeRoute, aborted: true };
|
|
204
275
|
}
|
|
205
276
|
}
|
|
206
|
-
//
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
277
|
+
// Model choice, with real ids and real prices.
|
|
278
|
+
//
|
|
279
|
+
// Aliases like "gpt" or "sonnet" hide what you're actually paying for, and
|
|
280
|
+
// the difference is not small: gpt-4o costs about 17x gpt-4o-mini. Since the
|
|
281
|
+
// credential is saved by now, ask the provider what this key can really use
|
|
282
|
+
// and show each model's rate, so nothing about the bill is implicit.
|
|
283
|
+
const { refreshPrices, priceFor, readPriceCache } = await import("./pricing.js");
|
|
284
|
+
if (!readPriceCache())
|
|
285
|
+
process.stdout.write(pc.dim(" fetching prices… "));
|
|
286
|
+
await refreshPrices(undefined, { baseUrl: profile.baseUrl ?? activeRoute.baseUrl });
|
|
287
|
+
process.stdout.write("\r\x1b[2K");
|
|
288
|
+
// Verify the credential before going further. A rejected key means setup
|
|
289
|
+
// did not succeed, so nothing is saved and the next run starts over — the
|
|
290
|
+
// same rule as skipping the key entirely.
|
|
291
|
+
const verdict = await verifyCredential(activeRoute, profile);
|
|
292
|
+
if (verdict === "rejected") {
|
|
293
|
+
if (activeRoute.keyEnv)
|
|
294
|
+
deleteCredential(activeRoute.keyEnv);
|
|
295
|
+
console.log();
|
|
296
|
+
console.log(pc.yellow(`${activeRoute.label} rejected that key.`));
|
|
297
|
+
console.log(pc.dim(" Nothing was saved. Run faber again with a working key."));
|
|
298
|
+
return { profile, route: activeRoute, aborted: true };
|
|
299
|
+
}
|
|
300
|
+
const { isChatModel, sortModels, requiresUnsupportedApi } = await import("./models.js");
|
|
301
|
+
const discovered = sortModels((await discoverModels(activeRoute, profile))
|
|
302
|
+
.filter((m) => isChatModel(m.id) && !requiresUnsupportedApi(m.id)), activeRoute.wire);
|
|
303
|
+
const fallback = modelsForRoute(activeRoute).map((m) => ({ id: m.id, name: m.blurb }));
|
|
304
|
+
const options = discovered.length ? discovered : fallback;
|
|
305
|
+
if (options.length) {
|
|
306
|
+
// Say which list this is. A built-in fallback and a live list look
|
|
307
|
+
// identical otherwise, and the user can't tell whether the models shown
|
|
308
|
+
// are the ones their key can actually reach.
|
|
309
|
+
console.log();
|
|
310
|
+
if (discovered.length) {
|
|
311
|
+
console.log(pc.dim(` ${discovered.length} models available to this key`));
|
|
312
|
+
}
|
|
313
|
+
else {
|
|
314
|
+
const { lastModelListError } = await import("./llm.js");
|
|
315
|
+
const why = lastModelListError();
|
|
316
|
+
console.log(pc.yellow(" Couldn't list models — showing built-in defaults."));
|
|
317
|
+
console.log(pc.dim(why ? ` ${why}` : " No response from the provider."));
|
|
318
|
+
console.log(pc.dim(" /model re-checks later."));
|
|
319
|
+
}
|
|
320
|
+
const width = Math.min(34, Math.max(...options.map((m) => m.id.length)) + 2);
|
|
321
|
+
const labels = options.map((m) => {
|
|
322
|
+
const p = priceFor(m.id);
|
|
323
|
+
const cost = p ? `$${p.in}/$${p.out} per Mtok` : "price unknown";
|
|
324
|
+
return `${m.id.padEnd(width)}${pc.dim(cost)}${m.name ? pc.dim(" " + m.name) : ""}`;
|
|
325
|
+
});
|
|
326
|
+
console.log();
|
|
327
|
+
const mi = await select(rl, "Which model? (input/output cost per million tokens)", labels);
|
|
328
|
+
profile.model = options[mi].id; // a concrete id, never an alias
|
|
211
329
|
}
|
|
212
330
|
else {
|
|
213
331
|
const hint = activeRoute.id === "ollama" ? "qwen2.5-coder" : "";
|
|
214
|
-
const ans = (await rl.question(`Model id${hint ? ` [${hint}]` : ""}
|
|
332
|
+
const ans = (await rl.question(`Model id${hint ? ` [${hint}]` : ""}: `)).trim();
|
|
215
333
|
profile.model = ans || hint || undefined;
|
|
216
334
|
}
|
|
217
|
-
// Fetch prices once during setup: costs are frozen per task, so starting
|
|
218
|
-
// with current rates keeps day-one history accurate.
|
|
219
|
-
const { refreshPrices } = await import("./pricing.js");
|
|
220
|
-
process.stdout.write(pc.dim(" fetching current prices… "));
|
|
221
|
-
const n = await refreshPrices(undefined, { baseUrl: profile.baseUrl ?? activeRoute.baseUrl });
|
|
222
|
-
process.stdout.write("\r\x1b[2K");
|
|
223
|
-
if (!n)
|
|
224
|
-
console.log(pc.dim(" (couldn't fetch prices — using built-in rates)"));
|
|
225
335
|
const settings = loadSettings();
|
|
226
336
|
settings.profiles[settings.activeProfile] = profile;
|
|
227
337
|
saveSettings(settings);
|