@bendyline/gezel-cli 1.2.12 → 1.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -2
- package/dist/app-author-command.js +7 -3
- package/dist/app-command.js +2 -0
- package/dist/app-guide.js +128 -0
- package/dist/bin/gezel-main.js +1896 -1660
- package/dist/handboek-export.js +175 -53
- package/dist/knowledge-command.js +2 -0
- package/dist/tui/index.js +62 -9
- package/package.json +9 -9
package/dist/bin/gezel-main.js
CHANGED
|
@@ -41,6 +41,7 @@ import { createHash, randomUUID } from "crypto";
|
|
|
41
41
|
import { chmod, mkdir, readFile, realpath, rm, writeFile } from "fs/promises";
|
|
42
42
|
import { homedir } from "os";
|
|
43
43
|
import { basename, dirname, join, resolve } from "path";
|
|
44
|
+
import { fileURLToPath } from "url";
|
|
44
45
|
import {
|
|
45
46
|
evaluateApiCompat,
|
|
46
47
|
getLogLevel,
|
|
@@ -58,6 +59,7 @@ import {
|
|
|
58
59
|
authorizeLocalOwner
|
|
59
60
|
} from "@bendyline/gezel-app-sdk";
|
|
60
61
|
import {
|
|
62
|
+
DaemonInAnotherNamespaceError,
|
|
61
63
|
GezelApiError,
|
|
62
64
|
GezelClient,
|
|
63
65
|
LiveDaemonUnhealthyError,
|
|
@@ -172,7 +174,7 @@ async function connectOwned(globals, options = {}) {
|
|
|
172
174
|
...process.env.GEZEL_HOME ? { home: process.env.GEZEL_HOME } : {}
|
|
173
175
|
}
|
|
174
176
|
}).catch((error) => {
|
|
175
|
-
throw isIncompatibleDaemonError(error) ? incompatibleDaemonCliError(error) : error;
|
|
177
|
+
throw isIncompatibleDaemonError(error) ? incompatibleDaemonCliError(error) : localDaemonCliError(error) ?? error;
|
|
176
178
|
});
|
|
177
179
|
if (authorized.daemon.mode === "spawned" && options.announceSpawn !== false) {
|
|
178
180
|
process.stderr.write(spawnedDaemonNotice(authorized.daemon.pid));
|
|
@@ -273,9 +275,8 @@ async function authorizeLocalOwnedDaemon(deps = {}) {
|
|
|
273
275
|
});
|
|
274
276
|
return await (deps.connect ?? defaultConnectAsOwner)(authorized);
|
|
275
277
|
} catch (error) {
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
}
|
|
278
|
+
const local = localDaemonCliError(error);
|
|
279
|
+
if (local) throw local;
|
|
279
280
|
if (!(error instanceof GezelSdkError) || error.code !== "daemon_not_running") {
|
|
280
281
|
throw cliGrantError(error, "the local Gezel daemon");
|
|
281
282
|
}
|
|
@@ -289,6 +290,13 @@ async function authorizeLocalOwnedDaemon(deps = {}) {
|
|
|
289
290
|
function defaultConnectAsOwner(authorized) {
|
|
290
291
|
return connectionFromAuthorization(authorized, "The local Gezel owner authorization");
|
|
291
292
|
}
|
|
293
|
+
function localDaemonCliError(error) {
|
|
294
|
+
if (error instanceof LiveDaemonUnhealthyError) {
|
|
295
|
+
return unhealthyLocalDaemonError(error.pid, error.baseUrl);
|
|
296
|
+
}
|
|
297
|
+
if (error instanceof DaemonInAnotherNamespaceError) return new CliError(error.message);
|
|
298
|
+
return null;
|
|
299
|
+
}
|
|
292
300
|
function unhealthyLocalDaemonError(pid, baseUrl) {
|
|
293
301
|
return new CliError(
|
|
294
302
|
`The local Gezel daemon (pid ${pid}) is running but did not answer at ${baseUrl}. Stop or repair it before retrying.`
|
|
@@ -566,12 +574,18 @@ var CLI_CLIENT_LABEL = "this gezel CLI";
|
|
|
566
574
|
function isIncompatibleDaemonError(error) {
|
|
567
575
|
return error instanceof GezelSdkError && error.code === "daemon_incompatible";
|
|
568
576
|
}
|
|
577
|
+
function cliUpdateCommand(moduleUrl = import.meta.url) {
|
|
578
|
+
const path = fileURLToPath(moduleUrl).replaceAll("\\", "/");
|
|
579
|
+
if (path.includes("/_npx/")) return "npx gezel-cli@latest";
|
|
580
|
+
const shortName = path.includes("/node_modules/gezel-cli/node_modules/@bendyline/gezel-cli/");
|
|
581
|
+
return `npm install -g ${shortName ? "gezel-cli" : "@bendyline/gezel-cli"}`;
|
|
582
|
+
}
|
|
569
583
|
function incompatibleDaemonCliError(source) {
|
|
570
584
|
const verdict = source instanceof GezelSdkError ? { ...source.cause, systemService: false } : source;
|
|
571
585
|
const service = `the Gezel service${verdict.version ? ` (${verdict.version})` : ""}`;
|
|
572
586
|
if (verdict.code === "client-too-old") {
|
|
573
587
|
return new CliError(
|
|
574
|
-
`This gezel CLI is older than ${service} supports. Update it with
|
|
588
|
+
`This gezel CLI is older than ${service} supports. Update it with \`${cliUpdateCommand()}\`, then try again.`
|
|
575
589
|
);
|
|
576
590
|
}
|
|
577
591
|
if (verdict.code === "machine-engine-role") {
|
|
@@ -938,1561 +952,1859 @@ function registerIntegrationCommands(program2, connect, output = console.log) {
|
|
|
938
952
|
});
|
|
939
953
|
}
|
|
940
954
|
|
|
941
|
-
// src/
|
|
942
|
-
import {
|
|
943
|
-
import { readFileSync } from "fs";
|
|
944
|
-
import { readFile as readFile2, writeFile as writeFile2 } from "fs/promises";
|
|
945
|
-
import { cpus } from "os";
|
|
946
|
-
import { join as join2 } from "path";
|
|
947
|
-
import { promisify } from "util";
|
|
948
|
-
import { ScorecardDatasetSchema } from "@bendyline/gezel";
|
|
949
|
-
import { gildePackageRoot } from "@bendyline/gezel-catalog";
|
|
955
|
+
// src/craftbook-setup.ts
|
|
956
|
+
import { Writable } from "stream";
|
|
950
957
|
import {
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
958
|
+
craftbookSetupGaps,
|
|
959
|
+
isCraftbookModelEngine,
|
|
960
|
+
resolveCraftbookModelNeeds,
|
|
961
|
+
resolveSecurityPolicy as resolveSecurityPolicy2
|
|
962
|
+
} from "@bendyline/gezel";
|
|
963
|
+
import { resolveOnDeviceProvider } from "@bendyline/gezel/native";
|
|
964
|
+
|
|
965
|
+
// src/model-pull.ts
|
|
966
|
+
async function pullChatModel(client, engine, id, writeProgress) {
|
|
967
|
+
let lastPct = -1;
|
|
968
|
+
let pullError;
|
|
969
|
+
const onEvent = (ev) => {
|
|
970
|
+
if (ev.type === "progress") {
|
|
971
|
+
const written = "bytesWrittenAll" in ev ? ev.bytesWrittenAll : ev.bytesWritten;
|
|
972
|
+
const total = "totalBytesAll" in ev ? ev.totalBytesAll : ev.totalBytes ?? 0;
|
|
973
|
+
const pct = total > 0 ? Math.floor(written / total * 100) : 0;
|
|
974
|
+
if (pct === lastPct) return;
|
|
975
|
+
lastPct = pct;
|
|
976
|
+
writeProgress(
|
|
977
|
+
`\rdownloading ${id} (${engine}): ${String(pct).padStart(3)}% ${formatGb(written)}/${formatGb(total)} GB`
|
|
978
|
+
);
|
|
979
|
+
} else if (ev.type === "retrying") {
|
|
980
|
+
writeProgress(`
|
|
981
|
+
retry ${ev.attempt}/${ev.maxAttempts}: ${ev.reason}
|
|
982
|
+
`);
|
|
983
|
+
} else if (ev.type === "error") {
|
|
984
|
+
pullError = ev.error;
|
|
985
|
+
writeProgress("\n");
|
|
986
|
+
} else if (ev.type === "done" && !pullError) {
|
|
987
|
+
writeProgress(`\rdownloaded ${id} (${engine})${" ".repeat(48)}
|
|
988
|
+
`);
|
|
989
|
+
}
|
|
990
|
+
};
|
|
991
|
+
if (engine === "mlx") await client.installMlxModel(id, onEvent);
|
|
992
|
+
else if (engine === "ds4") await client.installDs4Model(id, onEvent);
|
|
993
|
+
else await client.installLlamaCppModel(id, onEvent);
|
|
994
|
+
if (pullError) throw new CliError(`download failed: ${pullError}`);
|
|
976
995
|
}
|
|
977
|
-
function
|
|
978
|
-
const
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
`Unknown suite "${id}". Choose one of: ${SCORECARD_SUITE_IDS.join(", ")}, or all. \`gezel bench list\` shows every suite.`
|
|
996
|
+
async function waitForMlxRuntime(client, writeProgress, options = {}) {
|
|
997
|
+
const pollMs = options.pollMs ?? 1e3;
|
|
998
|
+
const now = options.now ?? Date.now;
|
|
999
|
+
const started = now();
|
|
1000
|
+
let shown = false;
|
|
1001
|
+
for (; ; ) {
|
|
1002
|
+
const status = await client.getMlxRuntimeStatus();
|
|
1003
|
+
if (status.phase === "provisioning") {
|
|
1004
|
+
const elapsed = Math.round((now() - started) / 1e3);
|
|
1005
|
+
writeProgress(
|
|
1006
|
+
`\rpreparing the MLX Python runtime (one-time, a few minutes): ${Math.floor(elapsed / 60)}m ${String(elapsed % 60).padStart(2, "0")}s`
|
|
989
1007
|
);
|
|
1008
|
+
shown = true;
|
|
1009
|
+
await new Promise((resolve7) => setTimeout(resolve7, pollMs));
|
|
1010
|
+
continue;
|
|
990
1011
|
}
|
|
991
|
-
if (
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
suites.push(smoke);
|
|
998
|
-
} else {
|
|
999
|
-
suites.push(base);
|
|
1012
|
+
if (status.phase === "error") {
|
|
1013
|
+
writeProgress(
|
|
1014
|
+
`${shown ? "\n" : ""}MLX runtime setup failed; the first chat will retry it: ${status.error ?? "unknown error"}
|
|
1015
|
+
`
|
|
1016
|
+
);
|
|
1017
|
+
return "error";
|
|
1000
1018
|
}
|
|
1019
|
+
if (shown) writeProgress(`\rMLX runtime ready${" ".repeat(56)}
|
|
1020
|
+
`);
|
|
1021
|
+
return status.phase === "ready" ? "ready" : "idle";
|
|
1001
1022
|
}
|
|
1002
|
-
return suites;
|
|
1003
|
-
}
|
|
1004
|
-
function resolveBenchTarget(catalog, targets, opts) {
|
|
1005
|
-
const available = targets.filter((t) => t.available);
|
|
1006
|
-
if (opts.model) {
|
|
1007
|
-
const matches = available.filter(
|
|
1008
|
-
(t) => t.modelId === opts.model && (!opts.provider || t.provider === opts.provider)
|
|
1009
|
-
);
|
|
1010
|
-
const preferred = matches.find((t) => t.provider === catalog.defaultProvider) ?? matches[0];
|
|
1011
|
-
if (preferred) return preferred;
|
|
1012
|
-
const unavailable = targets.find(
|
|
1013
|
-
(t) => t.modelId === opts.model && (!opts.provider || t.provider === opts.provider)
|
|
1014
|
-
);
|
|
1015
|
-
throw new CliError(
|
|
1016
|
-
unavailable?.unavailableReason ? `${opts.model} can't run here: ${unavailable.unavailableReason}` : `No model "${opts.model}"${opts.provider ? ` on ${opts.provider}` : ""} is available to evaluate. \`gezel model list\` shows what is installed.`
|
|
1017
|
-
);
|
|
1018
|
-
}
|
|
1019
|
-
const byProvider = opts.provider ? available.filter((t) => t.provider === opts.provider) : available;
|
|
1020
|
-
const chosen = byProvider.find((t) => t.isDefault && t.provider === catalog.defaultProvider) ?? byProvider.find((t) => t.isDefault) ?? byProvider.find((t) => t.provider === catalog.defaultProvider) ?? byProvider[0];
|
|
1021
|
-
if (!chosen) {
|
|
1022
|
-
throw new CliError(
|
|
1023
|
-
"No model is available to evaluate. Download one with `gezel model pull <id>` or sign in to a cloud provider, then try again."
|
|
1024
|
-
);
|
|
1025
|
-
}
|
|
1026
|
-
return chosen;
|
|
1027
1023
|
}
|
|
1028
|
-
function
|
|
1029
|
-
|
|
1030
|
-
if (!Number.isInteger(count) || count < 1 || count > 10) {
|
|
1031
|
-
throw new CliError(
|
|
1032
|
-
"--count must be a whole number from 1 to 10 (3 is what it takes to quote a rate)."
|
|
1033
|
-
);
|
|
1034
|
-
}
|
|
1035
|
-
const suites = resolveBenchSuites(catalog, opts);
|
|
1036
|
-
const target = resolveBenchTarget(catalog, targets, opts);
|
|
1037
|
-
const worstCaseMs = suites.reduce((sum, s) => sum + s.authoredCeilingMs * count, 0);
|
|
1038
|
-
return { suites, target, count, worstCaseMs, smoke: Boolean(opts.smoke) };
|
|
1024
|
+
function formatGb(bytes) {
|
|
1025
|
+
return (bytes / 1e9).toFixed(2);
|
|
1039
1026
|
}
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1027
|
+
|
|
1028
|
+
// src/native-command.ts
|
|
1029
|
+
import {
|
|
1030
|
+
NativeEngineNameSchema
|
|
1031
|
+
} from "@bendyline/gezel";
|
|
1032
|
+
|
|
1033
|
+
// src/tui/bootstrap.ts
|
|
1034
|
+
import {
|
|
1035
|
+
computeModelFit,
|
|
1036
|
+
formatModelAttribution,
|
|
1037
|
+
isMoEFromTags,
|
|
1038
|
+
isRecommendedModel,
|
|
1039
|
+
lowBitScoreFactor,
|
|
1040
|
+
pickRecommendedModel
|
|
1041
|
+
} from "@bendyline/gezel";
|
|
1042
|
+
var CHAT_MEMORY_OVERHEAD = {
|
|
1043
|
+
"llama-cpp": 1.2,
|
|
1044
|
+
mlx: 1.3
|
|
1045
|
+
};
|
|
1046
|
+
var NATIVE_TOOLKIT = [
|
|
1047
|
+
"uv",
|
|
1048
|
+
"sd-server",
|
|
1049
|
+
"whisper-server",
|
|
1050
|
+
"llama-server"
|
|
1051
|
+
];
|
|
1052
|
+
function recoDevice(memory) {
|
|
1053
|
+
return {
|
|
1054
|
+
platform: memory.platform,
|
|
1055
|
+
gpuVramBytes: memory.gpuVramBytes,
|
|
1056
|
+
...memory.gpuMemoryKind ? { gpuMemoryKind: memory.gpuMemoryKind } : {},
|
|
1057
|
+
totalRamBytes: memory.totalRamBytes,
|
|
1058
|
+
usableBytes: memory.usableBytes,
|
|
1059
|
+
...memory.budgetBytes !== void 0 ? { budgetBytes: memory.budgetBytes } : {}
|
|
1060
|
+
};
|
|
1057
1061
|
}
|
|
1058
|
-
function
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1062
|
+
function chatModelCandidates(items, device, provider) {
|
|
1063
|
+
const candidates = [];
|
|
1064
|
+
for (const item of items) {
|
|
1065
|
+
if (item.manifest.kind !== "chat-model") continue;
|
|
1066
|
+
const manifest = item.manifest;
|
|
1067
|
+
if (manifest.licenseClass !== "open") continue;
|
|
1068
|
+
const source = provider === "mlx" ? manifest.mlx : manifest.llamaCpp;
|
|
1069
|
+
if (!source || provider === "mlx" && manifest.mlx?.disabledReason) continue;
|
|
1070
|
+
const residentBytes = source.residentBytes ?? Math.round(source.approxSizeBytes * CHAT_MEMORY_OVERHEAD[provider]);
|
|
1071
|
+
const fit = computeModelFit({
|
|
1072
|
+
residentBytes,
|
|
1073
|
+
isMoE: isMoEFromTags(manifest.tags),
|
|
1074
|
+
usableBytes: device.usableBytes,
|
|
1075
|
+
totalRamBytes: device.totalRamBytes,
|
|
1076
|
+
gpuVramBytes: device.gpuVramBytes,
|
|
1077
|
+
...device.budgetBytes !== void 0 ? { admissibleBytes: device.budgetBytes } : {}
|
|
1078
|
+
}).tier;
|
|
1079
|
+
candidates.push({
|
|
1080
|
+
id: manifest.id,
|
|
1081
|
+
name: manifest.name,
|
|
1082
|
+
provider,
|
|
1083
|
+
approxSizeBytes: source.approxSizeBytes,
|
|
1084
|
+
residentBytes,
|
|
1085
|
+
// Effective score: a model the shared gate rejects (restricted license,
|
|
1086
|
+
// no tool support) sorts as unscored rather than leading the list.
|
|
1087
|
+
recoScore: isRecommendedModel(manifest) ? manifest.recoScore ?? 0 : 0,
|
|
1088
|
+
...source.quantization ? { quantization: source.quantization } : {},
|
|
1089
|
+
fit,
|
|
1090
|
+
attribution: formatModelAttribution(manifest)
|
|
1091
|
+
});
|
|
1070
1092
|
}
|
|
1093
|
+
return candidates;
|
|
1071
1094
|
}
|
|
1072
|
-
function
|
|
1073
|
-
const
|
|
1074
|
-
|
|
1075
|
-
|
|
1095
|
+
function rankChatModels(items, device, provider) {
|
|
1096
|
+
const candidates = chatModelCandidates(items, device, provider);
|
|
1097
|
+
if (candidates.length === 0) return [];
|
|
1098
|
+
const picked = pickRecommendedModel(
|
|
1099
|
+
candidates.map((model2) => ({
|
|
1100
|
+
id: model2.id,
|
|
1101
|
+
...model2.recoScore > 0 ? { recoScore: model2.recoScore } : {},
|
|
1102
|
+
licenseClass: "open",
|
|
1103
|
+
tags: items.find((item) => item.manifest.kind === "chat-model" && item.manifest.id === model2.id)?.manifest?.tags ?? [],
|
|
1104
|
+
residentBytes: model2.residentBytes,
|
|
1105
|
+
...model2.quantization ? { quantization: model2.quantization } : {}
|
|
1106
|
+
})),
|
|
1107
|
+
device
|
|
1076
1108
|
);
|
|
1077
|
-
const
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1109
|
+
const comfortable = candidates.filter(
|
|
1110
|
+
(model2) => model2.fit === "fits" || model2.fit === "fits-offload"
|
|
1111
|
+
);
|
|
1112
|
+
const pool = comfortable.length > 0 ? comfortable : candidates;
|
|
1113
|
+
const ranked = (model2) => model2.recoScore * lowBitScoreFactor(model2.quantization);
|
|
1114
|
+
pool.sort(
|
|
1115
|
+
(a, b) => ranked(b) - ranked(a) || fitOrder(a.fit) - fitOrder(b.fit) || b.residentBytes - a.residentBytes || a.name.localeCompare(b.name)
|
|
1116
|
+
);
|
|
1117
|
+
if (!picked) return pool;
|
|
1118
|
+
const bestIndex = pool.findIndex((model2) => model2.id === picked.id);
|
|
1119
|
+
if (bestIndex < 0) {
|
|
1120
|
+
const best = candidates.find((model2) => model2.id === picked.id);
|
|
1121
|
+
if (best) pool.unshift(best);
|
|
1122
|
+
return pool;
|
|
1084
1123
|
}
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
if (column && column.passRate === null && column.attributableTrials > 0) {
|
|
1089
|
-
lines.push(
|
|
1090
|
-
` A count, not a rate: ${minTrialsForRate} trials per task is the floor for quoting a percentage.`
|
|
1091
|
-
);
|
|
1124
|
+
if (bestIndex > 0) {
|
|
1125
|
+
const [best] = pool.splice(bestIndex, 1);
|
|
1126
|
+
if (best) pool.unshift(best);
|
|
1092
1127
|
}
|
|
1093
|
-
|
|
1094
|
-
lines.push(
|
|
1095
|
-
` Decode: ${decode ? `${decode.toFixed(1)} tok/s (median over finished tasks)` : "not recorded"}. Prefill: not recorded by the daemon's trial index.`
|
|
1096
|
-
);
|
|
1097
|
-
return [`${label} (${suite.id})`, ...lines];
|
|
1098
|
-
}
|
|
1099
|
-
function createJobFollower(suite, count) {
|
|
1100
|
-
let lastTrial = null;
|
|
1101
|
-
let lastCompleted = -1;
|
|
1102
|
-
let announcedWaiting = false;
|
|
1103
|
-
let announcedPreflight = false;
|
|
1104
|
-
return {
|
|
1105
|
-
update(job) {
|
|
1106
|
-
const lines = [];
|
|
1107
|
-
const target = job.targets[0];
|
|
1108
|
-
if (job.status === "waiting-for-device" && !announcedWaiting) {
|
|
1109
|
-
announcedWaiting = true;
|
|
1110
|
-
lines.push(
|
|
1111
|
-
`Waiting: another evaluation${job.waitingOn ? ` (${job.waitingOn})` : ""} holds this machine's engines.`
|
|
1112
|
-
);
|
|
1113
|
-
}
|
|
1114
|
-
if (!target) return lines;
|
|
1115
|
-
if (target.preflight && !announcedPreflight) {
|
|
1116
|
-
announcedPreflight = true;
|
|
1117
|
-
const p = target.preflight;
|
|
1118
|
-
lines.push(
|
|
1119
|
-
p.admitted === null ? `Fitness check skipped${p.skippedReason ? `: ${p.skippedReason}` : ""}.` : p.admitted ? `Fitness check passed${p.decodeTokensPerSec ? ` \xB7 ${p.decodeTokensPerSec.toFixed(1)} tok/s` : ""}.` : `Fitness check failed${p.skippedReason ? `: ${p.skippedReason}` : ""}.`
|
|
1120
|
-
);
|
|
1121
|
-
}
|
|
1122
|
-
const planned = target.plannedTrials ?? suite.scenarioIds.length * count;
|
|
1123
|
-
if (target.completedTrials !== lastCompleted && lastCompleted >= 0) {
|
|
1124
|
-
lines.push(` ${target.completedTrials}/${planned} done, ${target.passedTrials} passed`);
|
|
1125
|
-
}
|
|
1126
|
-
lastCompleted = target.completedTrials;
|
|
1127
|
-
const current = target.currentTrial;
|
|
1128
|
-
if (current && current.trialId !== lastTrial) {
|
|
1129
|
-
lastTrial = current.trialId;
|
|
1130
|
-
lines.push(` ${current.scenarioId} (${current.trialIndex}/${planned}) running\u2026`);
|
|
1131
|
-
}
|
|
1132
|
-
return lines;
|
|
1133
|
-
}
|
|
1134
|
-
};
|
|
1135
|
-
}
|
|
1136
|
-
function finishedJob(job) {
|
|
1137
|
-
return job.status === "completed" || job.status === "failed" || job.status === "cancelled" || job.status === "interrupted";
|
|
1128
|
+
return pool;
|
|
1138
1129
|
}
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
for (const line of follower.update(job)) io.note(line);
|
|
1143
|
-
await client.streamEvalJob(job.id, (event) => {
|
|
1144
|
-
if (event.type === "log") {
|
|
1145
|
-
if (verbose) io.note(` ${event.line}`);
|
|
1146
|
-
return;
|
|
1147
|
-
}
|
|
1148
|
-
latest = event.job;
|
|
1149
|
-
for (const line of follower.update(event.job)) io.note(line);
|
|
1150
|
-
});
|
|
1151
|
-
if (!finishedJob(latest)) latest = await client.getEvalJob(job.id);
|
|
1152
|
-
return latest;
|
|
1130
|
+
function formatDownloadSize(bytes) {
|
|
1131
|
+
if (bytes >= 1e9) return `${(bytes / 1e9).toFixed(1)} GB`;
|
|
1132
|
+
return `${Math.max(1, Math.round(bytes / 1e6))} MB`;
|
|
1153
1133
|
}
|
|
1154
|
-
function
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1134
|
+
function fitOrder(fit) {
|
|
1135
|
+
switch (fit) {
|
|
1136
|
+
case "fits":
|
|
1137
|
+
return 0;
|
|
1138
|
+
case "fits-offload":
|
|
1139
|
+
return 1;
|
|
1140
|
+
case "tight":
|
|
1141
|
+
return 2;
|
|
1142
|
+
case "too-big":
|
|
1143
|
+
return 3;
|
|
1160
1144
|
}
|
|
1161
1145
|
}
|
|
1162
|
-
|
|
1163
|
-
|
|
1146
|
+
|
|
1147
|
+
// src/native-command.ts
|
|
1148
|
+
var NATIVE_VARIANTS = ["cuda", "rocm", "vulkan", "metal", "cpu"];
|
|
1149
|
+
function parseNativeVariant(value) {
|
|
1150
|
+
if (value === void 0) return void 0;
|
|
1151
|
+
if (NATIVE_VARIANTS.includes(value)) return value;
|
|
1152
|
+
throw new CliError(`--variant must be one of ${NATIVE_VARIANTS.join(", ")} (got "${value}")`);
|
|
1164
1153
|
}
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
const plan = planBench(catalog, targetsResponse.targets, opts);
|
|
1172
|
-
const cpuModel = io.cpuModel ? io.cpuModel() : void 0;
|
|
1173
|
-
io.note(describeEvalHost(host, cpuModel));
|
|
1174
|
-
for (const line of describePlan(plan, catalog)) io.note(line);
|
|
1175
|
-
if (plan.worstCaseMs > BENCH_CONFIRM_ABOVE_MS && !opts.yes) {
|
|
1176
|
-
if (!io.confirm) {
|
|
1177
|
-
throw new CliError(
|
|
1178
|
-
`This run could take ${formatDuration(plan.worstCaseMs)}. Add --yes to start it without a prompt.`
|
|
1179
|
-
);
|
|
1180
|
-
}
|
|
1181
|
-
const go = await io.confirm(`Start a run of up to ${formatDuration(plan.worstCaseMs)}? [y/N] `);
|
|
1182
|
-
if (!go) throw new CliError("Not started.");
|
|
1183
|
-
}
|
|
1184
|
-
const startedAt = (io.now?.() ?? /* @__PURE__ */ new Date()).toISOString();
|
|
1185
|
-
const device = evalDeviceFromHost(host, cpuModel);
|
|
1186
|
-
const scenariosBySuite = {};
|
|
1187
|
-
const jobs = [];
|
|
1188
|
-
const results = [];
|
|
1189
|
-
const runId = evalRunId(startedAt, device, plan.target.modelId);
|
|
1190
|
-
const gildeVersion = (io.gildeVersion ?? defaultGildeVersion)();
|
|
1191
|
-
const run = evalRunProvenance({
|
|
1192
|
-
id: runId,
|
|
1193
|
-
startedAt,
|
|
1194
|
-
device,
|
|
1195
|
-
gezelVersion: host.version,
|
|
1196
|
-
gildeVersion,
|
|
1197
|
-
count: plan.count,
|
|
1198
|
-
engineBuilds: evalEngineBuilds(host),
|
|
1199
|
-
scenariosBySuite: Object.fromEntries(plan.suites.map((s) => [s.id, s.scenarioIds])),
|
|
1200
|
-
subset: false,
|
|
1201
|
-
...opts.note ? { note: opts.note } : {}
|
|
1202
|
-
});
|
|
1203
|
-
for (const suite of plan.suites) {
|
|
1204
|
-
const spec = {
|
|
1205
|
-
suiteId: suite.id,
|
|
1206
|
-
count: plan.count,
|
|
1207
|
-
...plan.count > 1 ? { countStrict: true } : {},
|
|
1208
|
-
targets: [{ provider: plan.target.provider, modelId: plan.target.modelId }],
|
|
1209
|
-
note: `gezel bench ${runId}`
|
|
1210
|
-
};
|
|
1211
|
-
io.note("");
|
|
1212
|
-
io.note(`Starting ${suite.id}\u2026`);
|
|
1213
|
-
const created = await client.createEvalJob(spec);
|
|
1214
|
-
const job = await followJob(client, created, suite, plan.count, io, opts.verbose);
|
|
1215
|
-
jobs.push(job);
|
|
1216
|
-
scenariosBySuite[suite.id] = suite.scenarioIds;
|
|
1217
|
-
const outcome = describeJobOutcome(job);
|
|
1218
|
-
if (outcome) io.note(outcome);
|
|
1219
|
-
const { trials } = await client.listEvalTrials({ jobId: job.id, limit: 2e3 });
|
|
1220
|
-
results.push(evalSuiteResult({ run, suite, target: plan.target, trials }));
|
|
1221
|
-
io.out("");
|
|
1222
|
-
for (const line of renderSuiteReport(suite, plan.target, trials, catalog.minTrialsForRate)) {
|
|
1223
|
-
io.out(line);
|
|
1224
|
-
}
|
|
1225
|
-
if (plan.smoke) io.out(` ${suite.description}`);
|
|
1226
|
-
}
|
|
1227
|
-
io.out("");
|
|
1228
|
-
io.out(describeEvalRun(run, [plan.target.provider]));
|
|
1229
|
-
io.out(
|
|
1230
|
-
`Results are kept in ${targetsResponse.environment.runsDir}; Settings \u2192 Benchmarks shows the same runs.`
|
|
1154
|
+
function parseNativeEngine(value) {
|
|
1155
|
+
if (value === void 0) return void 0;
|
|
1156
|
+
const parsed = NativeEngineNameSchema.safeParse(value);
|
|
1157
|
+
if (parsed.success) return parsed.data;
|
|
1158
|
+
throw new CliError(
|
|
1159
|
+
`--engine must be one of ${NativeEngineNameSchema.options.join(", ")} (got "${value}")`
|
|
1231
1160
|
);
|
|
1232
|
-
const shard = evalShard(run, results);
|
|
1233
|
-
if (opts.json) {
|
|
1234
|
-
await writeFile2(opts.json, `${JSON.stringify(shard, null, 2)}
|
|
1235
|
-
`, "utf8");
|
|
1236
|
-
io.out(`Wrote ${opts.json}. Share it with \`gezel bench submit ${opts.json}\`.`);
|
|
1237
|
-
}
|
|
1238
|
-
return { run, results, jobs, shard };
|
|
1239
1161
|
}
|
|
1240
|
-
function
|
|
1241
|
-
const
|
|
1242
|
-
|
|
1243
|
-
"Suites (`gezel bench --suite <id>`); times are the authored ceilings at one trial per task:"
|
|
1244
|
-
];
|
|
1245
|
-
const scorecard = catalog.suites.filter(
|
|
1246
|
-
(s) => SCORECARD_SUITE_IDS.includes(s.id)
|
|
1162
|
+
function formatNativeStatus(status) {
|
|
1163
|
+
const installed = new Set(
|
|
1164
|
+
status.engines.filter((engine) => engine.installed).map((engine) => engine.name)
|
|
1247
1165
|
);
|
|
1248
|
-
const
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1166
|
+
const toolkitInstalled = NATIVE_TOOLKIT.filter((engine) => installed.has(engine)).length;
|
|
1167
|
+
return [
|
|
1168
|
+
`release: native-v${status.release} (${status.pinned ? "verified pin" : "unpinned \u2014 downloads disabled"})`,
|
|
1169
|
+
`platform: ${status.platformKey ?? "unsupported \u2014 downloads unavailable"}`,
|
|
1170
|
+
`llama backend: ${status.llamaBackend ?? "automatic"}`,
|
|
1171
|
+
// Name the members: `native list` shows every engine the release carries
|
|
1172
|
+
// (ds4, duckdb too), so a bare "0/4" read as a miscount beside it.
|
|
1173
|
+
`toolkit: ${toolkitInstalled}/${NATIVE_TOOLKIT.length} installed (${NATIVE_TOOLKIT.join(", ")}; see \`gezel native list\` for every engine)`
|
|
1174
|
+
].join("\n");
|
|
1175
|
+
}
|
|
1176
|
+
function formatNativeList(status) {
|
|
1177
|
+
const rows = status.engines.map((engine) => {
|
|
1178
|
+
const state = engine.installed ? "installed" : "missing";
|
|
1179
|
+
return `${engine.name.padEnd(18)} ${state.padEnd(10)}${engine.path ? ` ${engine.path}` : ""}`;
|
|
1180
|
+
});
|
|
1181
|
+
return [
|
|
1182
|
+
`native-v${status.release} \xB7 ${status.platformKey ?? "unsupported platform"} \xB7 ${status.pinned ? "verified" : "unpinned"}`,
|
|
1183
|
+
"engine status path",
|
|
1184
|
+
...rows
|
|
1185
|
+
].join("\n");
|
|
1186
|
+
}
|
|
1187
|
+
async function installNativeToolkit(client, options) {
|
|
1188
|
+
const status = await client.getNativeEngineStatus();
|
|
1189
|
+
if (!status.pinned) {
|
|
1190
|
+
throw new CliError(
|
|
1191
|
+
"native downloads are disabled because the connected service has no verified native release pin."
|
|
1255
1192
|
);
|
|
1256
1193
|
}
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
);
|
|
1261
|
-
return lines;
|
|
1262
|
-
}
|
|
1263
|
-
async function renderResults(client, catalog, limit) {
|
|
1264
|
-
const { jobs } = await client.listEvalJobs();
|
|
1265
|
-
const finished = jobs.filter(finishedJob).slice(0, limit);
|
|
1266
|
-
if (finished.length === 0)
|
|
1267
|
-
return ["No finished runs on this machine yet. `gezel bench` starts one."];
|
|
1268
|
-
const lines = [];
|
|
1269
|
-
for (const job of finished) {
|
|
1270
|
-
const suite = job.spec.suiteId ? catalog.suites.find((s) => s.id === job.spec.suiteId) : void 0;
|
|
1271
|
-
const target = job.targets[0];
|
|
1272
|
-
const { trials } = await client.listEvalTrials({ jobId: job.id, limit: 2e3 });
|
|
1273
|
-
const order = suite?.scenarioIds ?? [...new Set(trials.map((t) => t.scenarioId))];
|
|
1274
|
-
const matrix = buildEvalMatrix(trials, { scenarioOrder: order });
|
|
1275
|
-
const column = target ? matrix.columns.find(
|
|
1276
|
-
(c) => c.modelId === target.modelId && (c.provider ?? target.provider) === target.provider
|
|
1277
|
-
) : matrix.columns[0];
|
|
1278
|
-
const subset = suite && job.spec.scenarioIds ? isSuiteSubset(suite, job.spec.scenarioIds) : false;
|
|
1279
|
-
const what = suite ? `${suite.id}${subset ? " (subset)" : ""}` : `${job.spec.scenarioIds?.length ?? 0} tasks`;
|
|
1280
|
-
lines.push(
|
|
1281
|
-
`${(job.finishedAt ?? job.createdAt).slice(0, 16).replace("T", " ")} ${(target?.modelId ?? "?").padEnd(24)} ${what.padEnd(22)} ${column?.claim ?? "not measured"}${job.status !== "completed" ? ` [${job.status}]` : ""}`
|
|
1194
|
+
if (!status.platformKey) {
|
|
1195
|
+
throw new CliError(
|
|
1196
|
+
`native downloads are unavailable on ${process.platform}/${process.arch}: no supported build exists.`
|
|
1282
1197
|
);
|
|
1283
|
-
if (subset) lines.push(` ${EVAL_SUBSET_NOTE}`);
|
|
1284
1198
|
}
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
const suites = run.suites.join(", ");
|
|
1291
|
-
const title = `Bench result: ${models.join(", ")} on ${run.provenance.device.label}`;
|
|
1292
|
-
const body = [
|
|
1293
|
-
"## Community bench result",
|
|
1294
|
-
"",
|
|
1295
|
-
`- Run: \`${run.id}\``,
|
|
1296
|
-
`- ${describeEvalRun(run, [...new Set(shard.results.map((r) => r.engine))])}`,
|
|
1297
|
-
`- Suites: ${suites}`,
|
|
1298
|
-
...run.note ? [`- Note: ${run.note}`] : [],
|
|
1299
|
-
"",
|
|
1300
|
-
"Results (count below three trials per task, rate otherwise):",
|
|
1301
|
-
...shard.results.map((r) => {
|
|
1302
|
-
const matrixLike = r.cells.reduce(
|
|
1303
|
-
(acc, c) => {
|
|
1304
|
-
acc.trials += Math.max(0, c.trials - c.nonModelFailures);
|
|
1305
|
-
acc.successes += c.successes;
|
|
1306
|
-
return acc;
|
|
1307
|
-
},
|
|
1308
|
-
{ trials: 0, successes: 0 }
|
|
1309
|
-
);
|
|
1310
|
-
return `- ${r.suiteId}: ${matrixLike.successes}/${matrixLike.trials}`;
|
|
1311
|
-
}),
|
|
1312
|
-
"",
|
|
1313
|
-
"The JSON shard produced by `gezel bench --json` is attached below. It is a `ScorecardDataset` and ingests with the same code as a published round; community rounds are shown separately from Bendyline rounds.",
|
|
1314
|
-
"",
|
|
1315
|
-
"<details><summary>shard.json</summary>",
|
|
1316
|
-
"",
|
|
1317
|
-
"```json",
|
|
1318
|
-
JSON.stringify(shard, null, 2),
|
|
1319
|
-
"```",
|
|
1320
|
-
"",
|
|
1321
|
-
"</details>"
|
|
1322
|
-
].join("\n");
|
|
1323
|
-
return { title, body };
|
|
1199
|
+
const llamaVariant = options.variant ?? status.llamaBackend;
|
|
1200
|
+
for (const engine of options.engines ?? NATIVE_TOOLKIT) {
|
|
1201
|
+
await ensureOneEngine(client, engine, llamaVariant, options.output);
|
|
1202
|
+
}
|
|
1203
|
+
return client.getNativeEngineStatus();
|
|
1324
1204
|
}
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1205
|
+
async function ensureOneEngine(client, engine, llamaVariant, output) {
|
|
1206
|
+
let terminalError;
|
|
1207
|
+
let progressLineOpen = false;
|
|
1208
|
+
const onEvent = (event) => {
|
|
1209
|
+
if (event.type === "progress") {
|
|
1210
|
+
progressLineOpen = true;
|
|
1211
|
+
output.writeProgress(`\r${formatProgress(engine, event.bytesWritten, event.totalBytes)}`);
|
|
1212
|
+
return;
|
|
1213
|
+
}
|
|
1214
|
+
if (progressLineOpen) {
|
|
1215
|
+
output.writeProgress("\n");
|
|
1216
|
+
progressLineOpen = false;
|
|
1217
|
+
}
|
|
1218
|
+
switch (event.type) {
|
|
1219
|
+
case "retrying":
|
|
1220
|
+
output.writeProgress(
|
|
1221
|
+
`${engine}: retry ${event.attempt}/${event.maxAttempts} in ${event.delayMs}ms \u2014 ${event.reason}
|
|
1222
|
+
`
|
|
1223
|
+
);
|
|
1224
|
+
break;
|
|
1225
|
+
case "verifying":
|
|
1226
|
+
output.writeProgress(`${engine}: verifying ${event.what}
|
|
1227
|
+
`);
|
|
1228
|
+
break;
|
|
1229
|
+
case "done":
|
|
1230
|
+
output.writeProgress(
|
|
1231
|
+
`${engine}: ready${event.cached ? " (cached)" : ` \u2014 ${event.binPath}`}
|
|
1232
|
+
`
|
|
1233
|
+
);
|
|
1234
|
+
break;
|
|
1235
|
+
case "error":
|
|
1236
|
+
terminalError = event.error;
|
|
1237
|
+
output.writeProgress(`${engine}: error \u2014 ${event.error}
|
|
1238
|
+
`);
|
|
1239
|
+
break;
|
|
1240
|
+
}
|
|
1241
|
+
};
|
|
1329
1242
|
try {
|
|
1330
|
-
|
|
1243
|
+
await client.ensureNativeEngine(
|
|
1244
|
+
engine,
|
|
1245
|
+
onEvent,
|
|
1246
|
+
engine === "llama-server" ? llamaVariant : void 0
|
|
1247
|
+
);
|
|
1331
1248
|
} catch (error) {
|
|
1249
|
+
if (progressLineOpen) output.writeProgress("\n");
|
|
1332
1250
|
throw new CliError(
|
|
1333
|
-
|
|
1251
|
+
`native install failed for ${engine}: ${terminalError ?? errorMessage(error)}`
|
|
1334
1252
|
);
|
|
1335
1253
|
}
|
|
1336
|
-
if (
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
const { title, body } = describeSubmission(shard);
|
|
1340
|
-
const params = new URLSearchParams({ title, labels: "bench-result" });
|
|
1341
|
-
const url = `https://github.com/${BENCH_SUBMIT_REPO}/issues/new?${params.toString()}`;
|
|
1342
|
-
const gh = deps.gh ?? (async (args) => (await execFileAsync("gh", args, { encoding: "utf8" })).stdout);
|
|
1343
|
-
try {
|
|
1344
|
-
const stdout = await gh([
|
|
1345
|
-
"issue",
|
|
1346
|
-
"create",
|
|
1347
|
-
"--repo",
|
|
1348
|
-
BENCH_SUBMIT_REPO,
|
|
1349
|
-
"--title",
|
|
1350
|
-
title,
|
|
1351
|
-
"--label",
|
|
1352
|
-
"bench-result",
|
|
1353
|
-
"--body",
|
|
1354
|
-
body
|
|
1355
|
-
]);
|
|
1356
|
-
const issueUrl = stdout.trim().split("\n").pop() ?? "";
|
|
1357
|
-
io.out(`Opened ${issueUrl}`);
|
|
1358
|
-
return { title, body, url, issueUrl };
|
|
1359
|
-
} catch {
|
|
1360
|
-
io.out("Open this page and attach the file to the issue:");
|
|
1361
|
-
io.out(` ${url}`);
|
|
1362
|
-
io.out(` file: ${file}`);
|
|
1363
|
-
io.note("(Install the GitHub CLI, `gh`, and sign in to have this done for you.)");
|
|
1364
|
-
return { title, body, url };
|
|
1365
|
-
}
|
|
1366
|
-
}
|
|
1367
|
-
function registerBenchCommands(program2, connect, io = {
|
|
1368
|
-
out: (line) => console.log(line),
|
|
1369
|
-
note: (line) => console.error(line),
|
|
1370
|
-
confirm: process.stdin.isTTY ? confirmOnTerminal : void 0,
|
|
1371
|
-
cpuModel: defaultCpuModel
|
|
1372
|
-
}) {
|
|
1373
|
-
const bench = program2.command("bench").description(
|
|
1374
|
-
"Run the model scorecard on this machine: the same tasks, checks and counting rules as the published scorecard"
|
|
1375
|
-
).option(
|
|
1376
|
-
"--suite <id>",
|
|
1377
|
-
"core (default), productivity, developer, complex-work, a comma list, or all"
|
|
1378
|
-
).option("--smoke", "Run the suite's pulse check instead (fast; not a suite score)").option("--model <id>", "Model to evaluate (default: the model chats use)").option("--provider <id>", "Provider for --model when a model is on more than one").option(
|
|
1379
|
-
"--count <n>",
|
|
1380
|
-
"Trials per task; 3 is what it takes to quote a rate (default 1)",
|
|
1381
|
-
(v) => Number.parseInt(v, 10)
|
|
1382
|
-
).option("--json <file>", "Also write the results as a scorecard shard").option("--note <text>", "A note kept with the run").option("--yes", "Start without asking, however long the run").option("--verbose", "Show the harness's own log lines while it runs").action(async (opts) => {
|
|
1383
|
-
const client = await connect();
|
|
1384
|
-
await runBench(client, opts, io);
|
|
1385
|
-
});
|
|
1386
|
-
bench.command("list").description("Show the suites, how many tasks each has, and how long it can take").action(async () => {
|
|
1387
|
-
const client = await connect();
|
|
1388
|
-
for (const line of renderSuiteList(await client.getEvalCatalog())) io.out(line);
|
|
1389
|
-
});
|
|
1390
|
-
bench.command("results").description("Show finished runs on this machine").option("--limit <n>", "How many runs to show (default 20)", (v) => Number.parseInt(v, 10)).action(async (opts) => {
|
|
1391
|
-
const client = await connect();
|
|
1392
|
-
const catalog = await client.getEvalCatalog();
|
|
1393
|
-
for (const line of await renderResults(client, catalog, opts.limit ?? 20)) io.out(line);
|
|
1394
|
-
});
|
|
1395
|
-
bench.command("submit <file>").description("Share a results shard from `gezel bench --json` as a GitHub issue").action(async (file) => {
|
|
1396
|
-
await submitBench(file, io);
|
|
1397
|
-
});
|
|
1398
|
-
}
|
|
1399
|
-
async function confirmOnTerminal(question) {
|
|
1400
|
-
const { createInterface } = await import("readline/promises");
|
|
1401
|
-
const rl = createInterface({ input: process.stdin, output: process.stderr });
|
|
1402
|
-
try {
|
|
1403
|
-
const answer = await rl.question(question);
|
|
1404
|
-
return /^y(es)?$/i.test(answer.trim());
|
|
1405
|
-
} finally {
|
|
1406
|
-
rl.close();
|
|
1254
|
+
if (progressLineOpen) output.writeProgress("\n");
|
|
1255
|
+
if (terminalError) {
|
|
1256
|
+
throw new CliError(`native install failed for ${engine}: ${terminalError}`);
|
|
1407
1257
|
}
|
|
1408
1258
|
}
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
if (err.name === "CliError") return err.message;
|
|
1414
|
-
if (err.name !== "GezelApiError" || env2.GEZEL_DEBUG === "1") return null;
|
|
1415
|
-
const { status, details } = err;
|
|
1416
|
-
const request = /\bon ([A-Z]+ \S+)/.exec(err.message)?.[1];
|
|
1417
|
-
const serviceText = detailText(details);
|
|
1418
|
-
if (status === 404) {
|
|
1419
|
-
const what = serviceText && serviceText.toLowerCase() !== "not found" ? serviceText : void 0;
|
|
1420
|
-
return `error: not found${what ? `: ${what}` : request ? ` (${request})` : ""}`;
|
|
1421
|
-
}
|
|
1422
|
-
const lines = [`error: ${serviceText ?? err.message}`];
|
|
1423
|
-
if (serviceText && request) lines.push(`(Gezel service answered ${status ?? "?"} to ${request})`);
|
|
1424
|
-
lines.push("Set GEZEL_DEBUG=1 to see the full error.");
|
|
1425
|
-
return lines.join("\n");
|
|
1259
|
+
function formatProgress(engine, written, total) {
|
|
1260
|
+
if (total <= 0) return `${engine}: ${formatBytes(written)} downloaded`;
|
|
1261
|
+
const pct = Math.min(100, Math.floor(written / total * 100));
|
|
1262
|
+
return `${engine}: ${String(pct).padStart(3)}% ${formatBytes(written)}/${formatBytes(total)}`;
|
|
1426
1263
|
}
|
|
1427
|
-
function
|
|
1428
|
-
|
|
1264
|
+
function formatBytes(bytes) {
|
|
1265
|
+
if (bytes < 1024) return `${bytes} B`;
|
|
1266
|
+
if (bytes < 1024 ** 2) return `${(bytes / 1024).toFixed(1)} KiB`;
|
|
1267
|
+
if (bytes < 1024 ** 3) return `${(bytes / 1024 ** 2).toFixed(1)} MiB`;
|
|
1268
|
+
return `${(bytes / 1024 ** 3).toFixed(2)} GiB`;
|
|
1429
1269
|
}
|
|
1430
|
-
function
|
|
1431
|
-
|
|
1432
|
-
if (details && typeof details === "object") {
|
|
1433
|
-
for (const key of ["message", "error"]) {
|
|
1434
|
-
const value = details[key];
|
|
1435
|
-
if (typeof value === "string" && value.trim()) return value.trim();
|
|
1436
|
-
}
|
|
1437
|
-
}
|
|
1438
|
-
return void 0;
|
|
1270
|
+
function errorMessage(error) {
|
|
1271
|
+
return error instanceof Error ? error.message : String(error);
|
|
1439
1272
|
}
|
|
1440
1273
|
|
|
1441
|
-
// src/
|
|
1442
|
-
import { setTimeout as delay } from "timers/promises";
|
|
1443
|
-
import {
|
|
1444
|
-
composeCraftbookLaunch,
|
|
1445
|
-
createAwakeTimeout,
|
|
1446
|
-
paramAsksUser
|
|
1447
|
-
} from "@bendyline/gezel";
|
|
1448
|
-
|
|
1449
|
-
// src/tui/craftbook-start.ts
|
|
1274
|
+
// src/settings-command.ts
|
|
1450
1275
|
import {
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
visibleCatalogItems
|
|
1276
|
+
classifySecurityLevel,
|
|
1277
|
+
resolveSecurityPolicy,
|
|
1278
|
+
securityCapabilitiesOf,
|
|
1279
|
+
securityPolicyForLevel
|
|
1456
1280
|
} from "@bendyline/gezel";
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
...manifest.description ? { description: manifest.description } : {},
|
|
1467
|
-
version: manifest.version,
|
|
1468
|
-
sourceId: item.sourceId,
|
|
1469
|
-
command: manifest.command ?? manifest.id,
|
|
1470
|
-
source: item.sourceId === "project" ? "project" : item.sourceId === "local" ? "local" : "bundled",
|
|
1471
|
-
stepCount: manifest.steps.length,
|
|
1472
|
-
tags: manifest.tags ?? [],
|
|
1473
|
-
role: manifest.role ?? "general",
|
|
1474
|
-
category: resolveCraftbookCategory(manifest)
|
|
1475
|
-
});
|
|
1476
|
-
}
|
|
1477
|
-
return [...byId.values()].sort((left, right) => left.name.localeCompare(right.name));
|
|
1478
|
-
}
|
|
1479
|
-
function findCraftbook(books, value) {
|
|
1480
|
-
const query = value.trim().toLowerCase();
|
|
1481
|
-
return books.find(
|
|
1482
|
-
(book) => book.id.toLowerCase() === query || book.command?.toLowerCase() === query || book.name.toLowerCase() === query
|
|
1483
|
-
);
|
|
1484
|
-
}
|
|
1485
|
-
function craftbookStartRequest(book) {
|
|
1486
|
-
const summary = book.description?.trim();
|
|
1487
|
-
const description = summary ? `Start the "${book.name}" craftbook for this project. ${summary}` : `Start the "${book.name}" craftbook for this project. Follow every step in the recipe and verify its required outcomes before completing the task.`;
|
|
1488
|
-
return {
|
|
1489
|
-
title: book.name,
|
|
1490
|
-
description,
|
|
1491
|
-
craftbookId: book.id,
|
|
1492
|
-
craftbookSourceId: book.sourceId,
|
|
1493
|
-
...book.version ? { craftbookVersion: book.version } : {},
|
|
1494
|
-
dispatchEntry: true,
|
|
1495
|
-
trustScripts: true
|
|
1496
|
-
};
|
|
1497
|
-
}
|
|
1498
|
-
|
|
1499
|
-
// src/craftbook-command.ts
|
|
1500
|
-
function resolveCraftbookInvocation(books, tokens) {
|
|
1501
|
-
for (let length = tokens.length; length > 0; length--) {
|
|
1502
|
-
const book = findCraftbook(books, tokens.slice(0, length).join(" "));
|
|
1503
|
-
if (book) return { book, args: tokens.slice(length) };
|
|
1504
|
-
}
|
|
1505
|
-
throw new CliError(`craftbook not found: ${tokens.join(" ")}`);
|
|
1281
|
+
var SECURITY_LEVEL_LABELS = {
|
|
1282
|
+
"super-lockdown": "Super Lockdown",
|
|
1283
|
+
lockdown: "Lockdown",
|
|
1284
|
+
free: "Unrestricted",
|
|
1285
|
+
custom: "Custom"
|
|
1286
|
+
};
|
|
1287
|
+
function policyWithCapability(config, key, enabled) {
|
|
1288
|
+
const caps = { ...securityCapabilitiesOf(resolveSecurityPolicy(config)), [key]: enabled };
|
|
1289
|
+
return { level: classifySecurityLevel(caps), ...caps };
|
|
1506
1290
|
}
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
if (Array.isArray(def.enum) && !def.enum.map(String).includes(value)) {
|
|
1510
|
-
throw new CliError(`${key} must be one of: ${def.enum.join(", ")}`);
|
|
1511
|
-
}
|
|
1512
|
-
if (def.type === "boolean" && !["true", "false"].includes(value))
|
|
1513
|
-
throw new CliError(`${key} must be true or false`);
|
|
1514
|
-
if (def.type === "number" || def.type === "integer") {
|
|
1515
|
-
const number = Number(value);
|
|
1516
|
-
if (!value.trim() || !Number.isFinite(number) || def.type === "integer" && !Number.isInteger(number))
|
|
1517
|
-
throw new CliError(`${key} must be a ${def.type}`);
|
|
1518
|
-
if (typeof def.minimum === "number" && number < def.minimum)
|
|
1519
|
-
throw new CliError(`${key} must be at least ${def.minimum}`);
|
|
1520
|
-
if (typeof def.maximum === "number" && number > def.maximum)
|
|
1521
|
-
throw new CliError(`${key} must be at most ${def.maximum}`);
|
|
1522
|
-
}
|
|
1523
|
-
if (typeof def.pattern === "string" && !new RegExp(def.pattern).test(value))
|
|
1524
|
-
throw new CliError(`${key} must match ${def.pattern}`);
|
|
1525
|
-
if (typeof def.minLength === "number" && value.trim().length < def.minLength)
|
|
1526
|
-
throw new CliError(`${key} is too short`);
|
|
1291
|
+
function policyWithExternalServices(config, enabled) {
|
|
1292
|
+
return policyWithCapability(config, "allowExternalServices", enabled);
|
|
1527
1293
|
}
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
set(match[1], match[2], token);
|
|
1547
|
-
continue;
|
|
1548
|
-
}
|
|
1549
|
-
const slot = slots.find((key) => !Object.hasOwn(params, key));
|
|
1550
|
-
if (slot) set(slot, token, token);
|
|
1551
|
-
else requestWords.push(token);
|
|
1552
|
-
}
|
|
1553
|
-
for (const token of named) {
|
|
1554
|
-
const match = NAMED_ARGUMENT.exec(token);
|
|
1555
|
-
if (!match) throw new CliError(`--param expects key=value: ${token}`);
|
|
1556
|
-
set(match[1], match[2], token);
|
|
1294
|
+
var SECURITY_SWITCHES = [
|
|
1295
|
+
{
|
|
1296
|
+
command: "external-services",
|
|
1297
|
+
key: "allowExternalServices",
|
|
1298
|
+
label: "External services",
|
|
1299
|
+
description: "Show or set on/off for model-initiated external services, including web search"
|
|
1300
|
+
},
|
|
1301
|
+
{
|
|
1302
|
+
command: "run-package-scripts",
|
|
1303
|
+
key: "allowPackageScriptExecution",
|
|
1304
|
+
label: "Run package scripts",
|
|
1305
|
+
description: "Show or set on/off for gezels running package scripts and installed package binaries (outside the sandbox, with network access)"
|
|
1306
|
+
},
|
|
1307
|
+
{
|
|
1308
|
+
command: "edit-package-scripts",
|
|
1309
|
+
key: "allowPackageScriptEdits",
|
|
1310
|
+
label: "Edit package scripts",
|
|
1311
|
+
description: "Show or set on/off for gezels adding or changing the scripts in package.json"
|
|
1557
1312
|
}
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1313
|
+
];
|
|
1314
|
+
var CAPABILITY_LABELS = [
|
|
1315
|
+
["allowExternalChat", "Cloud models"],
|
|
1316
|
+
["allowFileEdits", "File edits and the model\u2019s Git tools"],
|
|
1317
|
+
["allowScriptExecution", "Sandboxed scripts"],
|
|
1318
|
+
["allowAppNetwork", "Update checks and the connector syncs you start"],
|
|
1319
|
+
["allowExternalServices", "Web search, mail and other outside services"],
|
|
1320
|
+
["allowPackageScriptExecution", "Run package scripts"],
|
|
1321
|
+
["allowPackageScriptEdits", "Edit package scripts"]
|
|
1322
|
+
];
|
|
1323
|
+
var LEVEL_ALIASES = {
|
|
1324
|
+
"super-lockdown": "super-lockdown",
|
|
1325
|
+
superlockdown: "super-lockdown",
|
|
1326
|
+
lockdown: "lockdown",
|
|
1327
|
+
free: "free",
|
|
1328
|
+
unrestricted: "free"
|
|
1329
|
+
};
|
|
1330
|
+
function parseSecurityLevel(input) {
|
|
1331
|
+
const level = LEVEL_ALIASES[input.trim().toLowerCase()];
|
|
1332
|
+
if (!level) {
|
|
1333
|
+
throw new CliError(
|
|
1334
|
+
`Unknown security level "${input}". Choose super-lockdown, lockdown, or free (shown as Unrestricted in Settings).`
|
|
1335
|
+
);
|
|
1561
1336
|
}
|
|
1562
|
-
|
|
1563
|
-
return request ? { params, request } : { params };
|
|
1337
|
+
return level;
|
|
1564
1338
|
}
|
|
1565
|
-
function
|
|
1566
|
-
|
|
1567
|
-
const
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
}
|
|
1573
|
-
|
|
1339
|
+
function describeSecurityLevel(policy) {
|
|
1340
|
+
const caps = securityCapabilitiesOf(resolveSecurityPolicy({ securityPolicy: policy }));
|
|
1341
|
+
const level = classifySecurityLevel(caps);
|
|
1342
|
+
const width = Math.max(...CAPABILITY_LABELS.map(([, label]) => label.length));
|
|
1343
|
+
return [
|
|
1344
|
+
`Security level: ${SECURITY_LEVEL_LABELS[level]}${level === "custom" ? "" : ` (${level})`}`,
|
|
1345
|
+
...CAPABILITY_LABELS.map(
|
|
1346
|
+
([key, label]) => ` ${label.padEnd(width)} ${caps[key] ? "on" : "off"}`
|
|
1347
|
+
)
|
|
1348
|
+
];
|
|
1574
1349
|
}
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
if (
|
|
1579
|
-
|
|
1580
|
-
return apiError.status === 0 && apiError.details?.kind === "transport" && apiError.details.readRetryExhausted !== true && typeof apiError.details.cause === "string" && codes.test(apiError.details.cause);
|
|
1581
|
-
}
|
|
1582
|
-
return error instanceof TypeError && error.cause instanceof Error && codes.test(`${error.cause.code ?? ""} ${error.cause.message}`);
|
|
1350
|
+
function enabledValue(state) {
|
|
1351
|
+
if (state === void 0) return void 0;
|
|
1352
|
+
if (state === "on") return true;
|
|
1353
|
+
if (state === "off") return false;
|
|
1354
|
+
throw new CliError("Use on or off.");
|
|
1583
1355
|
}
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1356
|
+
function registerSecurityCommands(program2, connect) {
|
|
1357
|
+
const security = program2.command("security").description("Inspect or explicitly change security settings");
|
|
1358
|
+
security.command("level [action] [level]").description(
|
|
1359
|
+
"Show the security level, or set a preset: `level set super-lockdown|lockdown|free`"
|
|
1360
|
+
).option("--json", "Output the level and its capabilities as JSON").action(
|
|
1361
|
+
async (action, requested, options) => {
|
|
1362
|
+
if (action !== void 0 && action !== "show" && action !== "set") {
|
|
1363
|
+
throw new CliError("Use `gezel security level`, `level show`, or `level set <level>`.");
|
|
1364
|
+
}
|
|
1365
|
+
if (action === "set" && requested === void 0) {
|
|
1366
|
+
throw new CliError("Name a level: super-lockdown, lockdown, or free.");
|
|
1367
|
+
}
|
|
1368
|
+
if (action !== "set" && requested !== void 0) {
|
|
1369
|
+
throw new CliError("To change the level, use `gezel security level set <level>`.");
|
|
1370
|
+
}
|
|
1371
|
+
const next = action === "set" ? parseSecurityLevel(requested) : void 0;
|
|
1372
|
+
const client = await connect();
|
|
1373
|
+
const policy = next ? securityPolicyForLevel(next) : {
|
|
1374
|
+
level: "custom",
|
|
1375
|
+
...securityCapabilitiesOf(resolveSecurityPolicy(await client.getConfig()))
|
|
1376
|
+
};
|
|
1377
|
+
if (next) await client.updateConfig({ securityPolicy: policy });
|
|
1378
|
+
if (options.json) {
|
|
1379
|
+
const caps = securityCapabilitiesOf(resolveSecurityPolicy({ securityPolicy: policy }));
|
|
1380
|
+
console.log(JSON.stringify({ level: classifySecurityLevel(caps), ...caps }));
|
|
1381
|
+
return;
|
|
1382
|
+
}
|
|
1383
|
+
for (const line of describeSecurityLevel(policy)) console.log(line);
|
|
1595
1384
|
}
|
|
1385
|
+
);
|
|
1386
|
+
for (const setting of SECURITY_SWITCHES) {
|
|
1387
|
+
security.command(`${setting.command} [state]`).description(setting.description).option("--json", "Output the setting as JSON").action(async (state, options) => {
|
|
1388
|
+
const enabled = enabledValue(state);
|
|
1389
|
+
const client = await connect();
|
|
1390
|
+
const config = await client.getConfig();
|
|
1391
|
+
const policy = resolveSecurityPolicy(config);
|
|
1392
|
+
if (enabled !== void 0) {
|
|
1393
|
+
await client.updateConfig({
|
|
1394
|
+
securityPolicy: policyWithCapability(config, setting.key, enabled)
|
|
1395
|
+
});
|
|
1396
|
+
}
|
|
1397
|
+
const value = enabled ?? policy[setting.key];
|
|
1398
|
+
console.log(
|
|
1399
|
+
options.json ? JSON.stringify({ [setting.key]: value }) : `${setting.label}: ${value ? "on" : "off"}`
|
|
1400
|
+
);
|
|
1401
|
+
});
|
|
1596
1402
|
}
|
|
1597
1403
|
}
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1404
|
+
function registerGrowthCommand(program2, connect) {
|
|
1405
|
+
program2.command("growth [state]").description("Show or set on/off for gezel growth: XP and the level-ups it offers").option("--json", "Output the setting as JSON").action(async (state, options) => {
|
|
1406
|
+
const enabled = enabledValue(state);
|
|
1407
|
+
const client = await connect();
|
|
1408
|
+
const config = enabled === void 0 ? await client.getConfig() : await client.updateConfig({ growth: { enabled } });
|
|
1409
|
+
const value = config.growth?.enabled !== false;
|
|
1410
|
+
console.log(
|
|
1411
|
+
options.json ? JSON.stringify({ growthEnabled: value }) : `Growth (XP and level-ups): ${value ? "on" : "off"}`
|
|
1412
|
+
);
|
|
1413
|
+
});
|
|
1606
1414
|
}
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
lastTask = task2;
|
|
1618
|
-
const progress = `${task2.status}:${task2.activeStepId}`;
|
|
1619
|
-
if (progress !== previous) {
|
|
1620
|
-
options.onProgress?.(task2);
|
|
1621
|
-
previous = progress;
|
|
1622
|
-
}
|
|
1623
|
-
if (task2.status === "complete") return { task: task2, outcome: "complete", exitCode: 0 };
|
|
1624
|
-
if (task2.status === "canceled") return { task: task2, outcome: "canceled", exitCode: 1 };
|
|
1625
|
-
if (task2.status === "paused" || task2.status === "draft")
|
|
1626
|
-
return { task: task2, outcome: "blocked", exitCode: 2 };
|
|
1627
|
-
const refs = /* @__PURE__ */ new Set([task2.ref]);
|
|
1628
|
-
if (task2.fanout || task2.craftbook.spawn || task2.craftbook.cliWorkflow) {
|
|
1629
|
-
const { tasks } = await read(
|
|
1630
|
-
() => client.listTaskChildren(task2.projectId, task2.num, void 0, signal)
|
|
1631
|
-
);
|
|
1632
|
-
if (!task2.craftbook.cliWorkflow && !tasks.some((child) => child.status === "active") && tasks.some((child) => child.status === "paused" || child.status === "canceled"))
|
|
1633
|
-
return { task: task2, outcome: "blocked", exitCode: 2 };
|
|
1634
|
-
if (!task2.craftbook.cliWorkflow) for (const child of tasks) refs.add(child.ref);
|
|
1635
|
-
}
|
|
1636
|
-
if (client.listQuestions) {
|
|
1637
|
-
const { questions } = await read(
|
|
1638
|
-
() => client.listQuestions(
|
|
1639
|
-
{
|
|
1640
|
-
projectId: task2.projectId,
|
|
1641
|
-
pending: true,
|
|
1642
|
-
taskRefs: [...refs]
|
|
1643
|
-
},
|
|
1644
|
-
signal
|
|
1645
|
-
)
|
|
1646
|
-
);
|
|
1647
|
-
const questionIds = questions.filter(
|
|
1648
|
-
(q) => q.taskRef && refs.has(q.taskRef) && // A service pause card is a historical notification. Actual task
|
|
1649
|
-
// status above decides whether it is still paused; real session
|
|
1650
|
-
// questions and permissions must continue to block observation.
|
|
1651
|
-
!(q.sessionId === "" && q.intent?.kind === "task-paused")
|
|
1652
|
-
).map((q) => q.id);
|
|
1653
|
-
if (questionIds.length) return { task: task2, outcome: "blocked", exitCode: 2, questionIds };
|
|
1654
|
-
}
|
|
1655
|
-
if (budget.expired()) return { task: task2, outcome: "timeout", exitCode: 3 };
|
|
1656
|
-
await delay(Math.min(options.pollMs ?? 1e3, budget.remainingMs()), void 0, { signal });
|
|
1657
|
-
}
|
|
1658
|
-
} catch (error) {
|
|
1659
|
-
options.signal?.throwIfAborted();
|
|
1660
|
-
if (budget.expired() && lastTask) return { task: lastTask, outcome: "timeout", exitCode: 3 };
|
|
1661
|
-
throw error;
|
|
1662
|
-
} finally {
|
|
1663
|
-
timeout.dispose();
|
|
1664
|
-
}
|
|
1665
|
-
}
|
|
1666
|
-
function summarizeTaskNote(text, maxLength = 240) {
|
|
1667
|
-
const lines = text.split(/\r?\n/).map((line) => line.trim());
|
|
1668
|
-
const headingIndex = lines.findIndex((line) => line.startsWith("#"));
|
|
1669
|
-
const heading = headingIndex >= 0 ? lines[headingIndex].replace(/^#+\s*/, "") : "";
|
|
1670
|
-
const body = lines.slice(headingIndex + 1).find((line) => line.length > 0 && !line.startsWith("#"));
|
|
1671
|
-
const sentence = body?.match(/^.*?[.!?](?=\s|$)/)?.[0] ?? body ?? "";
|
|
1672
|
-
const summary = heading && sentence ? `${heading}: ${sentence}` : heading || sentence;
|
|
1673
|
-
return summary.length > maxLength ? `${summary.slice(0, maxLength - 1).trimEnd()}\u2026` : summary;
|
|
1674
|
-
}
|
|
1675
|
-
async function blockedTaskReason(client, result) {
|
|
1676
|
-
if (result.outcome !== "blocked") return void 0;
|
|
1677
|
-
if (result.questionIds?.length) {
|
|
1678
|
-
return "A gezel is waiting for your answer to a question. Answer it in the Gezel app or the interactive `gezel` terminal.";
|
|
1679
|
-
}
|
|
1680
|
-
if (result.task.status === "draft") return "The task is still a draft and has not started.";
|
|
1681
|
-
const { notes } = await client.listTaskNotes(result.task.projectId, result.task.num).catch(() => ({ notes: [] }));
|
|
1682
|
-
const latest = [...notes].sort((a, b) => a.at.localeCompare(b.at)).at(-1);
|
|
1683
|
-
return latest ? summarizeTaskNote(latest.text) || void 0 : void 0;
|
|
1684
|
-
}
|
|
1685
|
-
function filterCraftbooks(books, words) {
|
|
1686
|
-
const needles = words.map((word) => word.trim().toLowerCase()).filter(Boolean);
|
|
1687
|
-
if (needles.length === 0) return [...books];
|
|
1688
|
-
return books.filter((book) => {
|
|
1689
|
-
const haystack = [book.command, book.name, book.category, ...book.tags].join(" ").toLowerCase();
|
|
1690
|
-
return needles.every((needle) => haystack.includes(needle));
|
|
1415
|
+
function registerProjectSettingsCommands(env2, connect, projectFor) {
|
|
1416
|
+
env2.command("indexing [state]").description("Show or set on/off for background workspace indexing in the current project").option("--json", "Output the project setting as JSON").action(async (state, options) => {
|
|
1417
|
+
const enabled = enabledValue(state);
|
|
1418
|
+
const client = await connect();
|
|
1419
|
+
const projectId = await projectFor(client);
|
|
1420
|
+
const project = enabled === void 0 ? await client.getProject(projectId) : await client.updateProject(projectId, { indexingEnabled: enabled });
|
|
1421
|
+
const result = { projectId, indexingEnabled: project.indexingEnabled !== false };
|
|
1422
|
+
console.log(
|
|
1423
|
+
options.json ? JSON.stringify(result) : `${projectId}: workspace indexing ${result.indexingEnabled ? "on" : "off"}`
|
|
1424
|
+
);
|
|
1691
1425
|
});
|
|
1692
1426
|
}
|
|
1693
1427
|
|
|
1694
1428
|
// src/craftbook-setup.ts
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
}
|
|
1702
|
-
|
|
1429
|
+
var BRAVE_KEY_URL = "https://brave.com/search/api/";
|
|
1430
|
+
var SEARCH_CHECK = { query: "Gezel web search check", limit: 1 };
|
|
1431
|
+
var ENGINE_BINARY = {
|
|
1432
|
+
"llama-cpp": "llama-server",
|
|
1433
|
+
mlx: "uv",
|
|
1434
|
+
ds4: "ds4-server"
|
|
1435
|
+
};
|
|
1436
|
+
async function ensureCraftbookSetup(client, options) {
|
|
1437
|
+
const services = options.needs.services ?? [];
|
|
1438
|
+
const commands = options.needs.commands ?? [];
|
|
1439
|
+
if (!options.needs.models?.length && services.length === 0 && commands.length === 0) return;
|
|
1440
|
+
const config = await client.getConfig();
|
|
1441
|
+
const models = resolveCraftbookModelNeeds(options.needs.models, {
|
|
1442
|
+
...options.params ? { params: options.params } : {},
|
|
1443
|
+
...options.paramSchema ? { paramSchema: options.paramSchema } : {},
|
|
1444
|
+
defaultProvider: defaultModelEngine(config, options.platform, options.arch)
|
|
1445
|
+
});
|
|
1446
|
+
const gaps = craftbookSetupGaps(
|
|
1447
|
+
{ services, models, commands },
|
|
1448
|
+
await readSetupState(client, config, models)
|
|
1449
|
+
);
|
|
1450
|
+
if (gaps.length === 0) return;
|
|
1451
|
+
const sizes = await modelSizes(client, gaps);
|
|
1452
|
+
const lines = gaps.map((gap) => ` \u2022 ${describeGap(gap, sizes)}`);
|
|
1453
|
+
const onlyDownloads = gaps.every((gap) => gap.kind === "model");
|
|
1454
|
+
if (!options.prompter && !(options.autoDownload && onlyDownloads)) {
|
|
1455
|
+
throw new CliError(
|
|
1456
|
+
[
|
|
1457
|
+
`${options.label} is not set up to run yet:`,
|
|
1458
|
+
...lines,
|
|
1459
|
+
"",
|
|
1460
|
+
"Run the same command in a terminal to set it up step by step, or run:",
|
|
1461
|
+
...instructions(gaps).map((line) => ` ${line}`),
|
|
1462
|
+
...onlyDownloads ? ["", "Or run it again with --download to download them without asking."] : options.autoDownload ? ["", "--download only downloads; the other steps need a person."] : []
|
|
1463
|
+
].join("\n")
|
|
1464
|
+
);
|
|
1465
|
+
}
|
|
1466
|
+
options.write(`${options.label} needs some setup first:
|
|
1467
|
+
${lines.join("\n")}
|
|
1703
1468
|
|
|
1704
|
-
// src/model-pull.ts
|
|
1705
|
-
async function pullChatModel(client, engine, id, writeProgress) {
|
|
1706
|
-
let lastPct = -1;
|
|
1707
|
-
let pullError;
|
|
1708
|
-
const onEvent = (ev) => {
|
|
1709
|
-
if (ev.type === "progress") {
|
|
1710
|
-
const written = "bytesWrittenAll" in ev ? ev.bytesWrittenAll : ev.bytesWritten;
|
|
1711
|
-
const total = "totalBytesAll" in ev ? ev.totalBytesAll : ev.totalBytes ?? 0;
|
|
1712
|
-
const pct = total > 0 ? Math.floor(written / total * 100) : 0;
|
|
1713
|
-
if (pct === lastPct) return;
|
|
1714
|
-
lastPct = pct;
|
|
1715
|
-
writeProgress(
|
|
1716
|
-
`\rdownloading ${id} (${engine}): ${String(pct).padStart(3)}% ${formatGb(written)}/${formatGb(total)} GB`
|
|
1717
|
-
);
|
|
1718
|
-
} else if (ev.type === "retrying") {
|
|
1719
|
-
writeProgress(`
|
|
1720
|
-
retry ${ev.attempt}/${ev.maxAttempts}: ${ev.reason}
|
|
1721
|
-
`);
|
|
1722
|
-
} else if (ev.type === "error") {
|
|
1723
|
-
pullError = ev.error;
|
|
1724
|
-
writeProgress("\n");
|
|
1725
|
-
} else if (ev.type === "done" && !pullError) {
|
|
1726
|
-
writeProgress(`\rdownloaded ${id} (${engine})${" ".repeat(48)}
|
|
1727
1469
|
`);
|
|
1728
|
-
|
|
1470
|
+
const prompter = options.prompter ?? {
|
|
1471
|
+
confirm: async () => false,
|
|
1472
|
+
secret: async () => ""
|
|
1729
1473
|
};
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
}
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
const
|
|
1744
|
-
|
|
1745
|
-
|
|
1474
|
+
const stop = (from) => {
|
|
1475
|
+
const rest = gaps.slice(gaps.indexOf(from));
|
|
1476
|
+
throw new CliError(
|
|
1477
|
+
[
|
|
1478
|
+
`Not set up, so ${options.label} did not start. To finish later, run:`,
|
|
1479
|
+
...instructions(rest).map((line) => ` ${line}`)
|
|
1480
|
+
].join("\n")
|
|
1481
|
+
);
|
|
1482
|
+
};
|
|
1483
|
+
for (const gap of gaps) {
|
|
1484
|
+
if (gap.kind === "external-services") {
|
|
1485
|
+
const next = policyWithExternalServices(config, true);
|
|
1486
|
+
const from = SECURITY_LEVEL_LABELS[resolveSecurityPolicy2(config).level];
|
|
1487
|
+
const to = SECURITY_LEVEL_LABELS[next.level];
|
|
1488
|
+
const change = from === to ? "" : ` This changes your security level from ${from} to ${to}.`;
|
|
1489
|
+
const yes = await prompter.confirm(
|
|
1490
|
+
`Turn on External services, so gezels can search the web and read web pages?${change} [y/N] `
|
|
1746
1491
|
);
|
|
1747
|
-
|
|
1748
|
-
await
|
|
1749
|
-
|
|
1750
|
-
}
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1492
|
+
if (!yes) stop(gap);
|
|
1493
|
+
await client.updateConfig({ securityPolicy: next });
|
|
1494
|
+
options.write("External services: on\n");
|
|
1495
|
+
} else if (gap.kind === "package-scripts") {
|
|
1496
|
+
const current = await client.getConfig();
|
|
1497
|
+
const next = policyWithCapability(current, "allowPackageScriptExecution", true);
|
|
1498
|
+
const from = SECURITY_LEVEL_LABELS[resolveSecurityPolicy2(current).level];
|
|
1499
|
+
const to = SECURITY_LEVEL_LABELS[next.level];
|
|
1500
|
+
const change = from === to ? "" : ` This changes your security level from ${from} to ${to}.`;
|
|
1501
|
+
const yes = await prompter.confirm(
|
|
1502
|
+
`Let gezels run package scripts such as ${gap.commands[0]}? They run outside the sandbox, as your account, with network access, and Gezel asks before each new command.${change} [y/N] `
|
|
1755
1503
|
);
|
|
1756
|
-
|
|
1504
|
+
if (!yes) stop(gap);
|
|
1505
|
+
await client.updateConfig({ securityPolicy: next });
|
|
1506
|
+
options.write("Run package scripts: on\n");
|
|
1507
|
+
} else if (gap.kind === "web-search") {
|
|
1508
|
+
if (gap.keyMissing) {
|
|
1509
|
+
options.write(
|
|
1510
|
+
`Web search needs a Brave Search API key. Get one at ${BRAVE_KEY_URL} \u2014 Gezel keeps it in this computer's secret store.
|
|
1511
|
+
`
|
|
1512
|
+
);
|
|
1513
|
+
const key = (await prompter.secret(
|
|
1514
|
+
"Paste the key, then press Enter (Enter on its own stops here): ",
|
|
1515
|
+
(length) => `Key received (${length} characters, hidden). Press Enter to continue with this key. `
|
|
1516
|
+
)).trim();
|
|
1517
|
+
if (!key) stop(gap);
|
|
1518
|
+
options.write("Checking the key with one Brave search\u2026\n");
|
|
1519
|
+
await saveBraveKey(client, config, key, options.projectId);
|
|
1520
|
+
options.write("Web search: Brave (test search passed)\n");
|
|
1521
|
+
} else {
|
|
1522
|
+
const current = config.webSearch?.provider ?? "wikipedia";
|
|
1523
|
+
const yes = await prompter.confirm(
|
|
1524
|
+
`A Brave Search key is saved, but web search uses ${current === "wikipedia" ? "Wikipedia" : current}. Switch web search to Brave? [y/N] `
|
|
1525
|
+
);
|
|
1526
|
+
if (!yes) stop(gap);
|
|
1527
|
+
await client.updateConfig({ webSearch: { ...config.webSearch, provider: "brave" } });
|
|
1528
|
+
options.write("Web search: Brave\n");
|
|
1529
|
+
}
|
|
1757
1530
|
}
|
|
1758
|
-
if (shown) writeProgress(`\rMLX runtime ready${" ".repeat(56)}
|
|
1759
|
-
`);
|
|
1760
|
-
return status.phase === "ready" ? "ready" : "idle";
|
|
1761
1531
|
}
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
} from "@bendyline/gezel";
|
|
1771
|
-
|
|
1772
|
-
// src/tui/bootstrap.ts
|
|
1773
|
-
import {
|
|
1774
|
-
computeModelFit,
|
|
1775
|
-
formatModelAttribution,
|
|
1776
|
-
isMoEFromTags,
|
|
1777
|
-
isRecommendedModel,
|
|
1778
|
-
lowBitScoreFactor,
|
|
1779
|
-
pickRecommendedModel
|
|
1780
|
-
} from "@bendyline/gezel";
|
|
1781
|
-
var CHAT_MEMORY_OVERHEAD = {
|
|
1782
|
-
"llama-cpp": 1.2,
|
|
1783
|
-
mlx: 1.3
|
|
1784
|
-
};
|
|
1785
|
-
var NATIVE_TOOLKIT = [
|
|
1786
|
-
"uv",
|
|
1787
|
-
"sd-server",
|
|
1788
|
-
"whisper-server",
|
|
1789
|
-
"llama-server"
|
|
1790
|
-
];
|
|
1791
|
-
function recoDevice(memory) {
|
|
1792
|
-
return {
|
|
1793
|
-
platform: memory.platform,
|
|
1794
|
-
gpuVramBytes: memory.gpuVramBytes,
|
|
1795
|
-
...memory.gpuMemoryKind ? { gpuMemoryKind: memory.gpuMemoryKind } : {},
|
|
1796
|
-
totalRamBytes: memory.totalRamBytes,
|
|
1797
|
-
usableBytes: memory.usableBytes,
|
|
1798
|
-
...memory.budgetBytes !== void 0 ? { budgetBytes: memory.budgetBytes } : {}
|
|
1799
|
-
};
|
|
1800
|
-
}
|
|
1801
|
-
function chatModelCandidates(items, device, provider) {
|
|
1802
|
-
const candidates = [];
|
|
1803
|
-
for (const item of items) {
|
|
1804
|
-
if (item.manifest.kind !== "chat-model") continue;
|
|
1805
|
-
const manifest = item.manifest;
|
|
1806
|
-
if (manifest.licenseClass !== "open") continue;
|
|
1807
|
-
const source = provider === "mlx" ? manifest.mlx : manifest.llamaCpp;
|
|
1808
|
-
if (!source || provider === "mlx" && manifest.mlx?.disabledReason) continue;
|
|
1809
|
-
const residentBytes = source.residentBytes ?? Math.round(source.approxSizeBytes * CHAT_MEMORY_OVERHEAD[provider]);
|
|
1810
|
-
const fit = computeModelFit({
|
|
1811
|
-
residentBytes,
|
|
1812
|
-
isMoE: isMoEFromTags(manifest.tags),
|
|
1813
|
-
usableBytes: device.usableBytes,
|
|
1814
|
-
totalRamBytes: device.totalRamBytes,
|
|
1815
|
-
gpuVramBytes: device.gpuVramBytes,
|
|
1816
|
-
...device.budgetBytes !== void 0 ? { admissibleBytes: device.budgetBytes } : {}
|
|
1817
|
-
}).tier;
|
|
1818
|
-
candidates.push({
|
|
1819
|
-
id: manifest.id,
|
|
1820
|
-
name: manifest.name,
|
|
1821
|
-
provider,
|
|
1822
|
-
approxSizeBytes: source.approxSizeBytes,
|
|
1823
|
-
residentBytes,
|
|
1824
|
-
// Effective score: a model the shared gate rejects (restricted license,
|
|
1825
|
-
// no tool support) sorts as unscored rather than leading the list.
|
|
1826
|
-
recoScore: isRecommendedModel(manifest) ? manifest.recoScore ?? 0 : 0,
|
|
1827
|
-
...source.quantization ? { quantization: source.quantization } : {},
|
|
1828
|
-
fit,
|
|
1829
|
-
attribution: formatModelAttribution(manifest)
|
|
1830
|
-
});
|
|
1831
|
-
}
|
|
1832
|
-
return candidates;
|
|
1833
|
-
}
|
|
1834
|
-
function rankChatModels(items, device, provider) {
|
|
1835
|
-
const candidates = chatModelCandidates(items, device, provider);
|
|
1836
|
-
if (candidates.length === 0) return [];
|
|
1837
|
-
const picked = pickRecommendedModel(
|
|
1838
|
-
candidates.map((model2) => ({
|
|
1839
|
-
id: model2.id,
|
|
1840
|
-
...model2.recoScore > 0 ? { recoScore: model2.recoScore } : {},
|
|
1841
|
-
licenseClass: "open",
|
|
1842
|
-
tags: items.find((item) => item.manifest.kind === "chat-model" && item.manifest.id === model2.id)?.manifest?.tags ?? [],
|
|
1843
|
-
residentBytes: model2.residentBytes,
|
|
1844
|
-
...model2.quantization ? { quantization: model2.quantization } : {}
|
|
1845
|
-
})),
|
|
1846
|
-
device
|
|
1847
|
-
);
|
|
1848
|
-
const comfortable = candidates.filter(
|
|
1849
|
-
(model2) => model2.fit === "fits" || model2.fit === "fits-offload"
|
|
1850
|
-
);
|
|
1851
|
-
const pool = comfortable.length > 0 ? comfortable : candidates;
|
|
1852
|
-
const ranked = (model2) => model2.recoScore * lowBitScoreFactor(model2.quantization);
|
|
1853
|
-
pool.sort(
|
|
1854
|
-
(a, b) => ranked(b) - ranked(a) || fitOrder(a.fit) - fitOrder(b.fit) || b.residentBytes - a.residentBytes || a.name.localeCompare(b.name)
|
|
1855
|
-
);
|
|
1856
|
-
if (!picked) return pool;
|
|
1857
|
-
const bestIndex = pool.findIndex((model2) => model2.id === picked.id);
|
|
1858
|
-
if (bestIndex < 0) {
|
|
1859
|
-
const best = candidates.find((model2) => model2.id === picked.id);
|
|
1860
|
-
if (best) pool.unshift(best);
|
|
1861
|
-
return pool;
|
|
1862
|
-
}
|
|
1863
|
-
if (bestIndex > 0) {
|
|
1864
|
-
const [best] = pool.splice(bestIndex, 1);
|
|
1865
|
-
if (best) pool.unshift(best);
|
|
1866
|
-
}
|
|
1867
|
-
return pool;
|
|
1868
|
-
}
|
|
1869
|
-
function formatDownloadSize(bytes) {
|
|
1870
|
-
if (bytes >= 1e9) return `${(bytes / 1e9).toFixed(1)} GB`;
|
|
1871
|
-
return `${Math.max(1, Math.round(bytes / 1e6))} MB`;
|
|
1872
|
-
}
|
|
1873
|
-
function fitOrder(fit) {
|
|
1874
|
-
switch (fit) {
|
|
1875
|
-
case "fits":
|
|
1876
|
-
return 0;
|
|
1877
|
-
case "fits-offload":
|
|
1878
|
-
return 1;
|
|
1879
|
-
case "tight":
|
|
1880
|
-
return 2;
|
|
1881
|
-
case "too-big":
|
|
1882
|
-
return 3;
|
|
1883
|
-
}
|
|
1884
|
-
}
|
|
1885
|
-
|
|
1886
|
-
// src/native-command.ts
|
|
1887
|
-
var NATIVE_VARIANTS = ["cuda", "vulkan", "metal", "cpu"];
|
|
1888
|
-
function parseNativeVariant(value) {
|
|
1889
|
-
if (value === void 0) return void 0;
|
|
1890
|
-
if (NATIVE_VARIANTS.includes(value)) return value;
|
|
1891
|
-
throw new CliError(`--variant must be one of ${NATIVE_VARIANTS.join(", ")} (got "${value}")`);
|
|
1892
|
-
}
|
|
1893
|
-
function parseNativeEngine(value) {
|
|
1894
|
-
if (value === void 0) return void 0;
|
|
1895
|
-
const parsed = NativeEngineNameSchema.safeParse(value);
|
|
1896
|
-
if (parsed.success) return parsed.data;
|
|
1897
|
-
throw new CliError(
|
|
1898
|
-
`--engine must be one of ${NativeEngineNameSchema.options.join(", ")} (got "${value}")`
|
|
1899
|
-
);
|
|
1900
|
-
}
|
|
1901
|
-
function formatNativeStatus(status) {
|
|
1902
|
-
const installed = new Set(
|
|
1903
|
-
status.engines.filter((engine) => engine.installed).map((engine) => engine.name)
|
|
1904
|
-
);
|
|
1905
|
-
const toolkitInstalled = NATIVE_TOOLKIT.filter((engine) => installed.has(engine)).length;
|
|
1906
|
-
return [
|
|
1907
|
-
`release: native-v${status.release} (${status.pinned ? "verified pin" : "unpinned \u2014 downloads disabled"})`,
|
|
1908
|
-
`platform: ${status.platformKey ?? "unsupported \u2014 downloads unavailable"}`,
|
|
1909
|
-
`llama backend: ${status.llamaBackend ?? "automatic"}`,
|
|
1910
|
-
// Name the members: `native list` shows every engine the release carries
|
|
1911
|
-
// (ds4, duckdb too), so a bare "0/4" read as a miscount beside it.
|
|
1912
|
-
`toolkit: ${toolkitInstalled}/${NATIVE_TOOLKIT.length} installed (${NATIVE_TOOLKIT.join(", ")}; see \`gezel native list\` for every engine)`
|
|
1913
|
-
].join("\n");
|
|
1914
|
-
}
|
|
1915
|
-
function formatNativeList(status) {
|
|
1916
|
-
const rows = status.engines.map((engine) => {
|
|
1917
|
-
const state = engine.installed ? "installed" : "missing";
|
|
1918
|
-
return `${engine.name.padEnd(18)} ${state.padEnd(10)}${engine.path ? ` ${engine.path}` : ""}`;
|
|
1919
|
-
});
|
|
1920
|
-
return [
|
|
1921
|
-
`native-v${status.release} \xB7 ${status.platformKey ?? "unsupported platform"} \xB7 ${status.pinned ? "verified" : "unpinned"}`,
|
|
1922
|
-
"engine status path",
|
|
1923
|
-
...rows
|
|
1924
|
-
].join("\n");
|
|
1925
|
-
}
|
|
1926
|
-
async function installNativeToolkit(client, options) {
|
|
1927
|
-
const status = await client.getNativeEngineStatus();
|
|
1928
|
-
if (!status.pinned) {
|
|
1929
|
-
throw new CliError(
|
|
1930
|
-
"native downloads are disabled because the connected service has no verified native release pin."
|
|
1931
|
-
);
|
|
1932
|
-
}
|
|
1933
|
-
if (!status.platformKey) {
|
|
1934
|
-
throw new CliError(
|
|
1935
|
-
`native downloads are unavailable on ${process.platform}/${process.arch}: no supported build exists.`
|
|
1936
|
-
);
|
|
1937
|
-
}
|
|
1938
|
-
const llamaVariant = options.variant ?? status.llamaBackend;
|
|
1939
|
-
for (const engine of options.engines ?? NATIVE_TOOLKIT) {
|
|
1940
|
-
await ensureOneEngine(client, engine, llamaVariant, options.output);
|
|
1941
|
-
}
|
|
1942
|
-
return client.getNativeEngineStatus();
|
|
1943
|
-
}
|
|
1944
|
-
async function ensureOneEngine(client, engine, llamaVariant, output) {
|
|
1945
|
-
let terminalError;
|
|
1946
|
-
let progressLineOpen = false;
|
|
1947
|
-
const onEvent = (event) => {
|
|
1948
|
-
if (event.type === "progress") {
|
|
1949
|
-
progressLineOpen = true;
|
|
1950
|
-
output.writeProgress(`\r${formatProgress(engine, event.bytesWritten, event.totalBytes)}`);
|
|
1951
|
-
return;
|
|
1952
|
-
}
|
|
1953
|
-
if (progressLineOpen) {
|
|
1954
|
-
output.writeProgress("\n");
|
|
1955
|
-
progressLineOpen = false;
|
|
1956
|
-
}
|
|
1957
|
-
switch (event.type) {
|
|
1958
|
-
case "retrying":
|
|
1959
|
-
output.writeProgress(
|
|
1960
|
-
`${engine}: retry ${event.attempt}/${event.maxAttempts} in ${event.delayMs}ms \u2014 ${event.reason}
|
|
1961
|
-
`
|
|
1962
|
-
);
|
|
1963
|
-
break;
|
|
1964
|
-
case "verifying":
|
|
1965
|
-
output.writeProgress(`${engine}: verifying ${event.what}
|
|
1966
|
-
`);
|
|
1967
|
-
break;
|
|
1968
|
-
case "done":
|
|
1969
|
-
output.writeProgress(
|
|
1970
|
-
`${engine}: ready${event.cached ? " (cached)" : ` \u2014 ${event.binPath}`}
|
|
1971
|
-
`
|
|
1532
|
+
const downloads = gaps.filter((gap) => gap.kind === "model");
|
|
1533
|
+
if (downloads.length > 0) {
|
|
1534
|
+
const total = downloads.reduce((sum, gap) => sum + (sizes.get(modelKey(gap)) ?? 0), 0);
|
|
1535
|
+
if (total > 0) {
|
|
1536
|
+
const space = await client.checkModelDownloadSpace({ sizeBytes: total });
|
|
1537
|
+
if (space.known && !space.ok) {
|
|
1538
|
+
throw new CliError(
|
|
1539
|
+
`The models need ${formatGb(space.requiredBytes)} GB, and Gezel model storage has ${formatGb(space.freeBytes)} GB free. Free some space, then run again.`
|
|
1972
1540
|
);
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1541
|
+
}
|
|
1542
|
+
}
|
|
1543
|
+
const what = downloads.length === 1 ? "this model" : `these ${downloads.length} models`;
|
|
1544
|
+
const size = total > 0 ? ` of ${formatGb(total)} GB` : "";
|
|
1545
|
+
if (options.autoDownload) {
|
|
1546
|
+
options.write(`Downloading ${what} (--download), a one-time download${size}.
|
|
1977
1547
|
`);
|
|
1978
|
-
|
|
1548
|
+
} else {
|
|
1549
|
+
const yes = await prompter.confirm(
|
|
1550
|
+
`Download ${what} now? It is a one-time download${size}. [y/N] `
|
|
1551
|
+
);
|
|
1552
|
+
if (!yes) stop(downloads[0]);
|
|
1979
1553
|
}
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
onEvent,
|
|
1985
|
-
engine === "llama-server" ? llamaVariant : void 0
|
|
1554
|
+
await ensureModelEngines(
|
|
1555
|
+
client,
|
|
1556
|
+
downloads.map((gap) => gap.provider),
|
|
1557
|
+
options.write
|
|
1986
1558
|
);
|
|
1987
|
-
|
|
1988
|
-
|
|
1559
|
+
for (const gap of downloads) await pullChatModel(client, gap.provider, gap.id, options.write);
|
|
1560
|
+
}
|
|
1561
|
+
const after = craftbookSetupGaps(
|
|
1562
|
+
{ services, models, commands },
|
|
1563
|
+
await readSetupState(client, await client.getConfig(), models)
|
|
1564
|
+
);
|
|
1565
|
+
if (after.length > 0) {
|
|
1989
1566
|
throw new CliError(
|
|
1990
|
-
|
|
1567
|
+
[
|
|
1568
|
+
`${options.label} is still not set up:`,
|
|
1569
|
+
...after.map((gap) => ` \u2022 ${describeGap(gap, sizes)}`)
|
|
1570
|
+
].join("\n")
|
|
1991
1571
|
);
|
|
1992
1572
|
}
|
|
1993
|
-
|
|
1994
|
-
if (terminalError) {
|
|
1995
|
-
throw new CliError(`native install failed for ${engine}: ${terminalError}`);
|
|
1996
|
-
}
|
|
1573
|
+
options.write("\n");
|
|
1997
1574
|
}
|
|
1998
|
-
function
|
|
1999
|
-
if (
|
|
2000
|
-
|
|
2001
|
-
|
|
1575
|
+
function terminalPrompter(terminal = { input: process.stdin, output: process.stderr }) {
|
|
1576
|
+
if (!terminal.input.isTTY || !terminal.output.isTTY) return null;
|
|
1577
|
+
return {
|
|
1578
|
+
confirm: async (question) => {
|
|
1579
|
+
for (; ; ) {
|
|
1580
|
+
const answer = (await ask(terminal, question)).trim().toLowerCase();
|
|
1581
|
+
if (answer === "y" || answer === "yes") return true;
|
|
1582
|
+
if (answer === "" || answer === "n" || answer === "no") return false;
|
|
1583
|
+
terminal.output.write("Please answer y or n.\n");
|
|
1584
|
+
}
|
|
1585
|
+
},
|
|
1586
|
+
secret: (question, filled) => ask(terminal, question, { filled })
|
|
1587
|
+
};
|
|
2002
1588
|
}
|
|
2003
|
-
function
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
1589
|
+
function ask(terminal, question, secret) {
|
|
1590
|
+
const { input, output } = terminal;
|
|
1591
|
+
return import("readline").then(
|
|
1592
|
+
({ clearScreenDown, createInterface, cursorTo, moveCursor }) => new Promise((resolve7, reject) => {
|
|
1593
|
+
const rl = createInterface({
|
|
1594
|
+
input,
|
|
1595
|
+
output: secret ? new Writable({ write: (_chunk, _encoding, done) => done() }) : output,
|
|
1596
|
+
terminal: true,
|
|
1597
|
+
...secret ? { historySize: 0 } : {}
|
|
1598
|
+
});
|
|
1599
|
+
let settled = false;
|
|
1600
|
+
const settle = () => {
|
|
1601
|
+
settled = true;
|
|
1602
|
+
input.removeListener("keypress", onKeypress);
|
|
1603
|
+
};
|
|
1604
|
+
let drawn = "";
|
|
1605
|
+
let shownLength = -1;
|
|
1606
|
+
let pending = false;
|
|
1607
|
+
const draw = () => {
|
|
1608
|
+
pending = false;
|
|
1609
|
+
if (settled || !secret) return;
|
|
1610
|
+
const length = rl.line.length;
|
|
1611
|
+
if (length === shownLength) return;
|
|
1612
|
+
shownLength = length;
|
|
1613
|
+
const text = length > 0 && secret.filled ? secret.filled(length) : question;
|
|
1614
|
+
if (drawn) {
|
|
1615
|
+
const rows = Math.floor((drawn.length - 1) / (output.columns || 80));
|
|
1616
|
+
if (rows > 0) moveCursor(output, 0, -rows);
|
|
1617
|
+
cursorTo(output, 0);
|
|
1618
|
+
clearScreenDown(output);
|
|
1619
|
+
}
|
|
1620
|
+
output.write(text);
|
|
1621
|
+
drawn = text;
|
|
1622
|
+
};
|
|
1623
|
+
function onKeypress() {
|
|
1624
|
+
if (pending) return;
|
|
1625
|
+
pending = true;
|
|
1626
|
+
queueMicrotask(draw);
|
|
1627
|
+
}
|
|
1628
|
+
if (secret) input.on("keypress", onKeypress);
|
|
1629
|
+
rl.on("SIGINT", () => {
|
|
1630
|
+
settle();
|
|
1631
|
+
rl.close();
|
|
1632
|
+
output.write("\n");
|
|
1633
|
+
reject(new CliError("Cancelled."));
|
|
1634
|
+
});
|
|
1635
|
+
rl.on("close", () => {
|
|
1636
|
+
if (settled) return;
|
|
1637
|
+
settle();
|
|
1638
|
+
output.write("\n");
|
|
1639
|
+
resolve7("");
|
|
1640
|
+
});
|
|
1641
|
+
rl.question(question, (answer) => {
|
|
1642
|
+
settle();
|
|
1643
|
+
rl.close();
|
|
1644
|
+
if (secret) output.write("\n");
|
|
1645
|
+
resolve7(answer);
|
|
1646
|
+
});
|
|
1647
|
+
draw();
|
|
1648
|
+
})
|
|
1649
|
+
);
|
|
2008
1650
|
}
|
|
2009
|
-
function
|
|
1651
|
+
function defaultModelEngine(config, platform = process.platform, arch = process.arch) {
|
|
1652
|
+
const configured = config.provider;
|
|
1653
|
+
return configured && isCraftbookModelEngine(configured) ? configured : resolveOnDeviceProvider(platform, arch);
|
|
1654
|
+
}
|
|
1655
|
+
async function readSetupState(client, config, models) {
|
|
1656
|
+
const engines = [...new Set(models.map((model2) => model2.provider))];
|
|
1657
|
+
const installed = await Promise.all(
|
|
1658
|
+
engines.map(async (engine) => {
|
|
1659
|
+
const { models: list } = engine === "mlx" ? await client.listMlxModels() : engine === "ds4" ? await client.listDs4Models() : await client.listLlamaCppModels();
|
|
1660
|
+
return [engine, list.map((model2) => model2.id)];
|
|
1661
|
+
})
|
|
1662
|
+
);
|
|
1663
|
+
return {
|
|
1664
|
+
allowExternalServices: resolveSecurityPolicy2(config).allowExternalServices,
|
|
1665
|
+
allowPackageScriptExecution: resolveSecurityPolicy2(config).allowPackageScriptExecution,
|
|
1666
|
+
...config.webSearch?.provider ? { webSearchProvider: config.webSearch.provider } : {},
|
|
1667
|
+
hasBraveSearchApiKey: config.hasBraveSearchApiKey === true,
|
|
1668
|
+
installedModels: Object.fromEntries(installed)
|
|
1669
|
+
};
|
|
1670
|
+
}
|
|
1671
|
+
var modelKey = (model2) => `${model2.provider}:${model2.id}`;
|
|
1672
|
+
async function modelSizes(client, gaps) {
|
|
1673
|
+
const sizes = /* @__PURE__ */ new Map();
|
|
1674
|
+
for (const gap of gaps) {
|
|
1675
|
+
if (gap.kind !== "model") continue;
|
|
1676
|
+
let manifest;
|
|
1677
|
+
try {
|
|
1678
|
+
const item = await client.getCatalogItem("chat-model", gap.id);
|
|
1679
|
+
if (item.manifest.kind !== "chat-model") throw new Error("not a chat model");
|
|
1680
|
+
manifest = item.manifest;
|
|
1681
|
+
} catch {
|
|
1682
|
+
throw new CliError(
|
|
1683
|
+
`This run needs the model ${gap.id}, which is not installed and is not in the model catalog. Check the model name.`
|
|
1684
|
+
);
|
|
1685
|
+
}
|
|
1686
|
+
const source = gap.provider === "mlx" ? manifest.mlx : gap.provider === "ds4" ? manifest.ds4 : manifest.llamaCpp;
|
|
1687
|
+
if (!source) {
|
|
1688
|
+
throw new CliError(
|
|
1689
|
+
`This run needs the model ${gap.id} on ${gap.provider}, and the catalog has no ${gap.provider} download for it.`
|
|
1690
|
+
);
|
|
1691
|
+
}
|
|
1692
|
+
sizes.set(modelKey(gap), source.approxSizeBytes);
|
|
1693
|
+
}
|
|
1694
|
+
return sizes;
|
|
1695
|
+
}
|
|
1696
|
+
function describeGap(gap, sizes) {
|
|
1697
|
+
const why = gap.reasons.length > 0 ? ` (${gap.reasons.join("; ")})` : "";
|
|
1698
|
+
switch (gap.kind) {
|
|
1699
|
+
case "external-services":
|
|
1700
|
+
return `External services are off, so gezels cannot reach the web${why}.`;
|
|
1701
|
+
case "package-scripts":
|
|
1702
|
+
return `Run package scripts is off, so gezels cannot run ${gap.commands.join(", ")}${why}.`;
|
|
1703
|
+
case "web-search":
|
|
1704
|
+
return gap.keyMissing ? `Web search needs a Brave Search API key, and none is set${why}.` : `Web search is not set to Brave Search${why}.`;
|
|
1705
|
+
case "model": {
|
|
1706
|
+
const size = sizes.get(modelKey(gap));
|
|
1707
|
+
return `${gap.id} is not downloaded yet (${gap.provider}${size ? `, ${formatGb(size)} GB` : ""})${why}.`;
|
|
1708
|
+
}
|
|
1709
|
+
}
|
|
1710
|
+
}
|
|
1711
|
+
function instructions(gaps) {
|
|
1712
|
+
const out = [];
|
|
1713
|
+
for (const gap of gaps) {
|
|
1714
|
+
if (gap.kind === "external-services") out.push("gezel security external-services on");
|
|
1715
|
+
else if (gap.kind === "package-scripts") out.push("gezel security run-package-scripts on");
|
|
1716
|
+
else if (gap.kind === "web-search")
|
|
1717
|
+
out.push(
|
|
1718
|
+
`gezel secret set braveSearchApiKey --env BRAVE_SEARCH_API_KEY --use-for-search (get a key at ${BRAVE_KEY_URL})`
|
|
1719
|
+
);
|
|
1720
|
+
else out.push(`gezel model pull ${gap.id} --provider ${gap.provider}`);
|
|
1721
|
+
}
|
|
1722
|
+
return out;
|
|
1723
|
+
}
|
|
1724
|
+
async function saveBraveKey(client, config, key, projectId) {
|
|
1725
|
+
try {
|
|
1726
|
+
await client.updateConfig({
|
|
1727
|
+
braveSearchApiKey: key,
|
|
1728
|
+
webSearch: { ...config.webSearch, provider: "brave" }
|
|
1729
|
+
});
|
|
1730
|
+
} catch {
|
|
1731
|
+
throw new CliError("Could not save the key. Check the Gezel service, then run again.");
|
|
1732
|
+
}
|
|
1733
|
+
const failure = await client.toolWebSearch(projectId, SEARCH_CHECK).then(
|
|
1734
|
+
() => void 0,
|
|
1735
|
+
(error) => serviceErrorText(error)
|
|
1736
|
+
);
|
|
1737
|
+
if (failure === void 0) return;
|
|
1738
|
+
await client.updateConfig({ braveSearchApiKey: "", webSearch: { ...config.webSearch } }).catch(() => {
|
|
1739
|
+
});
|
|
1740
|
+
throw new CliError(
|
|
1741
|
+
`A test search with that key failed, so it was not kept: ${failure.replaceAll(key, "\u2026")}
|
|
1742
|
+
Check the key at ${BRAVE_KEY_URL}, then run again.`
|
|
1743
|
+
);
|
|
1744
|
+
}
|
|
1745
|
+
function serviceErrorText(error) {
|
|
1746
|
+
const details = error?.details;
|
|
1747
|
+
if (details && typeof details === "object") {
|
|
1748
|
+
const text = details.error;
|
|
1749
|
+
if (typeof text === "string" && text.trim()) return text.trim();
|
|
1750
|
+
}
|
|
2010
1751
|
return error instanceof Error ? error.message : String(error);
|
|
2011
1752
|
}
|
|
1753
|
+
async function ensureModelEngines(client, engines, write) {
|
|
1754
|
+
const status = await client.getNativeEngineStatus();
|
|
1755
|
+
if (!status.pinned || !status.platformKey) return;
|
|
1756
|
+
const missing = [...new Set(engines.map((engine) => ENGINE_BINARY[engine]))].filter(
|
|
1757
|
+
(binary) => !status.engines.some((entry) => entry.name === binary && entry.installed)
|
|
1758
|
+
);
|
|
1759
|
+
if (missing.length === 0) return;
|
|
1760
|
+
await installNativeToolkit(client, { engines: missing, output: { writeProgress: write } });
|
|
1761
|
+
}
|
|
2012
1762
|
|
|
2013
|
-
// src/
|
|
1763
|
+
// src/run-readiness.ts
|
|
1764
|
+
import { resolveOnDeviceProvider as resolveOnDeviceProvider2 } from "@bendyline/gezel/native";
|
|
1765
|
+
|
|
1766
|
+
// src/model-choice.ts
|
|
2014
1767
|
import {
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
securityCapabilitiesOf,
|
|
2018
|
-
securityPolicyForLevel
|
|
1768
|
+
measuredClaim,
|
|
1769
|
+
measuredResultFor
|
|
2019
1770
|
} from "@bendyline/gezel";
|
|
2020
|
-
|
|
2021
|
-
"
|
|
2022
|
-
|
|
2023
|
-
free: "Unrestricted",
|
|
2024
|
-
custom: "Custom"
|
|
2025
|
-
};
|
|
2026
|
-
function policyWithCapability(config, key, enabled) {
|
|
2027
|
-
const caps = { ...securityCapabilitiesOf(resolveSecurityPolicy(config)), [key]: enabled };
|
|
2028
|
-
return { level: classifySecurityLevel(caps), ...caps };
|
|
1771
|
+
async function installedChatModels(client, engine) {
|
|
1772
|
+
const { models } = engine === "mlx" ? await client.listMlxModels() : engine === "ds4" ? await client.listDs4Models() : await client.listLlamaCppModels();
|
|
1773
|
+
return models.map((model2) => model2.id);
|
|
2029
1774
|
}
|
|
2030
|
-
function
|
|
2031
|
-
return
|
|
1775
|
+
function providerFlag(engine, platformDefault) {
|
|
1776
|
+
return engine === platformDefault ? "" : ` --provider ${engine}`;
|
|
2032
1777
|
}
|
|
2033
|
-
var
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
description: "Show or set on/off for model-initiated external services, including web search"
|
|
2039
|
-
},
|
|
2040
|
-
{
|
|
2041
|
-
command: "run-package-scripts",
|
|
2042
|
-
key: "allowPackageScriptExecution",
|
|
2043
|
-
label: "Run package scripts",
|
|
2044
|
-
description: "Show or set on/off for gezels running package scripts and installed package binaries (outside the sandbox, with network access)"
|
|
2045
|
-
},
|
|
2046
|
-
{
|
|
2047
|
-
command: "edit-package-scripts",
|
|
2048
|
-
key: "allowPackageScriptEdits",
|
|
2049
|
-
label: "Edit package scripts",
|
|
2050
|
-
description: "Show or set on/off for gezels adding or changing the scripts in package.json"
|
|
2051
|
-
}
|
|
2052
|
-
];
|
|
2053
|
-
var CAPABILITY_LABELS = [
|
|
2054
|
-
["allowExternalChat", "Cloud models"],
|
|
2055
|
-
["allowFileEdits", "File edits and the model\u2019s Git tools"],
|
|
2056
|
-
["allowScriptExecution", "Sandboxed scripts"],
|
|
2057
|
-
["allowAppNetwork", "Update checks and the connector syncs you start"],
|
|
2058
|
-
["allowExternalServices", "Web search, mail and other outside services"],
|
|
2059
|
-
["allowPackageScriptExecution", "Run package scripts"],
|
|
2060
|
-
["allowPackageScriptEdits", "Edit package scripts"]
|
|
2061
|
-
];
|
|
2062
|
-
var LEVEL_ALIASES = {
|
|
2063
|
-
"super-lockdown": "super-lockdown",
|
|
2064
|
-
superlockdown: "super-lockdown",
|
|
2065
|
-
lockdown: "lockdown",
|
|
2066
|
-
free: "free",
|
|
2067
|
-
unrestricted: "free"
|
|
1778
|
+
var FIT_LABEL = {
|
|
1779
|
+
fits: "fits",
|
|
1780
|
+
"fits-offload": "fits, partly offloaded",
|
|
1781
|
+
tight: "tight",
|
|
1782
|
+
"too-big": "too big"
|
|
2068
1783
|
};
|
|
2069
|
-
function
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
throw new CliError(
|
|
2073
|
-
`Unknown security level "${input}". Choose super-lockdown, lockdown, or free (shown as Unrestricted in Settings).`
|
|
2074
|
-
);
|
|
1784
|
+
async function availableChatModels(client, engine) {
|
|
1785
|
+
if (engine === "ds4") {
|
|
1786
|
+
throw new CliError("`--available` lists mlx and llama-cpp models; pass one with --provider.");
|
|
2075
1787
|
}
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
1788
|
+
const [memory, catalog, installed, config] = await Promise.all([
|
|
1789
|
+
client.getMemoryProfile(),
|
|
1790
|
+
client.listCatalogItems("chat-model"),
|
|
1791
|
+
installedChatModels(client, engine),
|
|
1792
|
+
client.getConfig()
|
|
1793
|
+
]);
|
|
1794
|
+
const pinned = config.defaultModel?.[engine];
|
|
1795
|
+
return {
|
|
1796
|
+
ranked: rankChatModels(catalog.items, recoDevice(memory), engine),
|
|
1797
|
+
installed: new Set(installed),
|
|
1798
|
+
...pinned ? { pinned } : {}
|
|
1799
|
+
};
|
|
1800
|
+
}
|
|
1801
|
+
function formatAvailableChatModels(input) {
|
|
1802
|
+
const { engine, ranked, installed, pinned, measured } = input;
|
|
1803
|
+
if (ranked.length === 0) return `(no ${engine} chat models in the catalog for this machine)`;
|
|
1804
|
+
const width = Math.max(...ranked.map((model2) => model2.id.length));
|
|
1805
|
+
const measuredWidth = measured ? Math.max("not measured".length, ...ranked.map((m) => (measured.get(m.id) ?? "").length)) : 0;
|
|
1806
|
+
const lines = [
|
|
1807
|
+
`${engine} chat models for this machine, the recommended first download on top, then by catalog ranking:`
|
|
2087
1808
|
];
|
|
1809
|
+
ranked.forEach((model2, index) => {
|
|
1810
|
+
const notes = [
|
|
1811
|
+
index === 0 ? "recommended" : "",
|
|
1812
|
+
installed.has(model2.id) ? "installed" : "",
|
|
1813
|
+
model2.id === pinned ? "in use" : ""
|
|
1814
|
+
].filter(Boolean);
|
|
1815
|
+
const claim = measured ? ` ${(measured.get(model2.id) ?? "not measured").padEnd(measuredWidth)}` : "";
|
|
1816
|
+
lines.push(
|
|
1817
|
+
` ${model2.id.padEnd(width)} ${formatDownloadSize(model2.approxSizeBytes).padStart(7)} ${FIT_LABEL[model2.fit].padEnd(22)}${claim} ${model2.name}${notes.length > 0 ? ` (${notes.join(", ")})` : ""}`
|
|
1818
|
+
);
|
|
1819
|
+
});
|
|
1820
|
+
const flag = providerFlag(engine, input.platformDefault);
|
|
1821
|
+
const smallest = [...ranked].sort((a, b) => a.approxSizeBytes - b.approxSizeBytes)[0];
|
|
1822
|
+
lines.push(
|
|
1823
|
+
"",
|
|
1824
|
+
`Download one with \`gezel model pull <id>${flag}\`. The smallest is ${smallest.id} (${formatDownloadSize(smallest.approxSizeBytes)}).`,
|
|
1825
|
+
`\`gezel model use <id>${flag}\` picks the installed model chats use.`
|
|
1826
|
+
);
|
|
1827
|
+
if (measured) {
|
|
1828
|
+
lines.push(
|
|
1829
|
+
'"Fits" is about memory. The measured column is the published scorecard\'s pass rate on real jobs, on the hardware named; `gezel bench` measures a model on this machine.'
|
|
1830
|
+
);
|
|
1831
|
+
}
|
|
1832
|
+
return lines.join("\n");
|
|
2088
1833
|
}
|
|
2089
|
-
function
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
throw new CliError("Use on or off.");
|
|
1834
|
+
function measuredClaimsFor(dataset, ids, device) {
|
|
1835
|
+
const claims = /* @__PURE__ */ new Map();
|
|
1836
|
+
for (const id of ids) claims.set(id, measuredClaim(measuredResultFor(dataset, id, device)));
|
|
1837
|
+
return claims;
|
|
2094
1838
|
}
|
|
2095
|
-
function
|
|
2096
|
-
const
|
|
2097
|
-
|
|
2098
|
-
"Show the security level, or set a preset: `level set super-lockdown|lockdown|free`"
|
|
2099
|
-
).option("--json", "Output the level and its capabilities as JSON").action(
|
|
2100
|
-
async (action, requested, options) => {
|
|
2101
|
-
if (action !== void 0 && action !== "show" && action !== "set") {
|
|
2102
|
-
throw new CliError("Use `gezel security level`, `level show`, or `level set <level>`.");
|
|
2103
|
-
}
|
|
2104
|
-
if (action === "set" && requested === void 0) {
|
|
2105
|
-
throw new CliError("Name a level: super-lockdown, lockdown, or free.");
|
|
2106
|
-
}
|
|
2107
|
-
if (action !== "set" && requested !== void 0) {
|
|
2108
|
-
throw new CliError("To change the level, use `gezel security level set <level>`.");
|
|
2109
|
-
}
|
|
2110
|
-
const next = action === "set" ? parseSecurityLevel(requested) : void 0;
|
|
2111
|
-
const client = await connect();
|
|
2112
|
-
const policy = next ? securityPolicyForLevel(next) : {
|
|
2113
|
-
level: "custom",
|
|
2114
|
-
...securityCapabilitiesOf(resolveSecurityPolicy(await client.getConfig()))
|
|
2115
|
-
};
|
|
2116
|
-
if (next) await client.updateConfig({ securityPolicy: policy });
|
|
2117
|
-
if (options.json) {
|
|
2118
|
-
const caps = securityCapabilitiesOf(resolveSecurityPolicy({ securityPolicy: policy }));
|
|
2119
|
-
console.log(JSON.stringify({ level: classifySecurityLevel(caps), ...caps }));
|
|
2120
|
-
return;
|
|
2121
|
-
}
|
|
2122
|
-
for (const line of describeSecurityLevel(policy)) console.log(line);
|
|
2123
|
-
}
|
|
1839
|
+
async function useChatModel(client, id, options) {
|
|
1840
|
+
const order = options.requested ? [options.requested] : [options.platformDefault, ...["mlx", "llama-cpp", "ds4"]].filter(
|
|
1841
|
+
(engine2, index, all) => all.indexOf(engine2) === index
|
|
2124
1842
|
);
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
await client.updateConfig({
|
|
2133
|
-
securityPolicy: policyWithCapability(config, setting.key, enabled)
|
|
2134
|
-
});
|
|
2135
|
-
}
|
|
2136
|
-
const value = enabled ?? policy[setting.key];
|
|
2137
|
-
console.log(
|
|
2138
|
-
options.json ? JSON.stringify({ [setting.key]: value }) : `${setting.label}: ${value ? "on" : "off"}`
|
|
2139
|
-
);
|
|
2140
|
-
});
|
|
1843
|
+
let engine;
|
|
1844
|
+
for (const candidate of order) {
|
|
1845
|
+
const installed = await installedChatModels(client, candidate).catch(() => []);
|
|
1846
|
+
if (installed.includes(id)) {
|
|
1847
|
+
engine = candidate;
|
|
1848
|
+
break;
|
|
1849
|
+
}
|
|
2141
1850
|
}
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
const client = await connect();
|
|
2147
|
-
const config = enabled === void 0 ? await client.getConfig() : await client.updateConfig({ growth: { enabled } });
|
|
2148
|
-
const value = config.growth?.enabled !== false;
|
|
2149
|
-
console.log(
|
|
2150
|
-
options.json ? JSON.stringify({ growthEnabled: value }) : `Growth (XP and level-ups): ${value ? "on" : "off"}`
|
|
1851
|
+
if (!engine) {
|
|
1852
|
+
const target = options.requested ?? options.platformDefault;
|
|
1853
|
+
throw new CliError(
|
|
1854
|
+
`${id} isn't downloaded for ${target}. Download it with \`gezel model pull ${id}${providerFlag(target, options.platformDefault)}\`, or see \`gezel model list --available\`.`
|
|
2151
1855
|
);
|
|
1856
|
+
}
|
|
1857
|
+
const config = await client.getConfig();
|
|
1858
|
+
await client.updateConfig({
|
|
1859
|
+
provider: engine,
|
|
1860
|
+
defaultModel: { ...config.defaultModel ?? {}, [engine]: id },
|
|
1861
|
+
firstRunCompleted: true
|
|
2152
1862
|
});
|
|
1863
|
+
return engine;
|
|
2153
1864
|
}
|
|
2154
|
-
function
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
options.json ? JSON.stringify(result) : `${projectId}: workspace indexing ${result.indexingEnabled ? "on" : "off"}`
|
|
2163
|
-
);
|
|
1865
|
+
async function pinPulledModel(client, engine, id) {
|
|
1866
|
+
const config = await client.getConfig();
|
|
1867
|
+
const current = config.defaultModel?.[engine];
|
|
1868
|
+
if (current === id) return "already";
|
|
1869
|
+
if (current && (await installedChatModels(client, engine)).includes(current)) return "kept";
|
|
1870
|
+
await client.updateConfig({
|
|
1871
|
+
defaultModel: { ...config.defaultModel ?? {}, [engine]: id },
|
|
1872
|
+
firstRunCompleted: true
|
|
2164
1873
|
});
|
|
1874
|
+
return "pinned";
|
|
2165
1875
|
}
|
|
2166
1876
|
|
|
2167
|
-
// src/
|
|
2168
|
-
var
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
const commands = options.needs.commands ?? [];
|
|
2178
|
-
if (!options.needs.models?.length && services.length === 0 && commands.length === 0) return;
|
|
2179
|
-
const config = await client.getConfig();
|
|
2180
|
-
const models = resolveCraftbookModelNeeds(options.needs.models, {
|
|
2181
|
-
...options.params ? { params: options.params } : {},
|
|
2182
|
-
...options.paramSchema ? { paramSchema: options.paramSchema } : {},
|
|
2183
|
-
defaultProvider: defaultModelEngine(config, options.platform, options.arch)
|
|
2184
|
-
});
|
|
2185
|
-
const gaps = craftbookSetupGaps(
|
|
2186
|
-
{ services, models, commands },
|
|
2187
|
-
await readSetupState(client, config, models)
|
|
2188
|
-
);
|
|
2189
|
-
if (gaps.length === 0) return;
|
|
2190
|
-
const sizes = await modelSizes(client, gaps);
|
|
2191
|
-
const lines = gaps.map((gap) => ` \u2022 ${describeGap(gap, sizes)}`);
|
|
2192
|
-
if (!options.prompter) {
|
|
2193
|
-
throw new CliError(
|
|
2194
|
-
[
|
|
2195
|
-
`${options.label} is not set up to run yet:`,
|
|
2196
|
-
...lines,
|
|
2197
|
-
"",
|
|
2198
|
-
"Run the same command in a terminal to set it up step by step, or run:",
|
|
2199
|
-
...instructions(gaps).map((line) => ` ${line}`)
|
|
2200
|
-
].join("\n")
|
|
2201
|
-
);
|
|
1877
|
+
// src/run-readiness.ts
|
|
1878
|
+
var READY = { ready: true };
|
|
1879
|
+
async function settledFirstRunConfig(client, options = {}) {
|
|
1880
|
+
const attempts = options.attempts ?? 12;
|
|
1881
|
+
const delayMs = options.delayMs ?? 250;
|
|
1882
|
+
let config = await client.getConfig();
|
|
1883
|
+
for (let attempt = 0; attempt < attempts; attempt += 1) {
|
|
1884
|
+
if (config.firstRunCompleted || config.provider !== "copilot") return config;
|
|
1885
|
+
await new Promise((resolve7) => setTimeout(resolve7, delayMs));
|
|
1886
|
+
config = await client.getConfig();
|
|
2202
1887
|
}
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
const
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
1888
|
+
return config;
|
|
1889
|
+
}
|
|
1890
|
+
async function checkRunReadiness(client, gezelId, env2 = process.env) {
|
|
1891
|
+
if (env2.GEZEL_MOCK_PROVIDER === "1") return READY;
|
|
1892
|
+
const config = await settledFirstRunConfig(client);
|
|
1893
|
+
let provider = config.provider;
|
|
1894
|
+
if (gezelId) {
|
|
1895
|
+
try {
|
|
1896
|
+
const { parsed } = await client.getGezel(gezelId);
|
|
1897
|
+
if (parsed.frontmatter.provider) provider = parsed.frontmatter.provider;
|
|
1898
|
+
} catch {
|
|
1899
|
+
}
|
|
1900
|
+
}
|
|
1901
|
+
if (provider !== "llama-cpp" && provider !== "mlx") return READY;
|
|
1902
|
+
if (provider === "llama-cpp" && (config.llamaCppBaseUrl || config.llamaCppModelPath)) {
|
|
1903
|
+
return READY;
|
|
1904
|
+
}
|
|
1905
|
+
if (provider === "mlx" && (config.mlxBaseUrl || config.mlxModelPath)) return READY;
|
|
1906
|
+
const native2 = await client.getNativeEngineStatus();
|
|
1907
|
+
if (!native2.platformKey) return READY;
|
|
1908
|
+
const installed = provider === "mlx" ? await client.listMlxModels() : await client.listLlamaCppModels();
|
|
1909
|
+
const installedIds = installed.models.map((model2) => model2.id);
|
|
1910
|
+
const pinned = config.defaultModel?.[provider];
|
|
1911
|
+
const engineMissing = provider === "llama-cpp" && !native2.engines.some((engine) => engine.name === "llama-server" && engine.installed);
|
|
1912
|
+
const modelReady = pinned ? installedIds.includes(pinned) : installedIds.length > 0;
|
|
1913
|
+
if (!engineMissing && modelReady) return READY;
|
|
1914
|
+
return {
|
|
1915
|
+
ready: false,
|
|
1916
|
+
message: formatRunSetupMessage({
|
|
1917
|
+
provider,
|
|
1918
|
+
engineMissing,
|
|
1919
|
+
pinned,
|
|
1920
|
+
installedIds
|
|
1921
|
+
})
|
|
2216
1922
|
};
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
1923
|
+
}
|
|
1924
|
+
function canOfferRunSetup(io = process, env2 = process.env) {
|
|
1925
|
+
if (env2.CI) return false;
|
|
1926
|
+
return io.stdin.isTTY === true && io.stderr.isTTY === true;
|
|
1927
|
+
}
|
|
1928
|
+
function formatRunSetupMessage(input) {
|
|
1929
|
+
const { provider, engineMissing, pinned, installedIds } = input;
|
|
1930
|
+
const flag = providerFlag(
|
|
1931
|
+
provider,
|
|
1932
|
+
input.platformDefault ?? resolveOnDeviceProvider2(process.platform, process.arch)
|
|
1933
|
+
);
|
|
1934
|
+
const steps = [];
|
|
1935
|
+
if (engineMissing) steps.push("gezel native install");
|
|
1936
|
+
if (pinned && !installedIds.includes(pinned)) {
|
|
1937
|
+
if (installedIds.length > 0) {
|
|
1938
|
+
steps.push(`gezel model use ${installedIds[0]}${flag} (already downloaded)`);
|
|
1939
|
+
}
|
|
1940
|
+
steps.push(`gezel model pull ${pinned}${flag}`);
|
|
1941
|
+
if (installedIds.length === 0) {
|
|
1942
|
+
steps.push(`gezel model list --available${flag} (to pick a smaller one)`);
|
|
1943
|
+
}
|
|
1944
|
+
} else if (!pinned && installedIds.length === 0) {
|
|
1945
|
+
steps.push(`gezel model list --available${flag} (then: gezel model pull <id>${flag})`);
|
|
1946
|
+
}
|
|
1947
|
+
const lines = [];
|
|
1948
|
+
if (pinned && !installedIds.includes(pinned) && installedIds.length > 0) {
|
|
1949
|
+
lines.push(`Your selected on-device model, ${pinned}, isn't downloaded yet.`);
|
|
1950
|
+
} else if (engineMissing && installedIds.length > 0 && (!pinned || installedIds.includes(pinned))) {
|
|
1951
|
+
lines.push("Gezel's on-device engine isn't installed yet.");
|
|
1952
|
+
} else {
|
|
1953
|
+
lines.push("Gezel isn't set up to chat yet: this device doesn't have an on-device model.");
|
|
1954
|
+
}
|
|
1955
|
+
lines.push(
|
|
1956
|
+
"",
|
|
1957
|
+
"Run it again with --download to download what this computer needs and run your request.",
|
|
1958
|
+
"Or run `gezel` to choose a model and download it, step by step."
|
|
1959
|
+
);
|
|
1960
|
+
if (steps.length > 0) {
|
|
1961
|
+
lines.push("", "Or set it up directly:", ...steps.map((step) => ` ${step}`));
|
|
1962
|
+
}
|
|
1963
|
+
lines.push(
|
|
1964
|
+
"",
|
|
1965
|
+
"To use a cloud model instead, save a key (for example",
|
|
1966
|
+
"`gezel secret set openaiApiKey --stdin`), then run `gezel` and pick it with /model.",
|
|
1967
|
+
"",
|
|
1968
|
+
"Nothing was downloaded."
|
|
1969
|
+
);
|
|
1970
|
+
return lines.join("\n");
|
|
1971
|
+
}
|
|
1972
|
+
|
|
1973
|
+
// src/auto-download.ts
|
|
1974
|
+
var AUTO_DOWNLOAD_ENV = "GEZEL_AUTO_DOWNLOAD";
|
|
1975
|
+
function autoDownloadRequested(globals, env2 = process.env) {
|
|
1976
|
+
return globals.download === true || env2[AUTO_DOWNLOAD_ENV] === "1";
|
|
1977
|
+
}
|
|
1978
|
+
async function downloadForRun(client, options) {
|
|
1979
|
+
const { write } = options;
|
|
1980
|
+
const config = await settledFirstRunConfig(client);
|
|
1981
|
+
let provider = config.provider;
|
|
1982
|
+
if (options.gezelId) {
|
|
1983
|
+
try {
|
|
1984
|
+
const { parsed } = await client.getGezel(options.gezelId);
|
|
1985
|
+
if (parsed.frontmatter.provider) provider = parsed.frontmatter.provider;
|
|
1986
|
+
} catch {
|
|
1987
|
+
}
|
|
1988
|
+
}
|
|
1989
|
+
if (provider !== "llama-cpp" && provider !== "mlx") return;
|
|
1990
|
+
if (provider === "llama-cpp" && (config.llamaCppBaseUrl || config.llamaCppModelPath)) return;
|
|
1991
|
+
if (provider === "mlx" && (config.mlxBaseUrl || config.mlxModelPath)) return;
|
|
1992
|
+
if (!(await client.getNativeEngineStatus()).platformKey) return;
|
|
1993
|
+
write("Getting this computer ready for local models (--download)\u2026\n");
|
|
1994
|
+
await ensureModelEngines(client, [provider], write);
|
|
1995
|
+
const installed = provider === "mlx" ? await client.listMlxModels() : await client.listLlamaCppModels();
|
|
1996
|
+
const installedIds = new Set(installed.models.map((model2) => model2.id));
|
|
1997
|
+
const pinned = config.defaultModel?.[provider];
|
|
1998
|
+
if (pinned ? installedIds.has(pinned) : installedIds.size > 0) return;
|
|
1999
|
+
const recommend = options.recommend ?? (async (engine) => {
|
|
2000
|
+
const [memory, catalog] = await Promise.all([
|
|
2001
|
+
client.getMemoryProfile(),
|
|
2002
|
+
client.listCatalogItems("chat-model")
|
|
2003
|
+
]);
|
|
2004
|
+
return rankChatModels(catalog.items, recoDevice(memory), engine)[0];
|
|
2005
|
+
});
|
|
2006
|
+
const best = await recommend(provider);
|
|
2007
|
+
const target = pinned ?? best?.id;
|
|
2008
|
+
if (!target) {
|
|
2009
|
+
throw new CliError(
|
|
2010
|
+
"No open chat model in the catalog fits this computer. Save a cloud key instead (for example `gezel secret set openaiApiKey --stdin`) and pick it in `gezel` with /model."
|
|
2011
|
+
);
|
|
2012
|
+
}
|
|
2013
|
+
const size = best && best.id === target ? best.approxSizeBytes : 0;
|
|
2014
|
+
if (size > 0) {
|
|
2015
|
+
const space = await client.checkModelDownloadSpace({ sizeBytes: size });
|
|
2016
|
+
if (space.known && !space.ok) {
|
|
2017
|
+
throw new CliError(
|
|
2018
|
+
`${best?.name ?? target} needs ${formatGb(space.requiredBytes)} GB, and Gezel model storage has ${formatGb(space.freeBytes)} GB free. Free some space, then run again.`
|
|
2237
2019
|
);
|
|
2238
|
-
if (!yes) stop(gap);
|
|
2239
|
-
await client.updateConfig({ securityPolicy: next });
|
|
2240
|
-
options.write("Run package scripts: on\n");
|
|
2241
|
-
} else if (gap.kind === "web-search") {
|
|
2242
|
-
if (gap.keyMissing) {
|
|
2243
|
-
options.write(
|
|
2244
|
-
`Web search needs a Brave Search API key. Get one at ${BRAVE_KEY_URL} \u2014 Gezel keeps it in this computer's secret store.
|
|
2245
|
-
`
|
|
2246
|
-
);
|
|
2247
|
-
const key = (await prompter.secret(
|
|
2248
|
-
"Paste the key, then press Enter (Enter on its own stops here): ",
|
|
2249
|
-
(length) => `Key received (${length} characters, hidden). Press Enter to continue with this key. `
|
|
2250
|
-
)).trim();
|
|
2251
|
-
if (!key) stop(gap);
|
|
2252
|
-
options.write("Checking the key with one Brave search\u2026\n");
|
|
2253
|
-
await saveBraveKey(client, config, key, options.projectId);
|
|
2254
|
-
options.write("Web search: Brave (test search passed)\n");
|
|
2255
|
-
} else {
|
|
2256
|
-
const current = config.webSearch?.provider ?? "wikipedia";
|
|
2257
|
-
const yes = await prompter.confirm(
|
|
2258
|
-
`A Brave Search key is saved, but web search uses ${current === "wikipedia" ? "Wikipedia" : current}. Switch web search to Brave? [y/N] `
|
|
2259
|
-
);
|
|
2260
|
-
if (!yes) stop(gap);
|
|
2261
|
-
await client.updateConfig({ webSearch: { ...config.webSearch, provider: "brave" } });
|
|
2262
|
-
options.write("Web search: Brave\n");
|
|
2263
|
-
}
|
|
2264
2020
|
}
|
|
2265
2021
|
}
|
|
2266
|
-
|
|
2267
|
-
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2022
|
+
write(
|
|
2023
|
+
pinned ? `Downloading ${target}, the model this computer is set to use.
|
|
2024
|
+
` : `Downloading ${best?.name ?? target}${size > 0 ? ` (${formatGb(size)} GB)` : ""}, the model recommended for this computer.
|
|
2025
|
+
`
|
|
2026
|
+
);
|
|
2027
|
+
await pullChatModel(client, provider, target, write);
|
|
2028
|
+
if (provider === "mlx") await waitForMlxRuntime(client, write);
|
|
2029
|
+
await client.updateConfig({
|
|
2030
|
+
provider,
|
|
2031
|
+
defaultModel: { ...config.defaultModel, [provider]: target },
|
|
2032
|
+
firstRunInstallError: null
|
|
2033
|
+
});
|
|
2034
|
+
write("Ready.\n\n");
|
|
2035
|
+
}
|
|
2036
|
+
|
|
2037
|
+
// src/bench-command.ts
|
|
2038
|
+
import { execFile } from "child_process";
|
|
2039
|
+
import { readFileSync } from "fs";
|
|
2040
|
+
import { readFile as readFile2, writeFile as writeFile2 } from "fs/promises";
|
|
2041
|
+
import { cpus } from "os";
|
|
2042
|
+
import { join as join2 } from "path";
|
|
2043
|
+
import { promisify } from "util";
|
|
2044
|
+
import { ScorecardDatasetSchema } from "@bendyline/gezel";
|
|
2045
|
+
import { gildePackageRoot } from "@bendyline/gezel-catalog";
|
|
2046
|
+
import {
|
|
2047
|
+
EVAL_SUBSET_NOTE,
|
|
2048
|
+
SCORECARD_SUITE_IDS,
|
|
2049
|
+
SCORECARD_SUITE_LABELS,
|
|
2050
|
+
buildEvalMatrix,
|
|
2051
|
+
describeEvalHost,
|
|
2052
|
+
describeEvalRun,
|
|
2053
|
+
evalDeviceFromHost,
|
|
2054
|
+
evalEngineBuilds,
|
|
2055
|
+
evalRunId,
|
|
2056
|
+
evalRunProvenance,
|
|
2057
|
+
evalShard,
|
|
2058
|
+
evalSuiteResult,
|
|
2059
|
+
isSuiteSubset,
|
|
2060
|
+
medianDecodeTokensPerSec,
|
|
2061
|
+
smokeSuiteIdFor
|
|
2062
|
+
} from "@bendyline/gezel/eval";
|
|
2063
|
+
var BENCH_CONFIRM_ABOVE_MS = 60 * 6e4;
|
|
2064
|
+
var BENCH_SUBMIT_REPO = "bendyline/gezel";
|
|
2065
|
+
function formatDuration(ms) {
|
|
2066
|
+
if (ms < 6e4) return `${Math.max(1, Math.round(ms / 1e3))}s`;
|
|
2067
|
+
const minutes = Math.round(ms / 6e4);
|
|
2068
|
+
if (minutes < 60) return `${minutes}m`;
|
|
2069
|
+
const hours = Math.floor(minutes / 60);
|
|
2070
|
+
const rest = minutes % 60;
|
|
2071
|
+
return rest ? `${hours}h ${rest}m` : `${hours}h`;
|
|
2072
|
+
}
|
|
2073
|
+
function resolveBenchSuites(catalog, opts) {
|
|
2074
|
+
const wanted = !opts.suite || opts.suite === "core" || opts.suite === "default" ? ["core"] : opts.suite === "all" ? (
|
|
2075
|
+
// Every scorecard suite this daemon's catalog carries; an older
|
|
2076
|
+
// harness without one of them still runs the rest.
|
|
2077
|
+
SCORECARD_SUITE_IDS.filter((id) => catalog.suites.some((s) => s.id === id))
|
|
2078
|
+
) : opts.suite.split(",").map((s) => s.trim()).filter(Boolean);
|
|
2079
|
+
const suites = [];
|
|
2080
|
+
for (const id of wanted) {
|
|
2081
|
+
const base = catalog.suites.find((s) => s.id === id);
|
|
2082
|
+
if (!base) {
|
|
2083
|
+
throw new CliError(
|
|
2084
|
+
`Unknown suite "${id}". Choose one of: ${SCORECARD_SUITE_IDS.join(", ")}, or all. \`gezel bench list\` shows every suite.`
|
|
2085
|
+
);
|
|
2086
|
+
}
|
|
2087
|
+
if (opts.smoke) {
|
|
2088
|
+
const smokeId = smokeSuiteIdFor(id, catalog);
|
|
2089
|
+
const smoke = smokeId ? catalog.suites.find((s) => s.id === smokeId) : void 0;
|
|
2090
|
+
if (!smoke) {
|
|
2091
|
+
throw new CliError(`The ${id} suite has no smoke variant in this catalog.`);
|
|
2275
2092
|
}
|
|
2093
|
+
suites.push(smoke);
|
|
2094
|
+
} else {
|
|
2095
|
+
suites.push(base);
|
|
2276
2096
|
}
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
|
|
2097
|
+
}
|
|
2098
|
+
return suites;
|
|
2099
|
+
}
|
|
2100
|
+
function resolveBenchTarget(catalog, targets, opts) {
|
|
2101
|
+
const available = targets.filter((t) => t.available);
|
|
2102
|
+
if (opts.model) {
|
|
2103
|
+
const matches = available.filter(
|
|
2104
|
+
(t) => t.modelId === opts.model && (!opts.provider || t.provider === opts.provider)
|
|
2280
2105
|
);
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2106
|
+
const preferred = matches.find((t) => t.provider === catalog.defaultProvider) ?? matches[0];
|
|
2107
|
+
if (preferred) return preferred;
|
|
2108
|
+
const unavailable = targets.find(
|
|
2109
|
+
(t) => t.modelId === opts.model && (!opts.provider || t.provider === opts.provider)
|
|
2110
|
+
);
|
|
2111
|
+
throw new CliError(
|
|
2112
|
+
unavailable?.unavailableReason ? `${opts.model} can't run here: ${unavailable.unavailableReason}` : `No model "${opts.model}"${opts.provider ? ` on ${opts.provider}` : ""} is available to evaluate. \`gezel model list\` shows what is installed.`
|
|
2286
2113
|
);
|
|
2287
|
-
for (const gap of downloads) await pullChatModel(client, gap.provider, gap.id, options.write);
|
|
2288
2114
|
}
|
|
2289
|
-
const
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
);
|
|
2293
|
-
if (after.length > 0) {
|
|
2115
|
+
const byProvider = opts.provider ? available.filter((t) => t.provider === opts.provider) : available;
|
|
2116
|
+
const chosen = byProvider.find((t) => t.isDefault && t.provider === catalog.defaultProvider) ?? byProvider.find((t) => t.isDefault) ?? byProvider.find((t) => t.provider === catalog.defaultProvider) ?? byProvider[0];
|
|
2117
|
+
if (!chosen) {
|
|
2294
2118
|
throw new CliError(
|
|
2295
|
-
|
|
2296
|
-
`${options.label} is still not set up:`,
|
|
2297
|
-
...after.map((gap) => ` \u2022 ${describeGap(gap, sizes)}`)
|
|
2298
|
-
].join("\n")
|
|
2119
|
+
"No model is available to evaluate. Download one with `gezel model pull <id>` or sign in to a cloud provider, then try again."
|
|
2299
2120
|
);
|
|
2300
2121
|
}
|
|
2301
|
-
|
|
2122
|
+
return chosen;
|
|
2302
2123
|
}
|
|
2303
|
-
function
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
secret: (question, filled) => ask(terminal, question, { filled })
|
|
2315
|
-
};
|
|
2124
|
+
function planBench(catalog, targets, opts) {
|
|
2125
|
+
const count = opts.count ?? 1;
|
|
2126
|
+
if (!Number.isInteger(count) || count < 1 || count > 10) {
|
|
2127
|
+
throw new CliError(
|
|
2128
|
+
"--count must be a whole number from 1 to 10 (3 is what it takes to quote a rate)."
|
|
2129
|
+
);
|
|
2130
|
+
}
|
|
2131
|
+
const suites = resolveBenchSuites(catalog, opts);
|
|
2132
|
+
const target = resolveBenchTarget(catalog, targets, opts);
|
|
2133
|
+
const worstCaseMs = suites.reduce((sum, s) => sum + s.authoredCeilingMs * count, 0);
|
|
2134
|
+
return { suites, target, count, worstCaseMs, smoke: Boolean(opts.smoke) };
|
|
2316
2135
|
}
|
|
2317
|
-
function
|
|
2318
|
-
const
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
}
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2136
|
+
function describePlan(plan, catalog) {
|
|
2137
|
+
const lines = [
|
|
2138
|
+
`Model: ${plan.target.label} (${plan.target.modelId} on ${plan.target.provider})`,
|
|
2139
|
+
`Trials per task: ${plan.count}${plan.count < catalog.minTrialsForRate ? ` \u2014 results show as counts; ${catalog.minTrialsForRate} trials is what it takes to quote a rate` : ""}`
|
|
2140
|
+
];
|
|
2141
|
+
for (const suite of plan.suites) {
|
|
2142
|
+
lines.push(
|
|
2143
|
+
`Suite: ${suite.id} \u2014 ${suite.scenarioIds.length} tasks, at most ${formatDuration(
|
|
2144
|
+
suite.authoredCeilingMs * plan.count
|
|
2145
|
+
)}`
|
|
2146
|
+
);
|
|
2147
|
+
if (plan.smoke) lines.push(` ${suite.description}`);
|
|
2148
|
+
}
|
|
2149
|
+
lines.push(
|
|
2150
|
+
`At most ${formatDuration(plan.worstCaseMs)} in all. The run keeps going if this terminal closes.`
|
|
2151
|
+
);
|
|
2152
|
+
return lines;
|
|
2153
|
+
}
|
|
2154
|
+
function describeJobOutcome(job) {
|
|
2155
|
+
switch (job.status) {
|
|
2156
|
+
case "completed":
|
|
2157
|
+
return null;
|
|
2158
|
+
case "failed":
|
|
2159
|
+
return `The run could not start: ${job.error ?? "the harness reported a setup error"}.`;
|
|
2160
|
+
case "cancelled":
|
|
2161
|
+
return "The run was stopped before it finished.";
|
|
2162
|
+
case "interrupted":
|
|
2163
|
+
return "The Gezel service stopped while the run was in progress; the trials it finished are counted.";
|
|
2164
|
+
default:
|
|
2165
|
+
return `The run ended in state "${job.status}".`;
|
|
2166
|
+
}
|
|
2167
|
+
}
|
|
2168
|
+
function renderSuiteReport(suite, target, trials, minTrialsForRate) {
|
|
2169
|
+
const matrix = buildEvalMatrix(trials, { scenarioOrder: suite.scenarioIds });
|
|
2170
|
+
const column = matrix.columns.find(
|
|
2171
|
+
(c) => c.modelId === target.modelId && (c.provider ?? target.provider) === target.provider
|
|
2172
|
+
);
|
|
2173
|
+
const width = Math.min(40, Math.max(12, ...suite.scenarioIds.map((id) => id.length)) + 2);
|
|
2174
|
+
const lines = [];
|
|
2175
|
+
for (const scenarioId of suite.scenarioIds) {
|
|
2176
|
+
const cell = column ? matrix.cell(scenarioId, column.key) : void 0;
|
|
2177
|
+
const claim = cell ? cell.claim : "not run";
|
|
2178
|
+
const extra = cell?.nonModelFailures ? ` (${cell.nonModelFailures} not counted: not the model's fault)` : "";
|
|
2179
|
+
lines.push(` ${scenarioId.padEnd(width)}${claim}${extra}`);
|
|
2180
|
+
}
|
|
2181
|
+
const total = column ? column.claim : "not measured";
|
|
2182
|
+
const label = SCORECARD_SUITE_LABELS[suite.id] ?? suite.id;
|
|
2183
|
+
lines.push(` ${"suite".padEnd(width)}${total}`);
|
|
2184
|
+
if (column && column.passRate === null && column.attributableTrials > 0) {
|
|
2185
|
+
lines.push(
|
|
2186
|
+
` A count, not a rate: ${minTrialsForRate} trials per task is the floor for quoting a percentage.`
|
|
2187
|
+
);
|
|
2188
|
+
}
|
|
2189
|
+
const decode = medianDecodeTokensPerSec(trials);
|
|
2190
|
+
lines.push(
|
|
2191
|
+
` Decode: ${decode ? `${decode.toFixed(1)} tok/s (median over finished tasks)` : "not recorded"}. Prefill: not recorded by the daemon's trial index.`
|
|
2192
|
+
);
|
|
2193
|
+
return [`${label} (${suite.id})`, ...lines];
|
|
2194
|
+
}
|
|
2195
|
+
function createJobFollower(suite, count) {
|
|
2196
|
+
let lastTrial = null;
|
|
2197
|
+
let lastCompleted = -1;
|
|
2198
|
+
let announcedWaiting = false;
|
|
2199
|
+
let announcedPreflight = false;
|
|
2200
|
+
return {
|
|
2201
|
+
update(job) {
|
|
2202
|
+
const lines = [];
|
|
2203
|
+
const target = job.targets[0];
|
|
2204
|
+
if (job.status === "waiting-for-device" && !announcedWaiting) {
|
|
2205
|
+
announcedWaiting = true;
|
|
2206
|
+
lines.push(
|
|
2207
|
+
`Waiting: another evaluation${job.waitingOn ? ` (${job.waitingOn})` : ""} holds this machine's engines.`
|
|
2208
|
+
);
|
|
2355
2209
|
}
|
|
2356
|
-
if (
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
|
|
2210
|
+
if (!target) return lines;
|
|
2211
|
+
if (target.preflight && !announcedPreflight) {
|
|
2212
|
+
announcedPreflight = true;
|
|
2213
|
+
const p = target.preflight;
|
|
2214
|
+
lines.push(
|
|
2215
|
+
p.admitted === null ? `Fitness check skipped${p.skippedReason ? `: ${p.skippedReason}` : ""}.` : p.admitted ? `Fitness check passed${p.decodeTokensPerSec ? ` \xB7 ${p.decodeTokensPerSec.toFixed(1)} tok/s` : ""}.` : `Fitness check failed${p.skippedReason ? `: ${p.skippedReason}` : ""}.`
|
|
2216
|
+
);
|
|
2217
|
+
}
|
|
2218
|
+
const planned = target.plannedTrials ?? suite.scenarioIds.length * count;
|
|
2219
|
+
if (target.completedTrials !== lastCompleted && lastCompleted >= 0) {
|
|
2220
|
+
lines.push(` ${target.completedTrials}/${planned} done, ${target.passedTrials} passed`);
|
|
2221
|
+
}
|
|
2222
|
+
lastCompleted = target.completedTrials;
|
|
2223
|
+
const current = target.currentTrial;
|
|
2224
|
+
if (current && current.trialId !== lastTrial) {
|
|
2225
|
+
lastTrial = current.trialId;
|
|
2226
|
+
lines.push(` ${current.scenarioId} (${current.trialIndex}/${planned}) running\u2026`);
|
|
2227
|
+
}
|
|
2228
|
+
return lines;
|
|
2229
|
+
}
|
|
2230
|
+
};
|
|
2231
|
+
}
|
|
2232
|
+
function finishedJob(job) {
|
|
2233
|
+
return job.status === "completed" || job.status === "failed" || job.status === "cancelled" || job.status === "interrupted";
|
|
2234
|
+
}
|
|
2235
|
+
async function followJob(client, job, suite, count, io, verbose = false) {
|
|
2236
|
+
const follower = createJobFollower(suite, count);
|
|
2237
|
+
let latest = job;
|
|
2238
|
+
for (const line of follower.update(job)) io.note(line);
|
|
2239
|
+
await client.streamEvalJob(job.id, (event) => {
|
|
2240
|
+
if (event.type === "log") {
|
|
2241
|
+
if (verbose) io.note(` ${event.line}`);
|
|
2242
|
+
return;
|
|
2243
|
+
}
|
|
2244
|
+
latest = event.job;
|
|
2245
|
+
for (const line of follower.update(event.job)) io.note(line);
|
|
2246
|
+
});
|
|
2247
|
+
if (!finishedJob(latest)) latest = await client.getEvalJob(job.id);
|
|
2248
|
+
return latest;
|
|
2249
|
+
}
|
|
2250
|
+
function defaultGildeVersion() {
|
|
2251
|
+
try {
|
|
2252
|
+
const pkg = JSON.parse(readFileSync(join2(gildePackageRoot(), "package.json"), "utf8"));
|
|
2253
|
+
return pkg.version ?? "unknown";
|
|
2254
|
+
} catch {
|
|
2255
|
+
return "unknown";
|
|
2256
|
+
}
|
|
2257
|
+
}
|
|
2258
|
+
function defaultCpuModel() {
|
|
2259
|
+
return cpus()[0]?.model?.trim() || void 0;
|
|
2260
|
+
}
|
|
2261
|
+
async function runBench(client, opts, io) {
|
|
2262
|
+
const [catalog, targetsResponse, host] = await Promise.all([
|
|
2263
|
+
client.getEvalCatalog(),
|
|
2264
|
+
client.listEvalTargets(),
|
|
2265
|
+
client.getSystemDiagnostics()
|
|
2266
|
+
]);
|
|
2267
|
+
const plan = planBench(catalog, targetsResponse.targets, opts);
|
|
2268
|
+
const cpuModel = io.cpuModel ? io.cpuModel() : void 0;
|
|
2269
|
+
io.note(describeEvalHost(host, cpuModel));
|
|
2270
|
+
for (const line of describePlan(plan, catalog)) io.note(line);
|
|
2271
|
+
if (plan.worstCaseMs > BENCH_CONFIRM_ABOVE_MS && !opts.yes) {
|
|
2272
|
+
if (!io.confirm) {
|
|
2273
|
+
throw new CliError(
|
|
2274
|
+
`This run could take ${formatDuration(plan.worstCaseMs)}. Add --yes to start it without a prompt.`
|
|
2275
|
+
);
|
|
2276
|
+
}
|
|
2277
|
+
const go = await io.confirm(`Start a run of up to ${formatDuration(plan.worstCaseMs)}? [y/N] `);
|
|
2278
|
+
if (!go) throw new CliError("Not started.");
|
|
2279
|
+
}
|
|
2280
|
+
const startedAt = (io.now?.() ?? /* @__PURE__ */ new Date()).toISOString();
|
|
2281
|
+
const device = evalDeviceFromHost(host, cpuModel);
|
|
2282
|
+
const scenariosBySuite = {};
|
|
2283
|
+
const jobs = [];
|
|
2284
|
+
const results = [];
|
|
2285
|
+
const runId = evalRunId(startedAt, device, plan.target.modelId);
|
|
2286
|
+
const gildeVersion = (io.gildeVersion ?? defaultGildeVersion)();
|
|
2287
|
+
const run = evalRunProvenance({
|
|
2288
|
+
id: runId,
|
|
2289
|
+
startedAt,
|
|
2290
|
+
device,
|
|
2291
|
+
gezelVersion: host.version,
|
|
2292
|
+
gildeVersion,
|
|
2293
|
+
count: plan.count,
|
|
2294
|
+
engineBuilds: evalEngineBuilds(host),
|
|
2295
|
+
scenariosBySuite: Object.fromEntries(plan.suites.map((s) => [s.id, s.scenarioIds])),
|
|
2296
|
+
subset: false,
|
|
2297
|
+
...opts.note ? { note: opts.note } : {}
|
|
2298
|
+
});
|
|
2299
|
+
for (const suite of plan.suites) {
|
|
2300
|
+
const spec = {
|
|
2301
|
+
suiteId: suite.id,
|
|
2302
|
+
count: plan.count,
|
|
2303
|
+
...plan.count > 1 ? { countStrict: true } : {},
|
|
2304
|
+
targets: [{ provider: plan.target.provider, modelId: plan.target.modelId }],
|
|
2305
|
+
note: `gezel bench ${runId}`
|
|
2306
|
+
};
|
|
2307
|
+
io.note("");
|
|
2308
|
+
io.note(`Starting ${suite.id}\u2026`);
|
|
2309
|
+
const created = await client.createEvalJob(spec);
|
|
2310
|
+
const job = await followJob(client, created, suite, plan.count, io, opts.verbose);
|
|
2311
|
+
jobs.push(job);
|
|
2312
|
+
scenariosBySuite[suite.id] = suite.scenarioIds;
|
|
2313
|
+
const outcome = describeJobOutcome(job);
|
|
2314
|
+
if (outcome) io.note(outcome);
|
|
2315
|
+
const { trials } = await client.listEvalTrials({ jobId: job.id, limit: 2e3 });
|
|
2316
|
+
results.push(evalSuiteResult({ run, suite, target: plan.target, trials }));
|
|
2317
|
+
io.out("");
|
|
2318
|
+
for (const line of renderSuiteReport(suite, plan.target, trials, catalog.minTrialsForRate)) {
|
|
2319
|
+
io.out(line);
|
|
2320
|
+
}
|
|
2321
|
+
if (plan.smoke) io.out(` ${suite.description}`);
|
|
2322
|
+
}
|
|
2323
|
+
io.out("");
|
|
2324
|
+
io.out(describeEvalRun(run, [plan.target.provider]));
|
|
2325
|
+
io.out(
|
|
2326
|
+
`Results are kept in ${targetsResponse.environment.runsDir}; Settings \u2192 Benchmarks shows the same runs.`
|
|
2377
2327
|
);
|
|
2328
|
+
const shard = evalShard(run, results);
|
|
2329
|
+
if (opts.json) {
|
|
2330
|
+
await writeFile2(opts.json, `${JSON.stringify(shard, null, 2)}
|
|
2331
|
+
`, "utf8");
|
|
2332
|
+
io.out(`Wrote ${opts.json}. Share it with \`gezel bench submit ${opts.json}\`.`);
|
|
2333
|
+
}
|
|
2334
|
+
return { run, results, jobs, shard };
|
|
2378
2335
|
}
|
|
2379
|
-
function
|
|
2380
|
-
const
|
|
2381
|
-
|
|
2336
|
+
function renderSuiteList(catalog) {
|
|
2337
|
+
const width = Math.max(...catalog.suites.map((s) => s.id.length)) + 2;
|
|
2338
|
+
const lines = [
|
|
2339
|
+
"Suites (`gezel bench --suite <id>`); times are the authored ceilings at one trial per task:"
|
|
2340
|
+
];
|
|
2341
|
+
const scorecard = catalog.suites.filter(
|
|
2342
|
+
(s) => SCORECARD_SUITE_IDS.includes(s.id)
|
|
2343
|
+
);
|
|
2344
|
+
const rest = catalog.suites.filter((s) => !scorecard.includes(s));
|
|
2345
|
+
for (const suite of [...scorecard, ...rest]) {
|
|
2346
|
+
const label = SCORECARD_SUITE_LABELS[suite.id];
|
|
2347
|
+
lines.push(
|
|
2348
|
+
` ${suite.id.padEnd(width)}${String(suite.scenarioIds.length).padStart(2)} tasks ${formatDuration(
|
|
2349
|
+
suite.authoredCeilingMs
|
|
2350
|
+
).padStart(7)} ${label ?? suite.description}`
|
|
2351
|
+
);
|
|
2352
|
+
}
|
|
2353
|
+
lines.push(
|
|
2354
|
+
"",
|
|
2355
|
+
`The published scorecard runs ${SCORECARD_SUITE_IDS.join(", ")}. Add --smoke for a suite's pulse check; it is not a suite score.`
|
|
2356
|
+
);
|
|
2357
|
+
return lines;
|
|
2382
2358
|
}
|
|
2383
|
-
async function
|
|
2384
|
-
const
|
|
2385
|
-
const
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2359
|
+
async function renderResults(client, catalog, limit) {
|
|
2360
|
+
const { jobs } = await client.listEvalJobs();
|
|
2361
|
+
const finished = jobs.filter(finishedJob).slice(0, limit);
|
|
2362
|
+
if (finished.length === 0)
|
|
2363
|
+
return ["No finished runs on this machine yet. `gezel bench` starts one."];
|
|
2364
|
+
const lines = [];
|
|
2365
|
+
for (const job of finished) {
|
|
2366
|
+
const suite = job.spec.suiteId ? catalog.suites.find((s) => s.id === job.spec.suiteId) : void 0;
|
|
2367
|
+
const target = job.targets[0];
|
|
2368
|
+
const { trials } = await client.listEvalTrials({ jobId: job.id, limit: 2e3 });
|
|
2369
|
+
const order = suite?.scenarioIds ?? [...new Set(trials.map((t) => t.scenarioId))];
|
|
2370
|
+
const matrix = buildEvalMatrix(trials, { scenarioOrder: order });
|
|
2371
|
+
const column = target ? matrix.columns.find(
|
|
2372
|
+
(c) => c.modelId === target.modelId && (c.provider ?? target.provider) === target.provider
|
|
2373
|
+
) : matrix.columns[0];
|
|
2374
|
+
const subset = suite && job.spec.scenarioIds ? isSuiteSubset(suite, job.spec.scenarioIds) : false;
|
|
2375
|
+
const what = suite ? `${suite.id}${subset ? " (subset)" : ""}` : `${job.spec.scenarioIds?.length ?? 0} tasks`;
|
|
2376
|
+
lines.push(
|
|
2377
|
+
`${(job.finishedAt ?? job.createdAt).slice(0, 16).replace("T", " ")} ${(target?.modelId ?? "?").padEnd(24)} ${what.padEnd(22)} ${column?.claim ?? "not measured"}${job.status !== "completed" ? ` [${job.status}]` : ""}`
|
|
2378
|
+
);
|
|
2379
|
+
if (subset) lines.push(` ${EVAL_SUBSET_NOTE}`);
|
|
2380
|
+
}
|
|
2381
|
+
return lines;
|
|
2382
|
+
}
|
|
2383
|
+
function describeSubmission(shard) {
|
|
2384
|
+
const run = shard.runs[0];
|
|
2385
|
+
const models = [...new Set(shard.results.map((r) => `${r.modelId} (${r.engine})`))];
|
|
2386
|
+
const suites = run.suites.join(", ");
|
|
2387
|
+
const title = `Bench result: ${models.join(", ")} on ${run.provenance.device.label}`;
|
|
2388
|
+
const body = [
|
|
2389
|
+
"## Community bench result",
|
|
2390
|
+
"",
|
|
2391
|
+
`- Run: \`${run.id}\``,
|
|
2392
|
+
`- ${describeEvalRun(run, [...new Set(shard.results.map((r) => r.engine))])}`,
|
|
2393
|
+
`- Suites: ${suites}`,
|
|
2394
|
+
...run.note ? [`- Note: ${run.note}`] : [],
|
|
2395
|
+
"",
|
|
2396
|
+
"Results (count below three trials per task, rate otherwise):",
|
|
2397
|
+
...shard.results.map((r) => {
|
|
2398
|
+
const matrixLike = r.cells.reduce(
|
|
2399
|
+
(acc, c) => {
|
|
2400
|
+
acc.trials += Math.max(0, c.trials - c.nonModelFailures);
|
|
2401
|
+
acc.successes += c.successes;
|
|
2402
|
+
return acc;
|
|
2403
|
+
},
|
|
2404
|
+
{ trials: 0, successes: 0 }
|
|
2405
|
+
);
|
|
2406
|
+
return `- ${r.suiteId}: ${matrixLike.successes}/${matrixLike.trials}`;
|
|
2407
|
+
}),
|
|
2408
|
+
"",
|
|
2409
|
+
"The JSON shard produced by `gezel bench --json` is attached below. It is a `ScorecardDataset` and ingests with the same code as a published round; community rounds are shown separately from Bendyline rounds.",
|
|
2410
|
+
"",
|
|
2411
|
+
"<details><summary>shard.json</summary>",
|
|
2412
|
+
"",
|
|
2413
|
+
"```json",
|
|
2414
|
+
JSON.stringify(shard, null, 2),
|
|
2415
|
+
"```",
|
|
2416
|
+
"",
|
|
2417
|
+
"</details>"
|
|
2418
|
+
].join("\n");
|
|
2419
|
+
return { title, body };
|
|
2420
|
+
}
|
|
2421
|
+
var execFileAsync = promisify(execFile);
|
|
2422
|
+
var BENCH_RESULT_LABEL = "bench-result";
|
|
2423
|
+
function labelMissing(error) {
|
|
2424
|
+
const { stderr, message } = error ?? {};
|
|
2425
|
+
const text = typeof stderr === "string" && stderr ? stderr : String(message ?? error);
|
|
2426
|
+
return /could not add label/i.test(text) || text.includes(`'${BENCH_RESULT_LABEL}' not found`);
|
|
2427
|
+
}
|
|
2428
|
+
async function submitBench(file, io, deps = {}) {
|
|
2429
|
+
const text = await (deps.readFile ?? ((p) => readFile2(p, "utf8")))(file);
|
|
2430
|
+
let shard;
|
|
2431
|
+
try {
|
|
2432
|
+
shard = ScorecardDatasetSchema.parse(JSON.parse(text));
|
|
2433
|
+
} catch (error) {
|
|
2434
|
+
throw new CliError(
|
|
2435
|
+
`${file} is not a bench result file: ${error instanceof Error ? error.message.split("\n")[0] : String(error)}. Produce one with \`gezel bench --json <file>\`.`
|
|
2436
|
+
);
|
|
2437
|
+
}
|
|
2438
|
+
if (shard.runs.length !== 1) {
|
|
2439
|
+
throw new CliError(`${file} holds ${shard.runs.length} runs; submit one run per issue.`);
|
|
2440
|
+
}
|
|
2441
|
+
const { title, body } = describeSubmission(shard);
|
|
2442
|
+
const params = new URLSearchParams({ title, labels: BENCH_RESULT_LABEL });
|
|
2443
|
+
const url = `https://github.com/${BENCH_SUBMIT_REPO}/issues/new?${params.toString()}`;
|
|
2444
|
+
const gh = deps.gh ?? (async (args) => (await execFileAsync("gh", args, { encoding: "utf8" })).stdout);
|
|
2445
|
+
const createIssue = (labeled) => gh([
|
|
2446
|
+
"issue",
|
|
2447
|
+
"create",
|
|
2448
|
+
"--repo",
|
|
2449
|
+
BENCH_SUBMIT_REPO,
|
|
2450
|
+
"--title",
|
|
2451
|
+
title,
|
|
2452
|
+
...labeled ? ["--label", BENCH_RESULT_LABEL] : [],
|
|
2453
|
+
"--body",
|
|
2454
|
+
body
|
|
2455
|
+
]);
|
|
2456
|
+
try {
|
|
2457
|
+
let stdout;
|
|
2458
|
+
try {
|
|
2459
|
+
stdout = await createIssue(true);
|
|
2460
|
+
} catch (error) {
|
|
2461
|
+
if (!labelMissing(error)) throw error;
|
|
2462
|
+
stdout = await createIssue(false);
|
|
2463
|
+
}
|
|
2464
|
+
const issueUrl = stdout.trim().split("\n").pop() ?? "";
|
|
2465
|
+
io.out(`Opened ${issueUrl}`);
|
|
2466
|
+
return { title, body, url, issueUrl };
|
|
2467
|
+
} catch {
|
|
2468
|
+
io.out("Open this page and attach the file to the issue:");
|
|
2469
|
+
io.out(` ${url}`);
|
|
2470
|
+
io.out(` file: ${file}`);
|
|
2471
|
+
io.note("(Install the GitHub CLI, `gh`, and sign in to have this done for you.)");
|
|
2472
|
+
return { title, body, url };
|
|
2473
|
+
}
|
|
2474
|
+
}
|
|
2475
|
+
function registerBenchCommands(program2, connect, io = {
|
|
2476
|
+
out: (line) => console.log(line),
|
|
2477
|
+
note: (line) => console.error(line),
|
|
2478
|
+
confirm: process.stdin.isTTY ? confirmOnTerminal : void 0,
|
|
2479
|
+
cpuModel: defaultCpuModel
|
|
2480
|
+
}) {
|
|
2481
|
+
const bench = program2.command("bench").description(
|
|
2482
|
+
"Run the model scorecard on this machine: the same tasks, checks and counting rules as the published scorecard"
|
|
2483
|
+
).option(
|
|
2484
|
+
"--suite <id>",
|
|
2485
|
+
"core (default), productivity, developer, complex-work, a comma list, or all"
|
|
2486
|
+
).option("--smoke", "Run the suite's pulse check instead (fast; not a suite score)").option("--model <id>", "Model to evaluate (default: the model chats use)").option("--provider <id>", "Provider for --model when a model is on more than one").option(
|
|
2487
|
+
"--count <n>",
|
|
2488
|
+
"Trials per task; 3 is what it takes to quote a rate (default 1)",
|
|
2489
|
+
(v) => Number.parseInt(v, 10)
|
|
2490
|
+
).option("--json <file>", "Also write the results as a scorecard shard").option("--note <text>", "A note kept with the run").option("--yes", "Start without asking, however long the run").option("--verbose", "Show the harness's own log lines while it runs").action(async (opts) => {
|
|
2491
|
+
const client = await connect();
|
|
2492
|
+
await runBench(client, opts, io);
|
|
2493
|
+
});
|
|
2494
|
+
bench.command("list").description("Show the suites, how many tasks each has, and how long it can take").action(async () => {
|
|
2495
|
+
const client = await connect();
|
|
2496
|
+
for (const line of renderSuiteList(await client.getEvalCatalog())) io.out(line);
|
|
2497
|
+
});
|
|
2498
|
+
bench.command("results").description("Show finished runs on this machine").option("--limit <n>", "How many runs to show (default 20)", (v) => Number.parseInt(v, 10)).action(async (opts) => {
|
|
2499
|
+
const client = await connect();
|
|
2500
|
+
const catalog = await client.getEvalCatalog();
|
|
2501
|
+
for (const line of await renderResults(client, catalog, opts.limit ?? 20)) io.out(line);
|
|
2502
|
+
});
|
|
2503
|
+
bench.command("submit <file>").description("Share a results shard from `gezel bench --json` as a GitHub issue").action(async (file) => {
|
|
2504
|
+
await submitBench(file, io);
|
|
2505
|
+
});
|
|
2506
|
+
}
|
|
2507
|
+
async function confirmOnTerminal(question) {
|
|
2508
|
+
const { createInterface } = await import("readline/promises");
|
|
2509
|
+
const rl = createInterface({ input: process.stdin, output: process.stderr });
|
|
2510
|
+
try {
|
|
2511
|
+
const answer = await rl.question(question);
|
|
2512
|
+
return /^y(es)?$/i.test(answer.trim());
|
|
2513
|
+
} finally {
|
|
2514
|
+
rl.close();
|
|
2515
|
+
}
|
|
2516
|
+
}
|
|
2517
|
+
|
|
2518
|
+
// src/cli-errors.ts
|
|
2519
|
+
function formatCliFailure(err, env2 = process.env) {
|
|
2520
|
+
if (!(err instanceof Error)) return null;
|
|
2521
|
+
if (err.name === "CliError") return err.message;
|
|
2522
|
+
if (err.name !== "GezelApiError" || env2.GEZEL_DEBUG === "1") return null;
|
|
2523
|
+
const { status, details } = err;
|
|
2524
|
+
const request = /\bon ([A-Z]+ \S+)/.exec(err.message)?.[1];
|
|
2525
|
+
const serviceText = detailText(details);
|
|
2526
|
+
if (status === 404) {
|
|
2527
|
+
const what = serviceText && serviceText.toLowerCase() !== "not found" ? serviceText : void 0;
|
|
2528
|
+
return `error: not found${what ? `: ${what}` : request ? ` (${request})` : ""}`;
|
|
2529
|
+
}
|
|
2530
|
+
const lines = [`error: ${serviceText ?? err.message}`];
|
|
2531
|
+
if (serviceText && request) lines.push(`(Gezel service answered ${status ?? "?"} to ${request})`);
|
|
2532
|
+
lines.push("Set GEZEL_DEBUG=1 to see the full error.");
|
|
2533
|
+
return lines.join("\n");
|
|
2534
|
+
}
|
|
2535
|
+
function isNotFound(err) {
|
|
2536
|
+
return err instanceof Error && err.name === "GezelApiError" && err.status === 404;
|
|
2537
|
+
}
|
|
2538
|
+
function detailText(details) {
|
|
2539
|
+
if (typeof details === "string" && details.trim()) return details.trim();
|
|
2540
|
+
if (details && typeof details === "object") {
|
|
2541
|
+
for (const key of ["message", "error"]) {
|
|
2542
|
+
const value = details[key];
|
|
2543
|
+
if (typeof value === "string" && value.trim()) return value.trim();
|
|
2544
|
+
}
|
|
2545
|
+
}
|
|
2546
|
+
return void 0;
|
|
2547
|
+
}
|
|
2548
|
+
|
|
2549
|
+
// src/craftbook-command.ts
|
|
2550
|
+
import { setTimeout as delay } from "timers/promises";
|
|
2551
|
+
import {
|
|
2552
|
+
composeCraftbookLaunch,
|
|
2553
|
+
createAwakeTimeout,
|
|
2554
|
+
paramAsksUser
|
|
2555
|
+
} from "@bendyline/gezel";
|
|
2556
|
+
|
|
2557
|
+
// src/tui/craftbook-start.ts
|
|
2558
|
+
import {
|
|
2559
|
+
CRAFTBOOK_CATEGORY_FAMILY_META,
|
|
2560
|
+
CRAFTBOOK_CATEGORY_META,
|
|
2561
|
+
CRAFTBOOK_ROLE_META,
|
|
2562
|
+
resolveCraftbookCategory,
|
|
2563
|
+
visibleCatalogItems
|
|
2564
|
+
} from "@bendyline/gezel";
|
|
2565
|
+
function normalizeCraftbooks(items, showWorkInProgressFeatures = false) {
|
|
2566
|
+
const byId = /* @__PURE__ */ new Map();
|
|
2567
|
+
for (const item of visibleCatalogItems(items, showWorkInProgressFeatures)) {
|
|
2568
|
+
if (item.manifest.kind !== "craftbook-template") continue;
|
|
2569
|
+
const manifest = item.manifest;
|
|
2570
|
+
if (byId.has(manifest.id)) continue;
|
|
2571
|
+
byId.set(manifest.id, {
|
|
2572
|
+
id: manifest.id,
|
|
2573
|
+
name: manifest.name,
|
|
2574
|
+
...manifest.description ? { description: manifest.description } : {},
|
|
2575
|
+
version: manifest.version,
|
|
2576
|
+
sourceId: item.sourceId,
|
|
2577
|
+
command: manifest.command ?? manifest.id,
|
|
2578
|
+
source: item.sourceId === "project" ? "project" : item.sourceId === "local" ? "local" : "bundled",
|
|
2579
|
+
stepCount: manifest.steps.length,
|
|
2580
|
+
tags: manifest.tags ?? [],
|
|
2581
|
+
role: manifest.role ?? "general",
|
|
2582
|
+
category: resolveCraftbookCategory(manifest)
|
|
2583
|
+
});
|
|
2584
|
+
}
|
|
2585
|
+
return [...byId.values()].sort((left, right) => left.name.localeCompare(right.name));
|
|
2586
|
+
}
|
|
2587
|
+
function findCraftbook(books, value) {
|
|
2588
|
+
const query = value.trim().toLowerCase();
|
|
2589
|
+
return books.find(
|
|
2590
|
+
(book) => book.id.toLowerCase() === query || book.command?.toLowerCase() === query || book.name.toLowerCase() === query
|
|
2390
2591
|
);
|
|
2592
|
+
}
|
|
2593
|
+
function craftbookStartRequest(book) {
|
|
2594
|
+
const summary = book.description?.trim();
|
|
2595
|
+
const description = summary ? `Start the "${book.name}" craftbook for this project. ${summary}` : `Start the "${book.name}" craftbook for this project. Follow every step in the recipe and verify its required outcomes before completing the task.`;
|
|
2391
2596
|
return {
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2597
|
+
title: book.name,
|
|
2598
|
+
description,
|
|
2599
|
+
craftbookId: book.id,
|
|
2600
|
+
craftbookSourceId: book.sourceId,
|
|
2601
|
+
...book.version ? { craftbookVersion: book.version } : {},
|
|
2602
|
+
dispatchEntry: true,
|
|
2603
|
+
trustScripts: true
|
|
2397
2604
|
};
|
|
2398
2605
|
}
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
for (
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
);
|
|
2606
|
+
|
|
2607
|
+
// src/craftbook-command.ts
|
|
2608
|
+
function resolveCraftbookInvocation(books, tokens) {
|
|
2609
|
+
for (let length = tokens.length; length > 0; length--) {
|
|
2610
|
+
const book = findCraftbook(books, tokens.slice(0, length).join(" "));
|
|
2611
|
+
if (book) return { book, args: tokens.slice(length) };
|
|
2612
|
+
}
|
|
2613
|
+
throw new CliError(`craftbook not found: ${tokens.join(" ")}`);
|
|
2614
|
+
}
|
|
2615
|
+
var NAMED_ARGUMENT = /^([A-Za-z][\w.-]*)=([\s\S]*)$/;
|
|
2616
|
+
function checkParamValue(key, def, value) {
|
|
2617
|
+
if (Array.isArray(def.enum) && !def.enum.map(String).includes(value)) {
|
|
2618
|
+
throw new CliError(`${key} must be one of: ${def.enum.join(", ")}`);
|
|
2619
|
+
}
|
|
2620
|
+
if (def.type === "boolean" && !["true", "false"].includes(value))
|
|
2621
|
+
throw new CliError(`${key} must be true or false`);
|
|
2622
|
+
if (def.type === "number" || def.type === "integer") {
|
|
2623
|
+
const number = Number(value);
|
|
2624
|
+
if (!value.trim() || !Number.isFinite(number) || def.type === "integer" && !Number.isInteger(number))
|
|
2625
|
+
throw new CliError(`${key} must be a ${def.type}`);
|
|
2626
|
+
if (typeof def.minimum === "number" && number < def.minimum)
|
|
2627
|
+
throw new CliError(`${key} must be at least ${def.minimum}`);
|
|
2628
|
+
if (typeof def.maximum === "number" && number > def.maximum)
|
|
2629
|
+
throw new CliError(`${key} must be at most ${def.maximum}`);
|
|
2630
|
+
}
|
|
2631
|
+
if (typeof def.pattern === "string" && !new RegExp(def.pattern).test(value))
|
|
2632
|
+
throw new CliError(`${key} must match ${def.pattern}`);
|
|
2633
|
+
if (typeof def.minLength === "number" && value.trim().length < def.minLength)
|
|
2634
|
+
throw new CliError(`${key} is too short`);
|
|
2635
|
+
}
|
|
2636
|
+
function parseCraftbookArguments(book, tokens, named = []) {
|
|
2637
|
+
const schema = book.paramSchema ?? {};
|
|
2638
|
+
const properties = schema.properties ?? {};
|
|
2639
|
+
const required = new Set(Array.isArray(schema.required) ? schema.required : []);
|
|
2640
|
+
const slots = Object.keys(properties).filter(
|
|
2641
|
+
(key) => required.has(key) && properties[key].default === void 0 && paramAsksUser(properties[key], key)
|
|
2642
|
+
);
|
|
2643
|
+
const params = {};
|
|
2644
|
+
const requestWords = [];
|
|
2645
|
+
const set = (key, value, token) => {
|
|
2646
|
+
if (!Object.hasOwn(properties, key)) throw new CliError(`unknown craftbook argument: ${token}`);
|
|
2647
|
+
if (Object.hasOwn(params, key)) throw new CliError(`duplicate craftbook parameter: ${key}`);
|
|
2648
|
+
checkParamValue(key, properties[key], value);
|
|
2649
|
+
params[key] = value;
|
|
2650
|
+
};
|
|
2651
|
+
for (const token of tokens) {
|
|
2652
|
+
const match = NAMED_ARGUMENT.exec(token);
|
|
2653
|
+
if (match) {
|
|
2654
|
+
set(match[1], match[2], token);
|
|
2655
|
+
continue;
|
|
2419
2656
|
}
|
|
2420
|
-
|
|
2657
|
+
const slot = slots.find((key) => !Object.hasOwn(params, key));
|
|
2658
|
+
if (slot) set(slot, token, token);
|
|
2659
|
+
else requestWords.push(token);
|
|
2421
2660
|
}
|
|
2422
|
-
|
|
2661
|
+
for (const token of named) {
|
|
2662
|
+
const match = NAMED_ARGUMENT.exec(token);
|
|
2663
|
+
if (!match) throw new CliError(`--param expects key=value: ${token}`);
|
|
2664
|
+
set(match[1], match[2], token);
|
|
2665
|
+
}
|
|
2666
|
+
for (const key of required) {
|
|
2667
|
+
if (typeof key === "string" && !Object.hasOwn(params, key) && properties[key]?.default === void 0)
|
|
2668
|
+
throw new CliError(`missing required craftbook parameter: ${key}`);
|
|
2669
|
+
}
|
|
2670
|
+
const request = requestWords.join(" ").trim();
|
|
2671
|
+
return request ? { params, request } : { params };
|
|
2423
2672
|
}
|
|
2424
|
-
function
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2673
|
+
function craftbookDoLaunch(craftbook2, { params, request }) {
|
|
2674
|
+
if (!request) return { craftbookParams: params };
|
|
2675
|
+
const launch = composeCraftbookLaunch({
|
|
2676
|
+
message: request,
|
|
2677
|
+
craftbookName: craftbook2.name,
|
|
2678
|
+
paramSchema: craftbook2.paramSchema,
|
|
2679
|
+
params
|
|
2680
|
+
});
|
|
2681
|
+
return { description: launch.description, craftbookParams: launch.params };
|
|
2682
|
+
}
|
|
2683
|
+
var READ_RETRY_DELAYS = [250, 750, 1500];
|
|
2684
|
+
function transientTaskReadError(error) {
|
|
2685
|
+
const codes = /ECONNRESET|ECONNREFUSED|ECONNABORTED|EPIPE|ETIMEDOUT|EHOSTUNREACH|ENETUNREACH|EAI_AGAIN|UND_ERR_(?:SOCKET|CONNECT_TIMEOUT)|NGHTTP2_ENHANCE_YOUR_CALM|Connect Timeout Error|socket hang up|other side closed/;
|
|
2686
|
+
if (error instanceof Error && error.name === "GezelApiError") {
|
|
2687
|
+
const apiError = error;
|
|
2688
|
+
return apiError.status === 0 && apiError.details?.kind === "transport" && apiError.details.readRetryExhausted !== true && typeof apiError.details.cause === "string" && codes.test(apiError.details.cause);
|
|
2689
|
+
}
|
|
2690
|
+
return error instanceof TypeError && error.cause instanceof Error && codes.test(`${error.cause.code ?? ""} ${error.cause.message}`);
|
|
2691
|
+
}
|
|
2692
|
+
async function readTaskObservation(read, budget, signal) {
|
|
2693
|
+
for (let retries = 0; ; retries++) {
|
|
2694
|
+
signal.throwIfAborted();
|
|
2695
|
+
if (retries > 0 && budget.expired()) throw new CliError("Task observation timed out.");
|
|
2696
|
+
try {
|
|
2697
|
+
return await read();
|
|
2698
|
+
} catch (error) {
|
|
2699
|
+
signal.throwIfAborted();
|
|
2700
|
+
if (!transientTaskReadError(error) || retries >= READ_RETRY_DELAYS.length || budget.expired())
|
|
2701
|
+
throw error;
|
|
2702
|
+
await sleepWithinBudget(READ_RETRY_DELAYS[retries], budget, signal);
|
|
2436
2703
|
}
|
|
2437
2704
|
}
|
|
2438
2705
|
}
|
|
2439
|
-
function
|
|
2440
|
-
|
|
2441
|
-
|
|
2442
|
-
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
|
|
2446
|
-
`gezel secret set braveSearchApiKey --env BRAVE_SEARCH_API_KEY --use-for-search (get a key at ${BRAVE_KEY_URL})`
|
|
2447
|
-
);
|
|
2448
|
-
else out.push(`gezel model pull ${gap.id} --provider ${gap.provider}`);
|
|
2706
|
+
async function sleepWithinBudget(backoffMs, budget, signal) {
|
|
2707
|
+
if (budget.remainingMs() > backoffMs) {
|
|
2708
|
+
await delay(backoffMs, void 0, { signal });
|
|
2709
|
+
return;
|
|
2710
|
+
}
|
|
2711
|
+
while (!budget.expired()) {
|
|
2712
|
+
await delay(Math.max(1, budget.remainingMs()), void 0, { signal });
|
|
2449
2713
|
}
|
|
2450
|
-
return out;
|
|
2451
2714
|
}
|
|
2452
|
-
async function
|
|
2715
|
+
async function waitForTask(client, ref, options) {
|
|
2716
|
+
const timeout = createAwakeTimeout(options.timeoutMs, { pollMs: 100 });
|
|
2717
|
+
const budget = timeout.budget;
|
|
2718
|
+
const signal = options.signal ? AbortSignal.any([options.signal, timeout.signal]) : timeout.signal;
|
|
2719
|
+
let previous = "";
|
|
2720
|
+
let lastTask;
|
|
2721
|
+
const read = (operation) => readTaskObservation(operation, budget, signal);
|
|
2453
2722
|
try {
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
|
|
2723
|
+
for (; ; ) {
|
|
2724
|
+
const task2 = await read(() => client.getTaskByRef(ref, signal));
|
|
2725
|
+
lastTask = task2;
|
|
2726
|
+
const progress = `${task2.status}:${task2.activeStepId}`;
|
|
2727
|
+
if (progress !== previous) {
|
|
2728
|
+
options.onProgress?.(task2);
|
|
2729
|
+
previous = progress;
|
|
2730
|
+
}
|
|
2731
|
+
if (task2.status === "complete") return { task: task2, outcome: "complete", exitCode: 0 };
|
|
2732
|
+
if (task2.status === "canceled") return { task: task2, outcome: "canceled", exitCode: 1 };
|
|
2733
|
+
if (task2.status === "paused" || task2.status === "draft")
|
|
2734
|
+
return { task: task2, outcome: "blocked", exitCode: 2 };
|
|
2735
|
+
const refs = /* @__PURE__ */ new Set([task2.ref]);
|
|
2736
|
+
if (task2.fanout || task2.craftbook.spawn || task2.craftbook.cliWorkflow) {
|
|
2737
|
+
const { tasks } = await read(
|
|
2738
|
+
() => client.listTaskChildren(task2.projectId, task2.num, void 0, signal)
|
|
2739
|
+
);
|
|
2740
|
+
if (!task2.craftbook.cliWorkflow && !tasks.some((child) => child.status === "active") && tasks.some((child) => child.status === "paused" || child.status === "canceled"))
|
|
2741
|
+
return { task: task2, outcome: "blocked", exitCode: 2 };
|
|
2742
|
+
if (!task2.craftbook.cliWorkflow) for (const child of tasks) refs.add(child.ref);
|
|
2743
|
+
}
|
|
2744
|
+
if (client.listQuestions) {
|
|
2745
|
+
const { questions } = await read(
|
|
2746
|
+
() => client.listQuestions(
|
|
2747
|
+
{
|
|
2748
|
+
projectId: task2.projectId,
|
|
2749
|
+
pending: true,
|
|
2750
|
+
taskRefs: [...refs]
|
|
2751
|
+
},
|
|
2752
|
+
signal
|
|
2753
|
+
)
|
|
2754
|
+
);
|
|
2755
|
+
const questionIds = questions.filter(
|
|
2756
|
+
(q) => q.taskRef && refs.has(q.taskRef) && // A service pause card is a historical notification. Actual task
|
|
2757
|
+
// status above decides whether it is still paused; real session
|
|
2758
|
+
// questions and permissions must continue to block observation.
|
|
2759
|
+
!(q.sessionId === "" && q.intent?.kind === "task-paused")
|
|
2760
|
+
).map((q) => q.id);
|
|
2761
|
+
if (questionIds.length) return { task: task2, outcome: "blocked", exitCode: 2, questionIds };
|
|
2762
|
+
}
|
|
2763
|
+
if (budget.expired()) return { task: task2, outcome: "timeout", exitCode: 3 };
|
|
2764
|
+
await delay(Math.min(options.pollMs ?? 1e3, budget.remainingMs()), void 0, { signal });
|
|
2765
|
+
}
|
|
2766
|
+
} catch (error) {
|
|
2767
|
+
options.signal?.throwIfAborted();
|
|
2768
|
+
if (budget.expired() && lastTask) return { task: lastTask, outcome: "timeout", exitCode: 3 };
|
|
2769
|
+
throw error;
|
|
2770
|
+
} finally {
|
|
2771
|
+
timeout.dispose();
|
|
2460
2772
|
}
|
|
2461
|
-
const failure = await client.toolWebSearch(projectId, SEARCH_CHECK).then(
|
|
2462
|
-
() => void 0,
|
|
2463
|
-
(error) => serviceErrorText(error)
|
|
2464
|
-
);
|
|
2465
|
-
if (failure === void 0) return;
|
|
2466
|
-
await client.updateConfig({ braveSearchApiKey: "", webSearch: { ...config.webSearch } }).catch(() => {
|
|
2467
|
-
});
|
|
2468
|
-
throw new CliError(
|
|
2469
|
-
`A test search with that key failed, so it was not kept: ${failure.replaceAll(key, "\u2026")}
|
|
2470
|
-
Check the key at ${BRAVE_KEY_URL}, then run again.`
|
|
2471
|
-
);
|
|
2472
2773
|
}
|
|
2473
|
-
function
|
|
2474
|
-
const
|
|
2475
|
-
|
|
2476
|
-
|
|
2477
|
-
|
|
2774
|
+
function summarizeTaskNote(text, maxLength = 240) {
|
|
2775
|
+
const lines = text.split(/\r?\n/).map((line) => line.trim());
|
|
2776
|
+
const headingIndex = lines.findIndex((line) => line.startsWith("#"));
|
|
2777
|
+
const heading = headingIndex >= 0 ? lines[headingIndex].replace(/^#+\s*/, "") : "";
|
|
2778
|
+
const body = lines.slice(headingIndex + 1).find((line) => line.length > 0 && !line.startsWith("#"));
|
|
2779
|
+
const sentence = body?.match(/^.*?[.!?](?=\s|$)/)?.[0] ?? body ?? "";
|
|
2780
|
+
const summary = heading && sentence ? `${heading}: ${sentence}` : heading || sentence;
|
|
2781
|
+
return summary.length > maxLength ? `${summary.slice(0, maxLength - 1).trimEnd()}\u2026` : summary;
|
|
2782
|
+
}
|
|
2783
|
+
async function blockedTaskReason(client, result) {
|
|
2784
|
+
if (result.outcome !== "blocked") return void 0;
|
|
2785
|
+
if (result.questionIds?.length) {
|
|
2786
|
+
return "A gezel is waiting for your answer to a question. Answer it in the Gezel app or the interactive `gezel` terminal.";
|
|
2478
2787
|
}
|
|
2479
|
-
return
|
|
2788
|
+
if (result.task.status === "draft") return "The task is still a draft and has not started.";
|
|
2789
|
+
const { notes } = await client.listTaskNotes(result.task.projectId, result.task.num).catch(() => ({ notes: [] }));
|
|
2790
|
+
const latest = [...notes].sort((a, b) => a.at.localeCompare(b.at)).at(-1);
|
|
2791
|
+
return latest ? summarizeTaskNote(latest.text) || void 0 : void 0;
|
|
2480
2792
|
}
|
|
2481
|
-
|
|
2482
|
-
const
|
|
2483
|
-
if (
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
await installNativeToolkit(client, { engines: missing, output: { writeProgress: write } });
|
|
2793
|
+
function filterCraftbooks(books, words) {
|
|
2794
|
+
const needles = words.map((word) => word.trim().toLowerCase()).filter(Boolean);
|
|
2795
|
+
if (needles.length === 0) return [...books];
|
|
2796
|
+
return books.filter((book) => {
|
|
2797
|
+
const haystack = [book.command, book.name, book.category, ...book.tags].join(" ").toLowerCase();
|
|
2798
|
+
return needles.every((needle) => haystack.includes(needle));
|
|
2799
|
+
});
|
|
2489
2800
|
}
|
|
2490
2801
|
|
|
2491
2802
|
// src/craftbook-shorthand.ts
|
|
2492
2803
|
import { existsSync } from "fs";
|
|
2493
2804
|
import { readFile as readFile3, readdir } from "fs/promises";
|
|
2494
2805
|
import { join as join3 } from "path";
|
|
2495
|
-
import {
|
|
2806
|
+
import { CatalogService } from "@bendyline/gezel-catalog";
|
|
2807
|
+
import { gezelHome as gezelHome2, projectLocalCraftbooksRoot } from "@bendyline/gezel/paths";
|
|
2496
2808
|
var SHORTHAND_FLAG = "--via-craftbook-shorthand";
|
|
2497
2809
|
var VALUE_OPTIONS = /* @__PURE__ */ new Set(["--connect", "--token", "--home"]);
|
|
2498
2810
|
var OPTIONAL_VALUE_OPTIONS = /* @__PURE__ */ new Set(["--project"]);
|
|
@@ -2536,6 +2848,22 @@ async function workspaceCraftbookId(workspaceDir, word) {
|
|
|
2536
2848
|
}
|
|
2537
2849
|
return ids.find((id) => id.toLowerCase() === query);
|
|
2538
2850
|
}
|
|
2851
|
+
async function localCraftbookNamed(word, workspaceDir, catalogItems = bundledCraftbookItems) {
|
|
2852
|
+
try {
|
|
2853
|
+
if (craftbookForShorthand(normalizeCraftbooks(await catalogItems(), true), word)) return true;
|
|
2854
|
+
return await workspaceCraftbookId(workspaceDir, word) !== void 0;
|
|
2855
|
+
} catch {
|
|
2856
|
+
return true;
|
|
2857
|
+
}
|
|
2858
|
+
}
|
|
2859
|
+
async function bundledCraftbookItems() {
|
|
2860
|
+
const { bundledGildeDataDir } = await import("@bendyline/gezel-service/handboek");
|
|
2861
|
+
const catalog = new CatalogService(void 0, {
|
|
2862
|
+
localRoot: gezelHome2(),
|
|
2863
|
+
contentRoot: bundledGildeDataDir
|
|
2864
|
+
});
|
|
2865
|
+
return catalog.list("craftbook-template");
|
|
2866
|
+
}
|
|
2539
2867
|
function shellWord(token) {
|
|
2540
2868
|
return /^[\w@%+=:,./-]+$/.test(token) ? token : `'${token.replaceAll("'", `'\\''`)}'`;
|
|
2541
2869
|
}
|
|
@@ -2566,8 +2894,8 @@ function isGezeldCommand(command) {
|
|
|
2566
2894
|
// src/doctor-report.ts
|
|
2567
2895
|
import {
|
|
2568
2896
|
describeProvenance,
|
|
2569
|
-
measuredClaim,
|
|
2570
|
-
measuredResultFor,
|
|
2897
|
+
measuredClaim as measuredClaim2,
|
|
2898
|
+
measuredResultFor as measuredResultFor2,
|
|
2571
2899
|
measuredRowsFor,
|
|
2572
2900
|
scorecardHardwareKey,
|
|
2573
2901
|
scorecardHardwareLabel
|
|
@@ -2621,7 +2949,7 @@ function hardwareClassCoverage(dataset, modelId, device) {
|
|
|
2621
2949
|
return ` on ${name}: ${rounds} published ${roundWord}; ${mine} ${mine === 1 ? "result" : "results"} for this model`;
|
|
2622
2950
|
}
|
|
2623
2951
|
function renderMeasured(dataset, modelId, device, rows = 4) {
|
|
2624
|
-
const headline =
|
|
2952
|
+
const headline = measuredResultFor2(dataset, modelId, device);
|
|
2625
2953
|
const coverage = hardwareClassCoverage(dataset, modelId, device);
|
|
2626
2954
|
if (!headline) {
|
|
2627
2955
|
return [
|
|
@@ -2630,7 +2958,7 @@ function renderMeasured(dataset, modelId, device, rows = 4) {
|
|
|
2630
2958
|
];
|
|
2631
2959
|
}
|
|
2632
2960
|
const lines = [
|
|
2633
|
-
`measured: ${
|
|
2961
|
+
`measured: ${measuredClaim2(headline)}${headline.sameHardwareClass ? "" : " \u2014 a different platform; your numbers may differ"}`,
|
|
2634
2962
|
coverage
|
|
2635
2963
|
];
|
|
2636
2964
|
for (const row of measuredRowsFor(dataset, modelId).slice(0, rows)) {
|
|
@@ -2918,115 +3246,6 @@ function registerMcpCommands(program2, connect, output = console.log) {
|
|
|
2918
3246
|
});
|
|
2919
3247
|
}
|
|
2920
3248
|
|
|
2921
|
-
// src/model-choice.ts
|
|
2922
|
-
import {
|
|
2923
|
-
measuredClaim as measuredClaim2,
|
|
2924
|
-
measuredResultFor as measuredResultFor2
|
|
2925
|
-
} from "@bendyline/gezel";
|
|
2926
|
-
async function installedChatModels(client, engine) {
|
|
2927
|
-
const { models } = engine === "mlx" ? await client.listMlxModels() : engine === "ds4" ? await client.listDs4Models() : await client.listLlamaCppModels();
|
|
2928
|
-
return models.map((model2) => model2.id);
|
|
2929
|
-
}
|
|
2930
|
-
function providerFlag(engine, platformDefault) {
|
|
2931
|
-
return engine === platformDefault ? "" : ` --provider ${engine}`;
|
|
2932
|
-
}
|
|
2933
|
-
var FIT_LABEL = {
|
|
2934
|
-
fits: "fits",
|
|
2935
|
-
"fits-offload": "fits, partly offloaded",
|
|
2936
|
-
tight: "tight",
|
|
2937
|
-
"too-big": "too big"
|
|
2938
|
-
};
|
|
2939
|
-
async function availableChatModels(client, engine) {
|
|
2940
|
-
if (engine === "ds4") {
|
|
2941
|
-
throw new CliError("`--available` lists mlx and llama-cpp models; pass one with --provider.");
|
|
2942
|
-
}
|
|
2943
|
-
const [memory, catalog, installed, config] = await Promise.all([
|
|
2944
|
-
client.getMemoryProfile(),
|
|
2945
|
-
client.listCatalogItems("chat-model"),
|
|
2946
|
-
installedChatModels(client, engine),
|
|
2947
|
-
client.getConfig()
|
|
2948
|
-
]);
|
|
2949
|
-
const pinned = config.defaultModel?.[engine];
|
|
2950
|
-
return {
|
|
2951
|
-
ranked: rankChatModels(catalog.items, recoDevice(memory), engine),
|
|
2952
|
-
installed: new Set(installed),
|
|
2953
|
-
...pinned ? { pinned } : {}
|
|
2954
|
-
};
|
|
2955
|
-
}
|
|
2956
|
-
function formatAvailableChatModels(input) {
|
|
2957
|
-
const { engine, ranked, installed, pinned, measured } = input;
|
|
2958
|
-
if (ranked.length === 0) return `(no ${engine} chat models in the catalog for this machine)`;
|
|
2959
|
-
const width = Math.max(...ranked.map((model2) => model2.id.length));
|
|
2960
|
-
const measuredWidth = measured ? Math.max("not measured".length, ...ranked.map((m) => (measured.get(m.id) ?? "").length)) : 0;
|
|
2961
|
-
const lines = [`${engine} chat models for this machine, best first:`];
|
|
2962
|
-
ranked.forEach((model2, index) => {
|
|
2963
|
-
const notes = [
|
|
2964
|
-
index === 0 ? "recommended" : "",
|
|
2965
|
-
installed.has(model2.id) ? "installed" : "",
|
|
2966
|
-
model2.id === pinned ? "in use" : ""
|
|
2967
|
-
].filter(Boolean);
|
|
2968
|
-
const claim = measured ? ` ${(measured.get(model2.id) ?? "not measured").padEnd(measuredWidth)}` : "";
|
|
2969
|
-
lines.push(
|
|
2970
|
-
` ${model2.id.padEnd(width)} ${formatDownloadSize(model2.approxSizeBytes).padStart(7)} ${FIT_LABEL[model2.fit].padEnd(22)}${claim} ${model2.name}${notes.length > 0 ? ` (${notes.join(", ")})` : ""}`
|
|
2971
|
-
);
|
|
2972
|
-
});
|
|
2973
|
-
const flag = providerFlag(engine, input.platformDefault);
|
|
2974
|
-
const smallest = [...ranked].sort((a, b) => a.approxSizeBytes - b.approxSizeBytes)[0];
|
|
2975
|
-
lines.push(
|
|
2976
|
-
"",
|
|
2977
|
-
`Download one with \`gezel model pull <id>${flag}\`. The smallest is ${smallest.id} (${formatDownloadSize(smallest.approxSizeBytes)}).`,
|
|
2978
|
-
`\`gezel model use <id>${flag}\` picks the installed model chats use.`
|
|
2979
|
-
);
|
|
2980
|
-
if (measured) {
|
|
2981
|
-
lines.push(
|
|
2982
|
-
'"Fits" is about memory. The measured column is the published scorecard\'s pass rate on real jobs, on the hardware named; `gezel bench` measures a model on this machine.'
|
|
2983
|
-
);
|
|
2984
|
-
}
|
|
2985
|
-
return lines.join("\n");
|
|
2986
|
-
}
|
|
2987
|
-
function measuredClaimsFor(dataset, ids, device) {
|
|
2988
|
-
const claims = /* @__PURE__ */ new Map();
|
|
2989
|
-
for (const id of ids) claims.set(id, measuredClaim2(measuredResultFor2(dataset, id, device)));
|
|
2990
|
-
return claims;
|
|
2991
|
-
}
|
|
2992
|
-
async function useChatModel(client, id, options) {
|
|
2993
|
-
const order = options.requested ? [options.requested] : [options.platformDefault, ...["mlx", "llama-cpp", "ds4"]].filter(
|
|
2994
|
-
(engine2, index, all) => all.indexOf(engine2) === index
|
|
2995
|
-
);
|
|
2996
|
-
let engine;
|
|
2997
|
-
for (const candidate of order) {
|
|
2998
|
-
const installed = await installedChatModels(client, candidate).catch(() => []);
|
|
2999
|
-
if (installed.includes(id)) {
|
|
3000
|
-
engine = candidate;
|
|
3001
|
-
break;
|
|
3002
|
-
}
|
|
3003
|
-
}
|
|
3004
|
-
if (!engine) {
|
|
3005
|
-
const target = options.requested ?? options.platformDefault;
|
|
3006
|
-
throw new CliError(
|
|
3007
|
-
`${id} isn't downloaded for ${target}. Download it with \`gezel model pull ${id}${providerFlag(target, options.platformDefault)}\`, or see \`gezel model list --available\`.`
|
|
3008
|
-
);
|
|
3009
|
-
}
|
|
3010
|
-
const config = await client.getConfig();
|
|
3011
|
-
await client.updateConfig({
|
|
3012
|
-
provider: engine,
|
|
3013
|
-
defaultModel: { ...config.defaultModel ?? {}, [engine]: id },
|
|
3014
|
-
firstRunCompleted: true
|
|
3015
|
-
});
|
|
3016
|
-
return engine;
|
|
3017
|
-
}
|
|
3018
|
-
async function pinPulledModel(client, engine, id) {
|
|
3019
|
-
const config = await client.getConfig();
|
|
3020
|
-
const current = config.defaultModel?.[engine];
|
|
3021
|
-
if (current === id) return "already";
|
|
3022
|
-
if (current && (await installedChatModels(client, engine)).includes(current)) return "kept";
|
|
3023
|
-
await client.updateConfig({
|
|
3024
|
-
defaultModel: { ...config.defaultModel ?? {}, [engine]: id },
|
|
3025
|
-
firstRunCompleted: true
|
|
3026
|
-
});
|
|
3027
|
-
return "pinned";
|
|
3028
|
-
}
|
|
3029
|
-
|
|
3030
3249
|
// src/model-export.ts
|
|
3031
3250
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
3032
3251
|
import { rename, rm as rm2, stat } from "fs/promises";
|
|
@@ -3316,95 +3535,6 @@ async function resolveRunPrompt(parts, input = process.stdin) {
|
|
|
3316
3535
|
return prompt;
|
|
3317
3536
|
}
|
|
3318
3537
|
|
|
3319
|
-
// src/run-readiness.ts
|
|
3320
|
-
import { resolveOnDeviceProvider as resolveOnDeviceProvider2 } from "@bendyline/gezel/native";
|
|
3321
|
-
var READY = { ready: true };
|
|
3322
|
-
async function settledFirstRunConfig(client, options = {}) {
|
|
3323
|
-
const attempts = options.attempts ?? 12;
|
|
3324
|
-
const delayMs = options.delayMs ?? 250;
|
|
3325
|
-
let config = await client.getConfig();
|
|
3326
|
-
for (let attempt = 0; attempt < attempts; attempt += 1) {
|
|
3327
|
-
if (config.firstRunCompleted || config.provider !== "copilot") return config;
|
|
3328
|
-
await new Promise((resolve7) => setTimeout(resolve7, delayMs));
|
|
3329
|
-
config = await client.getConfig();
|
|
3330
|
-
}
|
|
3331
|
-
return config;
|
|
3332
|
-
}
|
|
3333
|
-
async function checkRunReadiness(client, gezelId, env2 = process.env) {
|
|
3334
|
-
if (env2.GEZEL_MOCK_PROVIDER === "1") return READY;
|
|
3335
|
-
const config = await settledFirstRunConfig(client);
|
|
3336
|
-
let provider = config.provider;
|
|
3337
|
-
if (gezelId) {
|
|
3338
|
-
try {
|
|
3339
|
-
const { parsed } = await client.getGezel(gezelId);
|
|
3340
|
-
if (parsed.frontmatter.provider) provider = parsed.frontmatter.provider;
|
|
3341
|
-
} catch {
|
|
3342
|
-
}
|
|
3343
|
-
}
|
|
3344
|
-
if (provider !== "llama-cpp" && provider !== "mlx") return READY;
|
|
3345
|
-
if (provider === "llama-cpp" && (config.llamaCppBaseUrl || config.llamaCppModelPath)) {
|
|
3346
|
-
return READY;
|
|
3347
|
-
}
|
|
3348
|
-
if (provider === "mlx" && (config.mlxBaseUrl || config.mlxModelPath)) return READY;
|
|
3349
|
-
const native2 = await client.getNativeEngineStatus();
|
|
3350
|
-
if (!native2.platformKey) return READY;
|
|
3351
|
-
const installed = provider === "mlx" ? await client.listMlxModels() : await client.listLlamaCppModels();
|
|
3352
|
-
const installedIds = installed.models.map((model2) => model2.id);
|
|
3353
|
-
const pinned = config.defaultModel?.[provider];
|
|
3354
|
-
const engineMissing = provider === "llama-cpp" && !native2.engines.some((engine) => engine.name === "llama-server" && engine.installed);
|
|
3355
|
-
const modelReady = pinned ? installedIds.includes(pinned) : installedIds.length > 0;
|
|
3356
|
-
if (!engineMissing && modelReady) return READY;
|
|
3357
|
-
return {
|
|
3358
|
-
ready: false,
|
|
3359
|
-
message: formatRunSetupMessage({
|
|
3360
|
-
provider,
|
|
3361
|
-
engineMissing,
|
|
3362
|
-
pinned,
|
|
3363
|
-
installedIds
|
|
3364
|
-
})
|
|
3365
|
-
};
|
|
3366
|
-
}
|
|
3367
|
-
function formatRunSetupMessage(input) {
|
|
3368
|
-
const { provider, engineMissing, pinned, installedIds } = input;
|
|
3369
|
-
const flag = providerFlag(
|
|
3370
|
-
provider,
|
|
3371
|
-
input.platformDefault ?? resolveOnDeviceProvider2(process.platform, process.arch)
|
|
3372
|
-
);
|
|
3373
|
-
const steps = [];
|
|
3374
|
-
if (engineMissing) steps.push("gezel native install");
|
|
3375
|
-
if (pinned && !installedIds.includes(pinned)) {
|
|
3376
|
-
if (installedIds.length > 0) {
|
|
3377
|
-
steps.push(`gezel model use ${installedIds[0]}${flag} (already downloaded)`);
|
|
3378
|
-
}
|
|
3379
|
-
steps.push(`gezel model pull ${pinned}${flag}`);
|
|
3380
|
-
if (installedIds.length === 0) {
|
|
3381
|
-
steps.push(`gezel model list --available${flag} (to pick a smaller one)`);
|
|
3382
|
-
}
|
|
3383
|
-
} else if (!pinned && installedIds.length === 0) {
|
|
3384
|
-
steps.push(`gezel model list --available${flag} (then: gezel model pull <id>${flag})`);
|
|
3385
|
-
}
|
|
3386
|
-
const lines = [];
|
|
3387
|
-
if (pinned && !installedIds.includes(pinned) && installedIds.length > 0) {
|
|
3388
|
-
lines.push(`Your selected on-device model, ${pinned}, isn't downloaded yet.`);
|
|
3389
|
-
} else if (engineMissing && installedIds.length > 0 && (!pinned || installedIds.includes(pinned))) {
|
|
3390
|
-
lines.push("Gezel's on-device engine isn't installed yet.");
|
|
3391
|
-
} else {
|
|
3392
|
-
lines.push("Gezel isn't set up to chat yet: this device doesn't have an on-device model.");
|
|
3393
|
-
}
|
|
3394
|
-
lines.push("", "Run `gezel` to choose a model and download it, step by step.");
|
|
3395
|
-
if (steps.length > 0) {
|
|
3396
|
-
lines.push("", "Or set it up directly:", ...steps.map((step) => ` ${step}`));
|
|
3397
|
-
}
|
|
3398
|
-
lines.push(
|
|
3399
|
-
"",
|
|
3400
|
-
"To use a cloud model instead, save a key (for example",
|
|
3401
|
-
"`gezel secret set openaiApiKey --stdin`), then run `gezel` and pick it with /model.",
|
|
3402
|
-
"",
|
|
3403
|
-
"Nothing was downloaded."
|
|
3404
|
-
);
|
|
3405
|
-
return lines.join("\n");
|
|
3406
|
-
}
|
|
3407
|
-
|
|
3408
3538
|
// src/run-reply.ts
|
|
3409
3539
|
var NAME = "[A-Za-z_][A-Za-z0-9_.-]*";
|
|
3410
3540
|
var CLOSED_CALLS = [
|
|
@@ -3466,6 +3596,15 @@ var ReplyPrinter = class {
|
|
|
3466
3596
|
this.raw += chunk;
|
|
3467
3597
|
this.extend(visibleReply(this.raw, true));
|
|
3468
3598
|
}
|
|
3599
|
+
/**
|
|
3600
|
+
* A tool ran. Providers join the text of each tool round with nothing
|
|
3601
|
+
* between, so narration ran into the answer: "…move date.Lisbon office…".
|
|
3602
|
+
* Marked like a cut-out call, it becomes a paragraph break only if text
|
|
3603
|
+
* follows, and nothing at all before the reply or at its end.
|
|
3604
|
+
*/
|
|
3605
|
+
toolCalled() {
|
|
3606
|
+
this.raw += CUT;
|
|
3607
|
+
}
|
|
3469
3608
|
/**
|
|
3470
3609
|
* End of the reply. `committed` is the saved message, printed when the
|
|
3471
3610
|
* stream carried no visible text (providers that send it whole). Returns
|
|
@@ -3508,6 +3647,9 @@ var RunOutput = class {
|
|
|
3508
3647
|
push(chunk) {
|
|
3509
3648
|
this.current.push(chunk);
|
|
3510
3649
|
}
|
|
3650
|
+
toolCalled() {
|
|
3651
|
+
this.current.toolCalled();
|
|
3652
|
+
}
|
|
3511
3653
|
/** One assistant message was saved; later deltas belong to the next one. */
|
|
3512
3654
|
complete(committed) {
|
|
3513
3655
|
this.current.finish(committed);
|
|
@@ -3634,6 +3776,50 @@ function registerSecretCommands(program2, connect, output = console.log) {
|
|
|
3634
3776
|
});
|
|
3635
3777
|
}
|
|
3636
3778
|
|
|
3779
|
+
// src/start-reuse.ts
|
|
3780
|
+
var SIGN_IN_STATE_TIMEOUT_MS = 3e3;
|
|
3781
|
+
async function startShortfall(runtime, want, deps) {
|
|
3782
|
+
if (want.port !== void 0 && runtime.port !== want.port) return "port";
|
|
3783
|
+
if (!want.web) return null;
|
|
3784
|
+
if (new URL(runtime.baseUrl).protocol !== "http:") return "web";
|
|
3785
|
+
if ((await deps.readWebUiToken()).trim().length === 0) return "web";
|
|
3786
|
+
if (want.accounts && !await requiresAccounts(runtime.baseUrl, deps.fetch ?? fetch)) {
|
|
3787
|
+
return "accounts";
|
|
3788
|
+
}
|
|
3789
|
+
return null;
|
|
3790
|
+
}
|
|
3791
|
+
async function requiresAccounts(baseUrl, fetchImpl) {
|
|
3792
|
+
const controller = new AbortController();
|
|
3793
|
+
const timer = setTimeout(() => controller.abort(), SIGN_IN_STATE_TIMEOUT_MS);
|
|
3794
|
+
timer.unref?.();
|
|
3795
|
+
try {
|
|
3796
|
+
const res = await fetchImpl(`${new URL(baseUrl).origin}/v1/apps/signin/state`, {
|
|
3797
|
+
signal: controller.signal
|
|
3798
|
+
});
|
|
3799
|
+
if (!res.ok) return false;
|
|
3800
|
+
const state = await res.json();
|
|
3801
|
+
return state.accountsRequired === true;
|
|
3802
|
+
} catch {
|
|
3803
|
+
return false;
|
|
3804
|
+
} finally {
|
|
3805
|
+
clearTimeout(timer);
|
|
3806
|
+
}
|
|
3807
|
+
}
|
|
3808
|
+
function startShortfallMessage(shortfall, running, want) {
|
|
3809
|
+
const has = shortfall === "port" ? `on port ${running.port}` : shortfall === "web" ? "without the web UI" : "with the web UI but without accounts";
|
|
3810
|
+
const retry = ["gezel start"];
|
|
3811
|
+
if (want.port !== void 0) retry.push(`--port ${want.port}`);
|
|
3812
|
+
if (want.web) retry.push("--web");
|
|
3813
|
+
if (want.accounts) retry.push("--accounts");
|
|
3814
|
+
retry.push("--restart");
|
|
3815
|
+
return `gezeld is already running for this home ${has} (pid ${running.pid}).
|
|
3816
|
+
Restart it ${startRestartWanted(want)} with \`${retry.join(" ")}\`, or stop it first with \`gezel stop\`.`;
|
|
3817
|
+
}
|
|
3818
|
+
function startRestartWanted(want) {
|
|
3819
|
+
if (want.accounts) return "with the web UI and accounts";
|
|
3820
|
+
return want.web ? "with the web UI" : `on port ${want.port}`;
|
|
3821
|
+
}
|
|
3822
|
+
|
|
3637
3823
|
// src/storage-format.ts
|
|
3638
3824
|
function formatBytes2(bytes) {
|
|
3639
3825
|
if (bytes >= 1024 ** 3) return `${(bytes / 1024 ** 3).toFixed(1)} GB`;
|
|
@@ -4015,6 +4201,9 @@ program.name("gezel").description("Gezel \u2014 assemble a team of AI companions
|
|
|
4015
4201
|
).option("--home <dir>", "Use this user-owned Gezel home (default: $GEZEL_HOME or ~/.gezel).").option(
|
|
4016
4202
|
"--project [folder]",
|
|
4017
4203
|
"Use this folder as the active project (default and bare flag: cwd)."
|
|
4204
|
+
).option(
|
|
4205
|
+
"--download",
|
|
4206
|
+
"Download what a command needs (the on-device engine and the recommended model) without asking, printing each step. Also GEZEL_AUTO_DOWNLOAD=1."
|
|
4018
4207
|
);
|
|
4019
4208
|
var cliGlobals = () => program.opts();
|
|
4020
4209
|
var connectForInspection = (opts) => opts.start === false ? connectRunning(cliGlobals()) : connectOwned(cliGlobals());
|
|
@@ -4037,11 +4226,19 @@ function builtinCommandNames() {
|
|
|
4037
4226
|
var shorthandTail = [];
|
|
4038
4227
|
async function resolveCraftbookShorthand(parts) {
|
|
4039
4228
|
const word = parts[0] ?? "";
|
|
4040
|
-
const
|
|
4229
|
+
const builtins = builtinCommandNames();
|
|
4230
|
+
const unknown = unknownCommandMessage(parts, builtins);
|
|
4231
|
+
let running = null;
|
|
4232
|
+
if (closestCommand(word, builtins)) {
|
|
4233
|
+
running = await connectRunning(cliGlobals()).catch(() => null);
|
|
4234
|
+
const project = cliGlobals().project;
|
|
4235
|
+
const folder = resolvePath(typeof project === "string" ? project : process.cwd());
|
|
4236
|
+
if (!running && !await localCraftbookNamed(word, folder)) throw new CliError(unknown);
|
|
4237
|
+
}
|
|
4041
4238
|
let client;
|
|
4042
4239
|
let id;
|
|
4043
4240
|
try {
|
|
4044
|
-
client = await connectOwned(cliGlobals());
|
|
4241
|
+
client = running ?? await connectOwned(cliGlobals());
|
|
4045
4242
|
const preview = await previewRunProject(client, cliGlobals());
|
|
4046
4243
|
const [config, items] = await Promise.all([
|
|
4047
4244
|
client.getConfig(),
|
|
@@ -4135,18 +4332,18 @@ program.command("start").description("Start the Gezel daemon (no-op if already r
|
|
|
4135
4332
|
if (port !== void 0 || opts.web) {
|
|
4136
4333
|
const runtime = await readRuntime2();
|
|
4137
4334
|
const running = runtime && isProcessAlive2(runtime.pid) ? runtime : null;
|
|
4138
|
-
const
|
|
4139
|
-
|
|
4140
|
-
|
|
4335
|
+
const want = {
|
|
4336
|
+
port,
|
|
4337
|
+
web: opts.web === true,
|
|
4338
|
+
accounts: opts.web === true && opts.accounts === true
|
|
4339
|
+
};
|
|
4340
|
+
const shortfall = running ? await startShortfall(running, want, { readWebUiToken }) : null;
|
|
4341
|
+
const reuse = running !== null && shortfall === null;
|
|
4342
|
+
if (running && shortfall) {
|
|
4141
4343
|
if (!opts.restart) {
|
|
4142
|
-
|
|
4143
|
-
if (opts.web) retry.push("--web");
|
|
4144
|
-
throw new CliError(
|
|
4145
|
-
`gezeld is already running for this home ${opts.web ? "without the web UI" : `on port ${running.port}`} (pid ${running.pid}).
|
|
4146
|
-
Restart it ${wanted} with \`${[...retry, "--restart"].join(" ")}\`, or stop it first with \`gezel stop\`.`
|
|
4147
|
-
);
|
|
4344
|
+
throw new CliError(startShortfallMessage(shortfall, running, want));
|
|
4148
4345
|
}
|
|
4149
|
-
console.error(`Restarting gezeld (pid ${running.pid}) ${
|
|
4346
|
+
console.error(`Restarting gezeld (pid ${running.pid}) ${startRestartWanted(want)}\u2026`);
|
|
4150
4347
|
if (!await stopDaemonPid(running.pid)) {
|
|
4151
4348
|
throw new CliError(`failed to confirm gezeld pid=${running.pid} stopped`);
|
|
4152
4349
|
}
|
|
@@ -4156,7 +4353,7 @@ Restart it ${wanted} with \`${[...retry, "--restart"].join(" ")}\`, or stop it f
|
|
|
4156
4353
|
daemonEntry: entry,
|
|
4157
4354
|
detached: true,
|
|
4158
4355
|
env: spawnEnv,
|
|
4159
|
-
// Never adopt a daemon that lacks the requested port or
|
|
4356
|
+
// Never adopt a daemon that lacks the requested port, transport or accounts.
|
|
4160
4357
|
// A forced spawn accepts only its own process, so losing a race
|
|
4161
4358
|
// fails here instead of reporting someone else's daemon.
|
|
4162
4359
|
forceSpawn: !reuse,
|
|
@@ -4183,14 +4380,10 @@ Restart it ${wanted} with \`${[...retry, "--restart"].join(" ")}\`, or stop it f
|
|
|
4183
4380
|
console.log(`gezeld running (version ${health.version})`);
|
|
4184
4381
|
}
|
|
4185
4382
|
);
|
|
4186
|
-
async function
|
|
4187
|
-
if (want.port !== void 0 && runtime.port !== want.port) return false;
|
|
4188
|
-
if (!want.web) return true;
|
|
4189
|
-
if (new URL(runtime.baseUrl).protocol !== "http:") return false;
|
|
4383
|
+
async function readWebUiToken() {
|
|
4190
4384
|
const { readFile: readFile5 } = await import("fs/promises");
|
|
4191
4385
|
const { gezelPaths: gezelPaths2 } = await import("@bendyline/gezel/paths");
|
|
4192
|
-
|
|
4193
|
-
return token.trim().length > 0;
|
|
4386
|
+
return readFile5(gezelPaths2().runtime.webUiToken, "utf8").catch(() => "");
|
|
4194
4387
|
}
|
|
4195
4388
|
async function printAccountsWebUrl(client, baseUrl, open) {
|
|
4196
4389
|
const origin = new URL(baseUrl).origin;
|
|
@@ -4522,7 +4715,9 @@ Examples:
|
|
|
4522
4715
|
gezel run - < prompt.txt
|
|
4523
4716
|
|
|
4524
4717
|
A sole - reads UTF-8 stdin to EOF. Empty input is rejected.
|
|
4525
|
-
Replies go to stdout; diagnostics go to stderr
|
|
4718
|
+
Replies go to stdout; diagnostics go to stderr.
|
|
4719
|
+
With no model set up yet, a run in a terminal first offers to download one,
|
|
4720
|
+
then runs the prompt.`
|
|
4526
4721
|
).action(async (promptParts, opts) => {
|
|
4527
4722
|
const prompt = await resolveRunPrompt(promptParts);
|
|
4528
4723
|
const previousLogOutput = getLogOutput2();
|
|
@@ -4543,7 +4738,28 @@ Replies go to stdout; diagnostics go to stderr.`
|
|
|
4543
4738
|
const { client } = conn;
|
|
4544
4739
|
const projectId = await resolveRunProject(client, cliGlobals());
|
|
4545
4740
|
const gezelId = opts.gezel ? await resolveRunGezel(client, opts.gezel) : await ensureCliProjectLead(client, projectId);
|
|
4546
|
-
|
|
4741
|
+
let readiness = await checkRunReadiness(client, gezelId);
|
|
4742
|
+
if (!readiness.ready && autoDownloadRequested(cliGlobals())) {
|
|
4743
|
+
await downloadForRun(client, { gezelId, write: (text) => process.stderr.write(text) });
|
|
4744
|
+
readiness = await checkRunReadiness(client, gezelId);
|
|
4745
|
+
} else if (!readiness.ready && canOfferRunSetup()) {
|
|
4746
|
+
const tuiModule = "../tui/index.js";
|
|
4747
|
+
const { launchRunSetup } = await import(tuiModule);
|
|
4748
|
+
const runLogLevel = getLogLevel2();
|
|
4749
|
+
if (!process.env.GEZEL_LOG_LEVEL) setLogLevel2("error");
|
|
4750
|
+
let finished;
|
|
4751
|
+
try {
|
|
4752
|
+
finished = await launchRunSetup({ client });
|
|
4753
|
+
} finally {
|
|
4754
|
+
setLogLevel2(runLogLevel);
|
|
4755
|
+
}
|
|
4756
|
+
if (!finished) {
|
|
4757
|
+
console.error("Setup stopped. Your request was not run.");
|
|
4758
|
+
process.exitCode = 130;
|
|
4759
|
+
return;
|
|
4760
|
+
}
|
|
4761
|
+
readiness = await checkRunReadiness(client, gezelId);
|
|
4762
|
+
}
|
|
4547
4763
|
if (!readiness.ready) {
|
|
4548
4764
|
console.error(readiness.message);
|
|
4549
4765
|
process.exitCode = 1;
|
|
@@ -4564,6 +4780,8 @@ Replies go to stdout; diagnostics go to stderr.`
|
|
|
4564
4780
|
} else if (ev.type === "delta") {
|
|
4565
4781
|
progress.settle();
|
|
4566
4782
|
reply.push(ev.content);
|
|
4783
|
+
} else if (ev.type === "tool") {
|
|
4784
|
+
reply.toolCalled();
|
|
4567
4785
|
} else if (ev.type === "complete") {
|
|
4568
4786
|
progress.settle();
|
|
4569
4787
|
reply.complete(ev.message.content);
|
|
@@ -4639,15 +4857,22 @@ Examples:
|
|
|
4639
4857
|
`);
|
|
4640
4858
|
}
|
|
4641
4859
|
const parsed = parseCraftbookArguments(craftbook2, args, opts.param);
|
|
4642
|
-
const
|
|
4643
|
-
|
|
4644
|
-
|
|
4645
|
-
|
|
4646
|
-
|
|
4647
|
-
|
|
4648
|
-
|
|
4649
|
-
|
|
4650
|
-
|
|
4860
|
+
const autoDownload = autoDownloadRequested(cliGlobals());
|
|
4861
|
+
const setUp = async () => {
|
|
4862
|
+
if (autoDownload && !(await checkRunReadiness(client, void 0)).ready) {
|
|
4863
|
+
await downloadForRun(client, { write: stderrProgress() });
|
|
4864
|
+
}
|
|
4865
|
+
await ensureCraftbookSetup(client, {
|
|
4866
|
+
projectId,
|
|
4867
|
+
label: craftbook2.name,
|
|
4868
|
+
needs: craftbook2,
|
|
4869
|
+
params: parsed.params,
|
|
4870
|
+
...craftbook2.paramSchema ? { paramSchema: craftbook2.paramSchema } : {},
|
|
4871
|
+
prompter: terminalPrompter(),
|
|
4872
|
+
autoDownload,
|
|
4873
|
+
write: stderrProgress()
|
|
4874
|
+
});
|
|
4875
|
+
};
|
|
4651
4876
|
if (craftbook2.cliWorkflow) {
|
|
4652
4877
|
if (book.source !== "project")
|
|
4653
4878
|
throw new CliError("CLI workflow craftbooks must belong to this project.");
|
|
@@ -4950,6 +5175,9 @@ handboek.command("export").description("Render the Handboek as a static HTML sit
|
|
|
4950
5175
|
).option(
|
|
4951
5176
|
"--apps-out <dir>",
|
|
4952
5177
|
"also write the apps library (gezel.com/apps) into this directory, outside --out"
|
|
5178
|
+
).option(
|
|
5179
|
+
"--open-in-gezel",
|
|
5180
|
+
'add gezel:// "Open in Gezel" buttons to craftbook and app pages (needs a desktop release that answers them)'
|
|
4953
5181
|
).action(
|
|
4954
5182
|
async (opts) => {
|
|
4955
5183
|
if (Boolean(opts.ogOut) !== Boolean(opts.ogBase)) {
|
|
@@ -4966,7 +5194,8 @@ handboek.command("export").description("Render the Handboek as a static HTML sit
|
|
|
4966
5194
|
canonicalBase: opts.canonicalBase,
|
|
4967
5195
|
ogOut: opts.ogOut,
|
|
4968
5196
|
ogBase: opts.ogBase,
|
|
4969
|
-
appsOut: opts.appsOut
|
|
5197
|
+
appsOut: opts.appsOut,
|
|
5198
|
+
openInGezel: opts.openInGezel === true
|
|
4970
5199
|
});
|
|
4971
5200
|
console.log(`Handboek exported: ${result.pages} pages \u2192 ${result.out}`);
|
|
4972
5201
|
if (result.apps) {
|
|
@@ -5212,6 +5441,13 @@ appCmd.command("demo <folder>").description("Export an offline page demo with sh
|
|
|
5212
5441
|
const { runAppDemo } = await loadAppAuthorCommand();
|
|
5213
5442
|
await runAppDemo(folder, opts);
|
|
5214
5443
|
});
|
|
5444
|
+
appCmd.command("guide").description(
|
|
5445
|
+
"Print the complete guide to building a Gezel app, for this version (give it to your AI agent)"
|
|
5446
|
+
).action(async () => {
|
|
5447
|
+
const appGuideModule = "../app-guide.js";
|
|
5448
|
+
const { runAppGuide } = await import(appGuideModule);
|
|
5449
|
+
await runAppGuide();
|
|
5450
|
+
});
|
|
5215
5451
|
appCmd.command("schemas").description("The JSON Schemas an AI App author codes against, rendered from this build").option("--out <dir>", "write the schema files into a directory").option("--json", "print one combined JSON object").action(async (opts) => {
|
|
5216
5452
|
const { runAppSchemas } = await loadAppAuthorCommand();
|
|
5217
5453
|
await runAppSchemas(opts);
|
|
@@ -5421,7 +5657,7 @@ program.command("create-audio [text...]").description("Synthesize speech from te
|
|
|
5421
5657
|
var native = program.command("native").description("Manage native engine binaries");
|
|
5422
5658
|
native.command("install").description(
|
|
5423
5659
|
"Download, verify, and activate the native inference toolkit (or one engine with --engine)"
|
|
5424
|
-
).option("--variant <backend>", "llama.cpp backend: cuda | vulkan | metal | cpu").option("--engine <name>", "install only this engine, e.g. duckdb or llama-server").action(async (opts) => {
|
|
5660
|
+
).option("--variant <backend>", "llama.cpp backend: cuda | rocm | vulkan | metal | cpu").option("--engine <name>", "install only this engine, e.g. duckdb or llama-server").action(async (opts) => {
|
|
5425
5661
|
const variant = parseNativeVariant(opts.variant);
|
|
5426
5662
|
const engine = parseNativeEngine(opts.engine);
|
|
5427
5663
|
const client = await connectOwned(cliGlobals());
|