@bendyline/gezel-cli 1.2.13 → 1.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/dist/app-author-command.js +2 -0
- package/dist/app-guide.js +128 -0
- package/dist/bin/gezel-main.js +1782 -1636
- package/dist/handboek-export.js +175 -53
- package/dist/tui/index.js +23 -7
- package/package.json +9 -9
package/dist/bin/gezel-main.js
CHANGED
|
@@ -952,1573 +952,1859 @@ function registerIntegrationCommands(program2, connect, output = console.log) {
|
|
|
952
952
|
});
|
|
953
953
|
}
|
|
954
954
|
|
|
955
|
-
// src/
|
|
956
|
-
import {
|
|
957
|
-
import { readFileSync } from "fs";
|
|
958
|
-
import { readFile as readFile2, writeFile as writeFile2 } from "fs/promises";
|
|
959
|
-
import { cpus } from "os";
|
|
960
|
-
import { join as join2 } from "path";
|
|
961
|
-
import { promisify } from "util";
|
|
962
|
-
import { ScorecardDatasetSchema } from "@bendyline/gezel";
|
|
963
|
-
import { gildePackageRoot } from "@bendyline/gezel-catalog";
|
|
955
|
+
// src/craftbook-setup.ts
|
|
956
|
+
import { Writable } from "stream";
|
|
964
957
|
import {
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
958
|
+
craftbookSetupGaps,
|
|
959
|
+
isCraftbookModelEngine,
|
|
960
|
+
resolveCraftbookModelNeeds,
|
|
961
|
+
resolveSecurityPolicy as resolveSecurityPolicy2
|
|
962
|
+
} from "@bendyline/gezel";
|
|
963
|
+
import { resolveOnDeviceProvider } from "@bendyline/gezel/native";
|
|
964
|
+
|
|
965
|
+
// src/model-pull.ts
|
|
966
|
+
async function pullChatModel(client, engine, id, writeProgress) {
|
|
967
|
+
let lastPct = -1;
|
|
968
|
+
let pullError;
|
|
969
|
+
const onEvent = (ev) => {
|
|
970
|
+
if (ev.type === "progress") {
|
|
971
|
+
const written = "bytesWrittenAll" in ev ? ev.bytesWrittenAll : ev.bytesWritten;
|
|
972
|
+
const total = "totalBytesAll" in ev ? ev.totalBytesAll : ev.totalBytes ?? 0;
|
|
973
|
+
const pct = total > 0 ? Math.floor(written / total * 100) : 0;
|
|
974
|
+
if (pct === lastPct) return;
|
|
975
|
+
lastPct = pct;
|
|
976
|
+
writeProgress(
|
|
977
|
+
`\rdownloading ${id} (${engine}): ${String(pct).padStart(3)}% ${formatGb(written)}/${formatGb(total)} GB`
|
|
978
|
+
);
|
|
979
|
+
} else if (ev.type === "retrying") {
|
|
980
|
+
writeProgress(`
|
|
981
|
+
retry ${ev.attempt}/${ev.maxAttempts}: ${ev.reason}
|
|
982
|
+
`);
|
|
983
|
+
} else if (ev.type === "error") {
|
|
984
|
+
pullError = ev.error;
|
|
985
|
+
writeProgress("\n");
|
|
986
|
+
} else if (ev.type === "done" && !pullError) {
|
|
987
|
+
writeProgress(`\rdownloaded ${id} (${engine})${" ".repeat(48)}
|
|
988
|
+
`);
|
|
989
|
+
}
|
|
990
|
+
};
|
|
991
|
+
if (engine === "mlx") await client.installMlxModel(id, onEvent);
|
|
992
|
+
else if (engine === "ds4") await client.installDs4Model(id, onEvent);
|
|
993
|
+
else await client.installLlamaCppModel(id, onEvent);
|
|
994
|
+
if (pullError) throw new CliError(`download failed: ${pullError}`);
|
|
990
995
|
}
|
|
991
|
-
function
|
|
992
|
-
const
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
`Unknown suite "${id}". Choose one of: ${SCORECARD_SUITE_IDS.join(", ")}, or all. \`gezel bench list\` shows every suite.`
|
|
996
|
+
async function waitForMlxRuntime(client, writeProgress, options = {}) {
|
|
997
|
+
const pollMs = options.pollMs ?? 1e3;
|
|
998
|
+
const now = options.now ?? Date.now;
|
|
999
|
+
const started = now();
|
|
1000
|
+
let shown = false;
|
|
1001
|
+
for (; ; ) {
|
|
1002
|
+
const status = await client.getMlxRuntimeStatus();
|
|
1003
|
+
if (status.phase === "provisioning") {
|
|
1004
|
+
const elapsed = Math.round((now() - started) / 1e3);
|
|
1005
|
+
writeProgress(
|
|
1006
|
+
`\rpreparing the MLX Python runtime (one-time, a few minutes): ${Math.floor(elapsed / 60)}m ${String(elapsed % 60).padStart(2, "0")}s`
|
|
1003
1007
|
);
|
|
1008
|
+
shown = true;
|
|
1009
|
+
await new Promise((resolve7) => setTimeout(resolve7, pollMs));
|
|
1010
|
+
continue;
|
|
1004
1011
|
}
|
|
1005
|
-
if (
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
suites.push(smoke);
|
|
1012
|
-
} else {
|
|
1013
|
-
suites.push(base);
|
|
1012
|
+
if (status.phase === "error") {
|
|
1013
|
+
writeProgress(
|
|
1014
|
+
`${shown ? "\n" : ""}MLX runtime setup failed; the first chat will retry it: ${status.error ?? "unknown error"}
|
|
1015
|
+
`
|
|
1016
|
+
);
|
|
1017
|
+
return "error";
|
|
1014
1018
|
}
|
|
1019
|
+
if (shown) writeProgress(`\rMLX runtime ready${" ".repeat(56)}
|
|
1020
|
+
`);
|
|
1021
|
+
return status.phase === "ready" ? "ready" : "idle";
|
|
1015
1022
|
}
|
|
1016
|
-
return suites;
|
|
1017
|
-
}
|
|
1018
|
-
function resolveBenchTarget(catalog, targets, opts) {
|
|
1019
|
-
const available = targets.filter((t) => t.available);
|
|
1020
|
-
if (opts.model) {
|
|
1021
|
-
const matches = available.filter(
|
|
1022
|
-
(t) => t.modelId === opts.model && (!opts.provider || t.provider === opts.provider)
|
|
1023
|
-
);
|
|
1024
|
-
const preferred = matches.find((t) => t.provider === catalog.defaultProvider) ?? matches[0];
|
|
1025
|
-
if (preferred) return preferred;
|
|
1026
|
-
const unavailable = targets.find(
|
|
1027
|
-
(t) => t.modelId === opts.model && (!opts.provider || t.provider === opts.provider)
|
|
1028
|
-
);
|
|
1029
|
-
throw new CliError(
|
|
1030
|
-
unavailable?.unavailableReason ? `${opts.model} can't run here: ${unavailable.unavailableReason}` : `No model "${opts.model}"${opts.provider ? ` on ${opts.provider}` : ""} is available to evaluate. \`gezel model list\` shows what is installed.`
|
|
1031
|
-
);
|
|
1032
|
-
}
|
|
1033
|
-
const byProvider = opts.provider ? available.filter((t) => t.provider === opts.provider) : available;
|
|
1034
|
-
const chosen = byProvider.find((t) => t.isDefault && t.provider === catalog.defaultProvider) ?? byProvider.find((t) => t.isDefault) ?? byProvider.find((t) => t.provider === catalog.defaultProvider) ?? byProvider[0];
|
|
1035
|
-
if (!chosen) {
|
|
1036
|
-
throw new CliError(
|
|
1037
|
-
"No model is available to evaluate. Download one with `gezel model pull <id>` or sign in to a cloud provider, then try again."
|
|
1038
|
-
);
|
|
1039
|
-
}
|
|
1040
|
-
return chosen;
|
|
1041
1023
|
}
|
|
1042
|
-
function
|
|
1043
|
-
|
|
1044
|
-
if (!Number.isInteger(count) || count < 1 || count > 10) {
|
|
1045
|
-
throw new CliError(
|
|
1046
|
-
"--count must be a whole number from 1 to 10 (3 is what it takes to quote a rate)."
|
|
1047
|
-
);
|
|
1048
|
-
}
|
|
1049
|
-
const suites = resolveBenchSuites(catalog, opts);
|
|
1050
|
-
const target = resolveBenchTarget(catalog, targets, opts);
|
|
1051
|
-
const worstCaseMs = suites.reduce((sum, s) => sum + s.authoredCeilingMs * count, 0);
|
|
1052
|
-
return { suites, target, count, worstCaseMs, smoke: Boolean(opts.smoke) };
|
|
1024
|
+
function formatGb(bytes) {
|
|
1025
|
+
return (bytes / 1e9).toFixed(2);
|
|
1053
1026
|
}
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1027
|
+
|
|
1028
|
+
// src/native-command.ts
|
|
1029
|
+
import {
|
|
1030
|
+
NativeEngineNameSchema
|
|
1031
|
+
} from "@bendyline/gezel";
|
|
1032
|
+
|
|
1033
|
+
// src/tui/bootstrap.ts
|
|
1034
|
+
import {
|
|
1035
|
+
computeModelFit,
|
|
1036
|
+
formatModelAttribution,
|
|
1037
|
+
isMoEFromTags,
|
|
1038
|
+
isRecommendedModel,
|
|
1039
|
+
lowBitScoreFactor,
|
|
1040
|
+
pickRecommendedModel
|
|
1041
|
+
} from "@bendyline/gezel";
|
|
1042
|
+
var CHAT_MEMORY_OVERHEAD = {
|
|
1043
|
+
"llama-cpp": 1.2,
|
|
1044
|
+
mlx: 1.3
|
|
1045
|
+
};
|
|
1046
|
+
var NATIVE_TOOLKIT = [
|
|
1047
|
+
"uv",
|
|
1048
|
+
"sd-server",
|
|
1049
|
+
"whisper-server",
|
|
1050
|
+
"llama-server"
|
|
1051
|
+
];
|
|
1052
|
+
function recoDevice(memory) {
|
|
1053
|
+
return {
|
|
1054
|
+
platform: memory.platform,
|
|
1055
|
+
gpuVramBytes: memory.gpuVramBytes,
|
|
1056
|
+
...memory.gpuMemoryKind ? { gpuMemoryKind: memory.gpuMemoryKind } : {},
|
|
1057
|
+
totalRamBytes: memory.totalRamBytes,
|
|
1058
|
+
usableBytes: memory.usableBytes,
|
|
1059
|
+
...memory.budgetBytes !== void 0 ? { budgetBytes: memory.budgetBytes } : {}
|
|
1060
|
+
};
|
|
1071
1061
|
}
|
|
1072
|
-
function
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1062
|
+
function chatModelCandidates(items, device, provider) {
|
|
1063
|
+
const candidates = [];
|
|
1064
|
+
for (const item of items) {
|
|
1065
|
+
if (item.manifest.kind !== "chat-model") continue;
|
|
1066
|
+
const manifest = item.manifest;
|
|
1067
|
+
if (manifest.licenseClass !== "open") continue;
|
|
1068
|
+
const source = provider === "mlx" ? manifest.mlx : manifest.llamaCpp;
|
|
1069
|
+
if (!source || provider === "mlx" && manifest.mlx?.disabledReason) continue;
|
|
1070
|
+
const residentBytes = source.residentBytes ?? Math.round(source.approxSizeBytes * CHAT_MEMORY_OVERHEAD[provider]);
|
|
1071
|
+
const fit = computeModelFit({
|
|
1072
|
+
residentBytes,
|
|
1073
|
+
isMoE: isMoEFromTags(manifest.tags),
|
|
1074
|
+
usableBytes: device.usableBytes,
|
|
1075
|
+
totalRamBytes: device.totalRamBytes,
|
|
1076
|
+
gpuVramBytes: device.gpuVramBytes,
|
|
1077
|
+
...device.budgetBytes !== void 0 ? { admissibleBytes: device.budgetBytes } : {}
|
|
1078
|
+
}).tier;
|
|
1079
|
+
candidates.push({
|
|
1080
|
+
id: manifest.id,
|
|
1081
|
+
name: manifest.name,
|
|
1082
|
+
provider,
|
|
1083
|
+
approxSizeBytes: source.approxSizeBytes,
|
|
1084
|
+
residentBytes,
|
|
1085
|
+
// Effective score: a model the shared gate rejects (restricted license,
|
|
1086
|
+
// no tool support) sorts as unscored rather than leading the list.
|
|
1087
|
+
recoScore: isRecommendedModel(manifest) ? manifest.recoScore ?? 0 : 0,
|
|
1088
|
+
...source.quantization ? { quantization: source.quantization } : {},
|
|
1089
|
+
fit,
|
|
1090
|
+
attribution: formatModelAttribution(manifest)
|
|
1091
|
+
});
|
|
1084
1092
|
}
|
|
1093
|
+
return candidates;
|
|
1085
1094
|
}
|
|
1086
|
-
function
|
|
1087
|
-
const
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1095
|
+
function rankChatModels(items, device, provider) {
|
|
1096
|
+
const candidates = chatModelCandidates(items, device, provider);
|
|
1097
|
+
if (candidates.length === 0) return [];
|
|
1098
|
+
const picked = pickRecommendedModel(
|
|
1099
|
+
candidates.map((model2) => ({
|
|
1100
|
+
id: model2.id,
|
|
1101
|
+
...model2.recoScore > 0 ? { recoScore: model2.recoScore } : {},
|
|
1102
|
+
licenseClass: "open",
|
|
1103
|
+
tags: items.find((item) => item.manifest.kind === "chat-model" && item.manifest.id === model2.id)?.manifest?.tags ?? [],
|
|
1104
|
+
residentBytes: model2.residentBytes,
|
|
1105
|
+
...model2.quantization ? { quantization: model2.quantization } : {}
|
|
1106
|
+
})),
|
|
1107
|
+
device
|
|
1108
|
+
);
|
|
1109
|
+
const comfortable = candidates.filter(
|
|
1110
|
+
(model2) => model2.fit === "fits" || model2.fit === "fits-offload"
|
|
1111
|
+
);
|
|
1112
|
+
const pool = comfortable.length > 0 ? comfortable : candidates;
|
|
1113
|
+
const ranked = (model2) => model2.recoScore * lowBitScoreFactor(model2.quantization);
|
|
1114
|
+
pool.sort(
|
|
1115
|
+
(a, b) => ranked(b) - ranked(a) || fitOrder(a.fit) - fitOrder(b.fit) || b.residentBytes - a.residentBytes || a.name.localeCompare(b.name)
|
|
1116
|
+
);
|
|
1117
|
+
if (!picked) return pool;
|
|
1118
|
+
const bestIndex = pool.findIndex((model2) => model2.id === picked.id);
|
|
1119
|
+
if (bestIndex < 0) {
|
|
1120
|
+
const best = candidates.find((model2) => model2.id === picked.id);
|
|
1121
|
+
if (best) pool.unshift(best);
|
|
1122
|
+
return pool;
|
|
1098
1123
|
}
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
if (column && column.passRate === null && column.attributableTrials > 0) {
|
|
1103
|
-
lines.push(
|
|
1104
|
-
` A count, not a rate: ${minTrialsForRate} trials per task is the floor for quoting a percentage.`
|
|
1105
|
-
);
|
|
1124
|
+
if (bestIndex > 0) {
|
|
1125
|
+
const [best] = pool.splice(bestIndex, 1);
|
|
1126
|
+
if (best) pool.unshift(best);
|
|
1106
1127
|
}
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1128
|
+
return pool;
|
|
1129
|
+
}
|
|
1130
|
+
function formatDownloadSize(bytes) {
|
|
1131
|
+
if (bytes >= 1e9) return `${(bytes / 1e9).toFixed(1)} GB`;
|
|
1132
|
+
return `${Math.max(1, Math.round(bytes / 1e6))} MB`;
|
|
1133
|
+
}
|
|
1134
|
+
function fitOrder(fit) {
|
|
1135
|
+
switch (fit) {
|
|
1136
|
+
case "fits":
|
|
1137
|
+
return 0;
|
|
1138
|
+
case "fits-offload":
|
|
1139
|
+
return 1;
|
|
1140
|
+
case "tight":
|
|
1141
|
+
return 2;
|
|
1142
|
+
case "too-big":
|
|
1143
|
+
return 3;
|
|
1144
|
+
}
|
|
1145
|
+
}
|
|
1146
|
+
|
|
1147
|
+
// src/native-command.ts
|
|
1148
|
+
var NATIVE_VARIANTS = ["cuda", "rocm", "vulkan", "metal", "cpu"];
|
|
1149
|
+
function parseNativeVariant(value) {
|
|
1150
|
+
if (value === void 0) return void 0;
|
|
1151
|
+
if (NATIVE_VARIANTS.includes(value)) return value;
|
|
1152
|
+
throw new CliError(`--variant must be one of ${NATIVE_VARIANTS.join(", ")} (got "${value}")`);
|
|
1153
|
+
}
|
|
1154
|
+
function parseNativeEngine(value) {
|
|
1155
|
+
if (value === void 0) return void 0;
|
|
1156
|
+
const parsed = NativeEngineNameSchema.safeParse(value);
|
|
1157
|
+
if (parsed.success) return parsed.data;
|
|
1158
|
+
throw new CliError(
|
|
1159
|
+
`--engine must be one of ${NativeEngineNameSchema.options.join(", ")} (got "${value}")`
|
|
1110
1160
|
);
|
|
1111
|
-
return [`${label} (${suite.id})`, ...lines];
|
|
1112
1161
|
}
|
|
1113
|
-
function
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
return
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1162
|
+
function formatNativeStatus(status) {
|
|
1163
|
+
const installed = new Set(
|
|
1164
|
+
status.engines.filter((engine) => engine.installed).map((engine) => engine.name)
|
|
1165
|
+
);
|
|
1166
|
+
const toolkitInstalled = NATIVE_TOOLKIT.filter((engine) => installed.has(engine)).length;
|
|
1167
|
+
return [
|
|
1168
|
+
`release: native-v${status.release} (${status.pinned ? "verified pin" : "unpinned \u2014 downloads disabled"})`,
|
|
1169
|
+
`platform: ${status.platformKey ?? "unsupported \u2014 downloads unavailable"}`,
|
|
1170
|
+
`llama backend: ${status.llamaBackend ?? "automatic"}`,
|
|
1171
|
+
// Name the members: `native list` shows every engine the release carries
|
|
1172
|
+
// (ds4, duckdb too), so a bare "0/4" read as a miscount beside it.
|
|
1173
|
+
`toolkit: ${toolkitInstalled}/${NATIVE_TOOLKIT.length} installed (${NATIVE_TOOLKIT.join(", ")}; see \`gezel native list\` for every engine)`
|
|
1174
|
+
].join("\n");
|
|
1175
|
+
}
|
|
1176
|
+
function formatNativeList(status) {
|
|
1177
|
+
const rows = status.engines.map((engine) => {
|
|
1178
|
+
const state = engine.installed ? "installed" : "missing";
|
|
1179
|
+
return `${engine.name.padEnd(18)} ${state.padEnd(10)}${engine.path ? ` ${engine.path}` : ""}`;
|
|
1180
|
+
});
|
|
1181
|
+
return [
|
|
1182
|
+
`native-v${status.release} \xB7 ${status.platformKey ?? "unsupported platform"} \xB7 ${status.pinned ? "verified" : "unpinned"}`,
|
|
1183
|
+
"engine status path",
|
|
1184
|
+
...rows
|
|
1185
|
+
].join("\n");
|
|
1186
|
+
}
|
|
1187
|
+
async function installNativeToolkit(client, options) {
|
|
1188
|
+
const status = await client.getNativeEngineStatus();
|
|
1189
|
+
if (!status.pinned) {
|
|
1190
|
+
throw new CliError(
|
|
1191
|
+
"native downloads are disabled because the connected service has no verified native release pin."
|
|
1192
|
+
);
|
|
1193
|
+
}
|
|
1194
|
+
if (!status.platformKey) {
|
|
1195
|
+
throw new CliError(
|
|
1196
|
+
`native downloads are unavailable on ${process.platform}/${process.arch}: no supported build exists.`
|
|
1197
|
+
);
|
|
1198
|
+
}
|
|
1199
|
+
const llamaVariant = options.variant ?? status.llamaBackend;
|
|
1200
|
+
for (const engine of options.engines ?? NATIVE_TOOLKIT) {
|
|
1201
|
+
await ensureOneEngine(client, engine, llamaVariant, options.output);
|
|
1202
|
+
}
|
|
1203
|
+
return client.getNativeEngineStatus();
|
|
1204
|
+
}
|
|
1205
|
+
async function ensureOneEngine(client, engine, llamaVariant, output) {
|
|
1206
|
+
let terminalError;
|
|
1207
|
+
let progressLineOpen = false;
|
|
1208
|
+
const onEvent = (event) => {
|
|
1209
|
+
if (event.type === "progress") {
|
|
1210
|
+
progressLineOpen = true;
|
|
1211
|
+
output.writeProgress(`\r${formatProgress(engine, event.bytesWritten, event.totalBytes)}`);
|
|
1212
|
+
return;
|
|
1213
|
+
}
|
|
1214
|
+
if (progressLineOpen) {
|
|
1215
|
+
output.writeProgress("\n");
|
|
1216
|
+
progressLineOpen = false;
|
|
1217
|
+
}
|
|
1218
|
+
switch (event.type) {
|
|
1219
|
+
case "retrying":
|
|
1220
|
+
output.writeProgress(
|
|
1221
|
+
`${engine}: retry ${event.attempt}/${event.maxAttempts} in ${event.delayMs}ms \u2014 ${event.reason}
|
|
1222
|
+
`
|
|
1126
1223
|
);
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1224
|
+
break;
|
|
1225
|
+
case "verifying":
|
|
1226
|
+
output.writeProgress(`${engine}: verifying ${event.what}
|
|
1227
|
+
`);
|
|
1228
|
+
break;
|
|
1229
|
+
case "done":
|
|
1230
|
+
output.writeProgress(
|
|
1231
|
+
`${engine}: ready${event.cached ? " (cached)" : ` \u2014 ${event.binPath}`}
|
|
1232
|
+
`
|
|
1134
1233
|
);
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
const current = target.currentTrial;
|
|
1142
|
-
if (current && current.trialId !== lastTrial) {
|
|
1143
|
-
lastTrial = current.trialId;
|
|
1144
|
-
lines.push(` ${current.scenarioId} (${current.trialIndex}/${planned}) running\u2026`);
|
|
1145
|
-
}
|
|
1146
|
-
return lines;
|
|
1234
|
+
break;
|
|
1235
|
+
case "error":
|
|
1236
|
+
terminalError = event.error;
|
|
1237
|
+
output.writeProgress(`${engine}: error \u2014 ${event.error}
|
|
1238
|
+
`);
|
|
1239
|
+
break;
|
|
1147
1240
|
}
|
|
1148
1241
|
};
|
|
1242
|
+
try {
|
|
1243
|
+
await client.ensureNativeEngine(
|
|
1244
|
+
engine,
|
|
1245
|
+
onEvent,
|
|
1246
|
+
engine === "llama-server" ? llamaVariant : void 0
|
|
1247
|
+
);
|
|
1248
|
+
} catch (error) {
|
|
1249
|
+
if (progressLineOpen) output.writeProgress("\n");
|
|
1250
|
+
throw new CliError(
|
|
1251
|
+
`native install failed for ${engine}: ${terminalError ?? errorMessage(error)}`
|
|
1252
|
+
);
|
|
1253
|
+
}
|
|
1254
|
+
if (progressLineOpen) output.writeProgress("\n");
|
|
1255
|
+
if (terminalError) {
|
|
1256
|
+
throw new CliError(`native install failed for ${engine}: ${terminalError}`);
|
|
1257
|
+
}
|
|
1149
1258
|
}
|
|
1150
|
-
function
|
|
1151
|
-
|
|
1259
|
+
function formatProgress(engine, written, total) {
|
|
1260
|
+
if (total <= 0) return `${engine}: ${formatBytes(written)} downloaded`;
|
|
1261
|
+
const pct = Math.min(100, Math.floor(written / total * 100));
|
|
1262
|
+
return `${engine}: ${String(pct).padStart(3)}% ${formatBytes(written)}/${formatBytes(total)}`;
|
|
1152
1263
|
}
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
if (event.type === "log") {
|
|
1159
|
-
if (verbose) io.note(` ${event.line}`);
|
|
1160
|
-
return;
|
|
1161
|
-
}
|
|
1162
|
-
latest = event.job;
|
|
1163
|
-
for (const line of follower.update(event.job)) io.note(line);
|
|
1164
|
-
});
|
|
1165
|
-
if (!finishedJob(latest)) latest = await client.getEvalJob(job.id);
|
|
1166
|
-
return latest;
|
|
1264
|
+
function formatBytes(bytes) {
|
|
1265
|
+
if (bytes < 1024) return `${bytes} B`;
|
|
1266
|
+
if (bytes < 1024 ** 2) return `${(bytes / 1024).toFixed(1)} KiB`;
|
|
1267
|
+
if (bytes < 1024 ** 3) return `${(bytes / 1024 ** 2).toFixed(1)} MiB`;
|
|
1268
|
+
return `${(bytes / 1024 ** 3).toFixed(2)} GiB`;
|
|
1167
1269
|
}
|
|
1168
|
-
function
|
|
1169
|
-
|
|
1170
|
-
const pkg = JSON.parse(readFileSync(join2(gildePackageRoot(), "package.json"), "utf8"));
|
|
1171
|
-
return pkg.version ?? "unknown";
|
|
1172
|
-
} catch {
|
|
1173
|
-
return "unknown";
|
|
1174
|
-
}
|
|
1270
|
+
function errorMessage(error) {
|
|
1271
|
+
return error instanceof Error ? error.message : String(error);
|
|
1175
1272
|
}
|
|
1176
|
-
|
|
1177
|
-
|
|
1273
|
+
|
|
1274
|
+
// src/settings-command.ts
|
|
1275
|
+
import {
|
|
1276
|
+
classifySecurityLevel,
|
|
1277
|
+
resolveSecurityPolicy,
|
|
1278
|
+
securityCapabilitiesOf,
|
|
1279
|
+
securityPolicyForLevel
|
|
1280
|
+
} from "@bendyline/gezel";
|
|
1281
|
+
var SECURITY_LEVEL_LABELS = {
|
|
1282
|
+
"super-lockdown": "Super Lockdown",
|
|
1283
|
+
lockdown: "Lockdown",
|
|
1284
|
+
free: "Unrestricted",
|
|
1285
|
+
custom: "Custom"
|
|
1286
|
+
};
|
|
1287
|
+
function policyWithCapability(config, key, enabled) {
|
|
1288
|
+
const caps = { ...securityCapabilitiesOf(resolveSecurityPolicy(config)), [key]: enabled };
|
|
1289
|
+
return { level: classifySecurityLevel(caps), ...caps };
|
|
1178
1290
|
}
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
const scenariosBySuite = {};
|
|
1201
|
-
const jobs = [];
|
|
1202
|
-
const results = [];
|
|
1203
|
-
const runId = evalRunId(startedAt, device, plan.target.modelId);
|
|
1204
|
-
const gildeVersion = (io.gildeVersion ?? defaultGildeVersion)();
|
|
1205
|
-
const run = evalRunProvenance({
|
|
1206
|
-
id: runId,
|
|
1207
|
-
startedAt,
|
|
1208
|
-
device,
|
|
1209
|
-
gezelVersion: host.version,
|
|
1210
|
-
gildeVersion,
|
|
1211
|
-
count: plan.count,
|
|
1212
|
-
engineBuilds: evalEngineBuilds(host),
|
|
1213
|
-
scenariosBySuite: Object.fromEntries(plan.suites.map((s) => [s.id, s.scenarioIds])),
|
|
1214
|
-
subset: false,
|
|
1215
|
-
...opts.note ? { note: opts.note } : {}
|
|
1216
|
-
});
|
|
1217
|
-
for (const suite of plan.suites) {
|
|
1218
|
-
const spec = {
|
|
1219
|
-
suiteId: suite.id,
|
|
1220
|
-
count: plan.count,
|
|
1221
|
-
...plan.count > 1 ? { countStrict: true } : {},
|
|
1222
|
-
targets: [{ provider: plan.target.provider, modelId: plan.target.modelId }],
|
|
1223
|
-
note: `gezel bench ${runId}`
|
|
1224
|
-
};
|
|
1225
|
-
io.note("");
|
|
1226
|
-
io.note(`Starting ${suite.id}\u2026`);
|
|
1227
|
-
const created = await client.createEvalJob(spec);
|
|
1228
|
-
const job = await followJob(client, created, suite, plan.count, io, opts.verbose);
|
|
1229
|
-
jobs.push(job);
|
|
1230
|
-
scenariosBySuite[suite.id] = suite.scenarioIds;
|
|
1231
|
-
const outcome = describeJobOutcome(job);
|
|
1232
|
-
if (outcome) io.note(outcome);
|
|
1233
|
-
const { trials } = await client.listEvalTrials({ jobId: job.id, limit: 2e3 });
|
|
1234
|
-
results.push(evalSuiteResult({ run, suite, target: plan.target, trials }));
|
|
1235
|
-
io.out("");
|
|
1236
|
-
for (const line of renderSuiteReport(suite, plan.target, trials, catalog.minTrialsForRate)) {
|
|
1237
|
-
io.out(line);
|
|
1238
|
-
}
|
|
1239
|
-
if (plan.smoke) io.out(` ${suite.description}`);
|
|
1291
|
+
function policyWithExternalServices(config, enabled) {
|
|
1292
|
+
return policyWithCapability(config, "allowExternalServices", enabled);
|
|
1293
|
+
}
|
|
1294
|
+
var SECURITY_SWITCHES = [
|
|
1295
|
+
{
|
|
1296
|
+
command: "external-services",
|
|
1297
|
+
key: "allowExternalServices",
|
|
1298
|
+
label: "External services",
|
|
1299
|
+
description: "Show or set on/off for model-initiated external services, including web search"
|
|
1300
|
+
},
|
|
1301
|
+
{
|
|
1302
|
+
command: "run-package-scripts",
|
|
1303
|
+
key: "allowPackageScriptExecution",
|
|
1304
|
+
label: "Run package scripts",
|
|
1305
|
+
description: "Show or set on/off for gezels running package scripts and installed package binaries (outside the sandbox, with network access)"
|
|
1306
|
+
},
|
|
1307
|
+
{
|
|
1308
|
+
command: "edit-package-scripts",
|
|
1309
|
+
key: "allowPackageScriptEdits",
|
|
1310
|
+
label: "Edit package scripts",
|
|
1311
|
+
description: "Show or set on/off for gezels adding or changing the scripts in package.json"
|
|
1240
1312
|
}
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1313
|
+
];
|
|
1314
|
+
var CAPABILITY_LABELS = [
|
|
1315
|
+
["allowExternalChat", "Cloud models"],
|
|
1316
|
+
["allowFileEdits", "File edits and the model\u2019s Git tools"],
|
|
1317
|
+
["allowScriptExecution", "Sandboxed scripts"],
|
|
1318
|
+
["allowAppNetwork", "Update checks and the connector syncs you start"],
|
|
1319
|
+
["allowExternalServices", "Web search, mail and other outside services"],
|
|
1320
|
+
["allowPackageScriptExecution", "Run package scripts"],
|
|
1321
|
+
["allowPackageScriptEdits", "Edit package scripts"]
|
|
1322
|
+
];
|
|
1323
|
+
var LEVEL_ALIASES = {
|
|
1324
|
+
"super-lockdown": "super-lockdown",
|
|
1325
|
+
superlockdown: "super-lockdown",
|
|
1326
|
+
lockdown: "lockdown",
|
|
1327
|
+
free: "free",
|
|
1328
|
+
unrestricted: "free"
|
|
1329
|
+
};
|
|
1330
|
+
function parseSecurityLevel(input) {
|
|
1331
|
+
const level = LEVEL_ALIASES[input.trim().toLowerCase()];
|
|
1332
|
+
if (!level) {
|
|
1333
|
+
throw new CliError(
|
|
1334
|
+
`Unknown security level "${input}". Choose super-lockdown, lockdown, or free (shown as Unrestricted in Settings).`
|
|
1335
|
+
);
|
|
1251
1336
|
}
|
|
1252
|
-
return
|
|
1337
|
+
return level;
|
|
1253
1338
|
}
|
|
1254
|
-
function
|
|
1255
|
-
const
|
|
1256
|
-
const
|
|
1257
|
-
|
|
1339
|
+
function describeSecurityLevel(policy) {
|
|
1340
|
+
const caps = securityCapabilitiesOf(resolveSecurityPolicy({ securityPolicy: policy }));
|
|
1341
|
+
const level = classifySecurityLevel(caps);
|
|
1342
|
+
const width = Math.max(...CAPABILITY_LABELS.map(([, label]) => label.length));
|
|
1343
|
+
return [
|
|
1344
|
+
`Security level: ${SECURITY_LEVEL_LABELS[level]}${level === "custom" ? "" : ` (${level})`}`,
|
|
1345
|
+
...CAPABILITY_LABELS.map(
|
|
1346
|
+
([key, label]) => ` ${label.padEnd(width)} ${caps[key] ? "on" : "off"}`
|
|
1347
|
+
)
|
|
1258
1348
|
];
|
|
1259
|
-
|
|
1260
|
-
|
|
1349
|
+
}
|
|
1350
|
+
function enabledValue(state) {
|
|
1351
|
+
if (state === void 0) return void 0;
|
|
1352
|
+
if (state === "on") return true;
|
|
1353
|
+
if (state === "off") return false;
|
|
1354
|
+
throw new CliError("Use on or off.");
|
|
1355
|
+
}
|
|
1356
|
+
function registerSecurityCommands(program2, connect) {
|
|
1357
|
+
const security = program2.command("security").description("Inspect or explicitly change security settings");
|
|
1358
|
+
security.command("level [action] [level]").description(
|
|
1359
|
+
"Show the security level, or set a preset: `level set super-lockdown|lockdown|free`"
|
|
1360
|
+
).option("--json", "Output the level and its capabilities as JSON").action(
|
|
1361
|
+
async (action, requested, options) => {
|
|
1362
|
+
if (action !== void 0 && action !== "show" && action !== "set") {
|
|
1363
|
+
throw new CliError("Use `gezel security level`, `level show`, or `level set <level>`.");
|
|
1364
|
+
}
|
|
1365
|
+
if (action === "set" && requested === void 0) {
|
|
1366
|
+
throw new CliError("Name a level: super-lockdown, lockdown, or free.");
|
|
1367
|
+
}
|
|
1368
|
+
if (action !== "set" && requested !== void 0) {
|
|
1369
|
+
throw new CliError("To change the level, use `gezel security level set <level>`.");
|
|
1370
|
+
}
|
|
1371
|
+
const next = action === "set" ? parseSecurityLevel(requested) : void 0;
|
|
1372
|
+
const client = await connect();
|
|
1373
|
+
const policy = next ? securityPolicyForLevel(next) : {
|
|
1374
|
+
level: "custom",
|
|
1375
|
+
...securityCapabilitiesOf(resolveSecurityPolicy(await client.getConfig()))
|
|
1376
|
+
};
|
|
1377
|
+
if (next) await client.updateConfig({ securityPolicy: policy });
|
|
1378
|
+
if (options.json) {
|
|
1379
|
+
const caps = securityCapabilitiesOf(resolveSecurityPolicy({ securityPolicy: policy }));
|
|
1380
|
+
console.log(JSON.stringify({ level: classifySecurityLevel(caps), ...caps }));
|
|
1381
|
+
return;
|
|
1382
|
+
}
|
|
1383
|
+
for (const line of describeSecurityLevel(policy)) console.log(line);
|
|
1384
|
+
}
|
|
1261
1385
|
);
|
|
1262
|
-
const
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1386
|
+
for (const setting of SECURITY_SWITCHES) {
|
|
1387
|
+
security.command(`${setting.command} [state]`).description(setting.description).option("--json", "Output the setting as JSON").action(async (state, options) => {
|
|
1388
|
+
const enabled = enabledValue(state);
|
|
1389
|
+
const client = await connect();
|
|
1390
|
+
const config = await client.getConfig();
|
|
1391
|
+
const policy = resolveSecurityPolicy(config);
|
|
1392
|
+
if (enabled !== void 0) {
|
|
1393
|
+
await client.updateConfig({
|
|
1394
|
+
securityPolicy: policyWithCapability(config, setting.key, enabled)
|
|
1395
|
+
});
|
|
1396
|
+
}
|
|
1397
|
+
const value = enabled ?? policy[setting.key];
|
|
1398
|
+
console.log(
|
|
1399
|
+
options.json ? JSON.stringify({ [setting.key]: value }) : `${setting.label}: ${value ? "on" : "off"}`
|
|
1400
|
+
);
|
|
1401
|
+
});
|
|
1270
1402
|
}
|
|
1271
|
-
lines.push(
|
|
1272
|
-
"",
|
|
1273
|
-
`The published scorecard runs ${SCORECARD_SUITE_IDS.join(", ")}. Add --smoke for a suite's pulse check; it is not a suite score.`
|
|
1274
|
-
);
|
|
1275
|
-
return lines;
|
|
1276
1403
|
}
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
const target = job.targets[0];
|
|
1286
|
-
const { trials } = await client.listEvalTrials({ jobId: job.id, limit: 2e3 });
|
|
1287
|
-
const order = suite?.scenarioIds ?? [...new Set(trials.map((t) => t.scenarioId))];
|
|
1288
|
-
const matrix = buildEvalMatrix(trials, { scenarioOrder: order });
|
|
1289
|
-
const column = target ? matrix.columns.find(
|
|
1290
|
-
(c) => c.modelId === target.modelId && (c.provider ?? target.provider) === target.provider
|
|
1291
|
-
) : matrix.columns[0];
|
|
1292
|
-
const subset = suite && job.spec.scenarioIds ? isSuiteSubset(suite, job.spec.scenarioIds) : false;
|
|
1293
|
-
const what = suite ? `${suite.id}${subset ? " (subset)" : ""}` : `${job.spec.scenarioIds?.length ?? 0} tasks`;
|
|
1294
|
-
lines.push(
|
|
1295
|
-
`${(job.finishedAt ?? job.createdAt).slice(0, 16).replace("T", " ")} ${(target?.modelId ?? "?").padEnd(24)} ${what.padEnd(22)} ${column?.claim ?? "not measured"}${job.status !== "completed" ? ` [${job.status}]` : ""}`
|
|
1404
|
+
function registerGrowthCommand(program2, connect) {
|
|
1405
|
+
program2.command("growth [state]").description("Show or set on/off for gezel growth: XP and the level-ups it offers").option("--json", "Output the setting as JSON").action(async (state, options) => {
|
|
1406
|
+
const enabled = enabledValue(state);
|
|
1407
|
+
const client = await connect();
|
|
1408
|
+
const config = enabled === void 0 ? await client.getConfig() : await client.updateConfig({ growth: { enabled } });
|
|
1409
|
+
const value = config.growth?.enabled !== false;
|
|
1410
|
+
console.log(
|
|
1411
|
+
options.json ? JSON.stringify({ growthEnabled: value }) : `Growth (XP and level-ups): ${value ? "on" : "off"}`
|
|
1296
1412
|
);
|
|
1297
|
-
|
|
1298
|
-
}
|
|
1299
|
-
return lines;
|
|
1413
|
+
});
|
|
1300
1414
|
}
|
|
1301
|
-
function
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
...run.note ? [`- Note: ${run.note}`] : [],
|
|
1313
|
-
"",
|
|
1314
|
-
"Results (count below three trials per task, rate otherwise):",
|
|
1315
|
-
...shard.results.map((r) => {
|
|
1316
|
-
const matrixLike = r.cells.reduce(
|
|
1317
|
-
(acc, c) => {
|
|
1318
|
-
acc.trials += Math.max(0, c.trials - c.nonModelFailures);
|
|
1319
|
-
acc.successes += c.successes;
|
|
1320
|
-
return acc;
|
|
1321
|
-
},
|
|
1322
|
-
{ trials: 0, successes: 0 }
|
|
1323
|
-
);
|
|
1324
|
-
return `- ${r.suiteId}: ${matrixLike.successes}/${matrixLike.trials}`;
|
|
1325
|
-
}),
|
|
1326
|
-
"",
|
|
1327
|
-
"The JSON shard produced by `gezel bench --json` is attached below. It is a `ScorecardDataset` and ingests with the same code as a published round; community rounds are shown separately from Bendyline rounds.",
|
|
1328
|
-
"",
|
|
1329
|
-
"<details><summary>shard.json</summary>",
|
|
1330
|
-
"",
|
|
1331
|
-
"```json",
|
|
1332
|
-
JSON.stringify(shard, null, 2),
|
|
1333
|
-
"```",
|
|
1334
|
-
"",
|
|
1335
|
-
"</details>"
|
|
1336
|
-
].join("\n");
|
|
1337
|
-
return { title, body };
|
|
1338
|
-
}
|
|
1339
|
-
var execFileAsync = promisify(execFile);
|
|
1340
|
-
var BENCH_RESULT_LABEL = "bench-result";
|
|
1341
|
-
function labelMissing(error) {
|
|
1342
|
-
const { stderr, message } = error ?? {};
|
|
1343
|
-
const text = typeof stderr === "string" && stderr ? stderr : String(message ?? error);
|
|
1344
|
-
return /could not add label/i.test(text) || text.includes(`'${BENCH_RESULT_LABEL}' not found`);
|
|
1415
|
+
function registerProjectSettingsCommands(env2, connect, projectFor) {
|
|
1416
|
+
env2.command("indexing [state]").description("Show or set on/off for background workspace indexing in the current project").option("--json", "Output the project setting as JSON").action(async (state, options) => {
|
|
1417
|
+
const enabled = enabledValue(state);
|
|
1418
|
+
const client = await connect();
|
|
1419
|
+
const projectId = await projectFor(client);
|
|
1420
|
+
const project = enabled === void 0 ? await client.getProject(projectId) : await client.updateProject(projectId, { indexingEnabled: enabled });
|
|
1421
|
+
const result = { projectId, indexingEnabled: project.indexingEnabled !== false };
|
|
1422
|
+
console.log(
|
|
1423
|
+
options.json ? JSON.stringify(result) : `${projectId}: workspace indexing ${result.indexingEnabled ? "on" : "off"}`
|
|
1424
|
+
);
|
|
1425
|
+
});
|
|
1345
1426
|
}
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1427
|
+
|
|
1428
|
+
// src/craftbook-setup.ts
|
|
1429
|
+
var BRAVE_KEY_URL = "https://brave.com/search/api/";
|
|
1430
|
+
var SEARCH_CHECK = { query: "Gezel web search check", limit: 1 };
|
|
1431
|
+
var ENGINE_BINARY = {
|
|
1432
|
+
"llama-cpp": "llama-server",
|
|
1433
|
+
mlx: "uv",
|
|
1434
|
+
ds4: "ds4-server"
|
|
1435
|
+
};
|
|
1436
|
+
async function ensureCraftbookSetup(client, options) {
|
|
1437
|
+
const services = options.needs.services ?? [];
|
|
1438
|
+
const commands = options.needs.commands ?? [];
|
|
1439
|
+
if (!options.needs.models?.length && services.length === 0 && commands.length === 0) return;
|
|
1440
|
+
const config = await client.getConfig();
|
|
1441
|
+
const models = resolveCraftbookModelNeeds(options.needs.models, {
|
|
1442
|
+
...options.params ? { params: options.params } : {},
|
|
1443
|
+
...options.paramSchema ? { paramSchema: options.paramSchema } : {},
|
|
1444
|
+
defaultProvider: defaultModelEngine(config, options.platform, options.arch)
|
|
1445
|
+
});
|
|
1446
|
+
const gaps = craftbookSetupGaps(
|
|
1447
|
+
{ services, models, commands },
|
|
1448
|
+
await readSetupState(client, config, models)
|
|
1449
|
+
);
|
|
1450
|
+
if (gaps.length === 0) return;
|
|
1451
|
+
const sizes = await modelSizes(client, gaps);
|
|
1452
|
+
const lines = gaps.map((gap) => ` \u2022 ${describeGap(gap, sizes)}`);
|
|
1453
|
+
const onlyDownloads = gaps.every((gap) => gap.kind === "model");
|
|
1454
|
+
if (!options.prompter && !(options.autoDownload && onlyDownloads)) {
|
|
1352
1455
|
throw new CliError(
|
|
1353
|
-
|
|
1456
|
+
[
|
|
1457
|
+
`${options.label} is not set up to run yet:`,
|
|
1458
|
+
...lines,
|
|
1459
|
+
"",
|
|
1460
|
+
"Run the same command in a terminal to set it up step by step, or run:",
|
|
1461
|
+
...instructions(gaps).map((line) => ` ${line}`),
|
|
1462
|
+
...onlyDownloads ? ["", "Or run it again with --download to download them without asking."] : options.autoDownload ? ["", "--download only downloads; the other steps need a person."] : []
|
|
1463
|
+
].join("\n")
|
|
1354
1464
|
);
|
|
1355
1465
|
}
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
const
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1466
|
+
options.write(`${options.label} needs some setup first:
|
|
1467
|
+
${lines.join("\n")}
|
|
1468
|
+
|
|
1469
|
+
`);
|
|
1470
|
+
const prompter = options.prompter ?? {
|
|
1471
|
+
confirm: async () => false,
|
|
1472
|
+
secret: async () => ""
|
|
1473
|
+
};
|
|
1474
|
+
const stop = (from) => {
|
|
1475
|
+
const rest = gaps.slice(gaps.indexOf(from));
|
|
1476
|
+
throw new CliError(
|
|
1477
|
+
[
|
|
1478
|
+
`Not set up, so ${options.label} did not start. To finish later, run:`,
|
|
1479
|
+
...instructions(rest).map((line) => ` ${line}`)
|
|
1480
|
+
].join("\n")
|
|
1481
|
+
);
|
|
1482
|
+
};
|
|
1483
|
+
for (const gap of gaps) {
|
|
1484
|
+
if (gap.kind === "external-services") {
|
|
1485
|
+
const next = policyWithExternalServices(config, true);
|
|
1486
|
+
const from = SECURITY_LEVEL_LABELS[resolveSecurityPolicy2(config).level];
|
|
1487
|
+
const to = SECURITY_LEVEL_LABELS[next.level];
|
|
1488
|
+
const change = from === to ? "" : ` This changes your security level from ${from} to ${to}.`;
|
|
1489
|
+
const yes = await prompter.confirm(
|
|
1490
|
+
`Turn on External services, so gezels can search the web and read web pages?${change} [y/N] `
|
|
1491
|
+
);
|
|
1492
|
+
if (!yes) stop(gap);
|
|
1493
|
+
await client.updateConfig({ securityPolicy: next });
|
|
1494
|
+
options.write("External services: on\n");
|
|
1495
|
+
} else if (gap.kind === "package-scripts") {
|
|
1496
|
+
const current = await client.getConfig();
|
|
1497
|
+
const next = policyWithCapability(current, "allowPackageScriptExecution", true);
|
|
1498
|
+
const from = SECURITY_LEVEL_LABELS[resolveSecurityPolicy2(current).level];
|
|
1499
|
+
const to = SECURITY_LEVEL_LABELS[next.level];
|
|
1500
|
+
const change = from === to ? "" : ` This changes your security level from ${from} to ${to}.`;
|
|
1501
|
+
const yes = await prompter.confirm(
|
|
1502
|
+
`Let gezels run package scripts such as ${gap.commands[0]}? They run outside the sandbox, as your account, with network access, and Gezel asks before each new command.${change} [y/N] `
|
|
1503
|
+
);
|
|
1504
|
+
if (!yes) stop(gap);
|
|
1505
|
+
await client.updateConfig({ securityPolicy: next });
|
|
1506
|
+
options.write("Run package scripts: on\n");
|
|
1507
|
+
} else if (gap.kind === "web-search") {
|
|
1508
|
+
if (gap.keyMissing) {
|
|
1509
|
+
options.write(
|
|
1510
|
+
`Web search needs a Brave Search API key. Get one at ${BRAVE_KEY_URL} \u2014 Gezel keeps it in this computer's secret store.
|
|
1511
|
+
`
|
|
1512
|
+
);
|
|
1513
|
+
const key = (await prompter.secret(
|
|
1514
|
+
"Paste the key, then press Enter (Enter on its own stops here): ",
|
|
1515
|
+
(length) => `Key received (${length} characters, hidden). Press Enter to continue with this key. `
|
|
1516
|
+
)).trim();
|
|
1517
|
+
if (!key) stop(gap);
|
|
1518
|
+
options.write("Checking the key with one Brave search\u2026\n");
|
|
1519
|
+
await saveBraveKey(client, config, key, options.projectId);
|
|
1520
|
+
options.write("Web search: Brave (test search passed)\n");
|
|
1521
|
+
} else {
|
|
1522
|
+
const current = config.webSearch?.provider ?? "wikipedia";
|
|
1523
|
+
const yes = await prompter.confirm(
|
|
1524
|
+
`A Brave Search key is saved, but web search uses ${current === "wikipedia" ? "Wikipedia" : current}. Switch web search to Brave? [y/N] `
|
|
1525
|
+
);
|
|
1526
|
+
if (!yes) stop(gap);
|
|
1527
|
+
await client.updateConfig({ webSearch: { ...config.webSearch, provider: "brave" } });
|
|
1528
|
+
options.write("Web search: Brave\n");
|
|
1529
|
+
}
|
|
1381
1530
|
}
|
|
1382
|
-
const issueUrl = stdout.trim().split("\n").pop() ?? "";
|
|
1383
|
-
io.out(`Opened ${issueUrl}`);
|
|
1384
|
-
return { title, body, url, issueUrl };
|
|
1385
|
-
} catch {
|
|
1386
|
-
io.out("Open this page and attach the file to the issue:");
|
|
1387
|
-
io.out(` ${url}`);
|
|
1388
|
-
io.out(` file: ${file}`);
|
|
1389
|
-
io.note("(Install the GitHub CLI, `gh`, and sign in to have this done for you.)");
|
|
1390
|
-
return { title, body, url };
|
|
1391
1531
|
}
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
for (const
|
|
1420
|
-
});
|
|
1421
|
-
bench.command("submit <file>").description("Share a results shard from `gezel bench --json` as a GitHub issue").action(async (file) => {
|
|
1422
|
-
await submitBench(file, io);
|
|
1423
|
-
});
|
|
1424
|
-
}
|
|
1425
|
-
async function confirmOnTerminal(question) {
|
|
1426
|
-
const { createInterface } = await import("readline/promises");
|
|
1427
|
-
const rl = createInterface({ input: process.stdin, output: process.stderr });
|
|
1428
|
-
try {
|
|
1429
|
-
const answer = await rl.question(question);
|
|
1430
|
-
return /^y(es)?$/i.test(answer.trim());
|
|
1431
|
-
} finally {
|
|
1432
|
-
rl.close();
|
|
1532
|
+
const downloads = gaps.filter((gap) => gap.kind === "model");
|
|
1533
|
+
if (downloads.length > 0) {
|
|
1534
|
+
const total = downloads.reduce((sum, gap) => sum + (sizes.get(modelKey(gap)) ?? 0), 0);
|
|
1535
|
+
if (total > 0) {
|
|
1536
|
+
const space = await client.checkModelDownloadSpace({ sizeBytes: total });
|
|
1537
|
+
if (space.known && !space.ok) {
|
|
1538
|
+
throw new CliError(
|
|
1539
|
+
`The models need ${formatGb(space.requiredBytes)} GB, and Gezel model storage has ${formatGb(space.freeBytes)} GB free. Free some space, then run again.`
|
|
1540
|
+
);
|
|
1541
|
+
}
|
|
1542
|
+
}
|
|
1543
|
+
const what = downloads.length === 1 ? "this model" : `these ${downloads.length} models`;
|
|
1544
|
+
const size = total > 0 ? ` of ${formatGb(total)} GB` : "";
|
|
1545
|
+
if (options.autoDownload) {
|
|
1546
|
+
options.write(`Downloading ${what} (--download), a one-time download${size}.
|
|
1547
|
+
`);
|
|
1548
|
+
} else {
|
|
1549
|
+
const yes = await prompter.confirm(
|
|
1550
|
+
`Download ${what} now? It is a one-time download${size}. [y/N] `
|
|
1551
|
+
);
|
|
1552
|
+
if (!yes) stop(downloads[0]);
|
|
1553
|
+
}
|
|
1554
|
+
await ensureModelEngines(
|
|
1555
|
+
client,
|
|
1556
|
+
downloads.map((gap) => gap.provider),
|
|
1557
|
+
options.write
|
|
1558
|
+
);
|
|
1559
|
+
for (const gap of downloads) await pullChatModel(client, gap.provider, gap.id, options.write);
|
|
1433
1560
|
}
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
if (
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
const what = serviceText && serviceText.toLowerCase() !== "not found" ? serviceText : void 0;
|
|
1446
|
-
return `error: not found${what ? `: ${what}` : request ? ` (${request})` : ""}`;
|
|
1561
|
+
const after = craftbookSetupGaps(
|
|
1562
|
+
{ services, models, commands },
|
|
1563
|
+
await readSetupState(client, await client.getConfig(), models)
|
|
1564
|
+
);
|
|
1565
|
+
if (after.length > 0) {
|
|
1566
|
+
throw new CliError(
|
|
1567
|
+
[
|
|
1568
|
+
`${options.label} is still not set up:`,
|
|
1569
|
+
...after.map((gap) => ` \u2022 ${describeGap(gap, sizes)}`)
|
|
1570
|
+
].join("\n")
|
|
1571
|
+
);
|
|
1447
1572
|
}
|
|
1448
|
-
|
|
1449
|
-
if (serviceText && request) lines.push(`(Gezel service answered ${status ?? "?"} to ${request})`);
|
|
1450
|
-
lines.push("Set GEZEL_DEBUG=1 to see the full error.");
|
|
1451
|
-
return lines.join("\n");
|
|
1452
|
-
}
|
|
1453
|
-
function isNotFound(err) {
|
|
1454
|
-
return err instanceof Error && err.name === "GezelApiError" && err.status === 404;
|
|
1573
|
+
options.write("\n");
|
|
1455
1574
|
}
|
|
1456
|
-
function
|
|
1457
|
-
if (
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
}
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
import {
|
|
1470
|
-
composeCraftbookLaunch,
|
|
1471
|
-
createAwakeTimeout,
|
|
1472
|
-
paramAsksUser
|
|
1473
|
-
} from "@bendyline/gezel";
|
|
1474
|
-
|
|
1475
|
-
// src/tui/craftbook-start.ts
|
|
1476
|
-
import {
|
|
1477
|
-
CRAFTBOOK_CATEGORY_FAMILY_META,
|
|
1478
|
-
CRAFTBOOK_CATEGORY_META,
|
|
1479
|
-
CRAFTBOOK_ROLE_META,
|
|
1480
|
-
resolveCraftbookCategory,
|
|
1481
|
-
visibleCatalogItems
|
|
1482
|
-
} from "@bendyline/gezel";
|
|
1483
|
-
function normalizeCraftbooks(items, showWorkInProgressFeatures = false) {
|
|
1484
|
-
const byId = /* @__PURE__ */ new Map();
|
|
1485
|
-
for (const item of visibleCatalogItems(items, showWorkInProgressFeatures)) {
|
|
1486
|
-
if (item.manifest.kind !== "craftbook-template") continue;
|
|
1487
|
-
const manifest = item.manifest;
|
|
1488
|
-
if (byId.has(manifest.id)) continue;
|
|
1489
|
-
byId.set(manifest.id, {
|
|
1490
|
-
id: manifest.id,
|
|
1491
|
-
name: manifest.name,
|
|
1492
|
-
...manifest.description ? { description: manifest.description } : {},
|
|
1493
|
-
version: manifest.version,
|
|
1494
|
-
sourceId: item.sourceId,
|
|
1495
|
-
command: manifest.command ?? manifest.id,
|
|
1496
|
-
source: item.sourceId === "project" ? "project" : item.sourceId === "local" ? "local" : "bundled",
|
|
1497
|
-
stepCount: manifest.steps.length,
|
|
1498
|
-
tags: manifest.tags ?? [],
|
|
1499
|
-
role: manifest.role ?? "general",
|
|
1500
|
-
category: resolveCraftbookCategory(manifest)
|
|
1501
|
-
});
|
|
1502
|
-
}
|
|
1503
|
-
return [...byId.values()].sort((left, right) => left.name.localeCompare(right.name));
|
|
1575
|
+
function terminalPrompter(terminal = { input: process.stdin, output: process.stderr }) {
|
|
1576
|
+
if (!terminal.input.isTTY || !terminal.output.isTTY) return null;
|
|
1577
|
+
return {
|
|
1578
|
+
confirm: async (question) => {
|
|
1579
|
+
for (; ; ) {
|
|
1580
|
+
const answer = (await ask(terminal, question)).trim().toLowerCase();
|
|
1581
|
+
if (answer === "y" || answer === "yes") return true;
|
|
1582
|
+
if (answer === "" || answer === "n" || answer === "no") return false;
|
|
1583
|
+
terminal.output.write("Please answer y or n.\n");
|
|
1584
|
+
}
|
|
1585
|
+
},
|
|
1586
|
+
secret: (question, filled) => ask(terminal, question, { filled })
|
|
1587
|
+
};
|
|
1504
1588
|
}
|
|
1505
|
-
function
|
|
1506
|
-
const
|
|
1507
|
-
return
|
|
1508
|
-
(
|
|
1589
|
+
function ask(terminal, question, secret) {
|
|
1590
|
+
const { input, output } = terminal;
|
|
1591
|
+
return import("readline").then(
|
|
1592
|
+
({ clearScreenDown, createInterface, cursorTo, moveCursor }) => new Promise((resolve7, reject) => {
|
|
1593
|
+
const rl = createInterface({
|
|
1594
|
+
input,
|
|
1595
|
+
output: secret ? new Writable({ write: (_chunk, _encoding, done) => done() }) : output,
|
|
1596
|
+
terminal: true,
|
|
1597
|
+
...secret ? { historySize: 0 } : {}
|
|
1598
|
+
});
|
|
1599
|
+
let settled = false;
|
|
1600
|
+
const settle = () => {
|
|
1601
|
+
settled = true;
|
|
1602
|
+
input.removeListener("keypress", onKeypress);
|
|
1603
|
+
};
|
|
1604
|
+
let drawn = "";
|
|
1605
|
+
let shownLength = -1;
|
|
1606
|
+
let pending = false;
|
|
1607
|
+
const draw = () => {
|
|
1608
|
+
pending = false;
|
|
1609
|
+
if (settled || !secret) return;
|
|
1610
|
+
const length = rl.line.length;
|
|
1611
|
+
if (length === shownLength) return;
|
|
1612
|
+
shownLength = length;
|
|
1613
|
+
const text = length > 0 && secret.filled ? secret.filled(length) : question;
|
|
1614
|
+
if (drawn) {
|
|
1615
|
+
const rows = Math.floor((drawn.length - 1) / (output.columns || 80));
|
|
1616
|
+
if (rows > 0) moveCursor(output, 0, -rows);
|
|
1617
|
+
cursorTo(output, 0);
|
|
1618
|
+
clearScreenDown(output);
|
|
1619
|
+
}
|
|
1620
|
+
output.write(text);
|
|
1621
|
+
drawn = text;
|
|
1622
|
+
};
|
|
1623
|
+
function onKeypress() {
|
|
1624
|
+
if (pending) return;
|
|
1625
|
+
pending = true;
|
|
1626
|
+
queueMicrotask(draw);
|
|
1627
|
+
}
|
|
1628
|
+
if (secret) input.on("keypress", onKeypress);
|
|
1629
|
+
rl.on("SIGINT", () => {
|
|
1630
|
+
settle();
|
|
1631
|
+
rl.close();
|
|
1632
|
+
output.write("\n");
|
|
1633
|
+
reject(new CliError("Cancelled."));
|
|
1634
|
+
});
|
|
1635
|
+
rl.on("close", () => {
|
|
1636
|
+
if (settled) return;
|
|
1637
|
+
settle();
|
|
1638
|
+
output.write("\n");
|
|
1639
|
+
resolve7("");
|
|
1640
|
+
});
|
|
1641
|
+
rl.question(question, (answer) => {
|
|
1642
|
+
settle();
|
|
1643
|
+
rl.close();
|
|
1644
|
+
if (secret) output.write("\n");
|
|
1645
|
+
resolve7(answer);
|
|
1646
|
+
});
|
|
1647
|
+
draw();
|
|
1648
|
+
})
|
|
1509
1649
|
);
|
|
1510
1650
|
}
|
|
1511
|
-
function
|
|
1512
|
-
const
|
|
1513
|
-
|
|
1651
|
+
function defaultModelEngine(config, platform = process.platform, arch = process.arch) {
|
|
1652
|
+
const configured = config.provider;
|
|
1653
|
+
return configured && isCraftbookModelEngine(configured) ? configured : resolveOnDeviceProvider(platform, arch);
|
|
1654
|
+
}
|
|
1655
|
+
async function readSetupState(client, config, models) {
|
|
1656
|
+
const engines = [...new Set(models.map((model2) => model2.provider))];
|
|
1657
|
+
const installed = await Promise.all(
|
|
1658
|
+
engines.map(async (engine) => {
|
|
1659
|
+
const { models: list } = engine === "mlx" ? await client.listMlxModels() : engine === "ds4" ? await client.listDs4Models() : await client.listLlamaCppModels();
|
|
1660
|
+
return [engine, list.map((model2) => model2.id)];
|
|
1661
|
+
})
|
|
1662
|
+
);
|
|
1514
1663
|
return {
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
dispatchEntry: true,
|
|
1521
|
-
trustScripts: true
|
|
1664
|
+
allowExternalServices: resolveSecurityPolicy2(config).allowExternalServices,
|
|
1665
|
+
allowPackageScriptExecution: resolveSecurityPolicy2(config).allowPackageScriptExecution,
|
|
1666
|
+
...config.webSearch?.provider ? { webSearchProvider: config.webSearch.provider } : {},
|
|
1667
|
+
hasBraveSearchApiKey: config.hasBraveSearchApiKey === true,
|
|
1668
|
+
installedModels: Object.fromEntries(installed)
|
|
1522
1669
|
};
|
|
1523
1670
|
}
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
for (
|
|
1528
|
-
|
|
1529
|
-
|
|
1671
|
+
var modelKey = (model2) => `${model2.provider}:${model2.id}`;
|
|
1672
|
+
async function modelSizes(client, gaps) {
|
|
1673
|
+
const sizes = /* @__PURE__ */ new Map();
|
|
1674
|
+
for (const gap of gaps) {
|
|
1675
|
+
if (gap.kind !== "model") continue;
|
|
1676
|
+
let manifest;
|
|
1677
|
+
try {
|
|
1678
|
+
const item = await client.getCatalogItem("chat-model", gap.id);
|
|
1679
|
+
if (item.manifest.kind !== "chat-model") throw new Error("not a chat model");
|
|
1680
|
+
manifest = item.manifest;
|
|
1681
|
+
} catch {
|
|
1682
|
+
throw new CliError(
|
|
1683
|
+
`This run needs the model ${gap.id}, which is not installed and is not in the model catalog. Check the model name.`
|
|
1684
|
+
);
|
|
1685
|
+
}
|
|
1686
|
+
const source = gap.provider === "mlx" ? manifest.mlx : gap.provider === "ds4" ? manifest.ds4 : manifest.llamaCpp;
|
|
1687
|
+
if (!source) {
|
|
1688
|
+
throw new CliError(
|
|
1689
|
+
`This run needs the model ${gap.id} on ${gap.provider}, and the catalog has no ${gap.provider} download for it.`
|
|
1690
|
+
);
|
|
1691
|
+
}
|
|
1692
|
+
sizes.set(modelKey(gap), source.approxSizeBytes);
|
|
1530
1693
|
}
|
|
1531
|
-
|
|
1694
|
+
return sizes;
|
|
1532
1695
|
}
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1696
|
+
function describeGap(gap, sizes) {
|
|
1697
|
+
const why = gap.reasons.length > 0 ? ` (${gap.reasons.join("; ")})` : "";
|
|
1698
|
+
switch (gap.kind) {
|
|
1699
|
+
case "external-services":
|
|
1700
|
+
return `External services are off, so gezels cannot reach the web${why}.`;
|
|
1701
|
+
case "package-scripts":
|
|
1702
|
+
return `Run package scripts is off, so gezels cannot run ${gap.commands.join(", ")}${why}.`;
|
|
1703
|
+
case "web-search":
|
|
1704
|
+
return gap.keyMissing ? `Web search needs a Brave Search API key, and none is set${why}.` : `Web search is not set to Brave Search${why}.`;
|
|
1705
|
+
case "model": {
|
|
1706
|
+
const size = sizes.get(modelKey(gap));
|
|
1707
|
+
return `${gap.id} is not downloaded yet (${gap.provider}${size ? `, ${formatGb(size)} GB` : ""})${why}.`;
|
|
1708
|
+
}
|
|
1537
1709
|
}
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
if (
|
|
1543
|
-
|
|
1544
|
-
if (
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1710
|
+
}
|
|
1711
|
+
function instructions(gaps) {
|
|
1712
|
+
const out = [];
|
|
1713
|
+
for (const gap of gaps) {
|
|
1714
|
+
if (gap.kind === "external-services") out.push("gezel security external-services on");
|
|
1715
|
+
else if (gap.kind === "package-scripts") out.push("gezel security run-package-scripts on");
|
|
1716
|
+
else if (gap.kind === "web-search")
|
|
1717
|
+
out.push(
|
|
1718
|
+
`gezel secret set braveSearchApiKey --env BRAVE_SEARCH_API_KEY --use-for-search (get a key at ${BRAVE_KEY_URL})`
|
|
1719
|
+
);
|
|
1720
|
+
else out.push(`gezel model pull ${gap.id} --provider ${gap.provider}`);
|
|
1548
1721
|
}
|
|
1549
|
-
|
|
1550
|
-
throw new CliError(`${key} must match ${def.pattern}`);
|
|
1551
|
-
if (typeof def.minLength === "number" && value.trim().length < def.minLength)
|
|
1552
|
-
throw new CliError(`${key} is too short`);
|
|
1722
|
+
return out;
|
|
1553
1723
|
}
|
|
1554
|
-
function
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
const requestWords = [];
|
|
1563
|
-
const set = (key, value, token) => {
|
|
1564
|
-
if (!Object.hasOwn(properties, key)) throw new CliError(`unknown craftbook argument: ${token}`);
|
|
1565
|
-
if (Object.hasOwn(params, key)) throw new CliError(`duplicate craftbook parameter: ${key}`);
|
|
1566
|
-
checkParamValue(key, properties[key], value);
|
|
1567
|
-
params[key] = value;
|
|
1568
|
-
};
|
|
1569
|
-
for (const token of tokens) {
|
|
1570
|
-
const match = NAMED_ARGUMENT.exec(token);
|
|
1571
|
-
if (match) {
|
|
1572
|
-
set(match[1], match[2], token);
|
|
1573
|
-
continue;
|
|
1574
|
-
}
|
|
1575
|
-
const slot = slots.find((key) => !Object.hasOwn(params, key));
|
|
1576
|
-
if (slot) set(slot, token, token);
|
|
1577
|
-
else requestWords.push(token);
|
|
1724
|
+
async function saveBraveKey(client, config, key, projectId) {
|
|
1725
|
+
try {
|
|
1726
|
+
await client.updateConfig({
|
|
1727
|
+
braveSearchApiKey: key,
|
|
1728
|
+
webSearch: { ...config.webSearch, provider: "brave" }
|
|
1729
|
+
});
|
|
1730
|
+
} catch {
|
|
1731
|
+
throw new CliError("Could not save the key. Check the Gezel service, then run again.");
|
|
1578
1732
|
}
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
if (typeof key === "string" && !Object.hasOwn(params, key) && properties[key]?.default === void 0)
|
|
1586
|
-
throw new CliError(`missing required craftbook parameter: ${key}`);
|
|
1587
|
-
}
|
|
1588
|
-
const request = requestWords.join(" ").trim();
|
|
1589
|
-
return request ? { params, request } : { params };
|
|
1590
|
-
}
|
|
1591
|
-
function craftbookDoLaunch(craftbook2, { params, request }) {
|
|
1592
|
-
if (!request) return { craftbookParams: params };
|
|
1593
|
-
const launch = composeCraftbookLaunch({
|
|
1594
|
-
message: request,
|
|
1595
|
-
craftbookName: craftbook2.name,
|
|
1596
|
-
paramSchema: craftbook2.paramSchema,
|
|
1597
|
-
params
|
|
1733
|
+
const failure = await client.toolWebSearch(projectId, SEARCH_CHECK).then(
|
|
1734
|
+
() => void 0,
|
|
1735
|
+
(error) => serviceErrorText(error)
|
|
1736
|
+
);
|
|
1737
|
+
if (failure === void 0) return;
|
|
1738
|
+
await client.updateConfig({ braveSearchApiKey: "", webSearch: { ...config.webSearch } }).catch(() => {
|
|
1598
1739
|
});
|
|
1599
|
-
|
|
1740
|
+
throw new CliError(
|
|
1741
|
+
`A test search with that key failed, so it was not kept: ${failure.replaceAll(key, "\u2026")}
|
|
1742
|
+
Check the key at ${BRAVE_KEY_URL}, then run again.`
|
|
1743
|
+
);
|
|
1600
1744
|
}
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
return apiError.status === 0 && apiError.details?.kind === "transport" && apiError.details.readRetryExhausted !== true && typeof apiError.details.cause === "string" && codes.test(apiError.details.cause);
|
|
1745
|
+
function serviceErrorText(error) {
|
|
1746
|
+
const details = error?.details;
|
|
1747
|
+
if (details && typeof details === "object") {
|
|
1748
|
+
const text = details.error;
|
|
1749
|
+
if (typeof text === "string" && text.trim()) return text.trim();
|
|
1607
1750
|
}
|
|
1608
|
-
return error instanceof
|
|
1751
|
+
return error instanceof Error ? error.message : String(error);
|
|
1609
1752
|
}
|
|
1610
|
-
async function
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
if (!transientTaskReadError(error) || retries >= READ_RETRY_DELAYS.length || budget.expired())
|
|
1619
|
-
throw error;
|
|
1620
|
-
await sleepWithinBudget(READ_RETRY_DELAYS[retries], budget, signal);
|
|
1621
|
-
}
|
|
1622
|
-
}
|
|
1753
|
+
async function ensureModelEngines(client, engines, write) {
|
|
1754
|
+
const status = await client.getNativeEngineStatus();
|
|
1755
|
+
if (!status.pinned || !status.platformKey) return;
|
|
1756
|
+
const missing = [...new Set(engines.map((engine) => ENGINE_BINARY[engine]))].filter(
|
|
1757
|
+
(binary) => !status.engines.some((entry) => entry.name === binary && entry.installed)
|
|
1758
|
+
);
|
|
1759
|
+
if (missing.length === 0) return;
|
|
1760
|
+
await installNativeToolkit(client, { engines: missing, output: { writeProgress: write } });
|
|
1623
1761
|
}
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1762
|
+
|
|
1763
|
+
// src/run-readiness.ts
|
|
1764
|
+
import { resolveOnDeviceProvider as resolveOnDeviceProvider2 } from "@bendyline/gezel/native";
|
|
1765
|
+
|
|
1766
|
+
// src/model-choice.ts
|
|
1767
|
+
import {
|
|
1768
|
+
measuredClaim,
|
|
1769
|
+
measuredResultFor
|
|
1770
|
+
} from "@bendyline/gezel";
|
|
1771
|
+
async function installedChatModels(client, engine) {
|
|
1772
|
+
const { models } = engine === "mlx" ? await client.listMlxModels() : engine === "ds4" ? await client.listDs4Models() : await client.listLlamaCppModels();
|
|
1773
|
+
return models.map((model2) => model2.id);
|
|
1774
|
+
}
|
|
1775
|
+
function providerFlag(engine, platformDefault) {
|
|
1776
|
+
return engine === platformDefault ? "" : ` --provider ${engine}`;
|
|
1777
|
+
}
|
|
1778
|
+
var FIT_LABEL = {
|
|
1779
|
+
fits: "fits",
|
|
1780
|
+
"fits-offload": "fits, partly offloaded",
|
|
1781
|
+
tight: "tight",
|
|
1782
|
+
"too-big": "too big"
|
|
1783
|
+
};
|
|
1784
|
+
async function availableChatModels(client, engine) {
|
|
1785
|
+
if (engine === "ds4") {
|
|
1786
|
+
throw new CliError("`--available` lists mlx and llama-cpp models; pass one with --provider.");
|
|
1631
1787
|
}
|
|
1788
|
+
const [memory, catalog, installed, config] = await Promise.all([
|
|
1789
|
+
client.getMemoryProfile(),
|
|
1790
|
+
client.listCatalogItems("chat-model"),
|
|
1791
|
+
installedChatModels(client, engine),
|
|
1792
|
+
client.getConfig()
|
|
1793
|
+
]);
|
|
1794
|
+
const pinned = config.defaultModel?.[engine];
|
|
1795
|
+
return {
|
|
1796
|
+
ranked: rankChatModels(catalog.items, recoDevice(memory), engine),
|
|
1797
|
+
installed: new Set(installed),
|
|
1798
|
+
...pinned ? { pinned } : {}
|
|
1799
|
+
};
|
|
1632
1800
|
}
|
|
1633
|
-
|
|
1634
|
-
const
|
|
1635
|
-
|
|
1636
|
-
const
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
const { questions } = await read(
|
|
1664
|
-
() => client.listQuestions(
|
|
1665
|
-
{
|
|
1666
|
-
projectId: task2.projectId,
|
|
1667
|
-
pending: true,
|
|
1668
|
-
taskRefs: [...refs]
|
|
1669
|
-
},
|
|
1670
|
-
signal
|
|
1671
|
-
)
|
|
1672
|
-
);
|
|
1673
|
-
const questionIds = questions.filter(
|
|
1674
|
-
(q) => q.taskRef && refs.has(q.taskRef) && // A service pause card is a historical notification. Actual task
|
|
1675
|
-
// status above decides whether it is still paused; real session
|
|
1676
|
-
// questions and permissions must continue to block observation.
|
|
1677
|
-
!(q.sessionId === "" && q.intent?.kind === "task-paused")
|
|
1678
|
-
).map((q) => q.id);
|
|
1679
|
-
if (questionIds.length) return { task: task2, outcome: "blocked", exitCode: 2, questionIds };
|
|
1680
|
-
}
|
|
1681
|
-
if (budget.expired()) return { task: task2, outcome: "timeout", exitCode: 3 };
|
|
1682
|
-
await delay(Math.min(options.pollMs ?? 1e3, budget.remainingMs()), void 0, { signal });
|
|
1683
|
-
}
|
|
1684
|
-
} catch (error) {
|
|
1685
|
-
options.signal?.throwIfAborted();
|
|
1686
|
-
if (budget.expired() && lastTask) return { task: lastTask, outcome: "timeout", exitCode: 3 };
|
|
1687
|
-
throw error;
|
|
1688
|
-
} finally {
|
|
1689
|
-
timeout.dispose();
|
|
1801
|
+
function formatAvailableChatModels(input) {
|
|
1802
|
+
const { engine, ranked, installed, pinned, measured } = input;
|
|
1803
|
+
if (ranked.length === 0) return `(no ${engine} chat models in the catalog for this machine)`;
|
|
1804
|
+
const width = Math.max(...ranked.map((model2) => model2.id.length));
|
|
1805
|
+
const measuredWidth = measured ? Math.max("not measured".length, ...ranked.map((m) => (measured.get(m.id) ?? "").length)) : 0;
|
|
1806
|
+
const lines = [
|
|
1807
|
+
`${engine} chat models for this machine, the recommended first download on top, then by catalog ranking:`
|
|
1808
|
+
];
|
|
1809
|
+
ranked.forEach((model2, index) => {
|
|
1810
|
+
const notes = [
|
|
1811
|
+
index === 0 ? "recommended" : "",
|
|
1812
|
+
installed.has(model2.id) ? "installed" : "",
|
|
1813
|
+
model2.id === pinned ? "in use" : ""
|
|
1814
|
+
].filter(Boolean);
|
|
1815
|
+
const claim = measured ? ` ${(measured.get(model2.id) ?? "not measured").padEnd(measuredWidth)}` : "";
|
|
1816
|
+
lines.push(
|
|
1817
|
+
` ${model2.id.padEnd(width)} ${formatDownloadSize(model2.approxSizeBytes).padStart(7)} ${FIT_LABEL[model2.fit].padEnd(22)}${claim} ${model2.name}${notes.length > 0 ? ` (${notes.join(", ")})` : ""}`
|
|
1818
|
+
);
|
|
1819
|
+
});
|
|
1820
|
+
const flag = providerFlag(engine, input.platformDefault);
|
|
1821
|
+
const smallest = [...ranked].sort((a, b) => a.approxSizeBytes - b.approxSizeBytes)[0];
|
|
1822
|
+
lines.push(
|
|
1823
|
+
"",
|
|
1824
|
+
`Download one with \`gezel model pull <id>${flag}\`. The smallest is ${smallest.id} (${formatDownloadSize(smallest.approxSizeBytes)}).`,
|
|
1825
|
+
`\`gezel model use <id>${flag}\` picks the installed model chats use.`
|
|
1826
|
+
);
|
|
1827
|
+
if (measured) {
|
|
1828
|
+
lines.push(
|
|
1829
|
+
'"Fits" is about memory. The measured column is the published scorecard\'s pass rate on real jobs, on the hardware named; `gezel bench` measures a model on this machine.'
|
|
1830
|
+
);
|
|
1690
1831
|
}
|
|
1832
|
+
return lines.join("\n");
|
|
1691
1833
|
}
|
|
1692
|
-
function
|
|
1693
|
-
const
|
|
1694
|
-
const
|
|
1695
|
-
|
|
1696
|
-
const body = lines.slice(headingIndex + 1).find((line) => line.length > 0 && !line.startsWith("#"));
|
|
1697
|
-
const sentence = body?.match(/^.*?[.!?](?=\s|$)/)?.[0] ?? body ?? "";
|
|
1698
|
-
const summary = heading && sentence ? `${heading}: ${sentence}` : heading || sentence;
|
|
1699
|
-
return summary.length > maxLength ? `${summary.slice(0, maxLength - 1).trimEnd()}\u2026` : summary;
|
|
1834
|
+
function measuredClaimsFor(dataset, ids, device) {
|
|
1835
|
+
const claims = /* @__PURE__ */ new Map();
|
|
1836
|
+
for (const id of ids) claims.set(id, measuredClaim(measuredResultFor(dataset, id, device)));
|
|
1837
|
+
return claims;
|
|
1700
1838
|
}
|
|
1701
|
-
async function
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1839
|
+
async function useChatModel(client, id, options) {
|
|
1840
|
+
const order = options.requested ? [options.requested] : [options.platformDefault, ...["mlx", "llama-cpp", "ds4"]].filter(
|
|
1841
|
+
(engine2, index, all) => all.indexOf(engine2) === index
|
|
1842
|
+
);
|
|
1843
|
+
let engine;
|
|
1844
|
+
for (const candidate of order) {
|
|
1845
|
+
const installed = await installedChatModels(client, candidate).catch(() => []);
|
|
1846
|
+
if (installed.includes(id)) {
|
|
1847
|
+
engine = candidate;
|
|
1848
|
+
break;
|
|
1849
|
+
}
|
|
1705
1850
|
}
|
|
1706
|
-
if (
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1851
|
+
if (!engine) {
|
|
1852
|
+
const target = options.requested ?? options.platformDefault;
|
|
1853
|
+
throw new CliError(
|
|
1854
|
+
`${id} isn't downloaded for ${target}. Download it with \`gezel model pull ${id}${providerFlag(target, options.platformDefault)}\`, or see \`gezel model list --available\`.`
|
|
1855
|
+
);
|
|
1856
|
+
}
|
|
1857
|
+
const config = await client.getConfig();
|
|
1858
|
+
await client.updateConfig({
|
|
1859
|
+
provider: engine,
|
|
1860
|
+
defaultModel: { ...config.defaultModel ?? {}, [engine]: id },
|
|
1861
|
+
firstRunCompleted: true
|
|
1862
|
+
});
|
|
1863
|
+
return engine;
|
|
1710
1864
|
}
|
|
1711
|
-
function
|
|
1712
|
-
const
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1865
|
+
async function pinPulledModel(client, engine, id) {
|
|
1866
|
+
const config = await client.getConfig();
|
|
1867
|
+
const current = config.defaultModel?.[engine];
|
|
1868
|
+
if (current === id) return "already";
|
|
1869
|
+
if (current && (await installedChatModels(client, engine)).includes(current)) return "kept";
|
|
1870
|
+
await client.updateConfig({
|
|
1871
|
+
defaultModel: { ...config.defaultModel ?? {}, [engine]: id },
|
|
1872
|
+
firstRunCompleted: true
|
|
1717
1873
|
});
|
|
1874
|
+
return "pinned";
|
|
1718
1875
|
}
|
|
1719
1876
|
|
|
1720
|
-
// src/
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
let lastPct = -1;
|
|
1733
|
-
let pullError;
|
|
1734
|
-
const onEvent = (ev) => {
|
|
1735
|
-
if (ev.type === "progress") {
|
|
1736
|
-
const written = "bytesWrittenAll" in ev ? ev.bytesWrittenAll : ev.bytesWritten;
|
|
1737
|
-
const total = "totalBytesAll" in ev ? ev.totalBytesAll : ev.totalBytes ?? 0;
|
|
1738
|
-
const pct = total > 0 ? Math.floor(written / total * 100) : 0;
|
|
1739
|
-
if (pct === lastPct) return;
|
|
1740
|
-
lastPct = pct;
|
|
1741
|
-
writeProgress(
|
|
1742
|
-
`\rdownloading ${id} (${engine}): ${String(pct).padStart(3)}% ${formatGb(written)}/${formatGb(total)} GB`
|
|
1743
|
-
);
|
|
1744
|
-
} else if (ev.type === "retrying") {
|
|
1745
|
-
writeProgress(`
|
|
1746
|
-
retry ${ev.attempt}/${ev.maxAttempts}: ${ev.reason}
|
|
1747
|
-
`);
|
|
1748
|
-
} else if (ev.type === "error") {
|
|
1749
|
-
pullError = ev.error;
|
|
1750
|
-
writeProgress("\n");
|
|
1751
|
-
} else if (ev.type === "done" && !pullError) {
|
|
1752
|
-
writeProgress(`\rdownloaded ${id} (${engine})${" ".repeat(48)}
|
|
1753
|
-
`);
|
|
1754
|
-
}
|
|
1755
|
-
};
|
|
1756
|
-
if (engine === "mlx") await client.installMlxModel(id, onEvent);
|
|
1757
|
-
else if (engine === "ds4") await client.installDs4Model(id, onEvent);
|
|
1758
|
-
else await client.installLlamaCppModel(id, onEvent);
|
|
1759
|
-
if (pullError) throw new CliError(`download failed: ${pullError}`);
|
|
1877
|
+
// src/run-readiness.ts
|
|
1878
|
+
var READY = { ready: true };
|
|
1879
|
+
async function settledFirstRunConfig(client, options = {}) {
|
|
1880
|
+
const attempts = options.attempts ?? 12;
|
|
1881
|
+
const delayMs = options.delayMs ?? 250;
|
|
1882
|
+
let config = await client.getConfig();
|
|
1883
|
+
for (let attempt = 0; attempt < attempts; attempt += 1) {
|
|
1884
|
+
if (config.firstRunCompleted || config.provider !== "copilot") return config;
|
|
1885
|
+
await new Promise((resolve7) => setTimeout(resolve7, delayMs));
|
|
1886
|
+
config = await client.getConfig();
|
|
1887
|
+
}
|
|
1888
|
+
return config;
|
|
1760
1889
|
}
|
|
1761
|
-
async function
|
|
1762
|
-
|
|
1763
|
-
const
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
writeProgress(
|
|
1771
|
-
`\rpreparing the MLX Python runtime (one-time, a few minutes): ${Math.floor(elapsed / 60)}m ${String(elapsed % 60).padStart(2, "0")}s`
|
|
1772
|
-
);
|
|
1773
|
-
shown = true;
|
|
1774
|
-
await new Promise((resolve7) => setTimeout(resolve7, pollMs));
|
|
1775
|
-
continue;
|
|
1776
|
-
}
|
|
1777
|
-
if (status.phase === "error") {
|
|
1778
|
-
writeProgress(
|
|
1779
|
-
`${shown ? "\n" : ""}MLX runtime setup failed; the first chat will retry it: ${status.error ?? "unknown error"}
|
|
1780
|
-
`
|
|
1781
|
-
);
|
|
1782
|
-
return "error";
|
|
1890
|
+
async function checkRunReadiness(client, gezelId, env2 = process.env) {
|
|
1891
|
+
if (env2.GEZEL_MOCK_PROVIDER === "1") return READY;
|
|
1892
|
+
const config = await settledFirstRunConfig(client);
|
|
1893
|
+
let provider = config.provider;
|
|
1894
|
+
if (gezelId) {
|
|
1895
|
+
try {
|
|
1896
|
+
const { parsed } = await client.getGezel(gezelId);
|
|
1897
|
+
if (parsed.frontmatter.provider) provider = parsed.frontmatter.provider;
|
|
1898
|
+
} catch {
|
|
1783
1899
|
}
|
|
1784
|
-
if (shown) writeProgress(`\rMLX runtime ready${" ".repeat(56)}
|
|
1785
|
-
`);
|
|
1786
|
-
return status.phase === "ready" ? "ready" : "idle";
|
|
1787
1900
|
}
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
}
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
formatModelAttribution,
|
|
1802
|
-
isMoEFromTags,
|
|
1803
|
-
isRecommendedModel,
|
|
1804
|
-
lowBitScoreFactor,
|
|
1805
|
-
pickRecommendedModel
|
|
1806
|
-
} from "@bendyline/gezel";
|
|
1807
|
-
var CHAT_MEMORY_OVERHEAD = {
|
|
1808
|
-
"llama-cpp": 1.2,
|
|
1809
|
-
mlx: 1.3
|
|
1810
|
-
};
|
|
1811
|
-
var NATIVE_TOOLKIT = [
|
|
1812
|
-
"uv",
|
|
1813
|
-
"sd-server",
|
|
1814
|
-
"whisper-server",
|
|
1815
|
-
"llama-server"
|
|
1816
|
-
];
|
|
1817
|
-
function recoDevice(memory) {
|
|
1901
|
+
if (provider !== "llama-cpp" && provider !== "mlx") return READY;
|
|
1902
|
+
if (provider === "llama-cpp" && (config.llamaCppBaseUrl || config.llamaCppModelPath)) {
|
|
1903
|
+
return READY;
|
|
1904
|
+
}
|
|
1905
|
+
if (provider === "mlx" && (config.mlxBaseUrl || config.mlxModelPath)) return READY;
|
|
1906
|
+
const native2 = await client.getNativeEngineStatus();
|
|
1907
|
+
if (!native2.platformKey) return READY;
|
|
1908
|
+
const installed = provider === "mlx" ? await client.listMlxModels() : await client.listLlamaCppModels();
|
|
1909
|
+
const installedIds = installed.models.map((model2) => model2.id);
|
|
1910
|
+
const pinned = config.defaultModel?.[provider];
|
|
1911
|
+
const engineMissing = provider === "llama-cpp" && !native2.engines.some((engine) => engine.name === "llama-server" && engine.installed);
|
|
1912
|
+
const modelReady = pinned ? installedIds.includes(pinned) : installedIds.length > 0;
|
|
1913
|
+
if (!engineMissing && modelReady) return READY;
|
|
1818
1914
|
return {
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1915
|
+
ready: false,
|
|
1916
|
+
message: formatRunSetupMessage({
|
|
1917
|
+
provider,
|
|
1918
|
+
engineMissing,
|
|
1919
|
+
pinned,
|
|
1920
|
+
installedIds
|
|
1921
|
+
})
|
|
1825
1922
|
};
|
|
1826
1923
|
}
|
|
1827
|
-
function
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
if (item.manifest.kind !== "chat-model") continue;
|
|
1831
|
-
const manifest = item.manifest;
|
|
1832
|
-
if (manifest.licenseClass !== "open") continue;
|
|
1833
|
-
const source = provider === "mlx" ? manifest.mlx : manifest.llamaCpp;
|
|
1834
|
-
if (!source || provider === "mlx" && manifest.mlx?.disabledReason) continue;
|
|
1835
|
-
const residentBytes = source.residentBytes ?? Math.round(source.approxSizeBytes * CHAT_MEMORY_OVERHEAD[provider]);
|
|
1836
|
-
const fit = computeModelFit({
|
|
1837
|
-
residentBytes,
|
|
1838
|
-
isMoE: isMoEFromTags(manifest.tags),
|
|
1839
|
-
usableBytes: device.usableBytes,
|
|
1840
|
-
totalRamBytes: device.totalRamBytes,
|
|
1841
|
-
gpuVramBytes: device.gpuVramBytes,
|
|
1842
|
-
...device.budgetBytes !== void 0 ? { admissibleBytes: device.budgetBytes } : {}
|
|
1843
|
-
}).tier;
|
|
1844
|
-
candidates.push({
|
|
1845
|
-
id: manifest.id,
|
|
1846
|
-
name: manifest.name,
|
|
1847
|
-
provider,
|
|
1848
|
-
approxSizeBytes: source.approxSizeBytes,
|
|
1849
|
-
residentBytes,
|
|
1850
|
-
// Effective score: a model the shared gate rejects (restricted license,
|
|
1851
|
-
// no tool support) sorts as unscored rather than leading the list.
|
|
1852
|
-
recoScore: isRecommendedModel(manifest) ? manifest.recoScore ?? 0 : 0,
|
|
1853
|
-
...source.quantization ? { quantization: source.quantization } : {},
|
|
1854
|
-
fit,
|
|
1855
|
-
attribution: formatModelAttribution(manifest)
|
|
1856
|
-
});
|
|
1857
|
-
}
|
|
1858
|
-
return candidates;
|
|
1924
|
+
function canOfferRunSetup(io = process, env2 = process.env) {
|
|
1925
|
+
if (env2.CI) return false;
|
|
1926
|
+
return io.stdin.isTTY === true && io.stderr.isTTY === true;
|
|
1859
1927
|
}
|
|
1860
|
-
function
|
|
1861
|
-
const
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
id: model2.id,
|
|
1866
|
-
...model2.recoScore > 0 ? { recoScore: model2.recoScore } : {},
|
|
1867
|
-
licenseClass: "open",
|
|
1868
|
-
tags: items.find((item) => item.manifest.kind === "chat-model" && item.manifest.id === model2.id)?.manifest?.tags ?? [],
|
|
1869
|
-
residentBytes: model2.residentBytes,
|
|
1870
|
-
...model2.quantization ? { quantization: model2.quantization } : {}
|
|
1871
|
-
})),
|
|
1872
|
-
device
|
|
1873
|
-
);
|
|
1874
|
-
const comfortable = candidates.filter(
|
|
1875
|
-
(model2) => model2.fit === "fits" || model2.fit === "fits-offload"
|
|
1876
|
-
);
|
|
1877
|
-
const pool = comfortable.length > 0 ? comfortable : candidates;
|
|
1878
|
-
const ranked = (model2) => model2.recoScore * lowBitScoreFactor(model2.quantization);
|
|
1879
|
-
pool.sort(
|
|
1880
|
-
(a, b) => ranked(b) - ranked(a) || fitOrder(a.fit) - fitOrder(b.fit) || b.residentBytes - a.residentBytes || a.name.localeCompare(b.name)
|
|
1928
|
+
function formatRunSetupMessage(input) {
|
|
1929
|
+
const { provider, engineMissing, pinned, installedIds } = input;
|
|
1930
|
+
const flag = providerFlag(
|
|
1931
|
+
provider,
|
|
1932
|
+
input.platformDefault ?? resolveOnDeviceProvider2(process.platform, process.arch)
|
|
1881
1933
|
);
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
if (
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1934
|
+
const steps = [];
|
|
1935
|
+
if (engineMissing) steps.push("gezel native install");
|
|
1936
|
+
if (pinned && !installedIds.includes(pinned)) {
|
|
1937
|
+
if (installedIds.length > 0) {
|
|
1938
|
+
steps.push(`gezel model use ${installedIds[0]}${flag} (already downloaded)`);
|
|
1939
|
+
}
|
|
1940
|
+
steps.push(`gezel model pull ${pinned}${flag}`);
|
|
1941
|
+
if (installedIds.length === 0) {
|
|
1942
|
+
steps.push(`gezel model list --available${flag} (to pick a smaller one)`);
|
|
1943
|
+
}
|
|
1944
|
+
} else if (!pinned && installedIds.length === 0) {
|
|
1945
|
+
steps.push(`gezel model list --available${flag} (then: gezel model pull <id>${flag})`);
|
|
1888
1946
|
}
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1947
|
+
const lines = [];
|
|
1948
|
+
if (pinned && !installedIds.includes(pinned) && installedIds.length > 0) {
|
|
1949
|
+
lines.push(`Your selected on-device model, ${pinned}, isn't downloaded yet.`);
|
|
1950
|
+
} else if (engineMissing && installedIds.length > 0 && (!pinned || installedIds.includes(pinned))) {
|
|
1951
|
+
lines.push("Gezel's on-device engine isn't installed yet.");
|
|
1952
|
+
} else {
|
|
1953
|
+
lines.push("Gezel isn't set up to chat yet: this device doesn't have an on-device model.");
|
|
1892
1954
|
}
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
switch (fit) {
|
|
1901
|
-
case "fits":
|
|
1902
|
-
return 0;
|
|
1903
|
-
case "fits-offload":
|
|
1904
|
-
return 1;
|
|
1905
|
-
case "tight":
|
|
1906
|
-
return 2;
|
|
1907
|
-
case "too-big":
|
|
1908
|
-
return 3;
|
|
1955
|
+
lines.push(
|
|
1956
|
+
"",
|
|
1957
|
+
"Run it again with --download to download what this computer needs and run your request.",
|
|
1958
|
+
"Or run `gezel` to choose a model and download it, step by step."
|
|
1959
|
+
);
|
|
1960
|
+
if (steps.length > 0) {
|
|
1961
|
+
lines.push("", "Or set it up directly:", ...steps.map((step) => ` ${step}`));
|
|
1909
1962
|
}
|
|
1963
|
+
lines.push(
|
|
1964
|
+
"",
|
|
1965
|
+
"To use a cloud model instead, save a key (for example",
|
|
1966
|
+
"`gezel secret set openaiApiKey --stdin`), then run `gezel` and pick it with /model.",
|
|
1967
|
+
"",
|
|
1968
|
+
"Nothing was downloaded."
|
|
1969
|
+
);
|
|
1970
|
+
return lines.join("\n");
|
|
1910
1971
|
}
|
|
1911
1972
|
|
|
1912
|
-
// src/
|
|
1913
|
-
var
|
|
1914
|
-
function
|
|
1915
|
-
|
|
1916
|
-
if (NATIVE_VARIANTS.includes(value)) return value;
|
|
1917
|
-
throw new CliError(`--variant must be one of ${NATIVE_VARIANTS.join(", ")} (got "${value}")`);
|
|
1918
|
-
}
|
|
1919
|
-
function parseNativeEngine(value) {
|
|
1920
|
-
if (value === void 0) return void 0;
|
|
1921
|
-
const parsed = NativeEngineNameSchema.safeParse(value);
|
|
1922
|
-
if (parsed.success) return parsed.data;
|
|
1923
|
-
throw new CliError(
|
|
1924
|
-
`--engine must be one of ${NativeEngineNameSchema.options.join(", ")} (got "${value}")`
|
|
1925
|
-
);
|
|
1926
|
-
}
|
|
1927
|
-
function formatNativeStatus(status) {
|
|
1928
|
-
const installed = new Set(
|
|
1929
|
-
status.engines.filter((engine) => engine.installed).map((engine) => engine.name)
|
|
1930
|
-
);
|
|
1931
|
-
const toolkitInstalled = NATIVE_TOOLKIT.filter((engine) => installed.has(engine)).length;
|
|
1932
|
-
return [
|
|
1933
|
-
`release: native-v${status.release} (${status.pinned ? "verified pin" : "unpinned \u2014 downloads disabled"})`,
|
|
1934
|
-
`platform: ${status.platformKey ?? "unsupported \u2014 downloads unavailable"}`,
|
|
1935
|
-
`llama backend: ${status.llamaBackend ?? "automatic"}`,
|
|
1936
|
-
// Name the members: `native list` shows every engine the release carries
|
|
1937
|
-
// (ds4, duckdb too), so a bare "0/4" read as a miscount beside it.
|
|
1938
|
-
`toolkit: ${toolkitInstalled}/${NATIVE_TOOLKIT.length} installed (${NATIVE_TOOLKIT.join(", ")}; see \`gezel native list\` for every engine)`
|
|
1939
|
-
].join("\n");
|
|
1940
|
-
}
|
|
1941
|
-
function formatNativeList(status) {
|
|
1942
|
-
const rows = status.engines.map((engine) => {
|
|
1943
|
-
const state = engine.installed ? "installed" : "missing";
|
|
1944
|
-
return `${engine.name.padEnd(18)} ${state.padEnd(10)}${engine.path ? ` ${engine.path}` : ""}`;
|
|
1945
|
-
});
|
|
1946
|
-
return [
|
|
1947
|
-
`native-v${status.release} \xB7 ${status.platformKey ?? "unsupported platform"} \xB7 ${status.pinned ? "verified" : "unpinned"}`,
|
|
1948
|
-
"engine status path",
|
|
1949
|
-
...rows
|
|
1950
|
-
].join("\n");
|
|
1973
|
+
// src/auto-download.ts
|
|
1974
|
+
var AUTO_DOWNLOAD_ENV = "GEZEL_AUTO_DOWNLOAD";
|
|
1975
|
+
function autoDownloadRequested(globals, env2 = process.env) {
|
|
1976
|
+
return globals.download === true || env2[AUTO_DOWNLOAD_ENV] === "1";
|
|
1951
1977
|
}
|
|
1952
|
-
async function
|
|
1953
|
-
const
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1978
|
+
async function downloadForRun(client, options) {
|
|
1979
|
+
const { write } = options;
|
|
1980
|
+
const config = await settledFirstRunConfig(client);
|
|
1981
|
+
let provider = config.provider;
|
|
1982
|
+
if (options.gezelId) {
|
|
1983
|
+
try {
|
|
1984
|
+
const { parsed } = await client.getGezel(options.gezelId);
|
|
1985
|
+
if (parsed.frontmatter.provider) provider = parsed.frontmatter.provider;
|
|
1986
|
+
} catch {
|
|
1987
|
+
}
|
|
1958
1988
|
}
|
|
1959
|
-
if (
|
|
1989
|
+
if (provider !== "llama-cpp" && provider !== "mlx") return;
|
|
1990
|
+
if (provider === "llama-cpp" && (config.llamaCppBaseUrl || config.llamaCppModelPath)) return;
|
|
1991
|
+
if (provider === "mlx" && (config.mlxBaseUrl || config.mlxModelPath)) return;
|
|
1992
|
+
if (!(await client.getNativeEngineStatus()).platformKey) return;
|
|
1993
|
+
write("Getting this computer ready for local models (--download)\u2026\n");
|
|
1994
|
+
await ensureModelEngines(client, [provider], write);
|
|
1995
|
+
const installed = provider === "mlx" ? await client.listMlxModels() : await client.listLlamaCppModels();
|
|
1996
|
+
const installedIds = new Set(installed.models.map((model2) => model2.id));
|
|
1997
|
+
const pinned = config.defaultModel?.[provider];
|
|
1998
|
+
if (pinned ? installedIds.has(pinned) : installedIds.size > 0) return;
|
|
1999
|
+
const recommend = options.recommend ?? (async (engine) => {
|
|
2000
|
+
const [memory, catalog] = await Promise.all([
|
|
2001
|
+
client.getMemoryProfile(),
|
|
2002
|
+
client.listCatalogItems("chat-model")
|
|
2003
|
+
]);
|
|
2004
|
+
return rankChatModels(catalog.items, recoDevice(memory), engine)[0];
|
|
2005
|
+
});
|
|
2006
|
+
const best = await recommend(provider);
|
|
2007
|
+
const target = pinned ?? best?.id;
|
|
2008
|
+
if (!target) {
|
|
1960
2009
|
throw new CliError(
|
|
1961
|
-
|
|
2010
|
+
"No open chat model in the catalog fits this computer. Save a cloud key instead (for example `gezel secret set openaiApiKey --stdin`) and pick it in `gezel` with /model."
|
|
1962
2011
|
);
|
|
1963
2012
|
}
|
|
1964
|
-
const
|
|
1965
|
-
|
|
1966
|
-
await
|
|
2013
|
+
const size = best && best.id === target ? best.approxSizeBytes : 0;
|
|
2014
|
+
if (size > 0) {
|
|
2015
|
+
const space = await client.checkModelDownloadSpace({ sizeBytes: size });
|
|
2016
|
+
if (space.known && !space.ok) {
|
|
2017
|
+
throw new CliError(
|
|
2018
|
+
`${best?.name ?? target} needs ${formatGb(space.requiredBytes)} GB, and Gezel model storage has ${formatGb(space.freeBytes)} GB free. Free some space, then run again.`
|
|
2019
|
+
);
|
|
2020
|
+
}
|
|
1967
2021
|
}
|
|
1968
|
-
|
|
2022
|
+
write(
|
|
2023
|
+
pinned ? `Downloading ${target}, the model this computer is set to use.
|
|
2024
|
+
` : `Downloading ${best?.name ?? target}${size > 0 ? ` (${formatGb(size)} GB)` : ""}, the model recommended for this computer.
|
|
2025
|
+
`
|
|
2026
|
+
);
|
|
2027
|
+
await pullChatModel(client, provider, target, write);
|
|
2028
|
+
if (provider === "mlx") await waitForMlxRuntime(client, write);
|
|
2029
|
+
await client.updateConfig({
|
|
2030
|
+
provider,
|
|
2031
|
+
defaultModel: { ...config.defaultModel, [provider]: target },
|
|
2032
|
+
firstRunInstallError: null
|
|
2033
|
+
});
|
|
2034
|
+
write("Ready.\n\n");
|
|
1969
2035
|
}
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
2036
|
+
|
|
2037
|
+
// src/bench-command.ts
|
|
2038
|
+
import { execFile } from "child_process";
|
|
2039
|
+
import { readFileSync } from "fs";
|
|
2040
|
+
import { readFile as readFile2, writeFile as writeFile2 } from "fs/promises";
|
|
2041
|
+
import { cpus } from "os";
|
|
2042
|
+
import { join as join2 } from "path";
|
|
2043
|
+
import { promisify } from "util";
|
|
2044
|
+
import { ScorecardDatasetSchema } from "@bendyline/gezel";
|
|
2045
|
+
import { gildePackageRoot } from "@bendyline/gezel-catalog";
|
|
2046
|
+
import {
|
|
2047
|
+
EVAL_SUBSET_NOTE,
|
|
2048
|
+
SCORECARD_SUITE_IDS,
|
|
2049
|
+
SCORECARD_SUITE_LABELS,
|
|
2050
|
+
buildEvalMatrix,
|
|
2051
|
+
describeEvalHost,
|
|
2052
|
+
describeEvalRun,
|
|
2053
|
+
evalDeviceFromHost,
|
|
2054
|
+
evalEngineBuilds,
|
|
2055
|
+
evalRunId,
|
|
2056
|
+
evalRunProvenance,
|
|
2057
|
+
evalShard,
|
|
2058
|
+
evalSuiteResult,
|
|
2059
|
+
isSuiteSubset,
|
|
2060
|
+
medianDecodeTokensPerSec,
|
|
2061
|
+
smokeSuiteIdFor
|
|
2062
|
+
} from "@bendyline/gezel/eval";
|
|
2063
|
+
var BENCH_CONFIRM_ABOVE_MS = 60 * 6e4;
|
|
2064
|
+
var BENCH_SUBMIT_REPO = "bendyline/gezel";
|
|
2065
|
+
function formatDuration(ms) {
|
|
2066
|
+
if (ms < 6e4) return `${Math.max(1, Math.round(ms / 1e3))}s`;
|
|
2067
|
+
const minutes = Math.round(ms / 6e4);
|
|
2068
|
+
if (minutes < 60) return `${minutes}m`;
|
|
2069
|
+
const hours = Math.floor(minutes / 60);
|
|
2070
|
+
const rest = minutes % 60;
|
|
2071
|
+
return rest ? `${hours}h ${rest}m` : `${hours}h`;
|
|
2072
|
+
}
|
|
2073
|
+
function resolveBenchSuites(catalog, opts) {
|
|
2074
|
+
const wanted = !opts.suite || opts.suite === "core" || opts.suite === "default" ? ["core"] : opts.suite === "all" ? (
|
|
2075
|
+
// Every scorecard suite this daemon's catalog carries; an older
|
|
2076
|
+
// harness without one of them still runs the rest.
|
|
2077
|
+
SCORECARD_SUITE_IDS.filter((id) => catalog.suites.some((s) => s.id === id))
|
|
2078
|
+
) : opts.suite.split(",").map((s) => s.trim()).filter(Boolean);
|
|
2079
|
+
const suites = [];
|
|
2080
|
+
for (const id of wanted) {
|
|
2081
|
+
const base = catalog.suites.find((s) => s.id === id);
|
|
2082
|
+
if (!base) {
|
|
2083
|
+
throw new CliError(
|
|
2084
|
+
`Unknown suite "${id}". Choose one of: ${SCORECARD_SUITE_IDS.join(", ")}, or all. \`gezel bench list\` shows every suite.`
|
|
2085
|
+
);
|
|
1982
2086
|
}
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
`
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
`);
|
|
1993
|
-
break;
|
|
1994
|
-
case "done":
|
|
1995
|
-
output.writeProgress(
|
|
1996
|
-
`${engine}: ready${event.cached ? " (cached)" : ` \u2014 ${event.binPath}`}
|
|
1997
|
-
`
|
|
1998
|
-
);
|
|
1999
|
-
break;
|
|
2000
|
-
case "error":
|
|
2001
|
-
terminalError = event.error;
|
|
2002
|
-
output.writeProgress(`${engine}: error \u2014 ${event.error}
|
|
2003
|
-
`);
|
|
2004
|
-
break;
|
|
2087
|
+
if (opts.smoke) {
|
|
2088
|
+
const smokeId = smokeSuiteIdFor(id, catalog);
|
|
2089
|
+
const smoke = smokeId ? catalog.suites.find((s) => s.id === smokeId) : void 0;
|
|
2090
|
+
if (!smoke) {
|
|
2091
|
+
throw new CliError(`The ${id} suite has no smoke variant in this catalog.`);
|
|
2092
|
+
}
|
|
2093
|
+
suites.push(smoke);
|
|
2094
|
+
} else {
|
|
2095
|
+
suites.push(base);
|
|
2005
2096
|
}
|
|
2006
|
-
}
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2097
|
+
}
|
|
2098
|
+
return suites;
|
|
2099
|
+
}
|
|
2100
|
+
function resolveBenchTarget(catalog, targets, opts) {
|
|
2101
|
+
const available = targets.filter((t) => t.available);
|
|
2102
|
+
if (opts.model) {
|
|
2103
|
+
const matches = available.filter(
|
|
2104
|
+
(t) => t.modelId === opts.model && (!opts.provider || t.provider === opts.provider)
|
|
2105
|
+
);
|
|
2106
|
+
const preferred = matches.find((t) => t.provider === catalog.defaultProvider) ?? matches[0];
|
|
2107
|
+
if (preferred) return preferred;
|
|
2108
|
+
const unavailable = targets.find(
|
|
2109
|
+
(t) => t.modelId === opts.model && (!opts.provider || t.provider === opts.provider)
|
|
2012
2110
|
);
|
|
2013
|
-
} catch (error) {
|
|
2014
|
-
if (progressLineOpen) output.writeProgress("\n");
|
|
2015
2111
|
throw new CliError(
|
|
2016
|
-
|
|
2112
|
+
unavailable?.unavailableReason ? `${opts.model} can't run here: ${unavailable.unavailableReason}` : `No model "${opts.model}"${opts.provider ? ` on ${opts.provider}` : ""} is available to evaluate. \`gezel model list\` shows what is installed.`
|
|
2017
2113
|
);
|
|
2018
2114
|
}
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2115
|
+
const byProvider = opts.provider ? available.filter((t) => t.provider === opts.provider) : available;
|
|
2116
|
+
const chosen = byProvider.find((t) => t.isDefault && t.provider === catalog.defaultProvider) ?? byProvider.find((t) => t.isDefault) ?? byProvider.find((t) => t.provider === catalog.defaultProvider) ?? byProvider[0];
|
|
2117
|
+
if (!chosen) {
|
|
2118
|
+
throw new CliError(
|
|
2119
|
+
"No model is available to evaluate. Download one with `gezel model pull <id>` or sign in to a cloud provider, then try again."
|
|
2120
|
+
);
|
|
2022
2121
|
}
|
|
2122
|
+
return chosen;
|
|
2023
2123
|
}
|
|
2024
|
-
function
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2124
|
+
function planBench(catalog, targets, opts) {
|
|
2125
|
+
const count = opts.count ?? 1;
|
|
2126
|
+
if (!Number.isInteger(count) || count < 1 || count > 10) {
|
|
2127
|
+
throw new CliError(
|
|
2128
|
+
"--count must be a whole number from 1 to 10 (3 is what it takes to quote a rate)."
|
|
2129
|
+
);
|
|
2130
|
+
}
|
|
2131
|
+
const suites = resolveBenchSuites(catalog, opts);
|
|
2132
|
+
const target = resolveBenchTarget(catalog, targets, opts);
|
|
2133
|
+
const worstCaseMs = suites.reduce((sum, s) => sum + s.authoredCeilingMs * count, 0);
|
|
2134
|
+
return { suites, target, count, worstCaseMs, smoke: Boolean(opts.smoke) };
|
|
2028
2135
|
}
|
|
2029
|
-
function
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2136
|
+
function describePlan(plan, catalog) {
|
|
2137
|
+
const lines = [
|
|
2138
|
+
`Model: ${plan.target.label} (${plan.target.modelId} on ${plan.target.provider})`,
|
|
2139
|
+
`Trials per task: ${plan.count}${plan.count < catalog.minTrialsForRate ? ` \u2014 results show as counts; ${catalog.minTrialsForRate} trials is what it takes to quote a rate` : ""}`
|
|
2140
|
+
];
|
|
2141
|
+
for (const suite of plan.suites) {
|
|
2142
|
+
lines.push(
|
|
2143
|
+
`Suite: ${suite.id} \u2014 ${suite.scenarioIds.length} tasks, at most ${formatDuration(
|
|
2144
|
+
suite.authoredCeilingMs * plan.count
|
|
2145
|
+
)}`
|
|
2146
|
+
);
|
|
2147
|
+
if (plan.smoke) lines.push(` ${suite.description}`);
|
|
2148
|
+
}
|
|
2149
|
+
lines.push(
|
|
2150
|
+
`At most ${formatDuration(plan.worstCaseMs)} in all. The run keeps going if this terminal closes.`
|
|
2151
|
+
);
|
|
2152
|
+
return lines;
|
|
2034
2153
|
}
|
|
2035
|
-
function
|
|
2036
|
-
|
|
2154
|
+
function describeJobOutcome(job) {
|
|
2155
|
+
switch (job.status) {
|
|
2156
|
+
case "completed":
|
|
2157
|
+
return null;
|
|
2158
|
+
case "failed":
|
|
2159
|
+
return `The run could not start: ${job.error ?? "the harness reported a setup error"}.`;
|
|
2160
|
+
case "cancelled":
|
|
2161
|
+
return "The run was stopped before it finished.";
|
|
2162
|
+
case "interrupted":
|
|
2163
|
+
return "The Gezel service stopped while the run was in progress; the trials it finished are counted.";
|
|
2164
|
+
default:
|
|
2165
|
+
return `The run ended in state "${job.status}".`;
|
|
2166
|
+
}
|
|
2037
2167
|
}
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
custom: "Custom"
|
|
2051
|
-
};
|
|
2052
|
-
function policyWithCapability(config, key, enabled) {
|
|
2053
|
-
const caps = { ...securityCapabilitiesOf(resolveSecurityPolicy(config)), [key]: enabled };
|
|
2054
|
-
return { level: classifySecurityLevel(caps), ...caps };
|
|
2055
|
-
}
|
|
2056
|
-
function policyWithExternalServices(config, enabled) {
|
|
2057
|
-
return policyWithCapability(config, "allowExternalServices", enabled);
|
|
2058
|
-
}
|
|
2059
|
-
var SECURITY_SWITCHES = [
|
|
2060
|
-
{
|
|
2061
|
-
command: "external-services",
|
|
2062
|
-
key: "allowExternalServices",
|
|
2063
|
-
label: "External services",
|
|
2064
|
-
description: "Show or set on/off for model-initiated external services, including web search"
|
|
2065
|
-
},
|
|
2066
|
-
{
|
|
2067
|
-
command: "run-package-scripts",
|
|
2068
|
-
key: "allowPackageScriptExecution",
|
|
2069
|
-
label: "Run package scripts",
|
|
2070
|
-
description: "Show or set on/off for gezels running package scripts and installed package binaries (outside the sandbox, with network access)"
|
|
2071
|
-
},
|
|
2072
|
-
{
|
|
2073
|
-
command: "edit-package-scripts",
|
|
2074
|
-
key: "allowPackageScriptEdits",
|
|
2075
|
-
label: "Edit package scripts",
|
|
2076
|
-
description: "Show or set on/off for gezels adding or changing the scripts in package.json"
|
|
2168
|
+
function renderSuiteReport(suite, target, trials, minTrialsForRate) {
|
|
2169
|
+
const matrix = buildEvalMatrix(trials, { scenarioOrder: suite.scenarioIds });
|
|
2170
|
+
const column = matrix.columns.find(
|
|
2171
|
+
(c) => c.modelId === target.modelId && (c.provider ?? target.provider) === target.provider
|
|
2172
|
+
);
|
|
2173
|
+
const width = Math.min(40, Math.max(12, ...suite.scenarioIds.map((id) => id.length)) + 2);
|
|
2174
|
+
const lines = [];
|
|
2175
|
+
for (const scenarioId of suite.scenarioIds) {
|
|
2176
|
+
const cell = column ? matrix.cell(scenarioId, column.key) : void 0;
|
|
2177
|
+
const claim = cell ? cell.claim : "not run";
|
|
2178
|
+
const extra = cell?.nonModelFailures ? ` (${cell.nonModelFailures} not counted: not the model's fault)` : "";
|
|
2179
|
+
lines.push(` ${scenarioId.padEnd(width)}${claim}${extra}`);
|
|
2077
2180
|
}
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
["allowExternalServices", "Web search, mail and other outside services"],
|
|
2085
|
-
["allowPackageScriptExecution", "Run package scripts"],
|
|
2086
|
-
["allowPackageScriptEdits", "Edit package scripts"]
|
|
2087
|
-
];
|
|
2088
|
-
var LEVEL_ALIASES = {
|
|
2089
|
-
"super-lockdown": "super-lockdown",
|
|
2090
|
-
superlockdown: "super-lockdown",
|
|
2091
|
-
lockdown: "lockdown",
|
|
2092
|
-
free: "free",
|
|
2093
|
-
unrestricted: "free"
|
|
2094
|
-
};
|
|
2095
|
-
function parseSecurityLevel(input) {
|
|
2096
|
-
const level = LEVEL_ALIASES[input.trim().toLowerCase()];
|
|
2097
|
-
if (!level) {
|
|
2098
|
-
throw new CliError(
|
|
2099
|
-
`Unknown security level "${input}". Choose super-lockdown, lockdown, or free (shown as Unrestricted in Settings).`
|
|
2181
|
+
const total = column ? column.claim : "not measured";
|
|
2182
|
+
const label = SCORECARD_SUITE_LABELS[suite.id] ?? suite.id;
|
|
2183
|
+
lines.push(` ${"suite".padEnd(width)}${total}`);
|
|
2184
|
+
if (column && column.passRate === null && column.attributableTrials > 0) {
|
|
2185
|
+
lines.push(
|
|
2186
|
+
` A count, not a rate: ${minTrialsForRate} trials per task is the floor for quoting a percentage.`
|
|
2100
2187
|
);
|
|
2101
2188
|
}
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
const width = Math.max(...CAPABILITY_LABELS.map(([, label]) => label.length));
|
|
2108
|
-
return [
|
|
2109
|
-
`Security level: ${SECURITY_LEVEL_LABELS[level]}${level === "custom" ? "" : ` (${level})`}`,
|
|
2110
|
-
...CAPABILITY_LABELS.map(
|
|
2111
|
-
([key, label]) => ` ${label.padEnd(width)} ${caps[key] ? "on" : "off"}`
|
|
2112
|
-
)
|
|
2113
|
-
];
|
|
2114
|
-
}
|
|
2115
|
-
function enabledValue(state) {
|
|
2116
|
-
if (state === void 0) return void 0;
|
|
2117
|
-
if (state === "on") return true;
|
|
2118
|
-
if (state === "off") return false;
|
|
2119
|
-
throw new CliError("Use on or off.");
|
|
2189
|
+
const decode = medianDecodeTokensPerSec(trials);
|
|
2190
|
+
lines.push(
|
|
2191
|
+
` Decode: ${decode ? `${decode.toFixed(1)} tok/s (median over finished tasks)` : "not recorded"}. Prefill: not recorded by the daemon's trial index.`
|
|
2192
|
+
);
|
|
2193
|
+
return [`${label} (${suite.id})`, ...lines];
|
|
2120
2194
|
}
|
|
2121
|
-
function
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2195
|
+
function createJobFollower(suite, count) {
|
|
2196
|
+
let lastTrial = null;
|
|
2197
|
+
let lastCompleted = -1;
|
|
2198
|
+
let announcedWaiting = false;
|
|
2199
|
+
let announcedPreflight = false;
|
|
2200
|
+
return {
|
|
2201
|
+
update(job) {
|
|
2202
|
+
const lines = [];
|
|
2203
|
+
const target = job.targets[0];
|
|
2204
|
+
if (job.status === "waiting-for-device" && !announcedWaiting) {
|
|
2205
|
+
announcedWaiting = true;
|
|
2206
|
+
lines.push(
|
|
2207
|
+
`Waiting: another evaluation${job.waitingOn ? ` (${job.waitingOn})` : ""} holds this machine's engines.`
|
|
2208
|
+
);
|
|
2129
2209
|
}
|
|
2130
|
-
if (
|
|
2131
|
-
|
|
2210
|
+
if (!target) return lines;
|
|
2211
|
+
if (target.preflight && !announcedPreflight) {
|
|
2212
|
+
announcedPreflight = true;
|
|
2213
|
+
const p = target.preflight;
|
|
2214
|
+
lines.push(
|
|
2215
|
+
p.admitted === null ? `Fitness check skipped${p.skippedReason ? `: ${p.skippedReason}` : ""}.` : p.admitted ? `Fitness check passed${p.decodeTokensPerSec ? ` \xB7 ${p.decodeTokensPerSec.toFixed(1)} tok/s` : ""}.` : `Fitness check failed${p.skippedReason ? `: ${p.skippedReason}` : ""}.`
|
|
2216
|
+
);
|
|
2132
2217
|
}
|
|
2133
|
-
|
|
2134
|
-
|
|
2218
|
+
const planned = target.plannedTrials ?? suite.scenarioIds.length * count;
|
|
2219
|
+
if (target.completedTrials !== lastCompleted && lastCompleted >= 0) {
|
|
2220
|
+
lines.push(` ${target.completedTrials}/${planned} done, ${target.passedTrials} passed`);
|
|
2135
2221
|
}
|
|
2136
|
-
|
|
2137
|
-
const
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
};
|
|
2142
|
-
if (next) await client.updateConfig({ securityPolicy: policy });
|
|
2143
|
-
if (options.json) {
|
|
2144
|
-
const caps = securityCapabilitiesOf(resolveSecurityPolicy({ securityPolicy: policy }));
|
|
2145
|
-
console.log(JSON.stringify({ level: classifySecurityLevel(caps), ...caps }));
|
|
2146
|
-
return;
|
|
2222
|
+
lastCompleted = target.completedTrials;
|
|
2223
|
+
const current = target.currentTrial;
|
|
2224
|
+
if (current && current.trialId !== lastTrial) {
|
|
2225
|
+
lastTrial = current.trialId;
|
|
2226
|
+
lines.push(` ${current.scenarioId} (${current.trialIndex}/${planned}) running\u2026`);
|
|
2147
2227
|
}
|
|
2148
|
-
|
|
2228
|
+
return lines;
|
|
2149
2229
|
}
|
|
2150
|
-
|
|
2151
|
-
for (const setting of SECURITY_SWITCHES) {
|
|
2152
|
-
security.command(`${setting.command} [state]`).description(setting.description).option("--json", "Output the setting as JSON").action(async (state, options) => {
|
|
2153
|
-
const enabled = enabledValue(state);
|
|
2154
|
-
const client = await connect();
|
|
2155
|
-
const config = await client.getConfig();
|
|
2156
|
-
const policy = resolveSecurityPolicy(config);
|
|
2157
|
-
if (enabled !== void 0) {
|
|
2158
|
-
await client.updateConfig({
|
|
2159
|
-
securityPolicy: policyWithCapability(config, setting.key, enabled)
|
|
2160
|
-
});
|
|
2161
|
-
}
|
|
2162
|
-
const value = enabled ?? policy[setting.key];
|
|
2163
|
-
console.log(
|
|
2164
|
-
options.json ? JSON.stringify({ [setting.key]: value }) : `${setting.label}: ${value ? "on" : "off"}`
|
|
2165
|
-
);
|
|
2166
|
-
});
|
|
2167
|
-
}
|
|
2230
|
+
};
|
|
2168
2231
|
}
|
|
2169
|
-
function
|
|
2170
|
-
|
|
2171
|
-
const enabled = enabledValue(state);
|
|
2172
|
-
const client = await connect();
|
|
2173
|
-
const config = enabled === void 0 ? await client.getConfig() : await client.updateConfig({ growth: { enabled } });
|
|
2174
|
-
const value = config.growth?.enabled !== false;
|
|
2175
|
-
console.log(
|
|
2176
|
-
options.json ? JSON.stringify({ growthEnabled: value }) : `Growth (XP and level-ups): ${value ? "on" : "off"}`
|
|
2177
|
-
);
|
|
2178
|
-
});
|
|
2232
|
+
function finishedJob(job) {
|
|
2233
|
+
return job.status === "completed" || job.status === "failed" || job.status === "cancelled" || job.status === "interrupted";
|
|
2179
2234
|
}
|
|
2180
|
-
function
|
|
2181
|
-
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2235
|
+
async function followJob(client, job, suite, count, io, verbose = false) {
|
|
2236
|
+
const follower = createJobFollower(suite, count);
|
|
2237
|
+
let latest = job;
|
|
2238
|
+
for (const line of follower.update(job)) io.note(line);
|
|
2239
|
+
await client.streamEvalJob(job.id, (event) => {
|
|
2240
|
+
if (event.type === "log") {
|
|
2241
|
+
if (verbose) io.note(` ${event.line}`);
|
|
2242
|
+
return;
|
|
2243
|
+
}
|
|
2244
|
+
latest = event.job;
|
|
2245
|
+
for (const line of follower.update(event.job)) io.note(line);
|
|
2190
2246
|
});
|
|
2247
|
+
if (!finishedJob(latest)) latest = await client.getEvalJob(job.id);
|
|
2248
|
+
return latest;
|
|
2191
2249
|
}
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
mlx: "uv",
|
|
2199
|
-
ds4: "ds4-server"
|
|
2200
|
-
};
|
|
2201
|
-
async function ensureCraftbookSetup(client, options) {
|
|
2202
|
-
const services = options.needs.services ?? [];
|
|
2203
|
-
const commands = options.needs.commands ?? [];
|
|
2204
|
-
if (!options.needs.models?.length && services.length === 0 && commands.length === 0) return;
|
|
2205
|
-
const config = await client.getConfig();
|
|
2206
|
-
const models = resolveCraftbookModelNeeds(options.needs.models, {
|
|
2207
|
-
...options.params ? { params: options.params } : {},
|
|
2208
|
-
...options.paramSchema ? { paramSchema: options.paramSchema } : {},
|
|
2209
|
-
defaultProvider: defaultModelEngine(config, options.platform, options.arch)
|
|
2210
|
-
});
|
|
2211
|
-
const gaps = craftbookSetupGaps(
|
|
2212
|
-
{ services, models, commands },
|
|
2213
|
-
await readSetupState(client, config, models)
|
|
2214
|
-
);
|
|
2215
|
-
if (gaps.length === 0) return;
|
|
2216
|
-
const sizes = await modelSizes(client, gaps);
|
|
2217
|
-
const lines = gaps.map((gap) => ` \u2022 ${describeGap(gap, sizes)}`);
|
|
2218
|
-
if (!options.prompter) {
|
|
2219
|
-
throw new CliError(
|
|
2220
|
-
[
|
|
2221
|
-
`${options.label} is not set up to run yet:`,
|
|
2222
|
-
...lines,
|
|
2223
|
-
"",
|
|
2224
|
-
"Run the same command in a terminal to set it up step by step, or run:",
|
|
2225
|
-
...instructions(gaps).map((line) => ` ${line}`)
|
|
2226
|
-
].join("\n")
|
|
2227
|
-
);
|
|
2250
|
+
function defaultGildeVersion() {
|
|
2251
|
+
try {
|
|
2252
|
+
const pkg = JSON.parse(readFileSync(join2(gildePackageRoot(), "package.json"), "utf8"));
|
|
2253
|
+
return pkg.version ?? "unknown";
|
|
2254
|
+
} catch {
|
|
2255
|
+
return "unknown";
|
|
2228
2256
|
}
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
const
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
if (
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
const to = SECURITY_LEVEL_LABELS[next.level];
|
|
2248
|
-
const change = from === to ? "" : ` This changes your security level from ${from} to ${to}.`;
|
|
2249
|
-
const yes = await prompter.confirm(
|
|
2250
|
-
`Turn on External services, so gezels can search the web and read web pages?${change} [y/N] `
|
|
2251
|
-
);
|
|
2252
|
-
if (!yes) stop(gap);
|
|
2253
|
-
await client.updateConfig({ securityPolicy: next });
|
|
2254
|
-
options.write("External services: on\n");
|
|
2255
|
-
} else if (gap.kind === "package-scripts") {
|
|
2256
|
-
const current = await client.getConfig();
|
|
2257
|
-
const next = policyWithCapability(current, "allowPackageScriptExecution", true);
|
|
2258
|
-
const from = SECURITY_LEVEL_LABELS[resolveSecurityPolicy2(current).level];
|
|
2259
|
-
const to = SECURITY_LEVEL_LABELS[next.level];
|
|
2260
|
-
const change = from === to ? "" : ` This changes your security level from ${from} to ${to}.`;
|
|
2261
|
-
const yes = await prompter.confirm(
|
|
2262
|
-
`Let gezels run package scripts such as ${gap.commands[0]}? They run outside the sandbox, as your account, with network access, and Gezel asks before each new command.${change} [y/N] `
|
|
2257
|
+
}
|
|
2258
|
+
function defaultCpuModel() {
|
|
2259
|
+
return cpus()[0]?.model?.trim() || void 0;
|
|
2260
|
+
}
|
|
2261
|
+
async function runBench(client, opts, io) {
|
|
2262
|
+
const [catalog, targetsResponse, host] = await Promise.all([
|
|
2263
|
+
client.getEvalCatalog(),
|
|
2264
|
+
client.listEvalTargets(),
|
|
2265
|
+
client.getSystemDiagnostics()
|
|
2266
|
+
]);
|
|
2267
|
+
const plan = planBench(catalog, targetsResponse.targets, opts);
|
|
2268
|
+
const cpuModel = io.cpuModel ? io.cpuModel() : void 0;
|
|
2269
|
+
io.note(describeEvalHost(host, cpuModel));
|
|
2270
|
+
for (const line of describePlan(plan, catalog)) io.note(line);
|
|
2271
|
+
if (plan.worstCaseMs > BENCH_CONFIRM_ABOVE_MS && !opts.yes) {
|
|
2272
|
+
if (!io.confirm) {
|
|
2273
|
+
throw new CliError(
|
|
2274
|
+
`This run could take ${formatDuration(plan.worstCaseMs)}. Add --yes to start it without a prompt.`
|
|
2263
2275
|
);
|
|
2264
|
-
if (!yes) stop(gap);
|
|
2265
|
-
await client.updateConfig({ securityPolicy: next });
|
|
2266
|
-
options.write("Run package scripts: on\n");
|
|
2267
|
-
} else if (gap.kind === "web-search") {
|
|
2268
|
-
if (gap.keyMissing) {
|
|
2269
|
-
options.write(
|
|
2270
|
-
`Web search needs a Brave Search API key. Get one at ${BRAVE_KEY_URL} \u2014 Gezel keeps it in this computer's secret store.
|
|
2271
|
-
`
|
|
2272
|
-
);
|
|
2273
|
-
const key = (await prompter.secret(
|
|
2274
|
-
"Paste the key, then press Enter (Enter on its own stops here): ",
|
|
2275
|
-
(length) => `Key received (${length} characters, hidden). Press Enter to continue with this key. `
|
|
2276
|
-
)).trim();
|
|
2277
|
-
if (!key) stop(gap);
|
|
2278
|
-
options.write("Checking the key with one Brave search\u2026\n");
|
|
2279
|
-
await saveBraveKey(client, config, key, options.projectId);
|
|
2280
|
-
options.write("Web search: Brave (test search passed)\n");
|
|
2281
|
-
} else {
|
|
2282
|
-
const current = config.webSearch?.provider ?? "wikipedia";
|
|
2283
|
-
const yes = await prompter.confirm(
|
|
2284
|
-
`A Brave Search key is saved, but web search uses ${current === "wikipedia" ? "Wikipedia" : current}. Switch web search to Brave? [y/N] `
|
|
2285
|
-
);
|
|
2286
|
-
if (!yes) stop(gap);
|
|
2287
|
-
await client.updateConfig({ webSearch: { ...config.webSearch, provider: "brave" } });
|
|
2288
|
-
options.write("Web search: Brave\n");
|
|
2289
|
-
}
|
|
2290
2276
|
}
|
|
2277
|
+
const go = await io.confirm(`Start a run of up to ${formatDuration(plan.worstCaseMs)}? [y/N] `);
|
|
2278
|
+
if (!go) throw new CliError("Not started.");
|
|
2291
2279
|
}
|
|
2292
|
-
const
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2280
|
+
const startedAt = (io.now?.() ?? /* @__PURE__ */ new Date()).toISOString();
|
|
2281
|
+
const device = evalDeviceFromHost(host, cpuModel);
|
|
2282
|
+
const scenariosBySuite = {};
|
|
2283
|
+
const jobs = [];
|
|
2284
|
+
const results = [];
|
|
2285
|
+
const runId = evalRunId(startedAt, device, plan.target.modelId);
|
|
2286
|
+
const gildeVersion = (io.gildeVersion ?? defaultGildeVersion)();
|
|
2287
|
+
const run = evalRunProvenance({
|
|
2288
|
+
id: runId,
|
|
2289
|
+
startedAt,
|
|
2290
|
+
device,
|
|
2291
|
+
gezelVersion: host.version,
|
|
2292
|
+
gildeVersion,
|
|
2293
|
+
count: plan.count,
|
|
2294
|
+
engineBuilds: evalEngineBuilds(host),
|
|
2295
|
+
scenariosBySuite: Object.fromEntries(plan.suites.map((s) => [s.id, s.scenarioIds])),
|
|
2296
|
+
subset: false,
|
|
2297
|
+
...opts.note ? { note: opts.note } : {}
|
|
2298
|
+
});
|
|
2299
|
+
for (const suite of plan.suites) {
|
|
2300
|
+
const spec = {
|
|
2301
|
+
suiteId: suite.id,
|
|
2302
|
+
count: plan.count,
|
|
2303
|
+
...plan.count > 1 ? { countStrict: true } : {},
|
|
2304
|
+
targets: [{ provider: plan.target.provider, modelId: plan.target.modelId }],
|
|
2305
|
+
note: `gezel bench ${runId}`
|
|
2306
|
+
};
|
|
2307
|
+
io.note("");
|
|
2308
|
+
io.note(`Starting ${suite.id}\u2026`);
|
|
2309
|
+
const created = await client.createEvalJob(spec);
|
|
2310
|
+
const job = await followJob(client, created, suite, plan.count, io, opts.verbose);
|
|
2311
|
+
jobs.push(job);
|
|
2312
|
+
scenariosBySuite[suite.id] = suite.scenarioIds;
|
|
2313
|
+
const outcome = describeJobOutcome(job);
|
|
2314
|
+
if (outcome) io.note(outcome);
|
|
2315
|
+
const { trials } = await client.listEvalTrials({ jobId: job.id, limit: 2e3 });
|
|
2316
|
+
results.push(evalSuiteResult({ run, suite, target: plan.target, trials }));
|
|
2317
|
+
io.out("");
|
|
2318
|
+
for (const line of renderSuiteReport(suite, plan.target, trials, catalog.minTrialsForRate)) {
|
|
2319
|
+
io.out(line);
|
|
2302
2320
|
}
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2321
|
+
if (plan.smoke) io.out(` ${suite.description}`);
|
|
2322
|
+
}
|
|
2323
|
+
io.out("");
|
|
2324
|
+
io.out(describeEvalRun(run, [plan.target.provider]));
|
|
2325
|
+
io.out(
|
|
2326
|
+
`Results are kept in ${targetsResponse.environment.runsDir}; Settings \u2192 Benchmarks shows the same runs.`
|
|
2327
|
+
);
|
|
2328
|
+
const shard = evalShard(run, results);
|
|
2329
|
+
if (opts.json) {
|
|
2330
|
+
await writeFile2(opts.json, `${JSON.stringify(shard, null, 2)}
|
|
2331
|
+
`, "utf8");
|
|
2332
|
+
io.out(`Wrote ${opts.json}. Share it with \`gezel bench submit ${opts.json}\`.`);
|
|
2333
|
+
}
|
|
2334
|
+
return { run, results, jobs, shard };
|
|
2335
|
+
}
|
|
2336
|
+
function renderSuiteList(catalog) {
|
|
2337
|
+
const width = Math.max(...catalog.suites.map((s) => s.id.length)) + 2;
|
|
2338
|
+
const lines = [
|
|
2339
|
+
"Suites (`gezel bench --suite <id>`); times are the authored ceilings at one trial per task:"
|
|
2340
|
+
];
|
|
2341
|
+
const scorecard = catalog.suites.filter(
|
|
2342
|
+
(s) => SCORECARD_SUITE_IDS.includes(s.id)
|
|
2343
|
+
);
|
|
2344
|
+
const rest = catalog.suites.filter((s) => !scorecard.includes(s));
|
|
2345
|
+
for (const suite of [...scorecard, ...rest]) {
|
|
2346
|
+
const label = SCORECARD_SUITE_LABELS[suite.id];
|
|
2347
|
+
lines.push(
|
|
2348
|
+
` ${suite.id.padEnd(width)}${String(suite.scenarioIds.length).padStart(2)} tasks ${formatDuration(
|
|
2349
|
+
suite.authoredCeilingMs
|
|
2350
|
+
).padStart(7)} ${label ?? suite.description}`
|
|
2312
2351
|
);
|
|
2313
|
-
for (const gap of downloads) await pullChatModel(client, gap.provider, gap.id, options.write);
|
|
2314
2352
|
}
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2353
|
+
lines.push(
|
|
2354
|
+
"",
|
|
2355
|
+
`The published scorecard runs ${SCORECARD_SUITE_IDS.join(", ")}. Add --smoke for a suite's pulse check; it is not a suite score.`
|
|
2318
2356
|
);
|
|
2319
|
-
|
|
2357
|
+
return lines;
|
|
2358
|
+
}
|
|
2359
|
+
async function renderResults(client, catalog, limit) {
|
|
2360
|
+
const { jobs } = await client.listEvalJobs();
|
|
2361
|
+
const finished = jobs.filter(finishedJob).slice(0, limit);
|
|
2362
|
+
if (finished.length === 0)
|
|
2363
|
+
return ["No finished runs on this machine yet. `gezel bench` starts one."];
|
|
2364
|
+
const lines = [];
|
|
2365
|
+
for (const job of finished) {
|
|
2366
|
+
const suite = job.spec.suiteId ? catalog.suites.find((s) => s.id === job.spec.suiteId) : void 0;
|
|
2367
|
+
const target = job.targets[0];
|
|
2368
|
+
const { trials } = await client.listEvalTrials({ jobId: job.id, limit: 2e3 });
|
|
2369
|
+
const order = suite?.scenarioIds ?? [...new Set(trials.map((t) => t.scenarioId))];
|
|
2370
|
+
const matrix = buildEvalMatrix(trials, { scenarioOrder: order });
|
|
2371
|
+
const column = target ? matrix.columns.find(
|
|
2372
|
+
(c) => c.modelId === target.modelId && (c.provider ?? target.provider) === target.provider
|
|
2373
|
+
) : matrix.columns[0];
|
|
2374
|
+
const subset = suite && job.spec.scenarioIds ? isSuiteSubset(suite, job.spec.scenarioIds) : false;
|
|
2375
|
+
const what = suite ? `${suite.id}${subset ? " (subset)" : ""}` : `${job.spec.scenarioIds?.length ?? 0} tasks`;
|
|
2376
|
+
lines.push(
|
|
2377
|
+
`${(job.finishedAt ?? job.createdAt).slice(0, 16).replace("T", " ")} ${(target?.modelId ?? "?").padEnd(24)} ${what.padEnd(22)} ${column?.claim ?? "not measured"}${job.status !== "completed" ? ` [${job.status}]` : ""}`
|
|
2378
|
+
);
|
|
2379
|
+
if (subset) lines.push(` ${EVAL_SUBSET_NOTE}`);
|
|
2380
|
+
}
|
|
2381
|
+
return lines;
|
|
2382
|
+
}
|
|
2383
|
+
function describeSubmission(shard) {
|
|
2384
|
+
const run = shard.runs[0];
|
|
2385
|
+
const models = [...new Set(shard.results.map((r) => `${r.modelId} (${r.engine})`))];
|
|
2386
|
+
const suites = run.suites.join(", ");
|
|
2387
|
+
const title = `Bench result: ${models.join(", ")} on ${run.provenance.device.label}`;
|
|
2388
|
+
const body = [
|
|
2389
|
+
"## Community bench result",
|
|
2390
|
+
"",
|
|
2391
|
+
`- Run: \`${run.id}\``,
|
|
2392
|
+
`- ${describeEvalRun(run, [...new Set(shard.results.map((r) => r.engine))])}`,
|
|
2393
|
+
`- Suites: ${suites}`,
|
|
2394
|
+
...run.note ? [`- Note: ${run.note}`] : [],
|
|
2395
|
+
"",
|
|
2396
|
+
"Results (count below three trials per task, rate otherwise):",
|
|
2397
|
+
...shard.results.map((r) => {
|
|
2398
|
+
const matrixLike = r.cells.reduce(
|
|
2399
|
+
(acc, c) => {
|
|
2400
|
+
acc.trials += Math.max(0, c.trials - c.nonModelFailures);
|
|
2401
|
+
acc.successes += c.successes;
|
|
2402
|
+
return acc;
|
|
2403
|
+
},
|
|
2404
|
+
{ trials: 0, successes: 0 }
|
|
2405
|
+
);
|
|
2406
|
+
return `- ${r.suiteId}: ${matrixLike.successes}/${matrixLike.trials}`;
|
|
2407
|
+
}),
|
|
2408
|
+
"",
|
|
2409
|
+
"The JSON shard produced by `gezel bench --json` is attached below. It is a `ScorecardDataset` and ingests with the same code as a published round; community rounds are shown separately from Bendyline rounds.",
|
|
2410
|
+
"",
|
|
2411
|
+
"<details><summary>shard.json</summary>",
|
|
2412
|
+
"",
|
|
2413
|
+
"```json",
|
|
2414
|
+
JSON.stringify(shard, null, 2),
|
|
2415
|
+
"```",
|
|
2416
|
+
"",
|
|
2417
|
+
"</details>"
|
|
2418
|
+
].join("\n");
|
|
2419
|
+
return { title, body };
|
|
2420
|
+
}
|
|
2421
|
+
var execFileAsync = promisify(execFile);
|
|
2422
|
+
var BENCH_RESULT_LABEL = "bench-result";
|
|
2423
|
+
function labelMissing(error) {
|
|
2424
|
+
const { stderr, message } = error ?? {};
|
|
2425
|
+
const text = typeof stderr === "string" && stderr ? stderr : String(message ?? error);
|
|
2426
|
+
return /could not add label/i.test(text) || text.includes(`'${BENCH_RESULT_LABEL}' not found`);
|
|
2427
|
+
}
|
|
2428
|
+
async function submitBench(file, io, deps = {}) {
|
|
2429
|
+
const text = await (deps.readFile ?? ((p) => readFile2(p, "utf8")))(file);
|
|
2430
|
+
let shard;
|
|
2431
|
+
try {
|
|
2432
|
+
shard = ScorecardDatasetSchema.parse(JSON.parse(text));
|
|
2433
|
+
} catch (error) {
|
|
2320
2434
|
throw new CliError(
|
|
2321
|
-
[
|
|
2322
|
-
`${options.label} is still not set up:`,
|
|
2323
|
-
...after.map((gap) => ` \u2022 ${describeGap(gap, sizes)}`)
|
|
2324
|
-
].join("\n")
|
|
2435
|
+
`${file} is not a bench result file: ${error instanceof Error ? error.message.split("\n")[0] : String(error)}. Produce one with \`gezel bench --json <file>\`.`
|
|
2325
2436
|
);
|
|
2326
2437
|
}
|
|
2327
|
-
|
|
2438
|
+
if (shard.runs.length !== 1) {
|
|
2439
|
+
throw new CliError(`${file} holds ${shard.runs.length} runs; submit one run per issue.`);
|
|
2440
|
+
}
|
|
2441
|
+
const { title, body } = describeSubmission(shard);
|
|
2442
|
+
const params = new URLSearchParams({ title, labels: BENCH_RESULT_LABEL });
|
|
2443
|
+
const url = `https://github.com/${BENCH_SUBMIT_REPO}/issues/new?${params.toString()}`;
|
|
2444
|
+
const gh = deps.gh ?? (async (args) => (await execFileAsync("gh", args, { encoding: "utf8" })).stdout);
|
|
2445
|
+
const createIssue = (labeled) => gh([
|
|
2446
|
+
"issue",
|
|
2447
|
+
"create",
|
|
2448
|
+
"--repo",
|
|
2449
|
+
BENCH_SUBMIT_REPO,
|
|
2450
|
+
"--title",
|
|
2451
|
+
title,
|
|
2452
|
+
...labeled ? ["--label", BENCH_RESULT_LABEL] : [],
|
|
2453
|
+
"--body",
|
|
2454
|
+
body
|
|
2455
|
+
]);
|
|
2456
|
+
try {
|
|
2457
|
+
let stdout;
|
|
2458
|
+
try {
|
|
2459
|
+
stdout = await createIssue(true);
|
|
2460
|
+
} catch (error) {
|
|
2461
|
+
if (!labelMissing(error)) throw error;
|
|
2462
|
+
stdout = await createIssue(false);
|
|
2463
|
+
}
|
|
2464
|
+
const issueUrl = stdout.trim().split("\n").pop() ?? "";
|
|
2465
|
+
io.out(`Opened ${issueUrl}`);
|
|
2466
|
+
return { title, body, url, issueUrl };
|
|
2467
|
+
} catch {
|
|
2468
|
+
io.out("Open this page and attach the file to the issue:");
|
|
2469
|
+
io.out(` ${url}`);
|
|
2470
|
+
io.out(` file: ${file}`);
|
|
2471
|
+
io.note("(Install the GitHub CLI, `gh`, and sign in to have this done for you.)");
|
|
2472
|
+
return { title, body, url };
|
|
2473
|
+
}
|
|
2328
2474
|
}
|
|
2329
|
-
function
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2475
|
+
function registerBenchCommands(program2, connect, io = {
|
|
2476
|
+
out: (line) => console.log(line),
|
|
2477
|
+
note: (line) => console.error(line),
|
|
2478
|
+
confirm: process.stdin.isTTY ? confirmOnTerminal : void 0,
|
|
2479
|
+
cpuModel: defaultCpuModel
|
|
2480
|
+
}) {
|
|
2481
|
+
const bench = program2.command("bench").description(
|
|
2482
|
+
"Run the model scorecard on this machine: the same tasks, checks and counting rules as the published scorecard"
|
|
2483
|
+
).option(
|
|
2484
|
+
"--suite <id>",
|
|
2485
|
+
"core (default), productivity, developer, complex-work, a comma list, or all"
|
|
2486
|
+
).option("--smoke", "Run the suite's pulse check instead (fast; not a suite score)").option("--model <id>", "Model to evaluate (default: the model chats use)").option("--provider <id>", "Provider for --model when a model is on more than one").option(
|
|
2487
|
+
"--count <n>",
|
|
2488
|
+
"Trials per task; 3 is what it takes to quote a rate (default 1)",
|
|
2489
|
+
(v) => Number.parseInt(v, 10)
|
|
2490
|
+
).option("--json <file>", "Also write the results as a scorecard shard").option("--note <text>", "A note kept with the run").option("--yes", "Start without asking, however long the run").option("--verbose", "Show the harness's own log lines while it runs").action(async (opts) => {
|
|
2491
|
+
const client = await connect();
|
|
2492
|
+
await runBench(client, opts, io);
|
|
2493
|
+
});
|
|
2494
|
+
bench.command("list").description("Show the suites, how many tasks each has, and how long it can take").action(async () => {
|
|
2495
|
+
const client = await connect();
|
|
2496
|
+
for (const line of renderSuiteList(await client.getEvalCatalog())) io.out(line);
|
|
2497
|
+
});
|
|
2498
|
+
bench.command("results").description("Show finished runs on this machine").option("--limit <n>", "How many runs to show (default 20)", (v) => Number.parseInt(v, 10)).action(async (opts) => {
|
|
2499
|
+
const client = await connect();
|
|
2500
|
+
const catalog = await client.getEvalCatalog();
|
|
2501
|
+
for (const line of await renderResults(client, catalog, opts.limit ?? 20)) io.out(line);
|
|
2502
|
+
});
|
|
2503
|
+
bench.command("submit <file>").description("Share a results shard from `gezel bench --json` as a GitHub issue").action(async (file) => {
|
|
2504
|
+
await submitBench(file, io);
|
|
2505
|
+
});
|
|
2342
2506
|
}
|
|
2343
|
-
function
|
|
2344
|
-
const {
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
});
|
|
2353
|
-
let settled = false;
|
|
2354
|
-
const settle = () => {
|
|
2355
|
-
settled = true;
|
|
2356
|
-
input.removeListener("keypress", onKeypress);
|
|
2357
|
-
};
|
|
2358
|
-
let drawn = "";
|
|
2359
|
-
let shownLength = -1;
|
|
2360
|
-
let pending = false;
|
|
2361
|
-
const draw = () => {
|
|
2362
|
-
pending = false;
|
|
2363
|
-
if (settled || !secret) return;
|
|
2364
|
-
const length = rl.line.length;
|
|
2365
|
-
if (length === shownLength) return;
|
|
2366
|
-
shownLength = length;
|
|
2367
|
-
const text = length > 0 && secret.filled ? secret.filled(length) : question;
|
|
2368
|
-
if (drawn) {
|
|
2369
|
-
const rows = Math.floor((drawn.length - 1) / (output.columns || 80));
|
|
2370
|
-
if (rows > 0) moveCursor(output, 0, -rows);
|
|
2371
|
-
cursorTo(output, 0);
|
|
2372
|
-
clearScreenDown(output);
|
|
2373
|
-
}
|
|
2374
|
-
output.write(text);
|
|
2375
|
-
drawn = text;
|
|
2376
|
-
};
|
|
2377
|
-
function onKeypress() {
|
|
2378
|
-
if (pending) return;
|
|
2379
|
-
pending = true;
|
|
2380
|
-
queueMicrotask(draw);
|
|
2381
|
-
}
|
|
2382
|
-
if (secret) input.on("keypress", onKeypress);
|
|
2383
|
-
rl.on("SIGINT", () => {
|
|
2384
|
-
settle();
|
|
2385
|
-
rl.close();
|
|
2386
|
-
output.write("\n");
|
|
2387
|
-
reject(new CliError("Cancelled."));
|
|
2388
|
-
});
|
|
2389
|
-
rl.on("close", () => {
|
|
2390
|
-
if (settled) return;
|
|
2391
|
-
settle();
|
|
2392
|
-
output.write("\n");
|
|
2393
|
-
resolve7("");
|
|
2394
|
-
});
|
|
2395
|
-
rl.question(question, (answer) => {
|
|
2396
|
-
settle();
|
|
2397
|
-
rl.close();
|
|
2398
|
-
if (secret) output.write("\n");
|
|
2399
|
-
resolve7(answer);
|
|
2400
|
-
});
|
|
2401
|
-
draw();
|
|
2402
|
-
})
|
|
2403
|
-
);
|
|
2507
|
+
async function confirmOnTerminal(question) {
|
|
2508
|
+
const { createInterface } = await import("readline/promises");
|
|
2509
|
+
const rl = createInterface({ input: process.stdin, output: process.stderr });
|
|
2510
|
+
try {
|
|
2511
|
+
const answer = await rl.question(question);
|
|
2512
|
+
return /^y(es)?$/i.test(answer.trim());
|
|
2513
|
+
} finally {
|
|
2514
|
+
rl.close();
|
|
2515
|
+
}
|
|
2404
2516
|
}
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2517
|
+
|
|
2518
|
+
// src/cli-errors.ts
|
|
2519
|
+
function formatCliFailure(err, env2 = process.env) {
|
|
2520
|
+
if (!(err instanceof Error)) return null;
|
|
2521
|
+
if (err.name === "CliError") return err.message;
|
|
2522
|
+
if (err.name !== "GezelApiError" || env2.GEZEL_DEBUG === "1") return null;
|
|
2523
|
+
const { status, details } = err;
|
|
2524
|
+
const request = /\bon ([A-Z]+ \S+)/.exec(err.message)?.[1];
|
|
2525
|
+
const serviceText = detailText(details);
|
|
2526
|
+
if (status === 404) {
|
|
2527
|
+
const what = serviceText && serviceText.toLowerCase() !== "not found" ? serviceText : void 0;
|
|
2528
|
+
return `error: not found${what ? `: ${what}` : request ? ` (${request})` : ""}`;
|
|
2529
|
+
}
|
|
2530
|
+
const lines = [`error: ${serviceText ?? err.message}`];
|
|
2531
|
+
if (serviceText && request) lines.push(`(Gezel service answered ${status ?? "?"} to ${request})`);
|
|
2532
|
+
lines.push("Set GEZEL_DEBUG=1 to see the full error.");
|
|
2533
|
+
return lines.join("\n");
|
|
2534
|
+
}
|
|
2535
|
+
function isNotFound(err) {
|
|
2536
|
+
return err instanceof Error && err.name === "GezelApiError" && err.status === 404;
|
|
2537
|
+
}
|
|
2538
|
+
function detailText(details) {
|
|
2539
|
+
if (typeof details === "string" && details.trim()) return details.trim();
|
|
2540
|
+
if (details && typeof details === "object") {
|
|
2541
|
+
for (const key of ["message", "error"]) {
|
|
2542
|
+
const value = details[key];
|
|
2543
|
+
if (typeof value === "string" && value.trim()) return value.trim();
|
|
2544
|
+
}
|
|
2545
|
+
}
|
|
2546
|
+
return void 0;
|
|
2547
|
+
}
|
|
2548
|
+
|
|
2549
|
+
// src/craftbook-command.ts
|
|
2550
|
+
import { setTimeout as delay } from "timers/promises";
|
|
2551
|
+
import {
|
|
2552
|
+
composeCraftbookLaunch,
|
|
2553
|
+
createAwakeTimeout,
|
|
2554
|
+
paramAsksUser
|
|
2555
|
+
} from "@bendyline/gezel";
|
|
2556
|
+
|
|
2557
|
+
// src/tui/craftbook-start.ts
|
|
2558
|
+
import {
|
|
2559
|
+
CRAFTBOOK_CATEGORY_FAMILY_META,
|
|
2560
|
+
CRAFTBOOK_CATEGORY_META,
|
|
2561
|
+
CRAFTBOOK_ROLE_META,
|
|
2562
|
+
resolveCraftbookCategory,
|
|
2563
|
+
visibleCatalogItems
|
|
2564
|
+
} from "@bendyline/gezel";
|
|
2565
|
+
function normalizeCraftbooks(items, showWorkInProgressFeatures = false) {
|
|
2566
|
+
const byId = /* @__PURE__ */ new Map();
|
|
2567
|
+
for (const item of visibleCatalogItems(items, showWorkInProgressFeatures)) {
|
|
2568
|
+
if (item.manifest.kind !== "craftbook-template") continue;
|
|
2569
|
+
const manifest = item.manifest;
|
|
2570
|
+
if (byId.has(manifest.id)) continue;
|
|
2571
|
+
byId.set(manifest.id, {
|
|
2572
|
+
id: manifest.id,
|
|
2573
|
+
name: manifest.name,
|
|
2574
|
+
...manifest.description ? { description: manifest.description } : {},
|
|
2575
|
+
version: manifest.version,
|
|
2576
|
+
sourceId: item.sourceId,
|
|
2577
|
+
command: manifest.command ?? manifest.id,
|
|
2578
|
+
source: item.sourceId === "project" ? "project" : item.sourceId === "local" ? "local" : "bundled",
|
|
2579
|
+
stepCount: manifest.steps.length,
|
|
2580
|
+
tags: manifest.tags ?? [],
|
|
2581
|
+
role: manifest.role ?? "general",
|
|
2582
|
+
category: resolveCraftbookCategory(manifest)
|
|
2583
|
+
});
|
|
2584
|
+
}
|
|
2585
|
+
return [...byId.values()].sort((left, right) => left.name.localeCompare(right.name));
|
|
2408
2586
|
}
|
|
2409
|
-
|
|
2410
|
-
const
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
const { models: list } = engine === "mlx" ? await client.listMlxModels() : engine === "ds4" ? await client.listDs4Models() : await client.listLlamaCppModels();
|
|
2414
|
-
return [engine, list.map((model2) => model2.id)];
|
|
2415
|
-
})
|
|
2587
|
+
function findCraftbook(books, value) {
|
|
2588
|
+
const query = value.trim().toLowerCase();
|
|
2589
|
+
return books.find(
|
|
2590
|
+
(book) => book.id.toLowerCase() === query || book.command?.toLowerCase() === query || book.name.toLowerCase() === query
|
|
2416
2591
|
);
|
|
2592
|
+
}
|
|
2593
|
+
function craftbookStartRequest(book) {
|
|
2594
|
+
const summary = book.description?.trim();
|
|
2595
|
+
const description = summary ? `Start the "${book.name}" craftbook for this project. ${summary}` : `Start the "${book.name}" craftbook for this project. Follow every step in the recipe and verify its required outcomes before completing the task.`;
|
|
2417
2596
|
return {
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2597
|
+
title: book.name,
|
|
2598
|
+
description,
|
|
2599
|
+
craftbookId: book.id,
|
|
2600
|
+
craftbookSourceId: book.sourceId,
|
|
2601
|
+
...book.version ? { craftbookVersion: book.version } : {},
|
|
2602
|
+
dispatchEntry: true,
|
|
2603
|
+
trustScripts: true
|
|
2423
2604
|
};
|
|
2424
2605
|
}
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
for (
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
|
|
2439
|
-
|
|
2440
|
-
|
|
2441
|
-
|
|
2442
|
-
|
|
2443
|
-
|
|
2444
|
-
);
|
|
2606
|
+
|
|
2607
|
+
// src/craftbook-command.ts
|
|
2608
|
+
function resolveCraftbookInvocation(books, tokens) {
|
|
2609
|
+
for (let length = tokens.length; length > 0; length--) {
|
|
2610
|
+
const book = findCraftbook(books, tokens.slice(0, length).join(" "));
|
|
2611
|
+
if (book) return { book, args: tokens.slice(length) };
|
|
2612
|
+
}
|
|
2613
|
+
throw new CliError(`craftbook not found: ${tokens.join(" ")}`);
|
|
2614
|
+
}
|
|
2615
|
+
var NAMED_ARGUMENT = /^([A-Za-z][\w.-]*)=([\s\S]*)$/;
|
|
2616
|
+
function checkParamValue(key, def, value) {
|
|
2617
|
+
if (Array.isArray(def.enum) && !def.enum.map(String).includes(value)) {
|
|
2618
|
+
throw new CliError(`${key} must be one of: ${def.enum.join(", ")}`);
|
|
2619
|
+
}
|
|
2620
|
+
if (def.type === "boolean" && !["true", "false"].includes(value))
|
|
2621
|
+
throw new CliError(`${key} must be true or false`);
|
|
2622
|
+
if (def.type === "number" || def.type === "integer") {
|
|
2623
|
+
const number = Number(value);
|
|
2624
|
+
if (!value.trim() || !Number.isFinite(number) || def.type === "integer" && !Number.isInteger(number))
|
|
2625
|
+
throw new CliError(`${key} must be a ${def.type}`);
|
|
2626
|
+
if (typeof def.minimum === "number" && number < def.minimum)
|
|
2627
|
+
throw new CliError(`${key} must be at least ${def.minimum}`);
|
|
2628
|
+
if (typeof def.maximum === "number" && number > def.maximum)
|
|
2629
|
+
throw new CliError(`${key} must be at most ${def.maximum}`);
|
|
2630
|
+
}
|
|
2631
|
+
if (typeof def.pattern === "string" && !new RegExp(def.pattern).test(value))
|
|
2632
|
+
throw new CliError(`${key} must match ${def.pattern}`);
|
|
2633
|
+
if (typeof def.minLength === "number" && value.trim().length < def.minLength)
|
|
2634
|
+
throw new CliError(`${key} is too short`);
|
|
2635
|
+
}
|
|
2636
|
+
function parseCraftbookArguments(book, tokens, named = []) {
|
|
2637
|
+
const schema = book.paramSchema ?? {};
|
|
2638
|
+
const properties = schema.properties ?? {};
|
|
2639
|
+
const required = new Set(Array.isArray(schema.required) ? schema.required : []);
|
|
2640
|
+
const slots = Object.keys(properties).filter(
|
|
2641
|
+
(key) => required.has(key) && properties[key].default === void 0 && paramAsksUser(properties[key], key)
|
|
2642
|
+
);
|
|
2643
|
+
const params = {};
|
|
2644
|
+
const requestWords = [];
|
|
2645
|
+
const set = (key, value, token) => {
|
|
2646
|
+
if (!Object.hasOwn(properties, key)) throw new CliError(`unknown craftbook argument: ${token}`);
|
|
2647
|
+
if (Object.hasOwn(params, key)) throw new CliError(`duplicate craftbook parameter: ${key}`);
|
|
2648
|
+
checkParamValue(key, properties[key], value);
|
|
2649
|
+
params[key] = value;
|
|
2650
|
+
};
|
|
2651
|
+
for (const token of tokens) {
|
|
2652
|
+
const match = NAMED_ARGUMENT.exec(token);
|
|
2653
|
+
if (match) {
|
|
2654
|
+
set(match[1], match[2], token);
|
|
2655
|
+
continue;
|
|
2445
2656
|
}
|
|
2446
|
-
|
|
2657
|
+
const slot = slots.find((key) => !Object.hasOwn(params, key));
|
|
2658
|
+
if (slot) set(slot, token, token);
|
|
2659
|
+
else requestWords.push(token);
|
|
2447
2660
|
}
|
|
2448
|
-
|
|
2661
|
+
for (const token of named) {
|
|
2662
|
+
const match = NAMED_ARGUMENT.exec(token);
|
|
2663
|
+
if (!match) throw new CliError(`--param expects key=value: ${token}`);
|
|
2664
|
+
set(match[1], match[2], token);
|
|
2665
|
+
}
|
|
2666
|
+
for (const key of required) {
|
|
2667
|
+
if (typeof key === "string" && !Object.hasOwn(params, key) && properties[key]?.default === void 0)
|
|
2668
|
+
throw new CliError(`missing required craftbook parameter: ${key}`);
|
|
2669
|
+
}
|
|
2670
|
+
const request = requestWords.join(" ").trim();
|
|
2671
|
+
return request ? { params, request } : { params };
|
|
2449
2672
|
}
|
|
2450
|
-
function
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
|
|
2673
|
+
function craftbookDoLaunch(craftbook2, { params, request }) {
|
|
2674
|
+
if (!request) return { craftbookParams: params };
|
|
2675
|
+
const launch = composeCraftbookLaunch({
|
|
2676
|
+
message: request,
|
|
2677
|
+
craftbookName: craftbook2.name,
|
|
2678
|
+
paramSchema: craftbook2.paramSchema,
|
|
2679
|
+
params
|
|
2680
|
+
});
|
|
2681
|
+
return { description: launch.description, craftbookParams: launch.params };
|
|
2682
|
+
}
|
|
2683
|
+
var READ_RETRY_DELAYS = [250, 750, 1500];
|
|
2684
|
+
function transientTaskReadError(error) {
|
|
2685
|
+
const codes = /ECONNRESET|ECONNREFUSED|ECONNABORTED|EPIPE|ETIMEDOUT|EHOSTUNREACH|ENETUNREACH|EAI_AGAIN|UND_ERR_(?:SOCKET|CONNECT_TIMEOUT)|NGHTTP2_ENHANCE_YOUR_CALM|Connect Timeout Error|socket hang up|other side closed/;
|
|
2686
|
+
if (error instanceof Error && error.name === "GezelApiError") {
|
|
2687
|
+
const apiError = error;
|
|
2688
|
+
return apiError.status === 0 && apiError.details?.kind === "transport" && apiError.details.readRetryExhausted !== true && typeof apiError.details.cause === "string" && codes.test(apiError.details.cause);
|
|
2689
|
+
}
|
|
2690
|
+
return error instanceof TypeError && error.cause instanceof Error && codes.test(`${error.cause.code ?? ""} ${error.cause.message}`);
|
|
2691
|
+
}
|
|
2692
|
+
async function readTaskObservation(read, budget, signal) {
|
|
2693
|
+
for (let retries = 0; ; retries++) {
|
|
2694
|
+
signal.throwIfAborted();
|
|
2695
|
+
if (retries > 0 && budget.expired()) throw new CliError("Task observation timed out.");
|
|
2696
|
+
try {
|
|
2697
|
+
return await read();
|
|
2698
|
+
} catch (error) {
|
|
2699
|
+
signal.throwIfAborted();
|
|
2700
|
+
if (!transientTaskReadError(error) || retries >= READ_RETRY_DELAYS.length || budget.expired())
|
|
2701
|
+
throw error;
|
|
2702
|
+
await sleepWithinBudget(READ_RETRY_DELAYS[retries], budget, signal);
|
|
2462
2703
|
}
|
|
2463
2704
|
}
|
|
2464
2705
|
}
|
|
2465
|
-
function
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
`gezel secret set braveSearchApiKey --env BRAVE_SEARCH_API_KEY --use-for-search (get a key at ${BRAVE_KEY_URL})`
|
|
2473
|
-
);
|
|
2474
|
-
else out.push(`gezel model pull ${gap.id} --provider ${gap.provider}`);
|
|
2706
|
+
async function sleepWithinBudget(backoffMs, budget, signal) {
|
|
2707
|
+
if (budget.remainingMs() > backoffMs) {
|
|
2708
|
+
await delay(backoffMs, void 0, { signal });
|
|
2709
|
+
return;
|
|
2710
|
+
}
|
|
2711
|
+
while (!budget.expired()) {
|
|
2712
|
+
await delay(Math.max(1, budget.remainingMs()), void 0, { signal });
|
|
2475
2713
|
}
|
|
2476
|
-
return out;
|
|
2477
2714
|
}
|
|
2478
|
-
async function
|
|
2715
|
+
async function waitForTask(client, ref, options) {
|
|
2716
|
+
const timeout = createAwakeTimeout(options.timeoutMs, { pollMs: 100 });
|
|
2717
|
+
const budget = timeout.budget;
|
|
2718
|
+
const signal = options.signal ? AbortSignal.any([options.signal, timeout.signal]) : timeout.signal;
|
|
2719
|
+
let previous = "";
|
|
2720
|
+
let lastTask;
|
|
2721
|
+
const read = (operation) => readTaskObservation(operation, budget, signal);
|
|
2479
2722
|
try {
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
|
|
2723
|
+
for (; ; ) {
|
|
2724
|
+
const task2 = await read(() => client.getTaskByRef(ref, signal));
|
|
2725
|
+
lastTask = task2;
|
|
2726
|
+
const progress = `${task2.status}:${task2.activeStepId}`;
|
|
2727
|
+
if (progress !== previous) {
|
|
2728
|
+
options.onProgress?.(task2);
|
|
2729
|
+
previous = progress;
|
|
2730
|
+
}
|
|
2731
|
+
if (task2.status === "complete") return { task: task2, outcome: "complete", exitCode: 0 };
|
|
2732
|
+
if (task2.status === "canceled") return { task: task2, outcome: "canceled", exitCode: 1 };
|
|
2733
|
+
if (task2.status === "paused" || task2.status === "draft")
|
|
2734
|
+
return { task: task2, outcome: "blocked", exitCode: 2 };
|
|
2735
|
+
const refs = /* @__PURE__ */ new Set([task2.ref]);
|
|
2736
|
+
if (task2.fanout || task2.craftbook.spawn || task2.craftbook.cliWorkflow) {
|
|
2737
|
+
const { tasks } = await read(
|
|
2738
|
+
() => client.listTaskChildren(task2.projectId, task2.num, void 0, signal)
|
|
2739
|
+
);
|
|
2740
|
+
if (!task2.craftbook.cliWorkflow && !tasks.some((child) => child.status === "active") && tasks.some((child) => child.status === "paused" || child.status === "canceled"))
|
|
2741
|
+
return { task: task2, outcome: "blocked", exitCode: 2 };
|
|
2742
|
+
if (!task2.craftbook.cliWorkflow) for (const child of tasks) refs.add(child.ref);
|
|
2743
|
+
}
|
|
2744
|
+
if (client.listQuestions) {
|
|
2745
|
+
const { questions } = await read(
|
|
2746
|
+
() => client.listQuestions(
|
|
2747
|
+
{
|
|
2748
|
+
projectId: task2.projectId,
|
|
2749
|
+
pending: true,
|
|
2750
|
+
taskRefs: [...refs]
|
|
2751
|
+
},
|
|
2752
|
+
signal
|
|
2753
|
+
)
|
|
2754
|
+
);
|
|
2755
|
+
const questionIds = questions.filter(
|
|
2756
|
+
(q) => q.taskRef && refs.has(q.taskRef) && // A service pause card is a historical notification. Actual task
|
|
2757
|
+
// status above decides whether it is still paused; real session
|
|
2758
|
+
// questions and permissions must continue to block observation.
|
|
2759
|
+
!(q.sessionId === "" && q.intent?.kind === "task-paused")
|
|
2760
|
+
).map((q) => q.id);
|
|
2761
|
+
if (questionIds.length) return { task: task2, outcome: "blocked", exitCode: 2, questionIds };
|
|
2762
|
+
}
|
|
2763
|
+
if (budget.expired()) return { task: task2, outcome: "timeout", exitCode: 3 };
|
|
2764
|
+
await delay(Math.min(options.pollMs ?? 1e3, budget.remainingMs()), void 0, { signal });
|
|
2765
|
+
}
|
|
2766
|
+
} catch (error) {
|
|
2767
|
+
options.signal?.throwIfAborted();
|
|
2768
|
+
if (budget.expired() && lastTask) return { task: lastTask, outcome: "timeout", exitCode: 3 };
|
|
2769
|
+
throw error;
|
|
2770
|
+
} finally {
|
|
2771
|
+
timeout.dispose();
|
|
2486
2772
|
}
|
|
2487
|
-
const failure = await client.toolWebSearch(projectId, SEARCH_CHECK).then(
|
|
2488
|
-
() => void 0,
|
|
2489
|
-
(error) => serviceErrorText(error)
|
|
2490
|
-
);
|
|
2491
|
-
if (failure === void 0) return;
|
|
2492
|
-
await client.updateConfig({ braveSearchApiKey: "", webSearch: { ...config.webSearch } }).catch(() => {
|
|
2493
|
-
});
|
|
2494
|
-
throw new CliError(
|
|
2495
|
-
`A test search with that key failed, so it was not kept: ${failure.replaceAll(key, "\u2026")}
|
|
2496
|
-
Check the key at ${BRAVE_KEY_URL}, then run again.`
|
|
2497
|
-
);
|
|
2498
2773
|
}
|
|
2499
|
-
function
|
|
2500
|
-
const
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2774
|
+
function summarizeTaskNote(text, maxLength = 240) {
|
|
2775
|
+
const lines = text.split(/\r?\n/).map((line) => line.trim());
|
|
2776
|
+
const headingIndex = lines.findIndex((line) => line.startsWith("#"));
|
|
2777
|
+
const heading = headingIndex >= 0 ? lines[headingIndex].replace(/^#+\s*/, "") : "";
|
|
2778
|
+
const body = lines.slice(headingIndex + 1).find((line) => line.length > 0 && !line.startsWith("#"));
|
|
2779
|
+
const sentence = body?.match(/^.*?[.!?](?=\s|$)/)?.[0] ?? body ?? "";
|
|
2780
|
+
const summary = heading && sentence ? `${heading}: ${sentence}` : heading || sentence;
|
|
2781
|
+
return summary.length > maxLength ? `${summary.slice(0, maxLength - 1).trimEnd()}\u2026` : summary;
|
|
2782
|
+
}
|
|
2783
|
+
async function blockedTaskReason(client, result) {
|
|
2784
|
+
if (result.outcome !== "blocked") return void 0;
|
|
2785
|
+
if (result.questionIds?.length) {
|
|
2786
|
+
return "A gezel is waiting for your answer to a question. Answer it in the Gezel app or the interactive `gezel` terminal.";
|
|
2504
2787
|
}
|
|
2505
|
-
return
|
|
2788
|
+
if (result.task.status === "draft") return "The task is still a draft and has not started.";
|
|
2789
|
+
const { notes } = await client.listTaskNotes(result.task.projectId, result.task.num).catch(() => ({ notes: [] }));
|
|
2790
|
+
const latest = [...notes].sort((a, b) => a.at.localeCompare(b.at)).at(-1);
|
|
2791
|
+
return latest ? summarizeTaskNote(latest.text) || void 0 : void 0;
|
|
2506
2792
|
}
|
|
2507
|
-
|
|
2508
|
-
const
|
|
2509
|
-
if (
|
|
2510
|
-
|
|
2511
|
-
|
|
2512
|
-
|
|
2513
|
-
|
|
2514
|
-
await installNativeToolkit(client, { engines: missing, output: { writeProgress: write } });
|
|
2793
|
+
function filterCraftbooks(books, words) {
|
|
2794
|
+
const needles = words.map((word) => word.trim().toLowerCase()).filter(Boolean);
|
|
2795
|
+
if (needles.length === 0) return [...books];
|
|
2796
|
+
return books.filter((book) => {
|
|
2797
|
+
const haystack = [book.command, book.name, book.category, ...book.tags].join(" ").toLowerCase();
|
|
2798
|
+
return needles.every((needle) => haystack.includes(needle));
|
|
2799
|
+
});
|
|
2515
2800
|
}
|
|
2516
2801
|
|
|
2517
2802
|
// src/craftbook-shorthand.ts
|
|
2518
2803
|
import { existsSync } from "fs";
|
|
2519
2804
|
import { readFile as readFile3, readdir } from "fs/promises";
|
|
2520
2805
|
import { join as join3 } from "path";
|
|
2521
|
-
import {
|
|
2806
|
+
import { CatalogService } from "@bendyline/gezel-catalog";
|
|
2807
|
+
import { gezelHome as gezelHome2, projectLocalCraftbooksRoot } from "@bendyline/gezel/paths";
|
|
2522
2808
|
var SHORTHAND_FLAG = "--via-craftbook-shorthand";
|
|
2523
2809
|
var VALUE_OPTIONS = /* @__PURE__ */ new Set(["--connect", "--token", "--home"]);
|
|
2524
2810
|
var OPTIONAL_VALUE_OPTIONS = /* @__PURE__ */ new Set(["--project"]);
|
|
@@ -2562,6 +2848,22 @@ async function workspaceCraftbookId(workspaceDir, word) {
|
|
|
2562
2848
|
}
|
|
2563
2849
|
return ids.find((id) => id.toLowerCase() === query);
|
|
2564
2850
|
}
|
|
2851
|
+
async function localCraftbookNamed(word, workspaceDir, catalogItems = bundledCraftbookItems) {
|
|
2852
|
+
try {
|
|
2853
|
+
if (craftbookForShorthand(normalizeCraftbooks(await catalogItems(), true), word)) return true;
|
|
2854
|
+
return await workspaceCraftbookId(workspaceDir, word) !== void 0;
|
|
2855
|
+
} catch {
|
|
2856
|
+
return true;
|
|
2857
|
+
}
|
|
2858
|
+
}
|
|
2859
|
+
async function bundledCraftbookItems() {
|
|
2860
|
+
const { bundledGildeDataDir } = await import("@bendyline/gezel-service/handboek");
|
|
2861
|
+
const catalog = new CatalogService(void 0, {
|
|
2862
|
+
localRoot: gezelHome2(),
|
|
2863
|
+
contentRoot: bundledGildeDataDir
|
|
2864
|
+
});
|
|
2865
|
+
return catalog.list("craftbook-template");
|
|
2866
|
+
}
|
|
2565
2867
|
function shellWord(token) {
|
|
2566
2868
|
return /^[\w@%+=:,./-]+$/.test(token) ? token : `'${token.replaceAll("'", `'\\''`)}'`;
|
|
2567
2869
|
}
|
|
@@ -2592,8 +2894,8 @@ function isGezeldCommand(command) {
|
|
|
2592
2894
|
// src/doctor-report.ts
|
|
2593
2895
|
import {
|
|
2594
2896
|
describeProvenance,
|
|
2595
|
-
measuredClaim,
|
|
2596
|
-
measuredResultFor,
|
|
2897
|
+
measuredClaim as measuredClaim2,
|
|
2898
|
+
measuredResultFor as measuredResultFor2,
|
|
2597
2899
|
measuredRowsFor,
|
|
2598
2900
|
scorecardHardwareKey,
|
|
2599
2901
|
scorecardHardwareLabel
|
|
@@ -2647,7 +2949,7 @@ function hardwareClassCoverage(dataset, modelId, device) {
|
|
|
2647
2949
|
return ` on ${name}: ${rounds} published ${roundWord}; ${mine} ${mine === 1 ? "result" : "results"} for this model`;
|
|
2648
2950
|
}
|
|
2649
2951
|
function renderMeasured(dataset, modelId, device, rows = 4) {
|
|
2650
|
-
const headline =
|
|
2952
|
+
const headline = measuredResultFor2(dataset, modelId, device);
|
|
2651
2953
|
const coverage = hardwareClassCoverage(dataset, modelId, device);
|
|
2652
2954
|
if (!headline) {
|
|
2653
2955
|
return [
|
|
@@ -2656,7 +2958,7 @@ function renderMeasured(dataset, modelId, device, rows = 4) {
|
|
|
2656
2958
|
];
|
|
2657
2959
|
}
|
|
2658
2960
|
const lines = [
|
|
2659
|
-
`measured: ${
|
|
2961
|
+
`measured: ${measuredClaim2(headline)}${headline.sameHardwareClass ? "" : " \u2014 a different platform; your numbers may differ"}`,
|
|
2660
2962
|
coverage
|
|
2661
2963
|
];
|
|
2662
2964
|
for (const row of measuredRowsFor(dataset, modelId).slice(0, rows)) {
|
|
@@ -2944,115 +3246,6 @@ function registerMcpCommands(program2, connect, output = console.log) {
|
|
|
2944
3246
|
});
|
|
2945
3247
|
}
|
|
2946
3248
|
|
|
2947
|
-
// src/model-choice.ts
|
|
2948
|
-
import {
|
|
2949
|
-
measuredClaim as measuredClaim2,
|
|
2950
|
-
measuredResultFor as measuredResultFor2
|
|
2951
|
-
} from "@bendyline/gezel";
|
|
2952
|
-
async function installedChatModels(client, engine) {
|
|
2953
|
-
const { models } = engine === "mlx" ? await client.listMlxModels() : engine === "ds4" ? await client.listDs4Models() : await client.listLlamaCppModels();
|
|
2954
|
-
return models.map((model2) => model2.id);
|
|
2955
|
-
}
|
|
2956
|
-
function providerFlag(engine, platformDefault) {
|
|
2957
|
-
return engine === platformDefault ? "" : ` --provider ${engine}`;
|
|
2958
|
-
}
|
|
2959
|
-
var FIT_LABEL = {
|
|
2960
|
-
fits: "fits",
|
|
2961
|
-
"fits-offload": "fits, partly offloaded",
|
|
2962
|
-
tight: "tight",
|
|
2963
|
-
"too-big": "too big"
|
|
2964
|
-
};
|
|
2965
|
-
async function availableChatModels(client, engine) {
|
|
2966
|
-
if (engine === "ds4") {
|
|
2967
|
-
throw new CliError("`--available` lists mlx and llama-cpp models; pass one with --provider.");
|
|
2968
|
-
}
|
|
2969
|
-
const [memory, catalog, installed, config] = await Promise.all([
|
|
2970
|
-
client.getMemoryProfile(),
|
|
2971
|
-
client.listCatalogItems("chat-model"),
|
|
2972
|
-
installedChatModels(client, engine),
|
|
2973
|
-
client.getConfig()
|
|
2974
|
-
]);
|
|
2975
|
-
const pinned = config.defaultModel?.[engine];
|
|
2976
|
-
return {
|
|
2977
|
-
ranked: rankChatModels(catalog.items, recoDevice(memory), engine),
|
|
2978
|
-
installed: new Set(installed),
|
|
2979
|
-
...pinned ? { pinned } : {}
|
|
2980
|
-
};
|
|
2981
|
-
}
|
|
2982
|
-
function formatAvailableChatModels(input) {
|
|
2983
|
-
const { engine, ranked, installed, pinned, measured } = input;
|
|
2984
|
-
if (ranked.length === 0) return `(no ${engine} chat models in the catalog for this machine)`;
|
|
2985
|
-
const width = Math.max(...ranked.map((model2) => model2.id.length));
|
|
2986
|
-
const measuredWidth = measured ? Math.max("not measured".length, ...ranked.map((m) => (measured.get(m.id) ?? "").length)) : 0;
|
|
2987
|
-
const lines = [`${engine} chat models for this machine, best first:`];
|
|
2988
|
-
ranked.forEach((model2, index) => {
|
|
2989
|
-
const notes = [
|
|
2990
|
-
index === 0 ? "recommended" : "",
|
|
2991
|
-
installed.has(model2.id) ? "installed" : "",
|
|
2992
|
-
model2.id === pinned ? "in use" : ""
|
|
2993
|
-
].filter(Boolean);
|
|
2994
|
-
const claim = measured ? ` ${(measured.get(model2.id) ?? "not measured").padEnd(measuredWidth)}` : "";
|
|
2995
|
-
lines.push(
|
|
2996
|
-
` ${model2.id.padEnd(width)} ${formatDownloadSize(model2.approxSizeBytes).padStart(7)} ${FIT_LABEL[model2.fit].padEnd(22)}${claim} ${model2.name}${notes.length > 0 ? ` (${notes.join(", ")})` : ""}`
|
|
2997
|
-
);
|
|
2998
|
-
});
|
|
2999
|
-
const flag = providerFlag(engine, input.platformDefault);
|
|
3000
|
-
const smallest = [...ranked].sort((a, b) => a.approxSizeBytes - b.approxSizeBytes)[0];
|
|
3001
|
-
lines.push(
|
|
3002
|
-
"",
|
|
3003
|
-
`Download one with \`gezel model pull <id>${flag}\`. The smallest is ${smallest.id} (${formatDownloadSize(smallest.approxSizeBytes)}).`,
|
|
3004
|
-
`\`gezel model use <id>${flag}\` picks the installed model chats use.`
|
|
3005
|
-
);
|
|
3006
|
-
if (measured) {
|
|
3007
|
-
lines.push(
|
|
3008
|
-
'"Fits" is about memory. The measured column is the published scorecard\'s pass rate on real jobs, on the hardware named; `gezel bench` measures a model on this machine.'
|
|
3009
|
-
);
|
|
3010
|
-
}
|
|
3011
|
-
return lines.join("\n");
|
|
3012
|
-
}
|
|
3013
|
-
function measuredClaimsFor(dataset, ids, device) {
|
|
3014
|
-
const claims = /* @__PURE__ */ new Map();
|
|
3015
|
-
for (const id of ids) claims.set(id, measuredClaim2(measuredResultFor2(dataset, id, device)));
|
|
3016
|
-
return claims;
|
|
3017
|
-
}
|
|
3018
|
-
async function useChatModel(client, id, options) {
|
|
3019
|
-
const order = options.requested ? [options.requested] : [options.platformDefault, ...["mlx", "llama-cpp", "ds4"]].filter(
|
|
3020
|
-
(engine2, index, all) => all.indexOf(engine2) === index
|
|
3021
|
-
);
|
|
3022
|
-
let engine;
|
|
3023
|
-
for (const candidate of order) {
|
|
3024
|
-
const installed = await installedChatModels(client, candidate).catch(() => []);
|
|
3025
|
-
if (installed.includes(id)) {
|
|
3026
|
-
engine = candidate;
|
|
3027
|
-
break;
|
|
3028
|
-
}
|
|
3029
|
-
}
|
|
3030
|
-
if (!engine) {
|
|
3031
|
-
const target = options.requested ?? options.platformDefault;
|
|
3032
|
-
throw new CliError(
|
|
3033
|
-
`${id} isn't downloaded for ${target}. Download it with \`gezel model pull ${id}${providerFlag(target, options.platformDefault)}\`, or see \`gezel model list --available\`.`
|
|
3034
|
-
);
|
|
3035
|
-
}
|
|
3036
|
-
const config = await client.getConfig();
|
|
3037
|
-
await client.updateConfig({
|
|
3038
|
-
provider: engine,
|
|
3039
|
-
defaultModel: { ...config.defaultModel ?? {}, [engine]: id },
|
|
3040
|
-
firstRunCompleted: true
|
|
3041
|
-
});
|
|
3042
|
-
return engine;
|
|
3043
|
-
}
|
|
3044
|
-
async function pinPulledModel(client, engine, id) {
|
|
3045
|
-
const config = await client.getConfig();
|
|
3046
|
-
const current = config.defaultModel?.[engine];
|
|
3047
|
-
if (current === id) return "already";
|
|
3048
|
-
if (current && (await installedChatModels(client, engine)).includes(current)) return "kept";
|
|
3049
|
-
await client.updateConfig({
|
|
3050
|
-
defaultModel: { ...config.defaultModel ?? {}, [engine]: id },
|
|
3051
|
-
firstRunCompleted: true
|
|
3052
|
-
});
|
|
3053
|
-
return "pinned";
|
|
3054
|
-
}
|
|
3055
|
-
|
|
3056
3249
|
// src/model-export.ts
|
|
3057
3250
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
3058
3251
|
import { rename, rm as rm2, stat } from "fs/promises";
|
|
@@ -3342,99 +3535,6 @@ async function resolveRunPrompt(parts, input = process.stdin) {
|
|
|
3342
3535
|
return prompt;
|
|
3343
3536
|
}
|
|
3344
3537
|
|
|
3345
|
-
// src/run-readiness.ts
|
|
3346
|
-
import { resolveOnDeviceProvider as resolveOnDeviceProvider2 } from "@bendyline/gezel/native";
|
|
3347
|
-
var READY = { ready: true };
|
|
3348
|
-
async function settledFirstRunConfig(client, options = {}) {
|
|
3349
|
-
const attempts = options.attempts ?? 12;
|
|
3350
|
-
const delayMs = options.delayMs ?? 250;
|
|
3351
|
-
let config = await client.getConfig();
|
|
3352
|
-
for (let attempt = 0; attempt < attempts; attempt += 1) {
|
|
3353
|
-
if (config.firstRunCompleted || config.provider !== "copilot") return config;
|
|
3354
|
-
await new Promise((resolve7) => setTimeout(resolve7, delayMs));
|
|
3355
|
-
config = await client.getConfig();
|
|
3356
|
-
}
|
|
3357
|
-
return config;
|
|
3358
|
-
}
|
|
3359
|
-
async function checkRunReadiness(client, gezelId, env2 = process.env) {
|
|
3360
|
-
if (env2.GEZEL_MOCK_PROVIDER === "1") return READY;
|
|
3361
|
-
const config = await settledFirstRunConfig(client);
|
|
3362
|
-
let provider = config.provider;
|
|
3363
|
-
if (gezelId) {
|
|
3364
|
-
try {
|
|
3365
|
-
const { parsed } = await client.getGezel(gezelId);
|
|
3366
|
-
if (parsed.frontmatter.provider) provider = parsed.frontmatter.provider;
|
|
3367
|
-
} catch {
|
|
3368
|
-
}
|
|
3369
|
-
}
|
|
3370
|
-
if (provider !== "llama-cpp" && provider !== "mlx") return READY;
|
|
3371
|
-
if (provider === "llama-cpp" && (config.llamaCppBaseUrl || config.llamaCppModelPath)) {
|
|
3372
|
-
return READY;
|
|
3373
|
-
}
|
|
3374
|
-
if (provider === "mlx" && (config.mlxBaseUrl || config.mlxModelPath)) return READY;
|
|
3375
|
-
const native2 = await client.getNativeEngineStatus();
|
|
3376
|
-
if (!native2.platformKey) return READY;
|
|
3377
|
-
const installed = provider === "mlx" ? await client.listMlxModels() : await client.listLlamaCppModels();
|
|
3378
|
-
const installedIds = installed.models.map((model2) => model2.id);
|
|
3379
|
-
const pinned = config.defaultModel?.[provider];
|
|
3380
|
-
const engineMissing = provider === "llama-cpp" && !native2.engines.some((engine) => engine.name === "llama-server" && engine.installed);
|
|
3381
|
-
const modelReady = pinned ? installedIds.includes(pinned) : installedIds.length > 0;
|
|
3382
|
-
if (!engineMissing && modelReady) return READY;
|
|
3383
|
-
return {
|
|
3384
|
-
ready: false,
|
|
3385
|
-
message: formatRunSetupMessage({
|
|
3386
|
-
provider,
|
|
3387
|
-
engineMissing,
|
|
3388
|
-
pinned,
|
|
3389
|
-
installedIds
|
|
3390
|
-
})
|
|
3391
|
-
};
|
|
3392
|
-
}
|
|
3393
|
-
function canOfferRunSetup(io = process, env2 = process.env) {
|
|
3394
|
-
if (env2.CI) return false;
|
|
3395
|
-
return io.stdin.isTTY === true && io.stderr.isTTY === true;
|
|
3396
|
-
}
|
|
3397
|
-
function formatRunSetupMessage(input) {
|
|
3398
|
-
const { provider, engineMissing, pinned, installedIds } = input;
|
|
3399
|
-
const flag = providerFlag(
|
|
3400
|
-
provider,
|
|
3401
|
-
input.platformDefault ?? resolveOnDeviceProvider2(process.platform, process.arch)
|
|
3402
|
-
);
|
|
3403
|
-
const steps = [];
|
|
3404
|
-
if (engineMissing) steps.push("gezel native install");
|
|
3405
|
-
if (pinned && !installedIds.includes(pinned)) {
|
|
3406
|
-
if (installedIds.length > 0) {
|
|
3407
|
-
steps.push(`gezel model use ${installedIds[0]}${flag} (already downloaded)`);
|
|
3408
|
-
}
|
|
3409
|
-
steps.push(`gezel model pull ${pinned}${flag}`);
|
|
3410
|
-
if (installedIds.length === 0) {
|
|
3411
|
-
steps.push(`gezel model list --available${flag} (to pick a smaller one)`);
|
|
3412
|
-
}
|
|
3413
|
-
} else if (!pinned && installedIds.length === 0) {
|
|
3414
|
-
steps.push(`gezel model list --available${flag} (then: gezel model pull <id>${flag})`);
|
|
3415
|
-
}
|
|
3416
|
-
const lines = [];
|
|
3417
|
-
if (pinned && !installedIds.includes(pinned) && installedIds.length > 0) {
|
|
3418
|
-
lines.push(`Your selected on-device model, ${pinned}, isn't downloaded yet.`);
|
|
3419
|
-
} else if (engineMissing && installedIds.length > 0 && (!pinned || installedIds.includes(pinned))) {
|
|
3420
|
-
lines.push("Gezel's on-device engine isn't installed yet.");
|
|
3421
|
-
} else {
|
|
3422
|
-
lines.push("Gezel isn't set up to chat yet: this device doesn't have an on-device model.");
|
|
3423
|
-
}
|
|
3424
|
-
lines.push("", "Run `gezel` to choose a model and download it, step by step.");
|
|
3425
|
-
if (steps.length > 0) {
|
|
3426
|
-
lines.push("", "Or set it up directly:", ...steps.map((step) => ` ${step}`));
|
|
3427
|
-
}
|
|
3428
|
-
lines.push(
|
|
3429
|
-
"",
|
|
3430
|
-
"To use a cloud model instead, save a key (for example",
|
|
3431
|
-
"`gezel secret set openaiApiKey --stdin`), then run `gezel` and pick it with /model.",
|
|
3432
|
-
"",
|
|
3433
|
-
"Nothing was downloaded."
|
|
3434
|
-
);
|
|
3435
|
-
return lines.join("\n");
|
|
3436
|
-
}
|
|
3437
|
-
|
|
3438
3538
|
// src/run-reply.ts
|
|
3439
3539
|
var NAME = "[A-Za-z_][A-Za-z0-9_.-]*";
|
|
3440
3540
|
var CLOSED_CALLS = [
|
|
@@ -3496,6 +3596,15 @@ var ReplyPrinter = class {
|
|
|
3496
3596
|
this.raw += chunk;
|
|
3497
3597
|
this.extend(visibleReply(this.raw, true));
|
|
3498
3598
|
}
|
|
3599
|
+
/**
|
|
3600
|
+
* A tool ran. Providers join the text of each tool round with nothing
|
|
3601
|
+
* between, so narration ran into the answer: "…move date.Lisbon office…".
|
|
3602
|
+
* Marked like a cut-out call, it becomes a paragraph break only if text
|
|
3603
|
+
* follows, and nothing at all before the reply or at its end.
|
|
3604
|
+
*/
|
|
3605
|
+
toolCalled() {
|
|
3606
|
+
this.raw += CUT;
|
|
3607
|
+
}
|
|
3499
3608
|
/**
|
|
3500
3609
|
* End of the reply. `committed` is the saved message, printed when the
|
|
3501
3610
|
* stream carried no visible text (providers that send it whole). Returns
|
|
@@ -3538,6 +3647,9 @@ var RunOutput = class {
|
|
|
3538
3647
|
push(chunk) {
|
|
3539
3648
|
this.current.push(chunk);
|
|
3540
3649
|
}
|
|
3650
|
+
toolCalled() {
|
|
3651
|
+
this.current.toolCalled();
|
|
3652
|
+
}
|
|
3541
3653
|
/** One assistant message was saved; later deltas belong to the next one. */
|
|
3542
3654
|
complete(committed) {
|
|
3543
3655
|
this.current.finish(committed);
|
|
@@ -4089,6 +4201,9 @@ program.name("gezel").description("Gezel \u2014 assemble a team of AI companions
|
|
|
4089
4201
|
).option("--home <dir>", "Use this user-owned Gezel home (default: $GEZEL_HOME or ~/.gezel).").option(
|
|
4090
4202
|
"--project [folder]",
|
|
4091
4203
|
"Use this folder as the active project (default and bare flag: cwd)."
|
|
4204
|
+
).option(
|
|
4205
|
+
"--download",
|
|
4206
|
+
"Download what a command needs (the on-device engine and the recommended model) without asking, printing each step. Also GEZEL_AUTO_DOWNLOAD=1."
|
|
4092
4207
|
);
|
|
4093
4208
|
var cliGlobals = () => program.opts();
|
|
4094
4209
|
var connectForInspection = (opts) => opts.start === false ? connectRunning(cliGlobals()) : connectOwned(cliGlobals());
|
|
@@ -4111,11 +4226,19 @@ function builtinCommandNames() {
|
|
|
4111
4226
|
var shorthandTail = [];
|
|
4112
4227
|
async function resolveCraftbookShorthand(parts) {
|
|
4113
4228
|
const word = parts[0] ?? "";
|
|
4114
|
-
const
|
|
4229
|
+
const builtins = builtinCommandNames();
|
|
4230
|
+
const unknown = unknownCommandMessage(parts, builtins);
|
|
4231
|
+
let running = null;
|
|
4232
|
+
if (closestCommand(word, builtins)) {
|
|
4233
|
+
running = await connectRunning(cliGlobals()).catch(() => null);
|
|
4234
|
+
const project = cliGlobals().project;
|
|
4235
|
+
const folder = resolvePath(typeof project === "string" ? project : process.cwd());
|
|
4236
|
+
if (!running && !await localCraftbookNamed(word, folder)) throw new CliError(unknown);
|
|
4237
|
+
}
|
|
4115
4238
|
let client;
|
|
4116
4239
|
let id;
|
|
4117
4240
|
try {
|
|
4118
|
-
client = await connectOwned(cliGlobals());
|
|
4241
|
+
client = running ?? await connectOwned(cliGlobals());
|
|
4119
4242
|
const preview = await previewRunProject(client, cliGlobals());
|
|
4120
4243
|
const [config, items] = await Promise.all([
|
|
4121
4244
|
client.getConfig(),
|
|
@@ -4616,7 +4739,10 @@ then runs the prompt.`
|
|
|
4616
4739
|
const projectId = await resolveRunProject(client, cliGlobals());
|
|
4617
4740
|
const gezelId = opts.gezel ? await resolveRunGezel(client, opts.gezel) : await ensureCliProjectLead(client, projectId);
|
|
4618
4741
|
let readiness = await checkRunReadiness(client, gezelId);
|
|
4619
|
-
if (!readiness.ready &&
|
|
4742
|
+
if (!readiness.ready && autoDownloadRequested(cliGlobals())) {
|
|
4743
|
+
await downloadForRun(client, { gezelId, write: (text) => process.stderr.write(text) });
|
|
4744
|
+
readiness = await checkRunReadiness(client, gezelId);
|
|
4745
|
+
} else if (!readiness.ready && canOfferRunSetup()) {
|
|
4620
4746
|
const tuiModule = "../tui/index.js";
|
|
4621
4747
|
const { launchRunSetup } = await import(tuiModule);
|
|
4622
4748
|
const runLogLevel = getLogLevel2();
|
|
@@ -4654,6 +4780,8 @@ then runs the prompt.`
|
|
|
4654
4780
|
} else if (ev.type === "delta") {
|
|
4655
4781
|
progress.settle();
|
|
4656
4782
|
reply.push(ev.content);
|
|
4783
|
+
} else if (ev.type === "tool") {
|
|
4784
|
+
reply.toolCalled();
|
|
4657
4785
|
} else if (ev.type === "complete") {
|
|
4658
4786
|
progress.settle();
|
|
4659
4787
|
reply.complete(ev.message.content);
|
|
@@ -4729,15 +4857,22 @@ Examples:
|
|
|
4729
4857
|
`);
|
|
4730
4858
|
}
|
|
4731
4859
|
const parsed = parseCraftbookArguments(craftbook2, args, opts.param);
|
|
4732
|
-
const
|
|
4733
|
-
|
|
4734
|
-
|
|
4735
|
-
|
|
4736
|
-
|
|
4737
|
-
|
|
4738
|
-
|
|
4739
|
-
|
|
4740
|
-
|
|
4860
|
+
const autoDownload = autoDownloadRequested(cliGlobals());
|
|
4861
|
+
const setUp = async () => {
|
|
4862
|
+
if (autoDownload && !(await checkRunReadiness(client, void 0)).ready) {
|
|
4863
|
+
await downloadForRun(client, { write: stderrProgress() });
|
|
4864
|
+
}
|
|
4865
|
+
await ensureCraftbookSetup(client, {
|
|
4866
|
+
projectId,
|
|
4867
|
+
label: craftbook2.name,
|
|
4868
|
+
needs: craftbook2,
|
|
4869
|
+
params: parsed.params,
|
|
4870
|
+
...craftbook2.paramSchema ? { paramSchema: craftbook2.paramSchema } : {},
|
|
4871
|
+
prompter: terminalPrompter(),
|
|
4872
|
+
autoDownload,
|
|
4873
|
+
write: stderrProgress()
|
|
4874
|
+
});
|
|
4875
|
+
};
|
|
4741
4876
|
if (craftbook2.cliWorkflow) {
|
|
4742
4877
|
if (book.source !== "project")
|
|
4743
4878
|
throw new CliError("CLI workflow craftbooks must belong to this project.");
|
|
@@ -5040,6 +5175,9 @@ handboek.command("export").description("Render the Handboek as a static HTML sit
|
|
|
5040
5175
|
).option(
|
|
5041
5176
|
"--apps-out <dir>",
|
|
5042
5177
|
"also write the apps library (gezel.com/apps) into this directory, outside --out"
|
|
5178
|
+
).option(
|
|
5179
|
+
"--open-in-gezel",
|
|
5180
|
+
'add gezel:// "Open in Gezel" buttons to craftbook and app pages (needs a desktop release that answers them)'
|
|
5043
5181
|
).action(
|
|
5044
5182
|
async (opts) => {
|
|
5045
5183
|
if (Boolean(opts.ogOut) !== Boolean(opts.ogBase)) {
|
|
@@ -5056,7 +5194,8 @@ handboek.command("export").description("Render the Handboek as a static HTML sit
|
|
|
5056
5194
|
canonicalBase: opts.canonicalBase,
|
|
5057
5195
|
ogOut: opts.ogOut,
|
|
5058
5196
|
ogBase: opts.ogBase,
|
|
5059
|
-
appsOut: opts.appsOut
|
|
5197
|
+
appsOut: opts.appsOut,
|
|
5198
|
+
openInGezel: opts.openInGezel === true
|
|
5060
5199
|
});
|
|
5061
5200
|
console.log(`Handboek exported: ${result.pages} pages \u2192 ${result.out}`);
|
|
5062
5201
|
if (result.apps) {
|
|
@@ -5302,6 +5441,13 @@ appCmd.command("demo <folder>").description("Export an offline page demo with sh
|
|
|
5302
5441
|
const { runAppDemo } = await loadAppAuthorCommand();
|
|
5303
5442
|
await runAppDemo(folder, opts);
|
|
5304
5443
|
});
|
|
5444
|
+
appCmd.command("guide").description(
|
|
5445
|
+
"Print the complete guide to building a Gezel app, for this version (give it to your AI agent)"
|
|
5446
|
+
).action(async () => {
|
|
5447
|
+
const appGuideModule = "../app-guide.js";
|
|
5448
|
+
const { runAppGuide } = await import(appGuideModule);
|
|
5449
|
+
await runAppGuide();
|
|
5450
|
+
});
|
|
5305
5451
|
appCmd.command("schemas").description("The JSON Schemas an AI App author codes against, rendered from this build").option("--out <dir>", "write the schema files into a directory").option("--json", "print one combined JSON object").action(async (opts) => {
|
|
5306
5452
|
const { runAppSchemas } = await loadAppAuthorCommand();
|
|
5307
5453
|
await runAppSchemas(opts);
|
|
@@ -5511,7 +5657,7 @@ program.command("create-audio [text...]").description("Synthesize speech from te
|
|
|
5511
5657
|
var native = program.command("native").description("Manage native engine binaries");
|
|
5512
5658
|
native.command("install").description(
|
|
5513
5659
|
"Download, verify, and activate the native inference toolkit (or one engine with --engine)"
|
|
5514
|
-
).option("--variant <backend>", "llama.cpp backend: cuda | vulkan | metal | cpu").option("--engine <name>", "install only this engine, e.g. duckdb or llama-server").action(async (opts) => {
|
|
5660
|
+
).option("--variant <backend>", "llama.cpp backend: cuda | rocm | vulkan | metal | cpu").option("--engine <name>", "install only this engine, e.g. duckdb or llama-server").action(async (opts) => {
|
|
5515
5661
|
const variant = parseNativeVariant(opts.variant);
|
|
5516
5662
|
const engine = parseNativeEngine(opts.engine);
|
|
5517
5663
|
const client = await connectOwned(cliGlobals());
|