@genex-ai/cli-demo 1.36.9 → 1.36.10-dev.959
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{blender-mcp-QVEGHE5L.js → blender-mcp-WSURPZET.js} +2 -2
- package/dist/{blender-serve-66OHC4ML.js → blender-serve-EH2MOW3U.js} +1 -1
- package/dist/{chunk-4BBTPRJW.js → chunk-EABIVAGT.js} +2 -2
- package/dist/{chunk-QI7FIYBY.js → chunk-K4EHXOZK.js} +1 -1
- package/dist/index.js +590 -316
- package/dist/sentry-reporter-FCSQOR4U.js +5442 -0
- package/package.json +2 -2
- package/templates/skills/genex-game-director/SKILL.md +1 -1
- package/templates/skills/genex-getting-started/SKILL.md +2 -2
- package/templates/skills/genex-llm-in-games/SKILL.md +209 -72
- package/templates/skills/genex-llm-in-games/references/pricing.md +126 -85
- package/templates/skills/genex-monetization/SKILL.md +1 -9
- package/templates/skills/genex-tool-llm/SKILL.md +11 -9
- package/templates/skills/genex-updates/SKILL.md +1 -1
package/dist/index.js
CHANGED
|
@@ -57,7 +57,7 @@ import {
|
|
|
57
57
|
writeSecretFile,
|
|
58
58
|
writeUserToken,
|
|
59
59
|
writeWorkspace
|
|
60
|
-
} from "./chunk-
|
|
60
|
+
} from "./chunk-EABIVAGT.js";
|
|
61
61
|
import {
|
|
62
62
|
CLI_CHANNEL,
|
|
63
63
|
DEFAULT_API_URL,
|
|
@@ -78,10 +78,10 @@ import {
|
|
|
78
78
|
normalizeApiOrigin,
|
|
79
79
|
originKey,
|
|
80
80
|
resolveAgentTargets
|
|
81
|
-
} from "./chunk-
|
|
81
|
+
} from "./chunk-K4EHXOZK.js";
|
|
82
82
|
|
|
83
83
|
// src/instrument.ts
|
|
84
|
-
import
|
|
84
|
+
import { setTimeout as sleep } from "timers/promises";
|
|
85
85
|
|
|
86
86
|
// src/lib/sentry-scrub.ts
|
|
87
87
|
import os from "os";
|
|
@@ -192,6 +192,9 @@ function groupCliEvent(event, hint = {}) {
|
|
|
192
192
|
|
|
193
193
|
// src/instrument.ts
|
|
194
194
|
var GENEX_CLI_DSN = "https://43efc6f7d16c3e67cad6c60fa8175c20@o4511115493900288.ingest.us.sentry.io/4511706579599360";
|
|
195
|
+
var FLUSH_TIMEOUT_MS = 2e3;
|
|
196
|
+
var IGNORED_REJECTION_NAMES = /* @__PURE__ */ new Set(["AbortError", "AI_NoOutputGeneratedError"]);
|
|
197
|
+
var REJECTION_WARNING = "This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). The promise rejected with the reason:";
|
|
195
198
|
function bakedDsn() {
|
|
196
199
|
return import.meta.url.includes("/dist/") ? GENEX_CLI_DSN : "";
|
|
197
200
|
}
|
|
@@ -209,40 +212,79 @@ function telemetryDisabled() {
|
|
|
209
212
|
}
|
|
210
213
|
var dsn = resolveDsn();
|
|
211
214
|
var sentryEnabled = !!dsn && !telemetryDisabled();
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
215
|
+
var tags = {};
|
|
216
|
+
var contexts = {};
|
|
217
|
+
var reporter;
|
|
218
|
+
var pending = /* @__PURE__ */ new Set();
|
|
219
|
+
function setSentryTag(key, value) {
|
|
220
|
+
tags[key] = value;
|
|
221
|
+
}
|
|
222
|
+
function setSentryContext(name, context3) {
|
|
223
|
+
contexts[name] = context3;
|
|
224
|
+
}
|
|
225
|
+
function beforeSend(event, hint) {
|
|
226
|
+
const e = event;
|
|
227
|
+
const h = hint;
|
|
228
|
+
if (isCliNoise(e, h)) return null;
|
|
229
|
+
groupCliEvent(e, h);
|
|
230
|
+
scrubEvent(e);
|
|
231
|
+
return event;
|
|
232
|
+
}
|
|
233
|
+
function loadReporter() {
|
|
234
|
+
reporter ??= import("./sentry-reporter-FCSQOR4U.js").then(
|
|
235
|
+
(m) => m.startReporter({
|
|
236
|
+
dsn,
|
|
237
|
+
// release aligns with the AG-757 `x-genex-cli-version` telemetry stream so
|
|
238
|
+
// the two correlate; environment is prod unless pointed at a non-default API.
|
|
239
|
+
release: `@genex-ai/cli-demo@${getCliVersion()}`,
|
|
240
|
+
environment: CLI_CHANNEL === "dev" ? "development" : getApiUrl() === DEFAULT_API_URL ? "production" : "development",
|
|
241
|
+
beforeSend
|
|
242
|
+
})
|
|
243
|
+
);
|
|
244
|
+
return reporter;
|
|
235
245
|
}
|
|
236
|
-
|
|
246
|
+
function reportException(err, hint = {}) {
|
|
237
247
|
if (!sentryEnabled) return;
|
|
248
|
+
const scope = { tags: { ...tags }, contexts: { ...contexts } };
|
|
249
|
+
const report = loadReporter().then((r) => r.capture(err, hint, scope)).catch(() => {
|
|
250
|
+
});
|
|
251
|
+
pending.add(report);
|
|
252
|
+
void report.finally(() => pending.delete(report));
|
|
253
|
+
}
|
|
254
|
+
async function flushSentry(timeoutMs = FLUSH_TIMEOUT_MS) {
|
|
255
|
+
if (!reporter) return;
|
|
256
|
+
const deadline = Date.now() + timeoutMs;
|
|
238
257
|
try {
|
|
239
|
-
await
|
|
258
|
+
await Promise.race([Promise.all(pending), sleep(timeoutMs, void 0, { ref: false })]);
|
|
259
|
+
await (await reporter).flush(Math.max(0, deadline - Date.now()));
|
|
240
260
|
} catch {
|
|
241
261
|
}
|
|
242
262
|
}
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
263
|
+
var crashing = false;
|
|
264
|
+
async function onUncaughtException(err) {
|
|
265
|
+
if (!crashing) {
|
|
266
|
+
crashing = true;
|
|
267
|
+
reportException(err, { level: "fatal", mechanism: { type: "auto.node.onuncaughtexception", handled: false } });
|
|
268
|
+
await flushSentry();
|
|
269
|
+
}
|
|
270
|
+
console.error(err);
|
|
271
|
+
process.exit(1);
|
|
272
|
+
}
|
|
273
|
+
function onUnhandledRejection(reason) {
|
|
274
|
+
const name = reason instanceof Error ? reason.name : "";
|
|
275
|
+
if (IGNORED_REJECTION_NAMES.has(name)) return;
|
|
276
|
+
reportException(reason, {
|
|
277
|
+
level: "error",
|
|
278
|
+
extra: { unhandledPromiseRejection: true },
|
|
279
|
+
mechanism: { type: "auto.node.onunhandledrejection", handled: false }
|
|
280
|
+
});
|
|
281
|
+
console.warn(REJECTION_WARNING);
|
|
282
|
+
console.error(reason instanceof Error && reason.stack ? reason.stack : reason);
|
|
283
|
+
}
|
|
284
|
+
if (sentryEnabled) {
|
|
285
|
+
process.on("uncaughtException", (err) => void onUncaughtException(err));
|
|
286
|
+
process.on("unhandledRejection", onUnhandledRejection);
|
|
287
|
+
}
|
|
246
288
|
|
|
247
289
|
// src/lib/auth.ts
|
|
248
290
|
import http from "http";
|
|
@@ -279,9 +321,9 @@ async function readPendingAuth(apiUrl, envPath) {
|
|
|
279
321
|
if (current) return current;
|
|
280
322
|
return defaultProfile(envPath) ? readRecord(legacyPath(), apiUrl) : null;
|
|
281
323
|
}
|
|
282
|
-
async function writePendingAuth(
|
|
283
|
-
const apiUrl = normalizeApiOrigin(
|
|
284
|
-
await writeSecretFile(pendingAuthPath(apiUrl, envPath), JSON.stringify({ ...
|
|
324
|
+
async function writePendingAuth(pending2, envPath) {
|
|
325
|
+
const apiUrl = normalizeApiOrigin(pending2.apiUrl);
|
|
326
|
+
await writeSecretFile(pendingAuthPath(apiUrl, envPath), JSON.stringify({ ...pending2, apiUrl }) + "\n");
|
|
285
327
|
}
|
|
286
328
|
async function clearPendingAuth(apiUrl = getApiUrl(), envPath) {
|
|
287
329
|
await fs.rm(pendingAuthPath(apiUrl, envPath), { force: true }).catch(() => {
|
|
@@ -316,7 +358,7 @@ var AuthPendingError = class extends Error {
|
|
|
316
358
|
function formatUserCode(code) {
|
|
317
359
|
return code.length === 8 ? `${code.slice(0, 4)}-${code.slice(4)}` : code;
|
|
318
360
|
}
|
|
319
|
-
var
|
|
361
|
+
var sleep2 = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
320
362
|
async function authorize(apiBaseUrl, authBaseUrl, options) {
|
|
321
363
|
apiBaseUrl = normalizeApiOrigin(apiBaseUrl);
|
|
322
364
|
const { log, inlineWaitMs = DEFAULT_INLINE_WAIT_MS, open = openBrowser, label, envPath } = options;
|
|
@@ -336,14 +378,14 @@ async function authorize(apiBaseUrl, authBaseUrl, options) {
|
|
|
336
378
|
}
|
|
337
379
|
throw err;
|
|
338
380
|
}
|
|
339
|
-
const
|
|
381
|
+
const pending2 = {
|
|
340
382
|
deviceCode: started.deviceCode,
|
|
341
383
|
userCode: started.userCode,
|
|
342
384
|
verifyUrl: started.verifyUrl,
|
|
343
385
|
expiresAt: Date.now() + started.expiresIn * 1e3,
|
|
344
386
|
apiUrl: apiBaseUrl
|
|
345
387
|
};
|
|
346
|
-
await writePendingAuth(
|
|
388
|
+
await writePendingAuth(pending2, envPath);
|
|
347
389
|
printCode(log, started.userCode, started.verifyUrl);
|
|
348
390
|
let warned = false;
|
|
349
391
|
const warnManual = () => {
|
|
@@ -352,7 +394,7 @@ async function authorize(apiBaseUrl, authBaseUrl, options) {
|
|
|
352
394
|
log.dim(" (couldn't open a browser here \u2014 open the link above yourself)");
|
|
353
395
|
};
|
|
354
396
|
if (!open(started.verifyUrl, warnManual)) warnManual();
|
|
355
|
-
return await pollForToken(apiBaseUrl,
|
|
397
|
+
return await pollForToken(apiBaseUrl, pending2, { log, inlineWaitMs, interval: started.interval, envPath });
|
|
356
398
|
}
|
|
357
399
|
var LegacyApiError = class extends Error {
|
|
358
400
|
};
|
|
@@ -398,7 +440,7 @@ function printCode(log, userCode, verifyUrl) {
|
|
|
398
440
|
].join("\n")
|
|
399
441
|
);
|
|
400
442
|
}
|
|
401
|
-
async function pollForToken(apiBaseUrl,
|
|
443
|
+
async function pollForToken(apiBaseUrl, pending2, opts) {
|
|
402
444
|
const { log, inlineWaitMs } = opts;
|
|
403
445
|
const deadline = Date.now() + inlineWaitMs;
|
|
404
446
|
let intervalMs = Math.max((opts.interval ?? 2) * 1e3, MIN_POLL_INTERVAL_MS);
|
|
@@ -409,7 +451,7 @@ async function pollForToken(apiBaseUrl, pending, opts) {
|
|
|
409
451
|
res = await apiFetch(`${apiBaseUrl}/api/cli/device/poll`, {
|
|
410
452
|
method: "POST",
|
|
411
453
|
headers: { "Content-Type": "application/json" },
|
|
412
|
-
body: JSON.stringify({ deviceCode:
|
|
454
|
+
body: JSON.stringify({ deviceCode: pending2.deviceCode })
|
|
413
455
|
});
|
|
414
456
|
} catch {
|
|
415
457
|
res = new Response(null, { status: 599 });
|
|
@@ -432,8 +474,8 @@ async function pollForToken(apiBaseUrl, pending, opts) {
|
|
|
432
474
|
}
|
|
433
475
|
if (Date.now() + intervalMs > deadline) {
|
|
434
476
|
throw new AuthPendingError(
|
|
435
|
-
|
|
436
|
-
|
|
477
|
+
pending2.userCode,
|
|
478
|
+
pending2.verifyUrl,
|
|
437
479
|
apiBaseUrl,
|
|
438
480
|
opts.envPath || process.env[ENV_FILE_ENV] ? getGenexEnvPath(opts.envPath) : void 0
|
|
439
481
|
);
|
|
@@ -442,7 +484,7 @@ async function pollForToken(apiBaseUrl, pending, opts) {
|
|
|
442
484
|
log.dim(" still waiting for you to approve\u2026");
|
|
443
485
|
nextReassureAt = Date.now() + REASSURE_EVERY_MS;
|
|
444
486
|
}
|
|
445
|
-
await
|
|
487
|
+
await sleep2(intervalMs);
|
|
446
488
|
}
|
|
447
489
|
}
|
|
448
490
|
function printAuthHandoff(log, err) {
|
|
@@ -459,11 +501,11 @@ function printAuthHandoff(log, err) {
|
|
|
459
501
|
);
|
|
460
502
|
}
|
|
461
503
|
async function resumeAuthorization(apiBaseUrl, options) {
|
|
462
|
-
const
|
|
463
|
-
if (!
|
|
504
|
+
const pending2 = await readPendingAuth(apiBaseUrl, options.envPath);
|
|
505
|
+
if (!pending2) return null;
|
|
464
506
|
const { log, inlineWaitMs = DEFAULT_INLINE_WAIT_MS } = options;
|
|
465
|
-
printCode(log,
|
|
466
|
-
return await pollForToken(apiBaseUrl,
|
|
507
|
+
printCode(log, pending2.userCode, pending2.verifyUrl);
|
|
508
|
+
return await pollForToken(apiBaseUrl, pending2, { log, inlineWaitMs, envPath: options.envPath });
|
|
467
509
|
}
|
|
468
510
|
var SUCCESS_HTML = `<!doctype html><html><head><meta charset="utf-8"><title>Genex</title>
|
|
469
511
|
<style>body{font-family:system-ui,sans-serif;background:#0b0b0f;color:#eaeaea;display:grid;place-items:center;height:100vh;margin:0}
|
|
@@ -1065,7 +1107,7 @@ Important note: put soul into your creations, with many details and love. Aim to
|
|
|
1065
1107
|
21. A turn ends in exactly one of two ways: on a question the player must answer before the next step can be chosen, or on a handoff \u2014 the draft link, what changed, what to try, and what is still open. Never close a turn on a promise. "I'll keep building", "while I keep working", "adding that now" are things you say and then DO before you stop; if you are stopping, say that you have stopped and what remains. While the Build plan's \`Now:\` line has open work and no answer is needed from the player, the turn is not over: preview, hand off, and start the next milestone in the same turn, until the requested outcome is reached or the platform's own budget ends the session. If a stop was forced on you anyway, the first line of your next turn is the promise you left and where it stands.
|
|
1066
1108
|
22. **What the player sees must read as the thing it is**, in this game's own style: a building as that building, a person as a person, a prop as that prop, a surface as its material \u2014 and a coloured box, a capsule, or a flat grey block standing in for one of them is never its finished version. Generation is the default route to that bar wherever code will not honestly reach it: the player's character, whatever the request names, whatever the player walks up to, enters, or interacts with, and the music and sound underneath \u2014 reach for the belt on your own judgment, without stopping to ask; a status line saying what you queued is enough, and \`--no-wait\` keeps you building while it lands. Procedural code is a first-class engine for what is structural, repeated, distant, or parametric \u2014 terrain, sky, fences, paving, modular kits, filler \u2014 and for anything else only when the result meets the same bar and you have looked at it in a capture before its row says \`landed\`. Start with one reusable implementation per distinct required gameplay or visual role. Background populations default to reusable procedural bodies and motion or compatible existing rigged assets. Catalog clips need a compatible skeleton and may be paid. Reserve paid custom bodies for the player and characters examined or interacted with closely, while honoring explicit user requests. Add variants only for an unmet requirement; unspent allowance is not unfinished work. Allowance questions use timeoutPolicy no-consent: silence does not raise the allowance. Record role, reuse and completion criteria in the existing Assets table. Decide per object, not per category, and write the route on each Assets row in \`DESIGN.md\` \u2014 the paid flow for generated pieces, the procedural flow with its local path for code-built ones. An Assets table with no generation in it is a decision, not a default: record it as \`Generation: none \u2014 <why code alone reaches the bar here>\` or generate.
|
|
1067
1109
|
23. **The screenshots you take to look at your own work go in \`.genex/scratch/\`.** Captures, render comparisons, before/after strips, traces and metrics dumps are how you SEE what you built \u2014 they are not part of what you built, and \`genex preview\` pushes the whole folder to the game's source, every time, forever. \`.genex/scratch/\` already exists for this and is already ignored, so there is nothing to set up: write the capture as \`.genex/scratch/arena-before.png\` and read it back from there. Never put them in a folder of your own at the top level: \`progress/\`, \`reports/\`, \`shots/\` and their kind are pushed like source and become permanent weight in the game and in every remix of it \u2014 one real game reached 11 GB and 10,332 committed screenshots exactly this way. If you find such a folder already there, add it to \`.gitignore\`; never delete the player's files (law 18). One file in \`.genex/scratch/\` IS read, by the platform: \`.genex/scratch/cover.png\` is the game's cover \u2014 the frame you would put on its poster: real gameplay at its signature moment, well lit, no menus, popups or busy HUD, landscape 16:9, and never any text (the page sets the title beside it). The next \`genex preview\` makes it the cover; save a better moment over it whenever one comes along, and if your browser tool can set the cover itself, use that instead. Before the first publish, and whenever the game's look changes, load \`$genex-cover\`: it chooses, stages, lights and judges that frame.
|
|
1068
|
-
24. **Benchmark before declaring an in-game LLM
|
|
1110
|
+
24. **Benchmark before declaring an in-game LLM ceiling.** If the game calls a language model while the player plays \u2014 an NPC answering in its own words, a quest written for this save, a judge reading what the player typed \u2014 the PLAYER pays for it from their Genex credits, billed as used, and the game declares no price, only a per-call CEILING: \`maxCredits\` on \`generate()\`, \`perCallMaxCredits\` on a standing budget. That ceiling bounds what one call may cost AND funds how long its answer may be, so never pick it by judgement or from a vendor's rate card. Run the real prompt on your own credits \u2014 \`npx genex llm bench "<the prompt>" --schema <file> --samples 3 --max-credits <n> --user-approved\` \u2014 and declare the figures it prints, recorded in \`DESIGN.md\` with its model and date; re-benchmark whenever the prompt, the schema or the model changes, because a prompt edit changes the ceiling. Check the lane first with \`npx genex llm models\`: where it is off the routes 404, so build the feature behind a graceful unavailable state and say so in the handoff. A repeated call is a standing budget the player approves once (\`requestSpendGrant()\`, then \`generate({ grantId })\`), never a popup per turn, and its disclosed call rate and credits per period are computed from the game's own loop and the bench's measured average. The coin spellings (\`estimateCoins\`, \`perCallMaxCoins\`) are deprecated \u2014 write the credit names. Model output is data the game validates against its own expectation \u2014 never executed, and never authority over coin, credits, items, entitlements or rewards. Load \`$genex-llm-in-games\` before writing any of it.
|
|
1069
1111
|
${CONTRACT_END}
|
|
1070
1112
|
`;
|
|
1071
1113
|
var TOOLS_CONTRACT_HEAD = `# Genex Tools (always in effect in this folder)
|
|
@@ -1079,7 +1121,7 @@ var TOOLS_CONTRACT_OFFER = `5. **When the game is built and playable \u2014 neve
|
|
|
1079
1121
|
var TOOLS_CONTRACT_HOSTED_LAWS = `5. **This game is hosted on Genex now, and publishing is glue \u2014 never a rebuild.** The game ships exactly as the user built it: \`initEmbed()\` first in the boot code (the \`genex-threejs-embed-auth\` card has the call; the renderer draws before any \`await waitForPlayer()\`), a static build into \`dist/\` with relative asset paths, then \`npx genex preview\`. Never restructure, reformat or "improve" the game to ship it, never scaffold a new app around it, and never start a design document or a build plan for it \u2014 the user's own process is theirs. Load the \`genex-tool-publish\` card before the first preview; it owns the vocabulary, the links and the limits.
|
|
1080
1122
|
6. **Ship first, then report.** \`preview\` prints a preflight \u2014 phone memory, a missing volume slider, an asset nobody wired, a viewport line. It never blocks a deploy, so push the build as it is, hand over the link, THEN relay each preflight line to the user in one plain sentence with an offer to fix it, and fix one only on their yes. The preflight is a report for the user, not a to-do list for you.
|
|
1081
1123
|
7. **The link is the game's page, and there are two versions.** After every preview give the user \`<dashboard>/draft/<slug>\` \u2014 \`dashboardOrigins[0]\` and \`slug\` from \`.genex/project.json\` \u2014 never localhost, a file path or the bare play origin. \`preview\` updates the draft and never touches what players are on; the first release is \`npx genex publish\` (it lists the game); after that "publish it", "update it" and "yes" all mean \`npx genex promote\` \u2014 the exact draft build, no rebuild. Ask once per round of work, in one line, and keep working while you wait.`;
|
|
1082
|
-
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with Genex
|
|
1124
|
+
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with their Genex credits or their own Claude/ChatGPT subscription, and approves it on a Genex sheet; your game just calls \`generate()\`. Want it that way?`;
|
|
1083
1125
|
function toolsContractLlmLaw(n, hosted) {
|
|
1084
1126
|
const onYes = hosted ? "On a yes, load `$genex-tool-llm` and then `$genex-llm-in-games`, which owns the build." : "On a yes, load `$genex-tool-llm`, which owns the rest: it checks the lane with `npx genex llm models` first, then runs `npx genex init --convert` (a hosted game is a static browser build \u2014 a local server holding a key can never ship).";
|
|
1085
1127
|
return `${n}. **A model running while people PLAY is a platform feature \u2014 never something you wire onto the user's own meter.** When a request implies one (NPCs that talk or decide in their own words, content written from what the player types, a prompt box in the game, "let the player pick a model"), recognise it, say in ONE line: "${TOOLS_CONTRACT_LLM_OFFER}" \u2014 then ASK and wait. ${onYes} On a no, build the authored version instead. Either way, never ship a game that calls a model on a key or an account of the user's own: every visitor would spend their money with nobody approving it, and the credential is readable in the bundle. Player-funded generation is text and JSON only \u2014 3D, images, video and audio stay the asset lanes above, on the user's meter.`;
|
|
@@ -1622,8 +1664,8 @@ async function fetchLatestFromRegistry(name) {
|
|
|
1622
1664
|
{ signal: AbortSignal.timeout(REGISTRY_TIMEOUT_MS) }
|
|
1623
1665
|
);
|
|
1624
1666
|
if (!res.ok) return null;
|
|
1625
|
-
const
|
|
1626
|
-
return typeof
|
|
1667
|
+
const tags2 = await res.json();
|
|
1668
|
+
return typeof tags2.latest === "string" && parseSemver(tags2.latest) ? tags2.latest : null;
|
|
1627
1669
|
} catch {
|
|
1628
1670
|
return null;
|
|
1629
1671
|
}
|
|
@@ -3299,10 +3341,10 @@ function hashReadsBeforeInit(content) {
|
|
|
3299
3341
|
else if (ch === "}") depth--;
|
|
3300
3342
|
}
|
|
3301
3343
|
if (depth <= 0) {
|
|
3302
|
-
return !inBracelessArrow(content, at) && !inits.some((
|
|
3344
|
+
return !inBracelessArrow(content, at) && !inits.some((init) => init.index < at);
|
|
3303
3345
|
}
|
|
3304
3346
|
const flowEnd = enclosingFlowEnd(content, at);
|
|
3305
|
-
return flowEnd !== content.length && inits.some((
|
|
3347
|
+
return flowEnd !== content.length && inits.some((init) => init.flowEnd === flowEnd && init.index > at);
|
|
3306
3348
|
});
|
|
3307
3349
|
}
|
|
3308
3350
|
async function detectShellApis(cwd = process.cwd()) {
|
|
@@ -4501,12 +4543,12 @@ function edgeOpaqueFraction(png, alphaThresh = 40) {
|
|
|
4501
4543
|
}
|
|
4502
4544
|
function assertNotFlush(png, label, maxFrac) {
|
|
4503
4545
|
const edges = edgeOpaqueFraction(png);
|
|
4504
|
-
const
|
|
4505
|
-
if (
|
|
4546
|
+
const flush = Object.entries(edges).filter(([, frac]) => frac > maxFrac);
|
|
4547
|
+
if (flush.length > 0) {
|
|
4506
4548
|
fail(
|
|
4507
4549
|
`${label}: the crop box is slicing the element \u2014 opaque content is flush against the crop edge. The sheet needs more margin here: widen the crop box if there is room, otherwise region-edit or regenerate the sheet with more spacing around this element (see genex-ai-hud troubleshooting), then re-run.`,
|
|
4508
4550
|
{
|
|
4509
|
-
flushEdges:
|
|
4551
|
+
flushEdges: flush.map(([edge, frac]) => ({ edge, opaqueFraction: Number(frac.toFixed(3)) })),
|
|
4510
4552
|
maxFraction: maxFrac
|
|
4511
4553
|
}
|
|
4512
4554
|
);
|
|
@@ -6357,7 +6399,7 @@ async function isDir(p) {
|
|
|
6357
6399
|
return false;
|
|
6358
6400
|
}
|
|
6359
6401
|
}
|
|
6360
|
-
var
|
|
6402
|
+
var sleep3 = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
6361
6403
|
function fingerprintOf(html) {
|
|
6362
6404
|
const m = html.match(/[A-Za-z0-9_]+-[A-Za-z0-9_-]{8}\.(?:js|css)/);
|
|
6363
6405
|
return m ? m[0] : null;
|
|
@@ -6378,7 +6420,7 @@ async function waitUntilLive(playUrl, fingerprint, timeoutMs, log) {
|
|
|
6378
6420
|
}
|
|
6379
6421
|
} catch {
|
|
6380
6422
|
}
|
|
6381
|
-
await
|
|
6423
|
+
await sleep3(1500);
|
|
6382
6424
|
}
|
|
6383
6425
|
log.success(`Published. \u{1F310} ${playUrl}`);
|
|
6384
6426
|
log.dim(" (If the previous build shows, hard-refresh in a few seconds.)");
|
|
@@ -8316,7 +8358,7 @@ async function runDomain(opts) {
|
|
|
8316
8358
|
}
|
|
8317
8359
|
const base = `${apiUrl}/api/projects/${encodeURIComponent(meta.id)}/domains`;
|
|
8318
8360
|
const auth = { Authorization: `Bearer ${token}` };
|
|
8319
|
-
const call = (url,
|
|
8361
|
+
const call = (url, init) => apiFetch(url, { ...init, headers: { ...auth, "Content-Type": "application/json", ...init?.headers ?? {} } });
|
|
8320
8362
|
let res;
|
|
8321
8363
|
try {
|
|
8322
8364
|
if (sub === "list") res = await call(base);
|
|
@@ -8464,7 +8506,7 @@ async function runShop(opts) {
|
|
|
8464
8506
|
await writeUserToken(token, opts.envPath, apiUrl);
|
|
8465
8507
|
}
|
|
8466
8508
|
const auth = { Authorization: `Bearer ${token}`, "Content-Type": "application/json" };
|
|
8467
|
-
const call = (url,
|
|
8509
|
+
const call = (url, init) => apiFetch(url, { ...init, headers: { ...auth, ...init?.headers ?? {} } });
|
|
8468
8510
|
const projectSkus = `${apiUrl}/api/coin/projects/${encodeURIComponent(meta.id)}/skus`;
|
|
8469
8511
|
async function fail5(res2, what) {
|
|
8470
8512
|
if (printedStructuredError(res2)) {
|
|
@@ -9586,13 +9628,13 @@ async function enqueueWithAllowance(input) {
|
|
|
9586
9628
|
const headers = { "Content-Type": "application/json", Authorization: `Bearer ${input.token}` };
|
|
9587
9629
|
const fingerprint = createHash("sha256").update(JSON.stringify([input.apiUrl, createPath, input.body])).digest("hex");
|
|
9588
9630
|
const rows = await readAdmissionRows(cwd);
|
|
9589
|
-
const
|
|
9631
|
+
const pending2 = /* @__PURE__ */ new Map();
|
|
9590
9632
|
for (const row of rows) {
|
|
9591
|
-
if (row.t === "reserve" && typeof row.id === "string")
|
|
9592
|
-
if (row.t === "release" && typeof row.id === "string")
|
|
9593
|
-
if (row.t === "q" && typeof row.reservationId === "string")
|
|
9633
|
+
if (row.t === "reserve" && typeof row.id === "string") pending2.set(row.id, row);
|
|
9634
|
+
if (row.t === "release" && typeof row.id === "string") pending2.delete(row.id);
|
|
9635
|
+
if (row.t === "q" && typeof row.reservationId === "string") pending2.delete(row.reservationId);
|
|
9594
9636
|
}
|
|
9595
|
-
for (const [requestId2, row] of
|
|
9637
|
+
for (const [requestId2, row] of pending2) {
|
|
9596
9638
|
if (row.apiUrl === input.apiUrl && row.generic === true) {
|
|
9597
9639
|
const response2 = await apiFetch(`${input.apiUrl}/api/generations/requests/${encodeURIComponent(requestId2)}`, { headers });
|
|
9598
9640
|
if (response2.ok) {
|
|
@@ -9811,7 +9853,7 @@ function reportMocked(view, log) {
|
|
|
9811
9853
|
if (view.mocked !== true) return;
|
|
9812
9854
|
log.dim(` ${view.provider === "fixture" ? FIXTURE_LINE : MOCKED_LINE}`);
|
|
9813
9855
|
}
|
|
9814
|
-
var
|
|
9856
|
+
var sleep4 = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
9815
9857
|
var INLINE_IMAGE_LIMIT_BYTES = 4 * 1024 * 1024;
|
|
9816
9858
|
var IMAGE_MIME_BY_EXT = {
|
|
9817
9859
|
".png": "image/png",
|
|
@@ -10600,14 +10642,14 @@ async function waitViaSSE(apiUrl, token, id, onProgress, deadline) {
|
|
|
10600
10642
|
});
|
|
10601
10643
|
} catch {
|
|
10602
10644
|
if (++connectFailures >= 3) return "unsupported";
|
|
10603
|
-
await
|
|
10645
|
+
await sleep4(2e3);
|
|
10604
10646
|
continue;
|
|
10605
10647
|
}
|
|
10606
10648
|
if (res.status === 404 || res.status === 405) return "unsupported";
|
|
10607
10649
|
const contentType = res.headers.get("content-type") ?? "";
|
|
10608
10650
|
if (!res.ok || !res.body || !contentType.includes("text/event-stream")) {
|
|
10609
10651
|
if (++connectFailures >= 3) return "unsupported";
|
|
10610
|
-
await
|
|
10652
|
+
await sleep4(2e3);
|
|
10611
10653
|
continue;
|
|
10612
10654
|
}
|
|
10613
10655
|
connectFailures = 0;
|
|
@@ -10628,7 +10670,7 @@ async function waitViaSSE(apiUrl, token, id, onProgress, deadline) {
|
|
|
10628
10670
|
}
|
|
10629
10671
|
} catch {
|
|
10630
10672
|
}
|
|
10631
|
-
await
|
|
10673
|
+
await sleep4(1e3);
|
|
10632
10674
|
} finally {
|
|
10633
10675
|
clearTimeout(timer);
|
|
10634
10676
|
abort.abort();
|
|
@@ -10658,7 +10700,7 @@ async function poll(apiUrl, token, id, onProgress, timeoutMs) {
|
|
|
10658
10700
|
}
|
|
10659
10701
|
const remaining = deadline - Date.now();
|
|
10660
10702
|
if (remaining <= 0) break;
|
|
10661
|
-
await
|
|
10703
|
+
await sleep4(Math.min(3e3, remaining));
|
|
10662
10704
|
}
|
|
10663
10705
|
return null;
|
|
10664
10706
|
}
|
|
@@ -20426,12 +20468,12 @@ async function loadManifest(baseOverride) {
|
|
|
20426
20468
|
function resolveSelectors(manifest, selectors) {
|
|
20427
20469
|
const byName = new Map(manifest.clips.map((entry) => [entry.name, entry]));
|
|
20428
20470
|
const byLowerName = new Map(manifest.clips.map((entry) => [entry.name.toLowerCase(), entry]));
|
|
20429
|
-
const
|
|
20471
|
+
const tags2 = /* @__PURE__ */ new Map();
|
|
20430
20472
|
for (const entry of manifest.clips) {
|
|
20431
20473
|
for (const tag of entry.tags) {
|
|
20432
|
-
const list =
|
|
20474
|
+
const list = tags2.get(tag) ?? [];
|
|
20433
20475
|
list.push(entry);
|
|
20434
|
-
|
|
20476
|
+
tags2.set(tag, list);
|
|
20435
20477
|
}
|
|
20436
20478
|
}
|
|
20437
20479
|
const out = /* @__PURE__ */ new Map();
|
|
@@ -20441,12 +20483,12 @@ function resolveSelectors(manifest, selectors) {
|
|
|
20441
20483
|
out.set(selector, [exact]);
|
|
20442
20484
|
continue;
|
|
20443
20485
|
}
|
|
20444
|
-
const tagHit =
|
|
20486
|
+
const tagHit = tags2.get(selector) ?? tags2.get(selector.toLowerCase());
|
|
20445
20487
|
if (tagHit) {
|
|
20446
20488
|
out.set(selector, tagHit);
|
|
20447
20489
|
continue;
|
|
20448
20490
|
}
|
|
20449
|
-
const candidates = [...
|
|
20491
|
+
const candidates = [...tags2.keys(), ...byName.keys()];
|
|
20450
20492
|
const close = suggest(selector, candidates);
|
|
20451
20493
|
throw new Error(
|
|
20452
20494
|
`unknown clip/tag "${selector}"${close.length > 0 ? ` \u2014 closest: ${close.join(", ")}` : ""}. Run ${c.cyan(
|
|
@@ -20512,13 +20554,13 @@ function printCatalog(log, manifest, selectors) {
|
|
|
20512
20554
|
}
|
|
20513
20555
|
return;
|
|
20514
20556
|
}
|
|
20515
|
-
const
|
|
20557
|
+
const tags2 = /* @__PURE__ */ new Map();
|
|
20516
20558
|
for (const entry of manifest.clips) {
|
|
20517
20559
|
for (const tag of entry.tags) {
|
|
20518
20560
|
if (tag === HIDDEN_TAG) continue;
|
|
20519
|
-
const list =
|
|
20561
|
+
const list = tags2.get(tag) ?? [];
|
|
20520
20562
|
list.push(entry);
|
|
20521
|
-
|
|
20563
|
+
tags2.set(tag, list);
|
|
20522
20564
|
}
|
|
20523
20565
|
}
|
|
20524
20566
|
log.plain(
|
|
@@ -20530,7 +20572,7 @@ function printCatalog(log, manifest, selectors) {
|
|
|
20530
20572
|
)}`
|
|
20531
20573
|
);
|
|
20532
20574
|
log.plain("");
|
|
20533
|
-
for (const [tag, entries] of [...
|
|
20575
|
+
for (const [tag, entries] of [...tags2.entries()].sort(([a], [b]) => a.localeCompare(b))) {
|
|
20534
20576
|
const bytes = entries.reduce((sum, entry) => sum + entry.bytes, 0);
|
|
20535
20577
|
const suffix = tag === "core" ? " \u2014 bundled in animation-library.glb" : ` (${formatMb(bytes)})`;
|
|
20536
20578
|
log.plain(` ${c.bold(tag.padEnd(18))} ${entries.map((e) => e.name).join(", ")}${suffix}`);
|
|
@@ -23736,7 +23778,7 @@ async function runBlender(opts) {
|
|
|
23736
23778
|
return 1;
|
|
23737
23779
|
}
|
|
23738
23780
|
if (sub === "serve") {
|
|
23739
|
-
const { serveLocalBlender } = await import("./blender-serve-
|
|
23781
|
+
const { serveLocalBlender } = await import("./blender-serve-EH2MOW3U.js");
|
|
23740
23782
|
const port = Number(process.env.GENEX_BLENDER_PORT ?? 8088);
|
|
23741
23783
|
log.step(`Starting a local Blender service on port ${port}`);
|
|
23742
23784
|
log.plain(
|
|
@@ -23745,7 +23787,7 @@ async function runBlender(opts) {
|
|
|
23745
23787
|
return serveLocalBlender({ port, log });
|
|
23746
23788
|
}
|
|
23747
23789
|
if (sub === "mcp") {
|
|
23748
|
-
const { runBlenderMcp } = await import("./blender-mcp-
|
|
23790
|
+
const { runBlenderMcp } = await import("./blender-mcp-WSURPZET.js");
|
|
23749
23791
|
return runBlenderMcp();
|
|
23750
23792
|
}
|
|
23751
23793
|
if (sub === "seat") {
|
|
@@ -24253,7 +24295,12 @@ import fs35 from "fs/promises";
|
|
|
24253
24295
|
import path36 from "path";
|
|
24254
24296
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
24255
24297
|
var SUBS3 = ["models", "bench", "price", "status", "cancel"];
|
|
24256
|
-
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN
|
|
24298
|
+
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN credits \u2014 your balance, not a player's, and a spend this build's asset allowance does not count. Re-run with --max-credits <n> --user-approved only after the person at the keyboard agreed to the number.";
|
|
24299
|
+
var LLM_MAX_COINS_DEPRECATED = "--max-coins is deprecated: it is read as --max-credits, the same number \u2014 a bench spends credits now.";
|
|
24300
|
+
function benchBalanceShortLine(spendable, maxCredits) {
|
|
24301
|
+
return `Your spendable credits (${spendable}) cannot cover one attempt at --max-credits ${maxCredits} \u2014 nothing was sent, nothing was spent. Add credits, or re-run with a lower --max-credits once the person at the keyboard agrees to it.`;
|
|
24302
|
+
}
|
|
24303
|
+
var LLM_BENCH_UNVERIFIED_LINE = "Credits pay only for a verified email, and this account's is not verified yet (credits_unverified) \u2014 nothing was sent, nothing was spent. Verify it, then re-run.";
|
|
24257
24304
|
var LLM_LANE_OFF_LINE = "The runtime LLM lane is off on this stand \u2014 in-game generate() answers 404 here, and nothing can be benchmarked.";
|
|
24258
24305
|
var LLM_TOOLS_CONVERT_HINT = [
|
|
24259
24306
|
"In-game model calls need a hosted game. This folder is a Genex Tools workspace, so nothing here",
|
|
@@ -24266,7 +24313,7 @@ function serverSlotCounts(body) {
|
|
|
24266
24313
|
return { held: body.slotsHeld, limit: body.slotLimit };
|
|
24267
24314
|
}
|
|
24268
24315
|
var PENDING_SLOT_RELEASE_MINUTES = 10;
|
|
24269
|
-
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so
|
|
24316
|
+
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so what it reserved stays held until the bill resolves; such a row stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`;
|
|
24270
24317
|
var LLM_STATUS_NO_COUNT_LINE = "This stand does not report how many slots are held across your projects, so no count is shown.";
|
|
24271
24318
|
var LLM_CANCEL_AWAITING_BILL = "this call already stopped; its bill is awaiting the provider";
|
|
24272
24319
|
var LLM_CANCEL_ALREADY_FINAL = "this call is already final";
|
|
@@ -24280,11 +24327,33 @@ var DEFAULT_SAMPLE_TIMEOUT_SEC = 180;
|
|
|
24280
24327
|
function parseProviderKey(value) {
|
|
24281
24328
|
return value === "ok" || value === "rejected" || value === "unknown" ? value : null;
|
|
24282
24329
|
}
|
|
24330
|
+
function bpsOrNull(value) {
|
|
24331
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
24332
|
+
}
|
|
24333
|
+
function standingCreditGrantOrNull(value) {
|
|
24334
|
+
if (!value || typeof value !== "object") return null;
|
|
24335
|
+
const v = value;
|
|
24336
|
+
const whole = (x) => typeof x === "number" && Number.isSafeInteger(x) && x >= 0;
|
|
24337
|
+
if (!whole(v.minCredits) || !whole(v.maxCredits) || !whole(v.stepCredits) || !whole(v.defaultCredits)) return null;
|
|
24338
|
+
return {
|
|
24339
|
+
minCredits: v.minCredits,
|
|
24340
|
+
maxCredits: v.maxCredits,
|
|
24341
|
+
stepCredits: v.stepCredits,
|
|
24342
|
+
defaultCredits: v.defaultCredits,
|
|
24343
|
+
...whole(v.minUsdCents) ? { minUsdCents: v.minUsdCents } : {},
|
|
24344
|
+
...whole(v.maxUsdCents) ? { maxUsdCents: v.maxUsdCents } : {},
|
|
24345
|
+
...whole(v.stepUsdCents) ? { stepUsdCents: v.stepUsdCents } : {}
|
|
24346
|
+
};
|
|
24347
|
+
}
|
|
24283
24348
|
async function readRuntimeLane(apiUrl, token) {
|
|
24284
24349
|
const empty = (state) => ({
|
|
24285
24350
|
state,
|
|
24286
24351
|
models: null,
|
|
24287
24352
|
recommendedDeclaredHeadroomBps: null,
|
|
24353
|
+
currency: null,
|
|
24354
|
+
feeBps: null,
|
|
24355
|
+
usdCentsPerCredit: null,
|
|
24356
|
+
standingCreditGrant: null,
|
|
24288
24357
|
externalProviders: null,
|
|
24289
24358
|
total: null,
|
|
24290
24359
|
featuredCount: null,
|
|
@@ -24303,7 +24372,11 @@ async function readRuntimeLane(apiUrl, token) {
|
|
|
24303
24372
|
return {
|
|
24304
24373
|
state: "live",
|
|
24305
24374
|
models: body.models,
|
|
24306
|
-
recommendedDeclaredHeadroomBps:
|
|
24375
|
+
recommendedDeclaredHeadroomBps: bpsOrNull(body.recommendedDeclaredHeadroomBps),
|
|
24376
|
+
currency: body.currency === "credits" || body.currency === "coins" ? body.currency : null,
|
|
24377
|
+
feeBps: bpsOrNull(body.feeBps),
|
|
24378
|
+
usdCentsPerCredit: typeof body.usdCentsPerCredit === "number" && Number.isSafeInteger(body.usdCentsPerCredit) && body.usdCentsPerCredit >= 1 ? body.usdCentsPerCredit : null,
|
|
24379
|
+
standingCreditGrant: standingCreditGrantOrNull(body.standingCreditGrant),
|
|
24307
24380
|
externalProviders: Array.isArray(body.externalProviders) ? body.externalProviders : null,
|
|
24308
24381
|
total: typeof body.total === "number" && Number.isFinite(body.total) ? body.total : null,
|
|
24309
24382
|
featuredCount: typeof body.featuredCount === "number" && Number.isFinite(body.featuredCount) ? body.featuredCount : null,
|
|
@@ -24319,18 +24392,45 @@ function percentile2(values, p) {
|
|
|
24319
24392
|
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
24320
24393
|
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
24321
24394
|
}
|
|
24322
|
-
function
|
|
24323
|
-
|
|
24395
|
+
function percentilePicos(values, p) {
|
|
24396
|
+
if (values.length === 0) return null;
|
|
24397
|
+
const sorted = [...values].sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
24398
|
+
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
24399
|
+
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
24324
24400
|
}
|
|
24325
|
-
|
|
24326
|
-
|
|
24401
|
+
var PICOS_PER_CENT = 10000000000n;
|
|
24402
|
+
function ceilingCreditsFor(costPicos, terms) {
|
|
24403
|
+
if (costPicos <= 0n) return 1;
|
|
24404
|
+
const num = costPicos * BigInt(1e4 + terms.feeBps) * BigInt(1e4 + terms.headroomBps);
|
|
24405
|
+
const den = 100000000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
24406
|
+
return Math.max(1, Number((num + den - 1n) / den));
|
|
24407
|
+
}
|
|
24408
|
+
function averageCreditsPerCall(costsPicos, terms) {
|
|
24409
|
+
if (costsPicos.length === 0) return null;
|
|
24410
|
+
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
24411
|
+
return Number(sum * BigInt(1e4 + terms.feeBps)) / Number(BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT);
|
|
24412
|
+
}
|
|
24413
|
+
function perCallEstimateCreditsFor(costsPicos, terms) {
|
|
24414
|
+
if (costsPicos.length === 0) return null;
|
|
24415
|
+
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
24416
|
+
const num = sum * BigInt(1e4 + terms.feeBps);
|
|
24417
|
+
const den = BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
24418
|
+
return Math.max(1, Number((num + den - 1n) / den));
|
|
24419
|
+
}
|
|
24420
|
+
function formatCredits(value) {
|
|
24421
|
+
if (!Number.isFinite(value)) return "\u2014";
|
|
24422
|
+
if (value >= 100) return String(Math.round(value));
|
|
24423
|
+
return String(Number(value.toPrecision(value < 1 ? 2 : 3)));
|
|
24424
|
+
}
|
|
24425
|
+
function usdFromPicos(picos) {
|
|
24426
|
+
if (picos === null || picos === void 0) return "unknown";
|
|
24427
|
+
const value = typeof picos === "bigint" ? picos : /^\d+$/.test(picos) ? BigInt(picos) : null;
|
|
24428
|
+
if (value === null) return "unknown";
|
|
24429
|
+
return `$${(Number(value) / 1e12).toFixed(6)}`;
|
|
24327
24430
|
}
|
|
24328
24431
|
function usdPerMillion(value) {
|
|
24329
24432
|
return `$${value.toFixed(value < 1 ? 4 : 2)}`;
|
|
24330
24433
|
}
|
|
24331
|
-
function usd(value) {
|
|
24332
|
-
return value === null ? "unknown" : `$${value.toFixed(6)}`;
|
|
24333
|
-
}
|
|
24334
24434
|
async function runLlm(opts = {}) {
|
|
24335
24435
|
const log = createLogger({ quiet: opts.quiet || opts.json });
|
|
24336
24436
|
const cwd = opts.cwd ?? process.cwd();
|
|
@@ -24356,16 +24456,23 @@ async function runLlm(opts = {}) {
|
|
|
24356
24456
|
return;
|
|
24357
24457
|
}
|
|
24358
24458
|
if (sub === "bench") {
|
|
24359
|
-
if (opts.maxCoins
|
|
24459
|
+
if (opts.maxCoins !== void 0) process.stderr.write(`${c.yellow("!")} ${LLM_MAX_COINS_DEPRECATED}
|
|
24460
|
+
`);
|
|
24461
|
+
if (opts.maxCoins !== void 0 && opts.maxCredits !== void 0 && opts.maxCoins !== opts.maxCredits) {
|
|
24462
|
+
fail5("llm bench", `Pass --max-credits alone \u2014 --max-coins is its deprecated name, and the two disagree (${opts.maxCredits} vs ${opts.maxCoins}).`);
|
|
24463
|
+
return;
|
|
24464
|
+
}
|
|
24465
|
+
const approvedCredits = opts.maxCredits ?? opts.maxCoins;
|
|
24466
|
+
if (approvedCredits === void 0 || opts.userApproved !== true) {
|
|
24360
24467
|
fail5("llm bench", LLM_BENCH_APPROVAL_REQUIRED);
|
|
24361
24468
|
return;
|
|
24362
24469
|
}
|
|
24363
|
-
if (!Number.isInteger(
|
|
24364
|
-
fail5("llm bench", `--max-
|
|
24470
|
+
if (!Number.isInteger(approvedCredits) || approvedCredits < 1) {
|
|
24471
|
+
fail5("llm bench", `--max-credits takes a whole number of credits, 1 or more (got ${String(approvedCredits)}).`);
|
|
24365
24472
|
return;
|
|
24366
24473
|
}
|
|
24367
24474
|
if (!opts.benchPrompt?.trim()) {
|
|
24368
|
-
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-
|
|
24475
|
+
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-credits <n> --user-approved.');
|
|
24369
24476
|
return;
|
|
24370
24477
|
}
|
|
24371
24478
|
if (opts.jsonOutput && opts.textOutput) {
|
|
@@ -24457,6 +24564,10 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24457
24564
|
models: null,
|
|
24458
24565
|
externalProviders: null,
|
|
24459
24566
|
recommendedDeclaredHeadroomBps: null,
|
|
24567
|
+
currency: null,
|
|
24568
|
+
feeBps: null,
|
|
24569
|
+
usdCentsPerCredit: null,
|
|
24570
|
+
standingCreditGrant: null,
|
|
24460
24571
|
total: null,
|
|
24461
24572
|
featuredCount: null,
|
|
24462
24573
|
// The stand was never asked, so the verdict is unknown — but the key is
|
|
@@ -24482,6 +24593,12 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24482
24593
|
models: lane.models,
|
|
24483
24594
|
externalProviders: lane.externalProviders,
|
|
24484
24595
|
recommendedDeclaredHeadroomBps: lane.recommendedDeclaredHeadroomBps,
|
|
24596
|
+
// The credit terms a call is billed on, the server's — null on a stand
|
|
24597
|
+
// that predates credit billing, never a number the CLI filled in.
|
|
24598
|
+
currency: lane.currency,
|
|
24599
|
+
feeBps: lane.feeBps,
|
|
24600
|
+
usdCentsPerCredit: lane.usdCentsPerCredit,
|
|
24601
|
+
standingCreditGrant: lane.standingCreditGrant,
|
|
24485
24602
|
total: lane.total ?? lane.models?.length ?? null,
|
|
24486
24603
|
featuredCount: lane.featuredCount ?? (lane.models ? lane.models.filter((m) => m.featured).length : null),
|
|
24487
24604
|
// `null` when the lane is not live or the server predates the probe.
|
|
@@ -24528,8 +24645,9 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24528
24645
|
);
|
|
24529
24646
|
for (const m of shown) {
|
|
24530
24647
|
const plan = m.personalPlan ? ` \xB7 personal plan: ${m.personalPlan}` : "";
|
|
24648
|
+
const kind = m.kind === "typesafe" ? " \xB7 judge (classifier): judge() under a budget, not generate() or bench" : "";
|
|
24531
24649
|
log.plain(
|
|
24532
|
-
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}`
|
|
24650
|
+
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}${kind}`
|
|
24533
24651
|
);
|
|
24534
24652
|
}
|
|
24535
24653
|
if (hiddenCount > 0) {
|
|
@@ -24545,74 +24663,104 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24545
24663
|
}
|
|
24546
24664
|
log.plain("");
|
|
24547
24665
|
if (lane.recommendedDeclaredHeadroomBps === null) {
|
|
24548
|
-
log.dim(" This stand serves no recommended headroom, so no
|
|
24666
|
+
log.dim(" This stand serves no recommended headroom, so no ceiling can be recommended from a bench.");
|
|
24549
24667
|
} else {
|
|
24550
|
-
log.dim(` Recommended headroom over benchmarked
|
|
24668
|
+
log.dim(` Recommended headroom over a benchmarked call's cost on this stand: ${lane.recommendedDeclaredHeadroomBps} bps.`);
|
|
24669
|
+
}
|
|
24670
|
+
if (lane.feeBps === null) {
|
|
24671
|
+
log.dim(" This stand serves no platform fee \u2014 it predates credit billing, so no ceiling can be recommended from a bench.");
|
|
24672
|
+
} else {
|
|
24673
|
+
log.dim(` In-game calls are billed in the player's credits, as used: the provider's cost plus this stand's platform fee (${lane.feeBps} bps).`);
|
|
24551
24674
|
}
|
|
24552
24675
|
if (toolsOnly) {
|
|
24553
24676
|
printConvertHint();
|
|
24554
24677
|
return;
|
|
24555
24678
|
}
|
|
24556
|
-
log.dim(" Those per-million rates are the PROVIDER's; what
|
|
24557
|
-
log.dim(`
|
|
24679
|
+
log.dim(" Those per-million rates are the PROVIDER's list; what one call of YOUR prompt really costs,");
|
|
24680
|
+
log.dim(` and the per-call ceiling to declare, only a real call measures: ${c.cyan('genex llm bench "<prompt>" --max-credits <n> --user-approved')}`);
|
|
24558
24681
|
}
|
|
24559
24682
|
function wholeOrNull(value) {
|
|
24560
24683
|
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
24561
24684
|
}
|
|
24685
|
+
function costPicosOf(usage) {
|
|
24686
|
+
if (typeof usage?.costUsdPicos === "string" && /^\d+$/.test(usage.costUsdPicos)) return usage.costUsdPicos;
|
|
24687
|
+
if (typeof usage?.costUsd === "number" && Number.isFinite(usage.costUsd) && usage.costUsd >= 0) {
|
|
24688
|
+
return String(BigInt(Math.round(usage.costUsd * 1e12)));
|
|
24689
|
+
}
|
|
24690
|
+
return null;
|
|
24691
|
+
}
|
|
24562
24692
|
function sampleFrom(id, settled) {
|
|
24563
24693
|
const usage = settled?.usage;
|
|
24694
|
+
const picos = costPicosOf(usage);
|
|
24564
24695
|
return {
|
|
24565
24696
|
id,
|
|
24566
24697
|
status: settled?.status ?? null,
|
|
24567
24698
|
billingStatus: settled?.billingStatus ?? null,
|
|
24568
|
-
|
|
24569
|
-
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd : null,
|
|
24699
|
+
chargedCredits: wholeOrNull(settled?.chargedCredits),
|
|
24700
|
+
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd : picos !== null ? Number(picos) / 1e12 : null,
|
|
24701
|
+
costUsdPicos: picos,
|
|
24570
24702
|
error: settled?.error ?? (settled ? null : "timed_out"),
|
|
24571
24703
|
providerMessage: typeof settled?.providerMessage === "string" && settled.providerMessage.trim() ? settled.providerMessage.trim() : null,
|
|
24572
24704
|
inputTokens: wholeOrNull(usage?.inputTokens),
|
|
24573
24705
|
outputTokens: wholeOrNull(usage?.outputTokens),
|
|
24574
24706
|
maxOutputTokens: wholeOrNull(usage?.maxOutputTokens),
|
|
24575
|
-
|
|
24707
|
+
fitCredits: typeof usage?.fitCredits === "number" && Number.isSafeInteger(usage.fitCredits) && usage.fitCredits >= 1 ? usage.fitCredits : null,
|
|
24576
24708
|
fitExceedsCap: typeof usage?.fitExceedsCap === "boolean" ? usage.fitExceedsCap : null,
|
|
24577
24709
|
truncated: typeof usage?.truncated === "boolean" ? usage.truncated : settled?.error === "provider_token_limit" ? true : null
|
|
24578
24710
|
};
|
|
24579
24711
|
}
|
|
24580
24712
|
var PROVIDER_TOKEN_LIMIT = "provider_token_limit";
|
|
24581
|
-
function benchRecommendation(results,
|
|
24582
|
-
const settled = results.filter(
|
|
24583
|
-
|
|
24584
|
-
|
|
24585
|
-
const
|
|
24586
|
-
const
|
|
24713
|
+
function benchRecommendation(results, terms) {
|
|
24714
|
+
const settled = results.filter(
|
|
24715
|
+
(r) => r.status === "succeeded" && r.billingStatus === "final" && r.chargedCredits !== null && r.costUsdPicos !== null
|
|
24716
|
+
);
|
|
24717
|
+
const costs = settled.map((r) => BigInt(r.costUsdPicos));
|
|
24718
|
+
const charged = settled.map((r) => r.chargedCredits);
|
|
24719
|
+
const fits = settled.map((r) => r.fitCredits).filter((v) => typeof v === "number");
|
|
24720
|
+
const costP95 = percentilePicos(costs, 95);
|
|
24721
|
+
const costMax = costs.length ? costs.reduce((a, b) => b > a ? b : a) : null;
|
|
24587
24722
|
const fitP95 = percentile2(fits, 95);
|
|
24588
24723
|
const fitMax = fits.length ? Math.max(...fits) : null;
|
|
24589
24724
|
const exceedsStandCap = results.some((r) => r.fitExceedsCap === true);
|
|
24590
24725
|
const cutOffSamples = results.filter((r) => r.error === PROVIDER_TOKEN_LIMIT && r.fitExceedsCap !== true).length;
|
|
24591
24726
|
const noRecommendationReason = exceedsStandCap ? "answer_exceeds_stand_cap" : cutOffSamples > 0 ? "answer_cut_off" : null;
|
|
24592
|
-
const
|
|
24593
|
-
const
|
|
24594
|
-
const
|
|
24727
|
+
const { headroomBps, feeBps, usdCentsPerCredit } = terms;
|
|
24728
|
+
const missingTerms = headroomBps === null ? "headroom" : feeBps === null || usdCentsPerCredit === null ? "credit_terms" : null;
|
|
24729
|
+
const full = headroomBps !== null && feeBps !== null && usdCentsPerCredit !== null ? { headroomBps, feeBps, usdCentsPerCredit } : null;
|
|
24730
|
+
const billing = feeBps !== null && usdCentsPerCredit !== null ? { feeBps, usdCentsPerCredit } : null;
|
|
24731
|
+
const costBased = full !== null && costP95 !== null ? ceilingCreditsFor(costP95, full) : null;
|
|
24732
|
+
const recommended = costBased === null || noRecommendationReason !== null ? null : Math.max(costBased, fitP95 ?? 0);
|
|
24733
|
+
const ceiling = recommended === null || costMax === null || full === null ? null : Math.max(ceilingCreditsFor(costMax, full), recommended, fitMax ?? 0);
|
|
24734
|
+
const average = billing !== null ? averageCreditsPerCall(costs, billing) : null;
|
|
24735
|
+
const estimate = ceiling === null || billing === null ? null : Math.min(ceiling, perCallEstimateCreditsFor(costs, billing) ?? 1);
|
|
24595
24736
|
return {
|
|
24737
|
+
costs,
|
|
24596
24738
|
charged,
|
|
24597
|
-
|
|
24598
|
-
|
|
24599
|
-
|
|
24739
|
+
costP50: percentilePicos(costs, 50),
|
|
24740
|
+
costP95,
|
|
24741
|
+
costMax,
|
|
24742
|
+
chargedP50: percentile2(charged, 50),
|
|
24743
|
+
chargedP95: percentile2(charged, 95),
|
|
24744
|
+
chargedMax: charged.length ? Math.max(...charged) : null,
|
|
24600
24745
|
fits,
|
|
24601
24746
|
fitP95,
|
|
24602
24747
|
fitMax,
|
|
24603
|
-
|
|
24604
|
-
|
|
24605
|
-
|
|
24606
|
-
|
|
24748
|
+
costBasedMaxCredits: costBased,
|
|
24749
|
+
recommendedMaxCredits: recommended,
|
|
24750
|
+
recommendedPerCallMaxCredits: ceiling,
|
|
24751
|
+
averageCreditsPerCall: average,
|
|
24752
|
+
recommendedPerCallEstimateCredits: estimate,
|
|
24753
|
+
lengthRaisedCeiling: recommended !== null && costBased !== null && recommended > costBased,
|
|
24607
24754
|
cutOffSamples,
|
|
24608
24755
|
exceedsStandCap,
|
|
24609
|
-
noRecommendationReason
|
|
24756
|
+
noRecommendationReason,
|
|
24757
|
+
missingTerms
|
|
24610
24758
|
};
|
|
24611
24759
|
}
|
|
24612
|
-
var LLM_LENGTH_RAISED_LINE = "The
|
|
24613
|
-
var LLM_EXCEEDS_STAND_CAP_LINE = "No
|
|
24614
|
-
function benchCutOffLine(cutOff,
|
|
24615
|
-
return `No
|
|
24760
|
+
var LLM_LENGTH_RAISED_LINE = "The ceiling also decides how long the answer may be: at the cost-based ceiling the answer would be cut off, so declare this one.";
|
|
24761
|
+
var LLM_EXCEEDS_STAND_CAP_LINE = "No ceiling recommended: this answer is longer than one call on this stand may produce. Ask for a shorter answer \u2014 fewer fields, shorter strings, a length the prompt states \u2014 and benchmark again.";
|
|
24762
|
+
function benchCutOffLine(cutOff, maxCredits) {
|
|
24763
|
+
return `No ceiling recommended: ${cutOff} sample${cutOff === 1 ? " was" : "s were"} cut off at this ceiling (--max-credits ${maxCredits}) before the answer was finished, so its real length is unknown. Re-run with a higher --max-credits.`;
|
|
24616
24764
|
}
|
|
24617
24765
|
var ACTIVE_STATUSES = /* @__PURE__ */ new Set(["requires_confirmation", "queued", "dispatching", "awaiting_external"]);
|
|
24618
24766
|
function rowState(row) {
|
|
@@ -24623,7 +24771,22 @@ function rowState(row) {
|
|
|
24623
24771
|
function rowHoldsSlot(row) {
|
|
24624
24772
|
if (typeof row.slotHeld === "boolean") return row.slotHeld;
|
|
24625
24773
|
const state = rowState(row);
|
|
24626
|
-
return state === "active" || state === "awaiting bill" && (row.reservedCoins ?? 0) > 0;
|
|
24774
|
+
return state === "active" || state === "awaiting bill" && ((row.reservedCoins ?? 0) > 0 || (row.reservedCredits ?? 0) > 0);
|
|
24775
|
+
}
|
|
24776
|
+
function rowMoney(row) {
|
|
24777
|
+
const credits = row.currency === "credits" || row.currency === void 0 && typeof row.reservedCredits === "number" && typeof row.reservedCoins !== "number";
|
|
24778
|
+
if (credits) {
|
|
24779
|
+
return {
|
|
24780
|
+
unit: "credits",
|
|
24781
|
+
reserved: typeof row.reservedCredits === "number" ? row.reservedCredits : null,
|
|
24782
|
+
charged: typeof row.chargedCredits === "number" ? row.chargedCredits : null
|
|
24783
|
+
};
|
|
24784
|
+
}
|
|
24785
|
+
return {
|
|
24786
|
+
unit: "coin",
|
|
24787
|
+
reserved: typeof row.reservedCoins === "number" ? row.reservedCoins : null,
|
|
24788
|
+
charged: typeof row.chargedCoins === "number" ? row.chargedCoins : null
|
|
24789
|
+
};
|
|
24627
24790
|
}
|
|
24628
24791
|
function providerRefusalStatus(error) {
|
|
24629
24792
|
const m = /^provider_http_(\d{3})$/.exec(error ?? "");
|
|
@@ -24642,11 +24805,11 @@ function formatAge(iso, now = Date.now()) {
|
|
|
24642
24805
|
}
|
|
24643
24806
|
async function runBench(args) {
|
|
24644
24807
|
const { apiUrl, token, projectId, cwd, opts, log } = args;
|
|
24645
|
-
const
|
|
24808
|
+
const maxCredits = opts.maxCredits ?? opts.maxCoins;
|
|
24646
24809
|
const samples = opts.samples ?? DEFAULT_BENCH_SAMPLES;
|
|
24647
24810
|
const prompt = opts.benchPrompt.trim();
|
|
24648
24811
|
const auth = { Authorization: `Bearer ${token}`, "Content-Type": "application/json" };
|
|
24649
|
-
const call = (url,
|
|
24812
|
+
const call = (url, init) => apiFetch(url, { ...init, headers: { ...auth, ...init?.headers ?? {} } });
|
|
24650
24813
|
let schema;
|
|
24651
24814
|
if (opts.schemaPath) {
|
|
24652
24815
|
try {
|
|
@@ -24664,7 +24827,7 @@ async function runBench(args) {
|
|
|
24664
24827
|
const lane = await readRuntimeLane(apiUrl, token);
|
|
24665
24828
|
if (lane.state !== "live" || !lane.models || lane.models.length === 0) {
|
|
24666
24829
|
if (lane.state === "off") {
|
|
24667
|
-
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
24830
|
+
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCredits, opts), status: "off", error: null });
|
|
24668
24831
|
else {
|
|
24669
24832
|
log.plain(LLM_LANE_OFF_LINE);
|
|
24670
24833
|
log.dim(" Nothing was spent.");
|
|
@@ -24691,17 +24854,27 @@ async function runBench(args) {
|
|
|
24691
24854
|
);
|
|
24692
24855
|
return;
|
|
24693
24856
|
}
|
|
24694
|
-
const
|
|
24857
|
+
const walletBefore = await fetchCreditsSnapshot(apiUrl, token);
|
|
24858
|
+
if (walletBefore?.emailVerified === false) {
|
|
24859
|
+
benchFailed(opts, log, LLM_BENCH_UNVERIFIED_LINE);
|
|
24860
|
+
return;
|
|
24861
|
+
}
|
|
24862
|
+
if (walletBefore !== null && walletBefore.spendable < maxCredits) {
|
|
24863
|
+
benchFailed(opts, log, benchBalanceShortLine(walletBefore.spendable, maxCredits));
|
|
24864
|
+
return;
|
|
24865
|
+
}
|
|
24866
|
+
const balanceBefore = walletBefore?.spendable ?? null;
|
|
24695
24867
|
if (!opts.json) {
|
|
24696
24868
|
log.plain(c.bold("genex llm bench"));
|
|
24697
24869
|
log.dim(` ${apiUrl}`);
|
|
24698
24870
|
log.plain("");
|
|
24699
24871
|
log.plain(` Model ${c.cyan(modelId)}`);
|
|
24700
24872
|
log.plain(` Samples ${samples} real attempt${samples === 1 ? "" : "s"}, ${outputFormat} output`);
|
|
24701
|
-
log.plain(` Approved ${
|
|
24873
|
+
log.plain(` Approved ${maxCredits} credit${maxCredits === 1 ? "" : "s"} per attempt \u2014 at worst ${maxCredits * samples} credits for this run, from your own balance`);
|
|
24702
24874
|
log.plain(
|
|
24703
|
-
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore}
|
|
24875
|
+
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore} credits spendable`}`
|
|
24704
24876
|
);
|
|
24877
|
+
log.dim(" Billed as used: each attempt is charged its real cost plus the platform fee, rounded up to a whole credit.");
|
|
24705
24878
|
log.plain("");
|
|
24706
24879
|
}
|
|
24707
24880
|
const base = `${apiUrl}/api/runtime/development/projects/${encodeURIComponent(projectId)}/generations`;
|
|
@@ -24727,7 +24900,7 @@ async function runBench(args) {
|
|
|
24727
24900
|
const res = await call(base, {
|
|
24728
24901
|
method: "POST",
|
|
24729
24902
|
body: JSON.stringify({
|
|
24730
|
-
|
|
24903
|
+
maxCredits,
|
|
24731
24904
|
request: {
|
|
24732
24905
|
idempotencyKey: `bench-${randomUUID2()}`,
|
|
24733
24906
|
modelId,
|
|
@@ -24757,13 +24930,14 @@ async function runBench(args) {
|
|
|
24757
24930
|
if (refusedAt !== null) providerRefused++;
|
|
24758
24931
|
if (!opts.json) {
|
|
24759
24932
|
const length = row.outputTokens !== null ? ` \xB7 ${row.outputTokens}${row.maxOutputTokens !== null ? ` of ${row.maxOutputTokens}` : ""} tokens out` : "";
|
|
24933
|
+
const charged = row.chargedCredits === null ? "\u2014" : String(row.chargedCredits);
|
|
24760
24934
|
log.plain(
|
|
24761
|
-
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${
|
|
24935
|
+
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${charged.padStart(4)} credit${row.chargedCredits === 1 ? "" : "s"} charged \xB7 cost ${usdFromPicos(row.costUsdPicos)}${length}${row.error ? ` \xB7 ${row.error}` : ""}`
|
|
24762
24936
|
);
|
|
24763
24937
|
if (refusedAt !== null) printProviderRefusal(log, refusedAt, row.providerMessage, row.status === "unknown");
|
|
24764
24938
|
if (row.error === PROVIDER_TOKEN_LIMIT) {
|
|
24765
24939
|
log.dim(
|
|
24766
|
-
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no
|
|
24940
|
+
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no ceiling makes room for it; it is not a sample." : ` The answer was cut off at this ceiling (--max-credits ${maxCredits}) before it was finished \u2014 it was charged, and it is not a sample.`
|
|
24767
24941
|
);
|
|
24768
24942
|
}
|
|
24769
24943
|
}
|
|
@@ -24771,16 +24945,19 @@ async function runBench(args) {
|
|
|
24771
24945
|
} finally {
|
|
24772
24946
|
process.removeListener("SIGINT", onSigint);
|
|
24773
24947
|
}
|
|
24774
|
-
const
|
|
24775
|
-
|
|
24776
|
-
|
|
24777
|
-
|
|
24778
|
-
|
|
24779
|
-
const
|
|
24948
|
+
const terms = {
|
|
24949
|
+
headroomBps: lane.recommendedDeclaredHeadroomBps,
|
|
24950
|
+
feeBps: lane.feeBps,
|
|
24951
|
+
usdCentsPerCredit: lane.usdCentsPerCredit
|
|
24952
|
+
};
|
|
24953
|
+
const rec = benchRecommendation(results, terms);
|
|
24954
|
+
const settledCount = rec.charged.length;
|
|
24780
24955
|
const lengthBlocked = rec.noRecommendationReason !== null;
|
|
24781
|
-
const balanceAfter = await
|
|
24956
|
+
const balanceAfter = (await fetchCreditsSnapshot(apiUrl, token))?.spendable ?? null;
|
|
24957
|
+
const picosOrNull = (v) => v === null ? null : String(v);
|
|
24782
24958
|
const record = {
|
|
24783
|
-
v:
|
|
24959
|
+
v: 2,
|
|
24960
|
+
currency: "credits",
|
|
24784
24961
|
ranAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
24785
24962
|
apiUrl,
|
|
24786
24963
|
projectId,
|
|
@@ -24789,28 +24966,34 @@ async function runBench(args) {
|
|
|
24789
24966
|
outputFormat,
|
|
24790
24967
|
samples,
|
|
24791
24968
|
completed: settledCount,
|
|
24792
|
-
|
|
24793
|
-
|
|
24794
|
-
|
|
24795
|
-
max,
|
|
24796
|
-
|
|
24797
|
-
|
|
24798
|
-
recommendedPerCallMaxCoins: ceiling,
|
|
24799
|
-
maxCoinsPerSample: maxCoins,
|
|
24800
|
-
fitCoins: rec.fits,
|
|
24969
|
+
maxCreditsPerSample: maxCredits,
|
|
24970
|
+
costUsdPicos: rec.costs.map(String),
|
|
24971
|
+
chargedCredits: rec.charged,
|
|
24972
|
+
cost: { p50: picosOrNull(rec.costP50), p95: picosOrNull(rec.costP95), max: picosOrNull(rec.costMax) },
|
|
24973
|
+
charged: { p50: rec.chargedP50, p95: rec.chargedP95, max: rec.chargedMax },
|
|
24974
|
+
fitCredits: rec.fits,
|
|
24801
24975
|
fitP95: rec.fitP95,
|
|
24802
|
-
|
|
24803
|
-
|
|
24976
|
+
fitMax: rec.fitMax,
|
|
24977
|
+
recommendedDeclaredHeadroomBps: terms.headroomBps,
|
|
24978
|
+
feeBps: terms.feeBps,
|
|
24979
|
+
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
24980
|
+
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
24981
|
+
recommendedMaxCredits: rec.recommendedMaxCredits,
|
|
24982
|
+
recommendedPerCallMaxCredits: rec.recommendedPerCallMaxCredits,
|
|
24983
|
+
averageCreditsPerCall: rec.averageCreditsPerCall,
|
|
24984
|
+
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
24985
|
+
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
24804
24986
|
cutOffSamples: rec.cutOffSamples,
|
|
24805
24987
|
noRecommendationReason: rec.noRecommendationReason
|
|
24806
24988
|
};
|
|
24807
24989
|
const savedTo = await saveBench(cwd, record);
|
|
24808
24990
|
if (opts.json) {
|
|
24809
|
-
const ok =
|
|
24991
|
+
const ok = settledCount > 0 && !lengthBlocked;
|
|
24810
24992
|
writeJsonLine({
|
|
24811
24993
|
command: "llm bench",
|
|
24812
24994
|
status: ok ? "ok" : "failed",
|
|
24813
24995
|
error: ok ? null : rec.noRecommendationReason ?? "no_sample_settled",
|
|
24996
|
+
currency: "credits",
|
|
24814
24997
|
apiUrl,
|
|
24815
24998
|
projectId,
|
|
24816
24999
|
modelId,
|
|
@@ -24819,35 +25002,54 @@ async function runBench(args) {
|
|
|
24819
25002
|
schemaPath: opts.schemaPath ?? null,
|
|
24820
25003
|
samples,
|
|
24821
25004
|
completed: settledCount,
|
|
24822
|
-
|
|
25005
|
+
maxCreditsPerSample: maxCredits,
|
|
25006
|
+
// Spendable credits before and after the run; null when unreadable.
|
|
24823
25007
|
balanceBefore,
|
|
24824
25008
|
balanceAfter,
|
|
24825
25009
|
results,
|
|
24826
25010
|
// The create-time refusal that ended the run, or null. Beside the
|
|
24827
25011
|
// samples, never among them: no attempt existed and nothing was spent.
|
|
24828
25012
|
refusal: refusal2,
|
|
24829
|
-
|
|
24830
|
-
//
|
|
25013
|
+
// Over the settled samples: the real provider cost (pico-dollar strings
|
|
25014
|
+
// and USD) and the whole credits charged.
|
|
25015
|
+
costUsdPicos: record.cost,
|
|
25016
|
+
costUsd: {
|
|
25017
|
+
p50: rec.costP50 === null ? null : Number(rec.costP50) / 1e12,
|
|
25018
|
+
p95: rec.costP95 === null ? null : Number(rec.costP95) / 1e12,
|
|
25019
|
+
max: rec.costMax === null ? null : Number(rec.costMax) / 1e12
|
|
25020
|
+
},
|
|
25021
|
+
chargedCredits: record.charged,
|
|
25022
|
+
// The answer's LENGTH, priced by the server per sample (`fitCredits`,
|
|
24831
25023
|
// headroom already on the tokens), over the settled samples.
|
|
24832
|
-
|
|
24833
|
-
|
|
24834
|
-
|
|
25024
|
+
fitCredits: { p95: rec.fitP95, max: rec.fitMax },
|
|
25025
|
+
recommendedDeclaredHeadroomBps: terms.headroomBps,
|
|
25026
|
+
feeBps: terms.feeBps,
|
|
25027
|
+
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
25028
|
+
missingTerms: rec.missingTerms,
|
|
25029
|
+
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
25030
|
+
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
24835
25031
|
cutOffSamples: rec.cutOffSamples,
|
|
24836
25032
|
exceedsStandCap: rec.exceedsStandCap,
|
|
24837
25033
|
noRecommendationReason: rec.noRecommendationReason,
|
|
24838
|
-
|
|
24839
|
-
|
|
24840
|
-
|
|
25034
|
+
recommendedMaxCredits: rec.recommendedMaxCredits,
|
|
25035
|
+
recommendedPerCallMaxCredits: rec.recommendedPerCallMaxCredits,
|
|
25036
|
+
averageCreditsPerCall: rec.averageCreditsPerCall,
|
|
25037
|
+
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
24841
25038
|
savedTo
|
|
24842
25039
|
});
|
|
24843
25040
|
if (!ok) process.exitCode = 1;
|
|
24844
25041
|
return;
|
|
24845
25042
|
}
|
|
24846
25043
|
log.plain("");
|
|
25044
|
+
const printMeasured = () => {
|
|
25045
|
+
log.plain(c.bold(" Real cost per call"));
|
|
25046
|
+
log.plain(` p50 ${usdFromPicos(rec.costP50)} p95 ${usdFromPicos(rec.costP95)} max ${usdFromPicos(rec.costMax)} (${settledCount} of ${samples} settled)`);
|
|
25047
|
+
log.plain(c.bold(" Charged credits"));
|
|
25048
|
+
log.plain(` p50 ${rec.chargedP50} p95 ${rec.chargedP95} max ${rec.chargedMax} (whole credits, rounded up per call)`);
|
|
25049
|
+
};
|
|
24847
25050
|
if (lengthBlocked) {
|
|
24848
|
-
if (
|
|
24849
|
-
|
|
24850
|
-
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
25051
|
+
if (settledCount > 0) {
|
|
25052
|
+
printMeasured();
|
|
24851
25053
|
log.plain("");
|
|
24852
25054
|
}
|
|
24853
25055
|
printRecommendation(log, record);
|
|
@@ -24855,17 +25057,16 @@ async function runBench(args) {
|
|
|
24855
25057
|
process.exitCode = 1;
|
|
24856
25058
|
return;
|
|
24857
25059
|
}
|
|
24858
|
-
if (
|
|
25060
|
+
if (settledCount === 0) {
|
|
24859
25061
|
log.error(
|
|
24860
|
-
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to
|
|
25062
|
+
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to measure from." : " No sample settled, so there is nothing to measure from."
|
|
24861
25063
|
);
|
|
24862
25064
|
process.exitCode = 1;
|
|
24863
25065
|
return;
|
|
24864
25066
|
}
|
|
24865
|
-
|
|
24866
|
-
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
25067
|
+
printMeasured();
|
|
24867
25068
|
if (balanceBefore !== null && balanceAfter !== null) {
|
|
24868
|
-
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter}
|
|
25069
|
+
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter} credits spendable`);
|
|
24869
25070
|
}
|
|
24870
25071
|
log.plain("");
|
|
24871
25072
|
printRecommendation(log, record);
|
|
@@ -24877,43 +25078,58 @@ function printRecommendation(log, record) {
|
|
|
24877
25078
|
return;
|
|
24878
25079
|
}
|
|
24879
25080
|
if (record.noRecommendationReason === "answer_cut_off") {
|
|
24880
|
-
log.warn(` ${benchCutOffLine(record.cutOffSamples
|
|
25081
|
+
log.warn(` ${benchCutOffLine(record.cutOffSamples || 1, record.maxCreditsPerSample)}`);
|
|
24881
25082
|
return;
|
|
24882
25083
|
}
|
|
24883
|
-
|
|
24884
|
-
|
|
25084
|
+
const attempts = `${record.completed} settled attempt${record.completed === 1 ? "" : "s"} on ${record.modelId}`;
|
|
25085
|
+
if (record.recommendedMaxCredits === null || record.cost.p95 === null) {
|
|
25086
|
+
log.warn(" No ceiling recommended.");
|
|
24885
25087
|
log.dim(
|
|
24886
|
-
record.recommendedDeclaredHeadroomBps === null ? " This stand served no recommended headroom, and the multiplier is the server's to set \u2014" : "
|
|
25088
|
+
record.cost.p95 === null ? " No sample settled, so there is no p95 to build a ceiling on \u2014" : record.recommendedDeclaredHeadroomBps === null ? " This stand served no recommended headroom, and the multiplier is the server's to set \u2014" : " This stand served no platform fee or credit value, and those are the server's to set \u2014"
|
|
24887
25089
|
);
|
|
24888
25090
|
log.dim(" declaring a number from this run would be a guess dressed as a measurement.");
|
|
25091
|
+
printAverage(log, record, attempts);
|
|
24889
25092
|
return;
|
|
24890
25093
|
}
|
|
24891
|
-
log.plain(c.bold(` Declare
|
|
24892
|
-
if (record.
|
|
25094
|
+
log.plain(c.bold(` Declare maxCredits: ${record.recommendedMaxCredits}`));
|
|
25095
|
+
if (record.lengthRaisedCeiling) {
|
|
24893
25096
|
log.plain(` ${LLM_LENGTH_RAISED_LINE}`);
|
|
24894
25097
|
log.dim(
|
|
24895
|
-
` = the smallest
|
|
25098
|
+
` = the smallest ceiling whose answer allowance holds the answer plus this stand's headroom (p95 ${record.fitP95 ?? "\u2014"}), above the`
|
|
24896
25099
|
);
|
|
24897
25100
|
log.dim(
|
|
24898
|
-
`
|
|
25101
|
+
` cost-based ${record.costBasedMaxCredits ?? "\u2014"} (p95 real cost ${usdFromPicos(record.cost.p95)} with the platform fee, ${record.feeBps} bps, and ${record.recommendedDeclaredHeadroomBps} bps of headroom), over ${attempts}.`
|
|
24899
25102
|
);
|
|
24900
25103
|
} else {
|
|
24901
25104
|
log.dim(
|
|
24902
|
-
` = p95
|
|
25105
|
+
` = p95 real cost (${usdFromPicos(record.cost.p95)}) with this stand's platform fee (${record.feeBps} bps) and recommended headroom (${record.recommendedDeclaredHeadroomBps} bps), in whole credits,`
|
|
24903
25106
|
);
|
|
24904
|
-
log.dim(` measured over ${
|
|
25107
|
+
log.dim(` measured over ${attempts}.`);
|
|
24905
25108
|
}
|
|
24906
|
-
log.dim(" That number is a
|
|
24907
|
-
|
|
24908
|
-
|
|
25109
|
+
log.dim(" That number is a CEILING, not a price: a call is billed its real cost as used, never more than it \u2014");
|
|
25110
|
+
log.dim(" and it also sizes how long the answer may be.");
|
|
25111
|
+
if (record.recommendedPerCallMaxCredits !== null) {
|
|
25112
|
+
log.plain(c.bold(` Grant perCallMaxCredits: ${record.recommendedPerCallMaxCredits}`));
|
|
24909
25113
|
log.dim(
|
|
24910
|
-
record.
|
|
25114
|
+
record.lengthRaisedCeiling ? ` = the same over the worst sample (${usdFromPicos(record.cost.max)}) or the longest answer's ceiling, whichever is larger, and never below the ceiling above it.` : ` = the same over the worst sample (${usdFromPicos(record.cost.max)}), and never below the ceiling above it.`
|
|
24911
25115
|
);
|
|
24912
|
-
|
|
25116
|
+
}
|
|
25117
|
+
printAverage(log, record, attempts);
|
|
25118
|
+
}
|
|
25119
|
+
function printAverage(log, record, attempts) {
|
|
25120
|
+
if (record.averageCreditsPerCall === null) return;
|
|
25121
|
+
log.plain(` Per-call estimate: about ${formatCredits(record.averageCreditsPerCall)} credits per call`);
|
|
25122
|
+
log.dim(` = the average real cost with the platform fee, no headroom, over ${attempts}.`);
|
|
25123
|
+
if (record.recommendedPerCallEstimateCredits !== null) {
|
|
25124
|
+
log.plain(c.bold(` Grant perCallEstimateCredits: ${record.recommendedPerCallEstimateCredits}`));
|
|
25125
|
+
log.dim(` = that average rounded up to a whole credit. disclosure.estimatedCreditsPerPeriod = your calls per period \xD7 ${formatCredits(record.averageCreditsPerCall)}, rounded up.`);
|
|
24913
25126
|
}
|
|
24914
25127
|
}
|
|
24915
25128
|
async function explainBenchRefusal(res, call, base, log, opts, index, settledSoFar) {
|
|
24916
|
-
if (printedStructuredError(res))
|
|
25129
|
+
if (printedStructuredError(res)) {
|
|
25130
|
+
const printed = await res.json().catch(() => ({}));
|
|
25131
|
+
return { stop: true, refusal: { status: res.status, error: typeof printed.error === "string" ? printed.error : null, slotsHeld: null, slotLimit: null } };
|
|
25132
|
+
}
|
|
24917
25133
|
const body = await res.json().catch(() => ({}));
|
|
24918
25134
|
const refusal2 = { status: res.status, error: body.error ?? null, slotsHeld: null, slotLimit: null };
|
|
24919
25135
|
if (res.status === 429 && body.error === "generation_limit") {
|
|
@@ -24923,6 +25139,10 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24923
25139
|
if (!opts.json) log.error(` Sample ${index + 1} ${benchSlotsHeldSentence(counts)}`);
|
|
24924
25140
|
return { stop: true, refusal: refusal2 };
|
|
24925
25141
|
}
|
|
25142
|
+
if (res.status === 403 && body.error === "credits_unverified") {
|
|
25143
|
+
if (!opts.json) log.error(` ${LLM_BENCH_UNVERIFIED_LINE}`);
|
|
25144
|
+
return { stop: true, refusal: refusal2 };
|
|
25145
|
+
}
|
|
24926
25146
|
if (opts.json) return { stop: res.status !== 429, refusal: refusal2 };
|
|
24927
25147
|
if (res.status === 404 && body.error === "not_found") {
|
|
24928
25148
|
log.error(` ${LLM_LANE_OFF_LINE}`);
|
|
@@ -24938,7 +25158,7 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24938
25158
|
return { stop: true, refusal: refusal2 };
|
|
24939
25159
|
}
|
|
24940
25160
|
if (res.status === 402) {
|
|
24941
|
-
log.error(" Not enough
|
|
25161
|
+
log.error(" Not enough credits to start the attempt \u2014 a bench is paid from your own balance.");
|
|
24942
25162
|
return { stop: true, refusal: refusal2 };
|
|
24943
25163
|
}
|
|
24944
25164
|
log.error(
|
|
@@ -24947,7 +25167,7 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24947
25167
|
return { stop: res.status >= 500 || res.status === 401 || res.status === 403, refusal: refusal2 };
|
|
24948
25168
|
}
|
|
24949
25169
|
function printProviderRefusal(log, status2, message, awaitingBill = false) {
|
|
24950
|
-
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so this call
|
|
25170
|
+
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so what this call reserved stays held until it resolves (see \`genex llm status\`); it is not a sample.`);
|
|
24951
25171
|
else log.dim(" The provider refused this call at its door \u2014 no inference ran, it cost nothing, and it is not a sample.");
|
|
24952
25172
|
if (message) log.dim(` Provider said: ${message}`);
|
|
24953
25173
|
if (status2 === 401 || status2 === 403) {
|
|
@@ -24973,19 +25193,6 @@ async function pollSettled(call, url, timeoutSec) {
|
|
|
24973
25193
|
}
|
|
24974
25194
|
return last;
|
|
24975
25195
|
}
|
|
24976
|
-
async function readCoinBalance(apiUrl, token) {
|
|
24977
|
-
try {
|
|
24978
|
-
const res = await apiFetch(`${apiUrl}/api/coin/balance`, {
|
|
24979
|
-
headers: { Authorization: `Bearer ${token}` },
|
|
24980
|
-
signal: AbortSignal.timeout(6e3)
|
|
24981
|
-
});
|
|
24982
|
-
if (!res.ok) return null;
|
|
24983
|
-
const body = await res.json().catch(() => null);
|
|
24984
|
-
return typeof body?.spendable === "number" ? body.spendable : null;
|
|
24985
|
-
} catch {
|
|
24986
|
-
return null;
|
|
24987
|
-
}
|
|
24988
|
-
}
|
|
24989
25196
|
async function saveBench(cwd, record) {
|
|
24990
25197
|
try {
|
|
24991
25198
|
const file = path36.join(cwd, LLM_BENCH_FILE);
|
|
@@ -25001,9 +25208,10 @@ function benchFailed(opts, log, message) {
|
|
|
25001
25208
|
else log.error(message);
|
|
25002
25209
|
process.exitCode = 1;
|
|
25003
25210
|
}
|
|
25004
|
-
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
25211
|
+
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCredits, opts) {
|
|
25005
25212
|
return {
|
|
25006
25213
|
command: "llm bench",
|
|
25214
|
+
currency: "credits",
|
|
25007
25215
|
apiUrl,
|
|
25008
25216
|
projectId,
|
|
25009
25217
|
modelId: opts.modelId ?? null,
|
|
@@ -25012,27 +25220,34 @@ function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCoi
|
|
|
25012
25220
|
schemaPath: opts.schemaPath ?? null,
|
|
25013
25221
|
samples,
|
|
25014
25222
|
completed: 0,
|
|
25015
|
-
|
|
25223
|
+
maxCreditsPerSample: maxCredits,
|
|
25016
25224
|
balanceBefore: null,
|
|
25017
25225
|
balanceAfter: null,
|
|
25018
25226
|
results: [],
|
|
25019
25227
|
refusal: null,
|
|
25020
|
-
|
|
25021
|
-
|
|
25022
|
-
|
|
25023
|
-
|
|
25228
|
+
costUsdPicos: { p50: null, p95: null, max: null },
|
|
25229
|
+
costUsd: { p50: null, p95: null, max: null },
|
|
25230
|
+
chargedCredits: { p50: null, p95: null, max: null },
|
|
25231
|
+
fitCredits: { p95: null, max: null },
|
|
25232
|
+
recommendedDeclaredHeadroomBps: null,
|
|
25233
|
+
feeBps: null,
|
|
25234
|
+
usdCentsPerCredit: null,
|
|
25235
|
+
missingTerms: null,
|
|
25236
|
+
costBasedMaxCredits: null,
|
|
25237
|
+
lengthRaisedCeiling: false,
|
|
25024
25238
|
cutOffSamples: 0,
|
|
25025
25239
|
exceedsStandCap: false,
|
|
25026
25240
|
noRecommendationReason: null,
|
|
25027
|
-
|
|
25028
|
-
|
|
25029
|
-
|
|
25241
|
+
recommendedMaxCredits: null,
|
|
25242
|
+
recommendedPerCallMaxCredits: null,
|
|
25243
|
+
averageCreditsPerCall: null,
|
|
25244
|
+
recommendedPerCallEstimateCredits: null,
|
|
25030
25245
|
savedTo: null
|
|
25031
25246
|
};
|
|
25032
25247
|
}
|
|
25033
25248
|
async function reportStatus(args) {
|
|
25034
25249
|
const { apiUrl, token, projectId, opts, log } = args;
|
|
25035
|
-
const call = (url,
|
|
25250
|
+
const call = (url, init) => apiFetch(url, { ...init, headers: { Authorization: `Bearer ${token}`, ...init?.headers ?? {} } });
|
|
25036
25251
|
const base = `${apiUrl}/api/runtime/development/projects/${encodeURIComponent(projectId)}/generations`;
|
|
25037
25252
|
const res = await call(`${base}?scope=open`).catch(() => null);
|
|
25038
25253
|
if (!res || !res.ok) {
|
|
@@ -25089,7 +25304,8 @@ async function reportStatus(args) {
|
|
|
25089
25304
|
for (const row of rows) {
|
|
25090
25305
|
const state = rowState(row);
|
|
25091
25306
|
const mark = state === "active" ? c.cyan("\u25CF") : state === "awaiting bill" ? c.yellow("\u25D0") : c.green("\u25CB");
|
|
25092
|
-
const
|
|
25307
|
+
const money2 = rowMoney(row);
|
|
25308
|
+
const reserved = money2.reserved === null ? "reserved \u2014" : `${money2.reserved} ${money2.unit} reserved`;
|
|
25093
25309
|
log.plain(
|
|
25094
25310
|
` ${mark} ${row.id ?? "?"} ${row.modelId ?? "?"} ${state.padEnd(13)} ${formatAge(row.createdAt, now).padStart(4)} old ${reserved}${row.error ? ` ${row.error}` : ""}${rowHoldsSlot(row) ? "" : c.dim(" (no slot)")}`
|
|
25095
25311
|
);
|
|
@@ -25124,7 +25340,7 @@ function statusJson(apiUrl, projectId, status2, error, rows, slotsHeld, slotLimi
|
|
|
25124
25340
|
}
|
|
25125
25341
|
async function cancelCall(args) {
|
|
25126
25342
|
const { apiUrl, token, projectId, id, opts, log } = args;
|
|
25127
|
-
const call = (url,
|
|
25343
|
+
const call = (url, init) => apiFetch(url, { ...init, headers: { Authorization: `Bearer ${token}`, ...init?.headers ?? {} } });
|
|
25128
25344
|
const base = `${apiUrl}/api/runtime/development/projects/${encodeURIComponent(projectId)}/generations`;
|
|
25129
25345
|
const rowUrl = `${base}/${encodeURIComponent(id)}`;
|
|
25130
25346
|
const done = (status2, error, reason, row2) => {
|
|
@@ -25161,10 +25377,11 @@ async function cancelCall(args) {
|
|
|
25161
25377
|
if (!opts.json) {
|
|
25162
25378
|
log.warn(` ${row.id} (${row.status ?? "?"}) \u2014 ${sentence}.`);
|
|
25163
25379
|
if (state === "awaiting bill") {
|
|
25164
|
-
log.dim(` Nothing to cancel:
|
|
25380
|
+
log.dim(` Nothing to cancel: what it reserved releases when the bill resolves, and it stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`);
|
|
25165
25381
|
if (row.providerMessage?.trim()) log.dim(` Provider said: ${row.providerMessage.trim()}`);
|
|
25166
25382
|
} else {
|
|
25167
|
-
|
|
25383
|
+
const money2 = rowMoney(row);
|
|
25384
|
+
log.dim(` ${money2.charged === null ? "charge unknown" : `${money2.charged} ${money2.unit} charged`}${row.error ? ` \xB7 ${row.error}` : ""}.`);
|
|
25168
25385
|
}
|
|
25169
25386
|
}
|
|
25170
25387
|
return done("ok", null, state === "awaiting bill" ? "awaiting_bill" : "already_final", row);
|
|
@@ -25186,78 +25403,126 @@ async function cancelCall(args) {
|
|
|
25186
25403
|
const after = await res.json().catch(() => null) ?? row;
|
|
25187
25404
|
if (!opts.json) {
|
|
25188
25405
|
log.success(` Cancelled ${row.id} (was ${row.status ?? "active"}).`);
|
|
25189
|
-
log.dim(`
|
|
25406
|
+
log.dim(` What it reserved releases once the bill settles; ${c.cyan("genex llm status")} shows it until then.`);
|
|
25190
25407
|
}
|
|
25191
25408
|
return done("ok", null, "cancelled", after);
|
|
25192
25409
|
}
|
|
25193
|
-
async function
|
|
25194
|
-
let
|
|
25410
|
+
async function readSavedBench(cwd) {
|
|
25411
|
+
let parsed;
|
|
25195
25412
|
try {
|
|
25196
|
-
|
|
25413
|
+
parsed = JSON.parse(await fs35.readFile(path36.join(cwd, LLM_BENCH_FILE), "utf8"));
|
|
25197
25414
|
} catch {
|
|
25198
|
-
|
|
25415
|
+
return null;
|
|
25199
25416
|
}
|
|
25200
|
-
if (!
|
|
25417
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null;
|
|
25418
|
+
const v = parsed;
|
|
25419
|
+
if (v.v === 2 && v.currency === "credits" && v.cost && v.charged) return { kind: "credits", record: parsed };
|
|
25420
|
+
return { kind: "coins", record: parsed };
|
|
25421
|
+
}
|
|
25422
|
+
var LLM_PRICE_COIN_TERMS_LINE = "This bench was measured under the old COIN terms, when a game declared a fixed price per call. In-game calls are now billed in the player's credits, as used, and a game declares a per-call CEILING (maxCredits) instead \u2014 a coin price is not that number. Re-run the bench for the figures to declare.";
|
|
25423
|
+
async function reportSavedPrice(cwd, opts, log) {
|
|
25424
|
+
const saved = await readSavedBench(cwd);
|
|
25425
|
+
const rerun = 'genex llm bench "<prompt>" --max-credits <n> --user-approved';
|
|
25426
|
+
const creditKeys = (record2) => ({
|
|
25427
|
+
costUsdPicos: record2?.cost ?? { p50: null, p95: null, max: null },
|
|
25428
|
+
chargedCredits: record2?.charged ?? { p50: null, p95: null, max: null },
|
|
25429
|
+
fitP95: record2?.fitP95 ?? null,
|
|
25430
|
+
fitMax: record2?.fitMax ?? null,
|
|
25431
|
+
recommendedDeclaredHeadroomBps: record2?.recommendedDeclaredHeadroomBps ?? null,
|
|
25432
|
+
feeBps: record2?.feeBps ?? null,
|
|
25433
|
+
usdCentsPerCredit: record2?.usdCentsPerCredit ?? null,
|
|
25434
|
+
costBasedMaxCredits: record2?.costBasedMaxCredits ?? null,
|
|
25435
|
+
lengthRaisedCeiling: record2?.lengthRaisedCeiling ?? false,
|
|
25436
|
+
cutOffSamples: record2?.cutOffSamples ?? 0,
|
|
25437
|
+
noRecommendationReason: record2?.noRecommendationReason ?? null,
|
|
25438
|
+
recommendedMaxCredits: record2?.recommendedMaxCredits ?? null,
|
|
25439
|
+
recommendedPerCallMaxCredits: record2?.recommendedPerCallMaxCredits ?? null,
|
|
25440
|
+
averageCreditsPerCall: record2?.averageCreditsPerCall ?? null,
|
|
25441
|
+
recommendedPerCallEstimateCredits: record2?.recommendedPerCallEstimateCredits ?? null
|
|
25442
|
+
});
|
|
25443
|
+
if (!saved) {
|
|
25201
25444
|
if (opts.json) {
|
|
25202
25445
|
writeJsonLine({
|
|
25203
25446
|
command: "llm price",
|
|
25204
25447
|
status: "failed",
|
|
25205
25448
|
error: "no_bench",
|
|
25449
|
+
currency: null,
|
|
25206
25450
|
ranAt: null,
|
|
25207
25451
|
modelId: null,
|
|
25208
25452
|
samples: null,
|
|
25209
25453
|
completed: null,
|
|
25210
|
-
|
|
25211
|
-
|
|
25212
|
-
max: null,
|
|
25213
|
-
fitP95: null,
|
|
25214
|
-
chargedBasedEstimateCoins: null,
|
|
25215
|
-
lengthRaisedPrice: false,
|
|
25216
|
-
cutOffSamples: 0,
|
|
25217
|
-
noRecommendationReason: null,
|
|
25218
|
-
recommendedDeclaredHeadroomBps: null,
|
|
25219
|
-
recommendedEstimateCoins: null,
|
|
25220
|
-
recommendedPerCallMaxCoins: null,
|
|
25454
|
+
...creditKeys(null),
|
|
25455
|
+
legacyCoinTerms: null,
|
|
25221
25456
|
savedTo: null
|
|
25222
25457
|
});
|
|
25223
25458
|
} else {
|
|
25224
25459
|
log.error("Nothing has been benchmarked in this folder yet.");
|
|
25225
|
-
log.dim(` Run ${c.cyan(
|
|
25460
|
+
log.dim(` Run ${c.cyan(rerun)} first.`);
|
|
25461
|
+
}
|
|
25462
|
+
process.exitCode = 1;
|
|
25463
|
+
return;
|
|
25464
|
+
}
|
|
25465
|
+
if (saved.kind === "coins") {
|
|
25466
|
+
const old = saved.record;
|
|
25467
|
+
const legacy = {
|
|
25468
|
+
p50: old.p50 ?? null,
|
|
25469
|
+
p95: old.p95 ?? null,
|
|
25470
|
+
max: old.max ?? null,
|
|
25471
|
+
recommendedEstimateCoins: old.recommendedEstimateCoins ?? null,
|
|
25472
|
+
recommendedPerCallMaxCoins: old.recommendedPerCallMaxCoins ?? null,
|
|
25473
|
+
maxCoinsPerSample: old.maxCoinsPerSample ?? null,
|
|
25474
|
+
noRecommendationReason: old.noRecommendationReason ?? null
|
|
25475
|
+
};
|
|
25476
|
+
if (opts.json) {
|
|
25477
|
+
writeJsonLine({
|
|
25478
|
+
command: "llm price",
|
|
25479
|
+
status: "failed",
|
|
25480
|
+
error: "coin_terms",
|
|
25481
|
+
currency: "coins",
|
|
25482
|
+
ranAt: old.ranAt ?? null,
|
|
25483
|
+
modelId: old.modelId ?? null,
|
|
25484
|
+
samples: old.samples ?? null,
|
|
25485
|
+
completed: old.completed ?? null,
|
|
25486
|
+
...creditKeys(null),
|
|
25487
|
+
legacyCoinTerms: legacy,
|
|
25488
|
+
savedTo: LLM_BENCH_FILE
|
|
25489
|
+
});
|
|
25490
|
+
} else {
|
|
25491
|
+
log.plain(c.bold("genex llm price"));
|
|
25492
|
+
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${old.ranAt ?? "at an unknown time"}${old.modelId ? ` on ${old.modelId}` : ""}`);
|
|
25493
|
+
log.plain("");
|
|
25494
|
+
log.warn(` ${LLM_PRICE_COIN_TERMS_LINE}`);
|
|
25495
|
+
const measured = legacy.p95 === null ? "no settled sample" : `charged coins p50 ${legacy.p50} \xB7 p95 ${legacy.p95} \xB7 max ${legacy.max}`;
|
|
25496
|
+
const recommended = legacy.recommendedEstimateCoins === null ? "no price recommended" : `it recommended estimateCoins ${legacy.recommendedEstimateCoins}${legacy.recommendedPerCallMaxCoins != null ? ` and perCallMaxCoins ${legacy.recommendedPerCallMaxCoins}` : ""}`;
|
|
25497
|
+
log.dim(` What it measured then: ${measured}; ${recommended}.`);
|
|
25498
|
+
log.dim(` Re-run: ${c.cyan(rerun)}`);
|
|
25226
25499
|
}
|
|
25227
25500
|
process.exitCode = 1;
|
|
25228
25501
|
return;
|
|
25229
25502
|
}
|
|
25503
|
+
const record = saved.record;
|
|
25230
25504
|
if (opts.json) {
|
|
25231
25505
|
writeJsonLine({
|
|
25232
25506
|
command: "llm price",
|
|
25233
|
-
status: record.
|
|
25234
|
-
error: record.
|
|
25507
|
+
status: record.recommendedMaxCredits === null ? "failed" : "ok",
|
|
25508
|
+
error: record.recommendedMaxCredits === null ? record.noRecommendationReason ?? "no_recommendation" : null,
|
|
25509
|
+
currency: "credits",
|
|
25235
25510
|
ranAt: record.ranAt ?? null,
|
|
25236
25511
|
modelId: record.modelId ?? null,
|
|
25237
25512
|
samples: record.samples ?? null,
|
|
25238
25513
|
completed: record.completed ?? null,
|
|
25239
|
-
|
|
25240
|
-
|
|
25241
|
-
max: record.max ?? null,
|
|
25242
|
-
// Absent from a file an older CLI wrote; null / false / 0 then, never missing.
|
|
25243
|
-
fitP95: record.fitP95 ?? null,
|
|
25244
|
-
chargedBasedEstimateCoins: record.chargedBasedEstimateCoins ?? null,
|
|
25245
|
-
lengthRaisedPrice: record.lengthRaisedPrice ?? false,
|
|
25246
|
-
cutOffSamples: record.cutOffSamples ?? 0,
|
|
25247
|
-
noRecommendationReason: record.noRecommendationReason ?? null,
|
|
25248
|
-
recommendedDeclaredHeadroomBps: record.recommendedDeclaredHeadroomBps ?? null,
|
|
25249
|
-
recommendedEstimateCoins: record.recommendedEstimateCoins ?? null,
|
|
25250
|
-
recommendedPerCallMaxCoins: record.recommendedPerCallMaxCoins ?? null,
|
|
25514
|
+
...creditKeys(record),
|
|
25515
|
+
legacyCoinTerms: null,
|
|
25251
25516
|
savedTo: LLM_BENCH_FILE
|
|
25252
25517
|
});
|
|
25253
|
-
if (record.
|
|
25518
|
+
if (record.recommendedMaxCredits === null) process.exitCode = 1;
|
|
25254
25519
|
return;
|
|
25255
25520
|
}
|
|
25256
25521
|
log.plain(c.bold("genex llm price"));
|
|
25257
25522
|
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${record.ranAt}`);
|
|
25258
25523
|
log.plain("");
|
|
25259
25524
|
printRecommendation(log, record);
|
|
25260
|
-
if (record.
|
|
25525
|
+
if (record.recommendedMaxCredits === null) process.exitCode = 1;
|
|
25261
25526
|
}
|
|
25262
25527
|
function writeJsonLine(value) {
|
|
25263
25528
|
process.stdout.write(`${JSON.stringify(value)}
|
|
@@ -25517,7 +25782,7 @@ function runtimeRow(state, lane, toolsOnly = false) {
|
|
|
25517
25782
|
return {
|
|
25518
25783
|
label: "Runtime",
|
|
25519
25784
|
value: `in-game LLM lane live \xB7 ${n} model${n === 1 ? "" : "s"}${keyRefused ? ` \xB7 ${c.yellow("provider key refused")}` : ""}`,
|
|
25520
|
-
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : '
|
|
25785
|
+
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : 'Measure a call before a game declares its per-call ceiling: `npx genex llm bench "<prompt>" --max-credits <n> --user-approved`.'
|
|
25521
25786
|
};
|
|
25522
25787
|
}
|
|
25523
25788
|
case "off":
|
|
@@ -25955,7 +26220,7 @@ var MIN_POLL_INTERVAL_MS2 = 1e3;
|
|
|
25955
26220
|
var DEFAULT_PLAYER_WAIT_MS = 10 * 60 * 1e3;
|
|
25956
26221
|
var REASSURE_EVERY_MS2 = 30 * 1e3;
|
|
25957
26222
|
var PLAYER_NOT_APPROVED = "Nobody approved this machine, so nothing was installed. Re-run `genex player install` when you can open the link.";
|
|
25958
|
-
var
|
|
26223
|
+
var sleep5 = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
25959
26224
|
async function authorizePlayer(apiBaseUrl, options) {
|
|
25960
26225
|
const apiUrl = normalizeApiOrigin(apiBaseUrl);
|
|
25961
26226
|
const { log, waitMs = DEFAULT_PLAYER_WAIT_MS, open = openBrowser } = options;
|
|
@@ -26012,7 +26277,7 @@ async function authorizePlayer(apiBaseUrl, options) {
|
|
|
26012
26277
|
log.dim(" still waiting for you to approve\u2026");
|
|
26013
26278
|
nextReassureAt = Date.now() + REASSURE_EVERY_MS2;
|
|
26014
26279
|
}
|
|
26015
|
-
await
|
|
26280
|
+
await sleep5(intervalMs);
|
|
26016
26281
|
}
|
|
26017
26282
|
}
|
|
26018
26283
|
async function startPlayerDeviceAuth(apiUrl, label) {
|
|
@@ -26077,15 +26342,15 @@ var PlayerClient = class {
|
|
|
26077
26342
|
this.version = options.version ?? getCliVersion();
|
|
26078
26343
|
this.timeoutMs = options.timeoutMs ?? 3e4;
|
|
26079
26344
|
}
|
|
26080
|
-
async call(path46,
|
|
26345
|
+
async call(path46, init = {}, provider) {
|
|
26081
26346
|
const headers = {
|
|
26082
26347
|
Authorization: `Bearer ${this.token}`,
|
|
26083
26348
|
[PLAYER_VERSION_HEADER]: this.version,
|
|
26084
26349
|
...provider ? { [EXTERNAL_PROVIDER_HEADER]: provider } : {},
|
|
26085
|
-
...
|
|
26350
|
+
...init.body === void 0 ? {} : { "Content-Type": "application/json" }
|
|
26086
26351
|
};
|
|
26087
26352
|
const res = await apiFetch(`${this.apiUrl}${path46}`, {
|
|
26088
|
-
...
|
|
26353
|
+
...init,
|
|
26089
26354
|
headers,
|
|
26090
26355
|
signal: AbortSignal.timeout(this.timeoutMs)
|
|
26091
26356
|
}, { noTermsWait: true });
|
|
@@ -26729,7 +26994,7 @@ var LEASE_RENEW_MS = 45e3;
|
|
|
26729
26994
|
var JOB_TIMEOUT_MS = 10 * 6e4;
|
|
26730
26995
|
var SHUTDOWN_RETRY_SECONDS = 5;
|
|
26731
26996
|
var MAX_PROVIDER_COOLDOWN_MS = 60 * 6e4;
|
|
26732
|
-
var
|
|
26997
|
+
var sleep6 = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
26733
26998
|
function platformTag(platform = process.platform) {
|
|
26734
26999
|
return platform.toLowerCase().replace(/[^a-z0-9_-]/g, "").slice(0, 40) || "unknown";
|
|
26735
27000
|
}
|
|
@@ -26856,7 +27121,7 @@ async function runWatcherLoop(options) {
|
|
|
26856
27121
|
state.lastError = describe(error);
|
|
26857
27122
|
heartbeatAt = now + backoffMs(policy, failures);
|
|
26858
27123
|
log(`player: heartbeat failed (${describe(error)})`);
|
|
26859
|
-
await
|
|
27124
|
+
await sleep6(backoffMs(policy, failures));
|
|
26860
27125
|
continue;
|
|
26861
27126
|
}
|
|
26862
27127
|
}
|
|
@@ -26871,9 +27136,9 @@ async function runWatcherLoop(options) {
|
|
|
26871
27136
|
const until = cooldown.get(provider) ?? 0;
|
|
26872
27137
|
if (until > Date.now()) continue;
|
|
26873
27138
|
if (until) cooldown.delete(provider);
|
|
26874
|
-
let
|
|
27139
|
+
let pending2;
|
|
26875
27140
|
try {
|
|
26876
|
-
|
|
27141
|
+
pending2 = await client.pending(provider);
|
|
26877
27142
|
state.lastPollAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
26878
27143
|
failures = 0;
|
|
26879
27144
|
} catch (error) {
|
|
@@ -26885,10 +27150,10 @@ async function runWatcherLoop(options) {
|
|
|
26885
27150
|
failures += 1;
|
|
26886
27151
|
state.lastError = describe(error);
|
|
26887
27152
|
log(`player: poll failed (${describe(error)})`);
|
|
26888
|
-
await
|
|
27153
|
+
await sleep6(backoffMs(policy, failures));
|
|
26889
27154
|
continue;
|
|
26890
27155
|
}
|
|
26891
|
-
for (const row of
|
|
27156
|
+
for (const row of pending2) {
|
|
26892
27157
|
if (stop2) break;
|
|
26893
27158
|
if (!workable(row, watcherId)) continue;
|
|
26894
27159
|
if (await handleOne(row, provider)) worked = true;
|
|
@@ -26902,7 +27167,7 @@ async function runWatcherLoop(options) {
|
|
|
26902
27167
|
break;
|
|
26903
27168
|
}
|
|
26904
27169
|
const interval = options.pollIntervalMsOverride ?? Math.max(1e3, (policy.pollIntervalSeconds || 5) * 1e3);
|
|
26905
|
-
if (!worked) await
|
|
27170
|
+
if (!worked) await sleep6(interval);
|
|
26906
27171
|
}
|
|
26907
27172
|
} finally {
|
|
26908
27173
|
process.off("SIGINT", sigint);
|
|
@@ -26939,7 +27204,7 @@ async function runWatcherLoop(options) {
|
|
|
26939
27204
|
}
|
|
26940
27205
|
failures += 1;
|
|
26941
27206
|
log(`player: could not claim ${row.id} (${describe(error)})`);
|
|
26942
|
-
await
|
|
27207
|
+
await sleep6(backoffMs(policy, failures));
|
|
26943
27208
|
return false;
|
|
26944
27209
|
}
|
|
26945
27210
|
held.row = { id: row.id, provider };
|
|
@@ -27284,11 +27549,11 @@ async function status(opts, log) {
|
|
|
27284
27549
|
server.policyVersion = policy.version ?? null;
|
|
27285
27550
|
server.minVersion = policy.minVersion ?? null;
|
|
27286
27551
|
server.latestVersion = policy.latestVersion ?? null;
|
|
27287
|
-
let
|
|
27552
|
+
let pending2 = 0;
|
|
27288
27553
|
for (const provider of new Set(runners.map((r) => PROVIDER_FOR_RUNNER[r]))) {
|
|
27289
|
-
|
|
27554
|
+
pending2 += (await client.pending(provider)).length;
|
|
27290
27555
|
}
|
|
27291
|
-
server.pending =
|
|
27556
|
+
server.pending = pending2;
|
|
27292
27557
|
} catch (error) {
|
|
27293
27558
|
server.error = error instanceof PlayerApiError ? `${error.status} ${error.code}`.trim() : error instanceof PlayerAuthError ? "credential_rejected" : error instanceof PlayerUpdateRequiredError ? "player_update_required" : "unreachable";
|
|
27294
27559
|
}
|
|
@@ -27473,16 +27738,16 @@ ${c.bold("Usage")}
|
|
|
27473
27738
|
status | cancel. "models" says whether this
|
|
27474
27739
|
stand serves the runtime LLM lane at all, and
|
|
27475
27740
|
at what provider rates. "bench" runs the real
|
|
27476
|
-
model on YOUR OWN
|
|
27477
|
-
attempt
|
|
27741
|
+
model on YOUR OWN credits and reports what each
|
|
27742
|
+
attempt really cost; it needs --max-credits <n>
|
|
27478
27743
|
--user-approved, like any spend the player has
|
|
27479
27744
|
to agree to. "price" reprints the last run's
|
|
27480
|
-
recommended
|
|
27745
|
+
recommended maxCredits. "status" lists this
|
|
27481
27746
|
project's open calls and the slots they hold;
|
|
27482
27747
|
"cancel <id>" stops an active one. Declare a
|
|
27483
|
-
game's
|
|
27484
|
-
a
|
|
27485
|
-
|
|
27748
|
+
game's per-call ceiling from a bench, never
|
|
27749
|
+
from a guess: the player is billed as used, and
|
|
27750
|
+
the ceiling also sizes how long an answer may be.
|
|
27486
27751
|
genex player <sub> [options] Answer the in-game model requests you approve, on your
|
|
27487
27752
|
OWN Claude or ChatGPT subscription: install | run |
|
|
27488
27753
|
status | stop | uninstall. "install" signs this machine
|
|
@@ -27944,7 +28209,7 @@ ${c.bold("Examples")}
|
|
|
27944
28209
|
genex shop remove sku_123
|
|
27945
28210
|
genex llm models
|
|
27946
28211
|
genex llm models --all
|
|
27947
|
-
genex llm bench "Reply with one short taunt." --samples 5 --max-
|
|
28212
|
+
genex llm bench "Reply with one short taunt." --samples 5 --max-credits 20 --user-approved
|
|
27948
28213
|
genex llm price
|
|
27949
28214
|
genex llm status
|
|
27950
28215
|
genex llm cancel <id>
|
|
@@ -28181,22 +28446,24 @@ ${c.bold("How the allowance works")}
|
|
|
28181
28446
|
|
|
28182
28447
|
Every number printed is live from your account; prices can change without a deploy.
|
|
28183
28448
|
`,
|
|
28184
|
-
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what should the game declare?
|
|
28449
|
+
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what ceiling should the game declare?
|
|
28185
28450
|
|
|
28186
28451
|
${c.bold("Usage")}
|
|
28187
28452
|
genex llm models [--all] [--json] Which models this stand serves, their provider rates per
|
|
28188
|
-
million tokens,
|
|
28189
|
-
|
|
28190
|
-
|
|
28191
|
-
|
|
28192
|
-
on this stand: it says so
|
|
28193
|
-
|
|
28194
|
-
|
|
28195
|
-
|
|
28196
|
-
|
|
28197
|
-
|
|
28198
|
-
|
|
28199
|
-
|
|
28453
|
+
million tokens, the platform fee a call is billed at, and
|
|
28454
|
+
the headroom it recommends over a benchmark. The catalog
|
|
28455
|
+
is synced from OpenRouter, so the FEATURED short list is
|
|
28456
|
+
printed by default and --all prints every row (--json
|
|
28457
|
+
always carries every row). Off on this stand: it says so
|
|
28458
|
+
and exits 0 \u2014 build the feature without a model rather
|
|
28459
|
+
than promising a 404.
|
|
28460
|
+
genex llm bench "<prompt>" --max-credits <n> --user-approved [options]
|
|
28461
|
+
Run the real model through the development lane and
|
|
28462
|
+
report what each attempt really COST and was charged.
|
|
28463
|
+
Refused without both flags, before any network call:
|
|
28464
|
+
this spends YOUR OWN CREDITS, a spend the build's asset
|
|
28465
|
+
allowance does not count. Needs a folder linked to a
|
|
28466
|
+
game you own, and a balance that covers one attempt.
|
|
28200
28467
|
genex llm price [--json] The recommendation from the last bench in this folder.
|
|
28201
28468
|
Reads a file; spends nothing.
|
|
28202
28469
|
genex llm status [--json] This project's open development calls: active, or
|
|
@@ -28213,29 +28480,32 @@ ${c.bold("Bench options")}
|
|
|
28213
28480
|
--samples <n> How many real attempts (default ${DEFAULT_BENCH_SAMPLES}). More samples, better p95.
|
|
28214
28481
|
--schema <file> A JSON file holding the output schema; implies JSON output.
|
|
28215
28482
|
--json-output Ask for JSON without a schema. --text Ask for text (the default).
|
|
28216
|
-
--max-
|
|
28483
|
+
--max-credits <n> The per-attempt ceiling YOU approved \u2014 a bench runs on your own credits,
|
|
28217
28484
|
never a player's. The run's worst case is that number times --samples,
|
|
28218
|
-
and it is printed before anything starts.
|
|
28485
|
+
and it is printed before anything starts. (--max-coins is its deprecated
|
|
28486
|
+
name, read as the same number.)
|
|
28219
28487
|
--timeout <seconds> Bounds the wait for each attempt.
|
|
28220
28488
|
--json One machine-readable object; every key is always present, null when
|
|
28221
28489
|
the CLI could not learn it.
|
|
28222
28490
|
|
|
28223
28491
|
${c.bold("Why a benchmark and not an estimate")}
|
|
28224
|
-
|
|
28225
|
-
|
|
28226
|
-
|
|
28227
|
-
|
|
28228
|
-
|
|
28229
|
-
|
|
28230
|
-
|
|
28231
|
-
|
|
28232
|
-
|
|
28492
|
+
An in-game call is paid from the PLAYER's credits and billed as used: its real provider cost
|
|
28493
|
+
plus the platform fee, never more than the per-call CEILING the game declares (\`maxCredits\`,
|
|
28494
|
+
or \`perCallMaxCredits\` on a standing budget). The ceiling is not a price, but it is still a
|
|
28495
|
+
number with two jobs: it bounds what one call may cost, and it funds how LONG the answer may
|
|
28496
|
+
be. Three layers sit between the provider's rate card and it \u2014 the provider's cost for YOUR
|
|
28497
|
+
prompt, the platform fee, and your headroom \u2014 and only a real call measures the first. The
|
|
28498
|
+
bench is billed to YOU, on the development lane; the player-funded lane is what the
|
|
28499
|
+
published game uses. The recommendation is the p95 real cost with the server's fee and
|
|
28500
|
+
headroom applied, in whole credits; a stand that serves no headroom or no fee gets no
|
|
28501
|
+
recommendation, because those multipliers are not the CLI's to invent.
|
|
28233
28502
|
|
|
28234
|
-
|
|
28235
|
-
|
|
28236
|
-
|
|
28237
|
-
|
|
28238
|
-
|
|
28503
|
+
Because the ceiling sizes each call's output allowance, the recommendation is never below
|
|
28504
|
+
the smallest ceiling that leaves room for the benchmarked answer, as the server computes it.
|
|
28505
|
+
A sample cut off at your --max-credits (provider_token_limit) is not a sample \u2014 re-run with a
|
|
28506
|
+
higher --max-credits; an answer longer than one call on this stand may produce gets no
|
|
28507
|
+
ceiling at all \u2014 ask for a shorter one. The run also prints the measured average per call,
|
|
28508
|
+
fee included, which is the honest estimate a budget's disclosure is built on.
|
|
28239
28509
|
|
|
28240
28510
|
Results land in ${LLM_BENCH_FILE} \u2014 gitignored, machine-local, re-read by \`genex llm price\`.
|
|
28241
28511
|
`,
|
|
@@ -28450,11 +28720,13 @@ function parseArgs(argv) {
|
|
|
28450
28720
|
"--limit",
|
|
28451
28721
|
// `genex budget --assets <credits>` — the allowance the player approved.
|
|
28452
28722
|
"--assets",
|
|
28453
|
-
// `genex llm bench` value flags. `--max-
|
|
28454
|
-
//
|
|
28723
|
+
// `genex llm bench` value flags. `--max-credits` is the same approval
|
|
28724
|
+
// shape as `--assets`, but its own gate: the asset allowance does not
|
|
28725
|
+
// count a bench. `--max-coins` is its deprecated name (same number).
|
|
28455
28726
|
"--model",
|
|
28456
28727
|
"--samples",
|
|
28457
28728
|
"--schema",
|
|
28729
|
+
"--max-credits",
|
|
28458
28730
|
"--max-coins",
|
|
28459
28731
|
// `genex player install --label <name>` — what this machine is called in
|
|
28460
28732
|
// the dashboard. Its own key rather than the generic 2nd positional,
|
|
@@ -28972,12 +29244,14 @@ function applyValueFlag(options, flag, value) {
|
|
|
28972
29244
|
options.samples = n;
|
|
28973
29245
|
break;
|
|
28974
29246
|
}
|
|
29247
|
+
case "--max-credits":
|
|
28975
29248
|
case "--max-coins": {
|
|
28976
29249
|
const n = Number(value);
|
|
28977
29250
|
if (!Number.isInteger(n) || n < 1) {
|
|
28978
|
-
throw new Error(`Invalid
|
|
29251
|
+
throw new Error(`Invalid ${flag} value: ${value} (whole credits, 1 or more)`);
|
|
28979
29252
|
}
|
|
28980
|
-
options.
|
|
29253
|
+
if (flag === "--max-credits") options.maxCredits = n;
|
|
29254
|
+
else options.maxCoins = n;
|
|
28981
29255
|
break;
|
|
28982
29256
|
}
|
|
28983
29257
|
case "--timeout": {
|
|
@@ -29205,11 +29479,11 @@ async function main() {
|
|
|
29205
29479
|
await syncSkills(updateLog, workspace);
|
|
29206
29480
|
}
|
|
29207
29481
|
const updateCheck = startUpdateCheck();
|
|
29208
|
-
|
|
29209
|
-
|
|
29210
|
-
|
|
29211
|
-
|
|
29212
|
-
|
|
29482
|
+
setSentryTag("command", parsed.command);
|
|
29483
|
+
setSentryTag("cli.version", getCliVersion());
|
|
29484
|
+
setSentryTag("node.version", process.versions.node);
|
|
29485
|
+
setSentryTag("os.platform", process.platform);
|
|
29486
|
+
setSentryContext("runtime", {
|
|
29213
29487
|
cliVersion: getCliVersion(),
|
|
29214
29488
|
node: process.version,
|
|
29215
29489
|
platform: process.platform,
|
|
@@ -29275,9 +29549,9 @@ async function main() {
|
|
|
29275
29549
|
await runBudget(parsed.options);
|
|
29276
29550
|
break;
|
|
29277
29551
|
// The in-game model lane: is it live here, what does one call really
|
|
29278
|
-
// cost, what should the game declare. `bench`
|
|
29279
|
-
//
|
|
29280
|
-
//
|
|
29552
|
+
// cost, what ceiling should the game declare. `bench` spends the
|
|
29553
|
+
// builder's own credits OUTSIDE the asset allowance, so it carries its
|
|
29554
|
+
// own --max-credits/--user-approved gate.
|
|
29281
29555
|
case "llm":
|
|
29282
29556
|
await runLlm(parsed.options);
|
|
29283
29557
|
break;
|
|
@@ -29457,7 +29731,7 @@ function checkNodeVersion() {
|
|
|
29457
29731
|
guardStdio();
|
|
29458
29732
|
if (checkNodeVersion()) {
|
|
29459
29733
|
main().catch(async (err) => {
|
|
29460
|
-
if (!isExpected(err))
|
|
29734
|
+
if (!isExpected(err)) reportException(err);
|
|
29461
29735
|
if (!wasPrinted(err)) {
|
|
29462
29736
|
const log = createLogger();
|
|
29463
29737
|
log.error(describeError(err));
|