@genex-ai/cli-demo 1.36.12 → 1.36.13-dev.1020
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/{blender-mcp-QVEGHE5L.js → blender-mcp-WSURPZET.js} +2 -2
- package/dist/{blender-serve-66OHC4ML.js → blender-serve-EH2MOW3U.js} +1 -1
- package/dist/{chunk-4BBTPRJW.js → chunk-EABIVAGT.js} +2 -2
- package/dist/{chunk-QI7FIYBY.js → chunk-K4EHXOZK.js} +1 -1
- package/dist/index.js +548 -228
- package/package.json +1 -1
- package/templates/skills/genex-game-director/SKILL.md +1 -1
- package/templates/skills/genex-getting-started/SKILL.md +2 -2
- package/templates/skills/genex-llm-in-games/SKILL.md +209 -72
- package/templates/skills/genex-llm-in-games/references/pricing.md +126 -85
- package/templates/skills/genex-monetization/SKILL.md +1 -9
- package/templates/skills/genex-tool-llm/SKILL.md +11 -9
- package/templates/skills/genex-updates/SKILL.md +1 -1
package/dist/index.js
CHANGED
|
@@ -57,7 +57,7 @@ import {
|
|
|
57
57
|
writeSecretFile,
|
|
58
58
|
writeUserToken,
|
|
59
59
|
writeWorkspace
|
|
60
|
-
} from "./chunk-
|
|
60
|
+
} from "./chunk-EABIVAGT.js";
|
|
61
61
|
import {
|
|
62
62
|
CLI_CHANNEL,
|
|
63
63
|
DEFAULT_API_URL,
|
|
@@ -78,7 +78,7 @@ import {
|
|
|
78
78
|
normalizeApiOrigin,
|
|
79
79
|
originKey,
|
|
80
80
|
resolveAgentTargets
|
|
81
|
-
} from "./chunk-
|
|
81
|
+
} from "./chunk-K4EHXOZK.js";
|
|
82
82
|
|
|
83
83
|
// src/instrument.ts
|
|
84
84
|
import { setTimeout as sleep } from "timers/promises";
|
|
@@ -1031,6 +1031,7 @@ var LFS_PATTERNS = [
|
|
|
1031
1031
|
"*.vrm",
|
|
1032
1032
|
"*.fbx",
|
|
1033
1033
|
"*.obj",
|
|
1034
|
+
"*.blend",
|
|
1034
1035
|
// textures / images
|
|
1035
1036
|
"*.png",
|
|
1036
1037
|
"*.jpg",
|
|
@@ -1041,13 +1042,39 @@ var LFS_PATTERNS = [
|
|
|
1041
1042
|
"*.hdr",
|
|
1042
1043
|
"*.exr",
|
|
1043
1044
|
"*.tga",
|
|
1045
|
+
"*.psd",
|
|
1046
|
+
"*.tif",
|
|
1047
|
+
"*.tiff",
|
|
1044
1048
|
// audio / video
|
|
1045
1049
|
"*.mp3",
|
|
1046
1050
|
"*.ogg",
|
|
1047
1051
|
"*.wav",
|
|
1048
1052
|
"*.m4a",
|
|
1049
1053
|
"*.mp4",
|
|
1050
|
-
"*.webm"
|
|
1054
|
+
"*.webm",
|
|
1055
|
+
"*.aif",
|
|
1056
|
+
"*.aiff",
|
|
1057
|
+
"*.flac",
|
|
1058
|
+
"*.mov",
|
|
1059
|
+
"*.avi",
|
|
1060
|
+
// fonts
|
|
1061
|
+
"*.ttf",
|
|
1062
|
+
"*.otf",
|
|
1063
|
+
// engine packages and packed assets (Unity, Unreal, Godot)
|
|
1064
|
+
"*.unitypackage",
|
|
1065
|
+
"*.bundle",
|
|
1066
|
+
"*.uasset",
|
|
1067
|
+
"*.umap",
|
|
1068
|
+
"*.pck",
|
|
1069
|
+
// native libraries and plugins
|
|
1070
|
+
"*.dll",
|
|
1071
|
+
"*.so",
|
|
1072
|
+
"*.dylib",
|
|
1073
|
+
"*.a",
|
|
1074
|
+
"*.lib",
|
|
1075
|
+
// archives
|
|
1076
|
+
"*.zip",
|
|
1077
|
+
"*.7z"
|
|
1051
1078
|
];
|
|
1052
1079
|
async function writeGitattributes(dir, log) {
|
|
1053
1080
|
const file = path4.join(dir, ".gitattributes");
|
|
@@ -1107,7 +1134,7 @@ Important note: put soul into your creations, with many details and love. Aim to
|
|
|
1107
1134
|
21. A turn ends in exactly one of two ways: on a question the player must answer before the next step can be chosen, or on a handoff \u2014 the draft link, what changed, what to try, and what is still open. Never close a turn on a promise. "I'll keep building", "while I keep working", "adding that now" are things you say and then DO before you stop; if you are stopping, say that you have stopped and what remains. While the Build plan's \`Now:\` line has open work and no answer is needed from the player, the turn is not over: preview, hand off, and start the next milestone in the same turn, until the requested outcome is reached or the platform's own budget ends the session. If a stop was forced on you anyway, the first line of your next turn is the promise you left and where it stands.
|
|
1108
1135
|
22. **What the player sees must read as the thing it is**, in this game's own style: a building as that building, a person as a person, a prop as that prop, a surface as its material \u2014 and a coloured box, a capsule, or a flat grey block standing in for one of them is never its finished version. Generation is the default route to that bar wherever code will not honestly reach it: the player's character, whatever the request names, whatever the player walks up to, enters, or interacts with, and the music and sound underneath \u2014 reach for the belt on your own judgment, without stopping to ask; a status line saying what you queued is enough, and \`--no-wait\` keeps you building while it lands. Procedural code is a first-class engine for what is structural, repeated, distant, or parametric \u2014 terrain, sky, fences, paving, modular kits, filler \u2014 and for anything else only when the result meets the same bar and you have looked at it in a capture before its row says \`landed\`. Start with one reusable implementation per distinct required gameplay or visual role. Background populations default to reusable procedural bodies and motion or compatible existing rigged assets. Catalog clips need a compatible skeleton and may be paid. Reserve paid custom bodies for the player and characters examined or interacted with closely, while honoring explicit user requests. Add variants only for an unmet requirement; unspent allowance is not unfinished work. Allowance questions use timeoutPolicy no-consent: silence does not raise the allowance. Record role, reuse and completion criteria in the existing Assets table. Decide per object, not per category, and write the route on each Assets row in \`DESIGN.md\` \u2014 the paid flow for generated pieces, the procedural flow with its local path for code-built ones. An Assets table with no generation in it is a decision, not a default: record it as \`Generation: none \u2014 <why code alone reaches the bar here>\` or generate.
|
|
1109
1136
|
23. **The screenshots you take to look at your own work go in \`.genex/scratch/\`.** Captures, render comparisons, before/after strips, traces and metrics dumps are how you SEE what you built \u2014 they are not part of what you built, and \`genex preview\` pushes the whole folder to the game's source, every time, forever. \`.genex/scratch/\` already exists for this and is already ignored, so there is nothing to set up: write the capture as \`.genex/scratch/arena-before.png\` and read it back from there. Never put them in a folder of your own at the top level: \`progress/\`, \`reports/\`, \`shots/\` and their kind are pushed like source and become permanent weight in the game and in every remix of it \u2014 one real game reached 11 GB and 10,332 committed screenshots exactly this way. If you find such a folder already there, add it to \`.gitignore\`; never delete the player's files (law 18). One file in \`.genex/scratch/\` IS read, by the platform: \`.genex/scratch/cover.png\` is the game's cover \u2014 the frame you would put on its poster: real gameplay at its signature moment, well lit, no menus, popups or busy HUD, landscape 16:9, and never any text (the page sets the title beside it). The next \`genex preview\` makes it the cover; save a better moment over it whenever one comes along, and if your browser tool can set the cover itself, use that instead. Before the first publish, and whenever the game's look changes, load \`$genex-cover\`: it chooses, stages, lights and judges that frame.
|
|
1110
|
-
24. **Benchmark before declaring an in-game LLM
|
|
1137
|
+
24. **Benchmark before declaring an in-game LLM ceiling.** If the game calls a language model while the player plays \u2014 an NPC answering in its own words, a quest written for this save, a judge reading what the player typed \u2014 the PLAYER pays for it from their Genex credits, billed as used, and the game declares no price, only a per-call CEILING: \`maxCredits\` on \`generate()\`, \`perCallMaxCredits\` on a standing budget. That ceiling bounds what one call may cost AND funds how long its answer may be, so never pick it by judgement or from a vendor's rate card. Run the real prompt on your own credits \u2014 \`npx genex llm bench "<the prompt>" --schema <file> --samples 3 --max-credits <n> --user-approved\` \u2014 and declare the figures it prints, recorded in \`DESIGN.md\` with its model and date; re-benchmark whenever the prompt, the schema or the model changes, because a prompt edit changes the ceiling. Check the lane first with \`npx genex llm models\`: where it is off the routes 404, so build the feature behind a graceful unavailable state and say so in the handoff. A repeated call is a standing budget the player approves once (\`requestSpendGrant()\`, then \`generate({ grantId })\`), never a popup per turn, and its disclosed call rate and credits per period are computed from the game's own loop and the bench's measured average. The coin spellings (\`estimateCoins\`, \`perCallMaxCoins\`) are deprecated \u2014 write the credit names. Model output is data the game validates against its own expectation \u2014 never executed, and never authority over coin, credits, items, entitlements or rewards. Load \`$genex-llm-in-games\` before writing any of it.
|
|
1111
1138
|
${CONTRACT_END}
|
|
1112
1139
|
`;
|
|
1113
1140
|
var TOOLS_CONTRACT_HEAD = `# Genex Tools (always in effect in this folder)
|
|
@@ -1121,7 +1148,7 @@ var TOOLS_CONTRACT_OFFER = `5. **When the game is built and playable \u2014 neve
|
|
|
1121
1148
|
var TOOLS_CONTRACT_HOSTED_LAWS = `5. **This game is hosted on Genex now, and publishing is glue \u2014 never a rebuild.** The game ships exactly as the user built it: \`initEmbed()\` first in the boot code (the \`genex-threejs-embed-auth\` card has the call; the renderer draws before any \`await waitForPlayer()\`), a static build into \`dist/\` with relative asset paths, then \`npx genex preview\`. Never restructure, reformat or "improve" the game to ship it, never scaffold a new app around it, and never start a design document or a build plan for it \u2014 the user's own process is theirs. Load the \`genex-tool-publish\` card before the first preview; it owns the vocabulary, the links and the limits.
|
|
1122
1149
|
6. **Ship first, then report.** \`preview\` prints a preflight \u2014 phone memory, a missing volume slider, an asset nobody wired, a viewport line. It never blocks a deploy, so push the build as it is, hand over the link, THEN relay each preflight line to the user in one plain sentence with an offer to fix it, and fix one only on their yes. The preflight is a report for the user, not a to-do list for you.
|
|
1123
1150
|
7. **The link is the game's page, and there are two versions.** After every preview give the user \`<dashboard>/draft/<slug>\` \u2014 \`dashboardOrigins[0]\` and \`slug\` from \`.genex/project.json\` \u2014 never localhost, a file path or the bare play origin. \`preview\` updates the draft and never touches what players are on; the first release is \`npx genex publish\` (it lists the game); after that "publish it", "update it" and "yes" all mean \`npx genex promote\` \u2014 the exact draft build, no rebuild. Ask once per round of work, in one line, and keep working while you wait.`;
|
|
1124
|
-
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with Genex
|
|
1151
|
+
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with their Genex credits or their own Claude/ChatGPT subscription, and approves it on a Genex sheet; your game just calls \`generate()\`. Want it that way?`;
|
|
1125
1152
|
function toolsContractLlmLaw(n, hosted) {
|
|
1126
1153
|
const onYes = hosted ? "On a yes, load `$genex-tool-llm` and then `$genex-llm-in-games`, which owns the build." : "On a yes, load `$genex-tool-llm`, which owns the rest: it checks the lane with `npx genex llm models` first, then runs `npx genex init --convert` (a hosted game is a static browser build \u2014 a local server holding a key can never ship).";
|
|
1127
1154
|
return `${n}. **A model running while people PLAY is a platform feature \u2014 never something you wire onto the user's own meter.** When a request implies one (NPCs that talk or decide in their own words, content written from what the player types, a prompt box in the game, "let the player pick a model"), recognise it, say in ONE line: "${TOOLS_CONTRACT_LLM_OFFER}" \u2014 then ASK and wait. ${onYes} On a no, build the authored version instead. Either way, never ship a game that calls a model on a key or an account of the user's own: every visitor would spend their money with nobody approving it, and the credential is readable in the bundle. Player-funded generation is text and JSON only \u2014 3D, images, video and audio stay the asset lanes above, on the user's meter.`;
|
|
@@ -5944,8 +5971,9 @@ async function deployGame(ctx, opts, log) {
|
|
|
5944
5971
|
return false;
|
|
5945
5972
|
}
|
|
5946
5973
|
let sourceCommitSha;
|
|
5974
|
+
const sourceDir = opts.sourceDir ?? cwd;
|
|
5947
5975
|
const pushed = await pushSource(
|
|
5948
|
-
|
|
5976
|
+
sourceDir,
|
|
5949
5977
|
ctx,
|
|
5950
5978
|
log,
|
|
5951
5979
|
opts.channel === "staging" ? "preview" : "main",
|
|
@@ -5959,11 +5987,14 @@ async function deployGame(ctx, opts, log) {
|
|
|
5959
5987
|
const published = await callPublish(ctx, commit, opts, log, sourceCommitSha, capabilities);
|
|
5960
5988
|
if (!published) return false;
|
|
5961
5989
|
if (opts.channel === "staging") {
|
|
5962
|
-
const
|
|
5990
|
+
const pushedHere = opts.sourceDir === void 0;
|
|
5991
|
+
const tree = pushedHere ? await sourceTreeHash(cwd) : null;
|
|
5963
5992
|
const current = await readProject(cwd);
|
|
5964
5993
|
if (current) {
|
|
5994
|
+
const { sourceTree: recorded, ...rest } = current;
|
|
5995
|
+
const kept = pushedHere && recorded ? { sourceTree: recorded } : {};
|
|
5965
5996
|
await writeProject(
|
|
5966
|
-
{ ...
|
|
5997
|
+
{ ...rest, ...kept, stagingCommit: commit, ...tree ? { sourceTree: tree } : {} },
|
|
5967
5998
|
cwd
|
|
5968
5999
|
).catch(() => {
|
|
5969
6000
|
});
|
|
@@ -6229,13 +6260,13 @@ async function callPublish(ctx, commit, opts, log, sourceCommitSha, capabilities
|
|
|
6229
6260
|
}
|
|
6230
6261
|
return { url: body?.url ?? "" };
|
|
6231
6262
|
}
|
|
6232
|
-
async function pushSource(
|
|
6263
|
+
async function pushSource(sourceDir, ctx, log, branch = "main", expectStagingCommit, onCommit) {
|
|
6233
6264
|
const target = await fetchPushUrl(ctx, log, expectStagingCommit);
|
|
6234
6265
|
if (target === "stale") return "stale";
|
|
6235
6266
|
if (!target) return false;
|
|
6236
6267
|
const ref = target.managed ? branch : "main";
|
|
6237
6268
|
const first = await pushWorktree(
|
|
6238
|
-
|
|
6269
|
+
sourceDir,
|
|
6239
6270
|
target.pushUrl,
|
|
6240
6271
|
target.managed,
|
|
6241
6272
|
log,
|
|
@@ -6253,7 +6284,7 @@ async function pushSource(cwd, ctx, log, branch = "main", expectStagingCommit, o
|
|
|
6253
6284
|
if (fresh === "stale") return "stale";
|
|
6254
6285
|
if (!fresh) return false;
|
|
6255
6286
|
return await pushWorktree(
|
|
6256
|
-
|
|
6287
|
+
sourceDir,
|
|
6257
6288
|
fresh.pushUrl,
|
|
6258
6289
|
fresh.managed,
|
|
6259
6290
|
log,
|
|
@@ -6263,6 +6294,34 @@ async function pushSource(cwd, ctx, log, branch = "main", expectStagingCommit, o
|
|
|
6263
6294
|
fresh.sourceLimits
|
|
6264
6295
|
) === true;
|
|
6265
6296
|
}
|
|
6297
|
+
async function resolveSourceDir(dir, cwd = process.cwd()) {
|
|
6298
|
+
const abs = path17.resolve(cwd, dir);
|
|
6299
|
+
const stat = await fs17.lstat(abs).catch(() => null);
|
|
6300
|
+
if (!stat) return { refused: `--source ${abs} does not exist.` };
|
|
6301
|
+
if (stat.isSymbolicLink()) {
|
|
6302
|
+
return { refused: `--source ${abs} is a symbolic link. Pass the folder itself.` };
|
|
6303
|
+
}
|
|
6304
|
+
if (!stat.isDirectory()) return { refused: `--source ${abs} is not a folder.` };
|
|
6305
|
+
const real = await fs17.realpath(abs);
|
|
6306
|
+
const project = await fs17.realpath(cwd).catch(() => path17.resolve(cwd));
|
|
6307
|
+
const genex = path17.join(project, ".genex");
|
|
6308
|
+
const genexTarget = await fs17.realpath(genex).catch(() => genex);
|
|
6309
|
+
for (const meta of /* @__PURE__ */ new Set([genex, genexTarget])) {
|
|
6310
|
+
if (real === meta) {
|
|
6311
|
+
return { refused: `--source ${abs} is this project's .genex folder, not the game's source.` };
|
|
6312
|
+
}
|
|
6313
|
+
if (isInside(meta, real)) {
|
|
6314
|
+
return {
|
|
6315
|
+
refused: `--source ${abs} holds this project's .genex folder. Pass a separate folder with the game's own files.`
|
|
6316
|
+
};
|
|
6317
|
+
}
|
|
6318
|
+
}
|
|
6319
|
+
return { dir: real };
|
|
6320
|
+
}
|
|
6321
|
+
function isInside(child, parent) {
|
|
6322
|
+
const rel = path17.relative(parent, child);
|
|
6323
|
+
return rel !== "" && rel !== ".." && !rel.startsWith(`..${path17.sep}`) && !path17.isAbsolute(rel);
|
|
6324
|
+
}
|
|
6266
6325
|
var WORKSPACE_SOURCE_REF = "workspace";
|
|
6267
6326
|
function gitLfsInstallCommand(platform) {
|
|
6268
6327
|
if (platform === "darwin") {
|
|
@@ -6274,7 +6333,7 @@ function gitLfsInstallCommand(platform) {
|
|
|
6274
6333
|
}
|
|
6275
6334
|
return { bin: "sudo", args: ["-n", "apt-get", "install", "-y", "git-lfs"], env: {} };
|
|
6276
6335
|
}
|
|
6277
|
-
async function pushWorktree(
|
|
6336
|
+
async function pushWorktree(workTree, pushUrl, managed, log, branch = "main", onCommit, skipSizeCheck = false, limits) {
|
|
6278
6337
|
const EMPTY_TREE = "4b825dc642cb6eb9a060e54bf8d69288fbee4904";
|
|
6279
6338
|
const failed = () => {
|
|
6280
6339
|
log.error("Couldn't save your game's source \u2014 please try again.");
|
|
@@ -6296,7 +6355,7 @@ async function pushWorktree(cwd, pushUrl, managed, log, branch = "main", onCommi
|
|
|
6296
6355
|
try {
|
|
6297
6356
|
if ((await run("git", ["init", "-q"], base)).code !== 0) return failed();
|
|
6298
6357
|
await fs17.writeFile(path17.join(gitDir, "info", "exclude"), excludeFile());
|
|
6299
|
-
const env = { ...base, GIT_WORK_TREE:
|
|
6358
|
+
const env = { ...base, GIT_WORK_TREE: workTree, GIT_INDEX_FILE: path17.join(gitDir, "index-source") };
|
|
6300
6359
|
let lfs = (await run("git", ["lfs", "version"], base)).code !== 0 ? false : true;
|
|
6301
6360
|
if (!lfs) {
|
|
6302
6361
|
log.step("Installing git-lfs (keeps large binary assets out of the source push)\u2026");
|
|
@@ -6310,7 +6369,7 @@ async function pushWorktree(cwd, pushUrl, managed, log, branch = "main", onCommi
|
|
|
6310
6369
|
}
|
|
6311
6370
|
}
|
|
6312
6371
|
if (lfs) {
|
|
6313
|
-
await writeGitattributes(
|
|
6372
|
+
await writeGitattributes(workTree, log);
|
|
6314
6373
|
const filters = [
|
|
6315
6374
|
["filter.lfs.clean", "git-lfs clean -- %f"],
|
|
6316
6375
|
["filter.lfs.smudge", "git-lfs smudge -- %f"],
|
|
@@ -6325,7 +6384,7 @@ async function pushWorktree(cwd, pushUrl, managed, log, branch = "main", onCommi
|
|
|
6325
6384
|
log.error("Nothing to publish \u2014 the project has no files.");
|
|
6326
6385
|
return "failed";
|
|
6327
6386
|
}
|
|
6328
|
-
const payload = await measurePayload(
|
|
6387
|
+
const payload = await measurePayload(workTree, staged);
|
|
6329
6388
|
log.dim(`Source: ${payload.files} files, ${formatBytes2(payload.bytes)}.`);
|
|
6330
6389
|
const ceiling = limits ?? FALLBACK_SOURCE_LIMITS;
|
|
6331
6390
|
if (!skipSizeCheck && isOversized(payload, ceiling)) {
|
|
@@ -6939,6 +6998,12 @@ async function runPreview(opts) {
|
|
|
6939
6998
|
process.exitCode = 1;
|
|
6940
6999
|
return;
|
|
6941
7000
|
}
|
|
7001
|
+
const source = opts.sourceDir === void 0 ? void 0 : await resolveSourceDir(opts.sourceDir);
|
|
7002
|
+
if (source && "refused" in source) {
|
|
7003
|
+
log.error(`${source.refused} Nothing was uploaded.`);
|
|
7004
|
+
process.exitCode = 1;
|
|
7005
|
+
return;
|
|
7006
|
+
}
|
|
6942
7007
|
const apiUrl = getApiUrl(meta.apiUrl);
|
|
6943
7008
|
const token = await readUserToken(opts.envPath, apiUrl);
|
|
6944
7009
|
if (!token) {
|
|
@@ -6965,6 +7030,7 @@ async function runPreview(opts) {
|
|
|
6965
7030
|
channel: "staging",
|
|
6966
7031
|
noBuild: opts.noBuild,
|
|
6967
7032
|
force: opts.force,
|
|
7033
|
+
sourceDir: source?.dir,
|
|
6968
7034
|
matchmaking: detections.matchmaking,
|
|
6969
7035
|
embedSdkVersion: detections.embedSdkVersion,
|
|
6970
7036
|
multiplayer: detections.multiplayer,
|
|
@@ -8271,6 +8337,12 @@ async function runMakeRemixable(opts) {
|
|
|
8271
8337
|
process.exitCode = 1;
|
|
8272
8338
|
return;
|
|
8273
8339
|
}
|
|
8340
|
+
const source = opts.sourceDir === void 0 ? void 0 : await resolveSourceDir(opts.sourceDir);
|
|
8341
|
+
if (source && "refused" in source) {
|
|
8342
|
+
log.error(`${source.refused} Nothing was pushed.`);
|
|
8343
|
+
process.exitCode = 1;
|
|
8344
|
+
return;
|
|
8345
|
+
}
|
|
8274
8346
|
const apiUrl = getApiUrl(opts.apiUrl ?? meta.apiUrl);
|
|
8275
8347
|
const token = await readUserToken(opts.envPath, apiUrl);
|
|
8276
8348
|
if (!token) {
|
|
@@ -8312,7 +8384,7 @@ async function runMakeRemixable(opts) {
|
|
|
8312
8384
|
return;
|
|
8313
8385
|
}
|
|
8314
8386
|
log.step("Copying your game's source to the public repo\u2026");
|
|
8315
|
-
if (await pushWorktree(process.cwd(), data.pushUrl, true, log) !== true) {
|
|
8387
|
+
if (await pushWorktree(source?.dir ?? process.cwd(), data.pushUrl, true, log) !== true) {
|
|
8316
8388
|
process.exitCode = 1;
|
|
8317
8389
|
return;
|
|
8318
8390
|
}
|
|
@@ -8763,6 +8835,12 @@ async function runPublish(opts) {
|
|
|
8763
8835
|
process.exitCode = 1;
|
|
8764
8836
|
return;
|
|
8765
8837
|
}
|
|
8838
|
+
const source = opts.sourceDir === void 0 ? void 0 : await resolveSourceDir(opts.sourceDir);
|
|
8839
|
+
if (source && "refused" in source) {
|
|
8840
|
+
log.error(`${source.refused} Nothing was uploaded.`);
|
|
8841
|
+
process.exitCode = 1;
|
|
8842
|
+
return;
|
|
8843
|
+
}
|
|
8766
8844
|
const apiUrl = getApiUrl(opts.apiUrl ?? meta.apiUrl);
|
|
8767
8845
|
const token = opts.token ?? await readUserToken(opts.envPath, apiUrl);
|
|
8768
8846
|
if (!token) {
|
|
@@ -8783,6 +8861,7 @@ async function runPublish(opts) {
|
|
|
8783
8861
|
channel: "staging",
|
|
8784
8862
|
noBuild: opts.noBuild,
|
|
8785
8863
|
force: opts.force,
|
|
8864
|
+
sourceDir: source?.dir,
|
|
8786
8865
|
matchmaking: detections.matchmaking,
|
|
8787
8866
|
embedSdkVersion: detections.embedSdkVersion,
|
|
8788
8867
|
multiplayer: detections.multiplayer,
|
|
@@ -23807,7 +23886,7 @@ async function runBlender(opts) {
|
|
|
23807
23886
|
return 1;
|
|
23808
23887
|
}
|
|
23809
23888
|
if (sub === "serve") {
|
|
23810
|
-
const { serveLocalBlender } = await import("./blender-serve-
|
|
23889
|
+
const { serveLocalBlender } = await import("./blender-serve-EH2MOW3U.js");
|
|
23811
23890
|
const port = Number(process.env.GENEX_BLENDER_PORT ?? 8088);
|
|
23812
23891
|
log.step(`Starting a local Blender service on port ${port}`);
|
|
23813
23892
|
log.plain(
|
|
@@ -23816,7 +23895,7 @@ async function runBlender(opts) {
|
|
|
23816
23895
|
return serveLocalBlender({ port, log });
|
|
23817
23896
|
}
|
|
23818
23897
|
if (sub === "mcp") {
|
|
23819
|
-
const { runBlenderMcp } = await import("./blender-mcp-
|
|
23898
|
+
const { runBlenderMcp } = await import("./blender-mcp-WSURPZET.js");
|
|
23820
23899
|
return runBlenderMcp();
|
|
23821
23900
|
}
|
|
23822
23901
|
if (sub === "seat") {
|
|
@@ -24324,7 +24403,12 @@ import fs35 from "fs/promises";
|
|
|
24324
24403
|
import path36 from "path";
|
|
24325
24404
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
24326
24405
|
var SUBS3 = ["models", "bench", "price", "status", "cancel"];
|
|
24327
|
-
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN
|
|
24406
|
+
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN credits \u2014 your balance, not a player's, and a spend this build's asset allowance does not count. Re-run with --max-credits <n> --user-approved only after the person at the keyboard agreed to the number.";
|
|
24407
|
+
var LLM_MAX_COINS_DEPRECATED = "--max-coins is deprecated: it is read as --max-credits, the same number \u2014 a bench spends credits now.";
|
|
24408
|
+
function benchBalanceShortLine(spendable, maxCredits) {
|
|
24409
|
+
return `Your spendable credits (${spendable}) cannot cover one attempt at --max-credits ${maxCredits} \u2014 nothing was sent, nothing was spent. Add credits, or re-run with a lower --max-credits once the person at the keyboard agrees to it.`;
|
|
24410
|
+
}
|
|
24411
|
+
var LLM_BENCH_UNVERIFIED_LINE = "Credits pay only for a verified email, and this account's is not verified yet (credits_unverified) \u2014 nothing was sent, nothing was spent. Verify it, then re-run.";
|
|
24328
24412
|
var LLM_LANE_OFF_LINE = "The runtime LLM lane is off on this stand \u2014 in-game generate() answers 404 here, and nothing can be benchmarked.";
|
|
24329
24413
|
var LLM_TOOLS_CONVERT_HINT = [
|
|
24330
24414
|
"In-game model calls need a hosted game. This folder is a Genex Tools workspace, so nothing here",
|
|
@@ -24337,7 +24421,7 @@ function serverSlotCounts(body) {
|
|
|
24337
24421
|
return { held: body.slotsHeld, limit: body.slotLimit };
|
|
24338
24422
|
}
|
|
24339
24423
|
var PENDING_SLOT_RELEASE_MINUTES = 10;
|
|
24340
|
-
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so
|
|
24424
|
+
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so what it reserved stays held until the bill resolves; such a row stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`;
|
|
24341
24425
|
var LLM_STATUS_NO_COUNT_LINE = "This stand does not report how many slots are held across your projects, so no count is shown.";
|
|
24342
24426
|
var LLM_CANCEL_AWAITING_BILL = "this call already stopped; its bill is awaiting the provider";
|
|
24343
24427
|
var LLM_CANCEL_ALREADY_FINAL = "this call is already final";
|
|
@@ -24351,11 +24435,33 @@ var DEFAULT_SAMPLE_TIMEOUT_SEC = 180;
|
|
|
24351
24435
|
function parseProviderKey(value) {
|
|
24352
24436
|
return value === "ok" || value === "rejected" || value === "unknown" ? value : null;
|
|
24353
24437
|
}
|
|
24438
|
+
function bpsOrNull(value) {
|
|
24439
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
24440
|
+
}
|
|
24441
|
+
function standingCreditGrantOrNull(value) {
|
|
24442
|
+
if (!value || typeof value !== "object") return null;
|
|
24443
|
+
const v = value;
|
|
24444
|
+
const whole = (x) => typeof x === "number" && Number.isSafeInteger(x) && x >= 0;
|
|
24445
|
+
if (!whole(v.minCredits) || !whole(v.maxCredits) || !whole(v.stepCredits) || !whole(v.defaultCredits)) return null;
|
|
24446
|
+
return {
|
|
24447
|
+
minCredits: v.minCredits,
|
|
24448
|
+
maxCredits: v.maxCredits,
|
|
24449
|
+
stepCredits: v.stepCredits,
|
|
24450
|
+
defaultCredits: v.defaultCredits,
|
|
24451
|
+
...whole(v.minUsdCents) ? { minUsdCents: v.minUsdCents } : {},
|
|
24452
|
+
...whole(v.maxUsdCents) ? { maxUsdCents: v.maxUsdCents } : {},
|
|
24453
|
+
...whole(v.stepUsdCents) ? { stepUsdCents: v.stepUsdCents } : {}
|
|
24454
|
+
};
|
|
24455
|
+
}
|
|
24354
24456
|
async function readRuntimeLane(apiUrl, token) {
|
|
24355
24457
|
const empty = (state) => ({
|
|
24356
24458
|
state,
|
|
24357
24459
|
models: null,
|
|
24358
24460
|
recommendedDeclaredHeadroomBps: null,
|
|
24461
|
+
currency: null,
|
|
24462
|
+
feeBps: null,
|
|
24463
|
+
usdCentsPerCredit: null,
|
|
24464
|
+
standingCreditGrant: null,
|
|
24359
24465
|
externalProviders: null,
|
|
24360
24466
|
total: null,
|
|
24361
24467
|
featuredCount: null,
|
|
@@ -24374,7 +24480,11 @@ async function readRuntimeLane(apiUrl, token) {
|
|
|
24374
24480
|
return {
|
|
24375
24481
|
state: "live",
|
|
24376
24482
|
models: body.models,
|
|
24377
|
-
recommendedDeclaredHeadroomBps:
|
|
24483
|
+
recommendedDeclaredHeadroomBps: bpsOrNull(body.recommendedDeclaredHeadroomBps),
|
|
24484
|
+
currency: body.currency === "credits" || body.currency === "coins" ? body.currency : null,
|
|
24485
|
+
feeBps: bpsOrNull(body.feeBps),
|
|
24486
|
+
usdCentsPerCredit: typeof body.usdCentsPerCredit === "number" && Number.isSafeInteger(body.usdCentsPerCredit) && body.usdCentsPerCredit >= 1 ? body.usdCentsPerCredit : null,
|
|
24487
|
+
standingCreditGrant: standingCreditGrantOrNull(body.standingCreditGrant),
|
|
24378
24488
|
externalProviders: Array.isArray(body.externalProviders) ? body.externalProviders : null,
|
|
24379
24489
|
total: typeof body.total === "number" && Number.isFinite(body.total) ? body.total : null,
|
|
24380
24490
|
featuredCount: typeof body.featuredCount === "number" && Number.isFinite(body.featuredCount) ? body.featuredCount : null,
|
|
@@ -24390,18 +24500,45 @@ function percentile2(values, p) {
|
|
|
24390
24500
|
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
24391
24501
|
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
24392
24502
|
}
|
|
24393
|
-
function
|
|
24394
|
-
|
|
24503
|
+
function percentilePicos(values, p) {
|
|
24504
|
+
if (values.length === 0) return null;
|
|
24505
|
+
const sorted = [...values].sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
24506
|
+
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
24507
|
+
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
24395
24508
|
}
|
|
24396
|
-
|
|
24397
|
-
|
|
24509
|
+
var PICOS_PER_CENT = 10000000000n;
|
|
24510
|
+
function ceilingCreditsFor(costPicos, terms) {
|
|
24511
|
+
if (costPicos <= 0n) return 1;
|
|
24512
|
+
const num = costPicos * BigInt(1e4 + terms.feeBps) * BigInt(1e4 + terms.headroomBps);
|
|
24513
|
+
const den = 100000000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
24514
|
+
return Math.max(1, Number((num + den - 1n) / den));
|
|
24515
|
+
}
|
|
24516
|
+
function averageCreditsPerCall(costsPicos, terms) {
|
|
24517
|
+
if (costsPicos.length === 0) return null;
|
|
24518
|
+
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
24519
|
+
return Number(sum * BigInt(1e4 + terms.feeBps)) / Number(BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT);
|
|
24520
|
+
}
|
|
24521
|
+
function perCallEstimateCreditsFor(costsPicos, terms) {
|
|
24522
|
+
if (costsPicos.length === 0) return null;
|
|
24523
|
+
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
24524
|
+
const num = sum * BigInt(1e4 + terms.feeBps);
|
|
24525
|
+
const den = BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
24526
|
+
return Math.max(1, Number((num + den - 1n) / den));
|
|
24527
|
+
}
|
|
24528
|
+
function formatCredits(value) {
|
|
24529
|
+
if (!Number.isFinite(value)) return "\u2014";
|
|
24530
|
+
if (value >= 100) return String(Math.round(value));
|
|
24531
|
+
return String(Number(value.toPrecision(value < 1 ? 2 : 3)));
|
|
24532
|
+
}
|
|
24533
|
+
function usdFromPicos(picos) {
|
|
24534
|
+
if (picos === null || picos === void 0) return "unknown";
|
|
24535
|
+
const value = typeof picos === "bigint" ? picos : /^\d+$/.test(picos) ? BigInt(picos) : null;
|
|
24536
|
+
if (value === null) return "unknown";
|
|
24537
|
+
return `$${(Number(value) / 1e12).toFixed(6)}`;
|
|
24398
24538
|
}
|
|
24399
24539
|
function usdPerMillion(value) {
|
|
24400
24540
|
return `$${value.toFixed(value < 1 ? 4 : 2)}`;
|
|
24401
24541
|
}
|
|
24402
|
-
function usd(value) {
|
|
24403
|
-
return value === null ? "unknown" : `$${value.toFixed(6)}`;
|
|
24404
|
-
}
|
|
24405
24542
|
async function runLlm(opts = {}) {
|
|
24406
24543
|
const log = createLogger({ quiet: opts.quiet || opts.json });
|
|
24407
24544
|
const cwd = opts.cwd ?? process.cwd();
|
|
@@ -24427,16 +24564,23 @@ async function runLlm(opts = {}) {
|
|
|
24427
24564
|
return;
|
|
24428
24565
|
}
|
|
24429
24566
|
if (sub === "bench") {
|
|
24430
|
-
if (opts.maxCoins
|
|
24567
|
+
if (opts.maxCoins !== void 0) process.stderr.write(`${c.yellow("!")} ${LLM_MAX_COINS_DEPRECATED}
|
|
24568
|
+
`);
|
|
24569
|
+
if (opts.maxCoins !== void 0 && opts.maxCredits !== void 0 && opts.maxCoins !== opts.maxCredits) {
|
|
24570
|
+
fail5("llm bench", `Pass --max-credits alone \u2014 --max-coins is its deprecated name, and the two disagree (${opts.maxCredits} vs ${opts.maxCoins}).`);
|
|
24571
|
+
return;
|
|
24572
|
+
}
|
|
24573
|
+
const approvedCredits = opts.maxCredits ?? opts.maxCoins;
|
|
24574
|
+
if (approvedCredits === void 0 || opts.userApproved !== true) {
|
|
24431
24575
|
fail5("llm bench", LLM_BENCH_APPROVAL_REQUIRED);
|
|
24432
24576
|
return;
|
|
24433
24577
|
}
|
|
24434
|
-
if (!Number.isInteger(
|
|
24435
|
-
fail5("llm bench", `--max-
|
|
24578
|
+
if (!Number.isInteger(approvedCredits) || approvedCredits < 1) {
|
|
24579
|
+
fail5("llm bench", `--max-credits takes a whole number of credits, 1 or more (got ${String(approvedCredits)}).`);
|
|
24436
24580
|
return;
|
|
24437
24581
|
}
|
|
24438
24582
|
if (!opts.benchPrompt?.trim()) {
|
|
24439
|
-
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-
|
|
24583
|
+
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-credits <n> --user-approved.');
|
|
24440
24584
|
return;
|
|
24441
24585
|
}
|
|
24442
24586
|
if (opts.jsonOutput && opts.textOutput) {
|
|
@@ -24528,6 +24672,10 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24528
24672
|
models: null,
|
|
24529
24673
|
externalProviders: null,
|
|
24530
24674
|
recommendedDeclaredHeadroomBps: null,
|
|
24675
|
+
currency: null,
|
|
24676
|
+
feeBps: null,
|
|
24677
|
+
usdCentsPerCredit: null,
|
|
24678
|
+
standingCreditGrant: null,
|
|
24531
24679
|
total: null,
|
|
24532
24680
|
featuredCount: null,
|
|
24533
24681
|
// The stand was never asked, so the verdict is unknown — but the key is
|
|
@@ -24553,6 +24701,12 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24553
24701
|
models: lane.models,
|
|
24554
24702
|
externalProviders: lane.externalProviders,
|
|
24555
24703
|
recommendedDeclaredHeadroomBps: lane.recommendedDeclaredHeadroomBps,
|
|
24704
|
+
// The credit terms a call is billed on, the server's — null on a stand
|
|
24705
|
+
// that predates credit billing, never a number the CLI filled in.
|
|
24706
|
+
currency: lane.currency,
|
|
24707
|
+
feeBps: lane.feeBps,
|
|
24708
|
+
usdCentsPerCredit: lane.usdCentsPerCredit,
|
|
24709
|
+
standingCreditGrant: lane.standingCreditGrant,
|
|
24556
24710
|
total: lane.total ?? lane.models?.length ?? null,
|
|
24557
24711
|
featuredCount: lane.featuredCount ?? (lane.models ? lane.models.filter((m) => m.featured).length : null),
|
|
24558
24712
|
// `null` when the lane is not live or the server predates the probe.
|
|
@@ -24599,8 +24753,9 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24599
24753
|
);
|
|
24600
24754
|
for (const m of shown) {
|
|
24601
24755
|
const plan = m.personalPlan ? ` \xB7 personal plan: ${m.personalPlan}` : "";
|
|
24756
|
+
const kind = m.kind === "typesafe" ? " \xB7 judge (classifier): judge() under a budget, not generate() or bench" : "";
|
|
24602
24757
|
log.plain(
|
|
24603
|
-
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}`
|
|
24758
|
+
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}${kind}`
|
|
24604
24759
|
);
|
|
24605
24760
|
}
|
|
24606
24761
|
if (hiddenCount > 0) {
|
|
@@ -24616,74 +24771,104 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24616
24771
|
}
|
|
24617
24772
|
log.plain("");
|
|
24618
24773
|
if (lane.recommendedDeclaredHeadroomBps === null) {
|
|
24619
|
-
log.dim(" This stand serves no recommended headroom, so no
|
|
24774
|
+
log.dim(" This stand serves no recommended headroom, so no ceiling can be recommended from a bench.");
|
|
24620
24775
|
} else {
|
|
24621
|
-
log.dim(` Recommended headroom over benchmarked
|
|
24776
|
+
log.dim(` Recommended headroom over a benchmarked call's cost on this stand: ${lane.recommendedDeclaredHeadroomBps} bps.`);
|
|
24777
|
+
}
|
|
24778
|
+
if (lane.feeBps === null) {
|
|
24779
|
+
log.dim(" This stand serves no platform fee \u2014 it predates credit billing, so no ceiling can be recommended from a bench.");
|
|
24780
|
+
} else {
|
|
24781
|
+
log.dim(` In-game calls are billed in the player's credits, as used: the provider's cost plus this stand's platform fee (${lane.feeBps} bps).`);
|
|
24622
24782
|
}
|
|
24623
24783
|
if (toolsOnly) {
|
|
24624
24784
|
printConvertHint();
|
|
24625
24785
|
return;
|
|
24626
24786
|
}
|
|
24627
|
-
log.dim(" Those per-million rates are the PROVIDER's; what
|
|
24628
|
-
log.dim(`
|
|
24787
|
+
log.dim(" Those per-million rates are the PROVIDER's list; what one call of YOUR prompt really costs,");
|
|
24788
|
+
log.dim(` and the per-call ceiling to declare, only a real call measures: ${c.cyan('genex llm bench "<prompt>" --max-credits <n> --user-approved')}`);
|
|
24629
24789
|
}
|
|
24630
24790
|
function wholeOrNull(value) {
|
|
24631
24791
|
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
24632
24792
|
}
|
|
24793
|
+
function costPicosOf(usage) {
|
|
24794
|
+
if (typeof usage?.costUsdPicos === "string" && /^\d+$/.test(usage.costUsdPicos)) return usage.costUsdPicos;
|
|
24795
|
+
if (typeof usage?.costUsd === "number" && Number.isFinite(usage.costUsd) && usage.costUsd >= 0) {
|
|
24796
|
+
return String(BigInt(Math.round(usage.costUsd * 1e12)));
|
|
24797
|
+
}
|
|
24798
|
+
return null;
|
|
24799
|
+
}
|
|
24633
24800
|
function sampleFrom(id, settled) {
|
|
24634
24801
|
const usage = settled?.usage;
|
|
24802
|
+
const picos = costPicosOf(usage);
|
|
24635
24803
|
return {
|
|
24636
24804
|
id,
|
|
24637
24805
|
status: settled?.status ?? null,
|
|
24638
24806
|
billingStatus: settled?.billingStatus ?? null,
|
|
24639
|
-
|
|
24640
|
-
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd : null,
|
|
24807
|
+
chargedCredits: wholeOrNull(settled?.chargedCredits),
|
|
24808
|
+
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd : picos !== null ? Number(picos) / 1e12 : null,
|
|
24809
|
+
costUsdPicos: picos,
|
|
24641
24810
|
error: settled?.error ?? (settled ? null : "timed_out"),
|
|
24642
24811
|
providerMessage: typeof settled?.providerMessage === "string" && settled.providerMessage.trim() ? settled.providerMessage.trim() : null,
|
|
24643
24812
|
inputTokens: wholeOrNull(usage?.inputTokens),
|
|
24644
24813
|
outputTokens: wholeOrNull(usage?.outputTokens),
|
|
24645
24814
|
maxOutputTokens: wholeOrNull(usage?.maxOutputTokens),
|
|
24646
|
-
|
|
24815
|
+
fitCredits: typeof usage?.fitCredits === "number" && Number.isSafeInteger(usage.fitCredits) && usage.fitCredits >= 1 ? usage.fitCredits : null,
|
|
24647
24816
|
fitExceedsCap: typeof usage?.fitExceedsCap === "boolean" ? usage.fitExceedsCap : null,
|
|
24648
24817
|
truncated: typeof usage?.truncated === "boolean" ? usage.truncated : settled?.error === "provider_token_limit" ? true : null
|
|
24649
24818
|
};
|
|
24650
24819
|
}
|
|
24651
24820
|
var PROVIDER_TOKEN_LIMIT = "provider_token_limit";
|
|
24652
|
-
function benchRecommendation(results,
|
|
24653
|
-
const settled = results.filter(
|
|
24654
|
-
|
|
24655
|
-
|
|
24656
|
-
const
|
|
24657
|
-
const
|
|
24821
|
+
function benchRecommendation(results, terms) {
|
|
24822
|
+
const settled = results.filter(
|
|
24823
|
+
(r) => r.status === "succeeded" && r.billingStatus === "final" && r.chargedCredits !== null && r.costUsdPicos !== null
|
|
24824
|
+
);
|
|
24825
|
+
const costs = settled.map((r) => BigInt(r.costUsdPicos));
|
|
24826
|
+
const charged = settled.map((r) => r.chargedCredits);
|
|
24827
|
+
const fits = settled.map((r) => r.fitCredits).filter((v) => typeof v === "number");
|
|
24828
|
+
const costP95 = percentilePicos(costs, 95);
|
|
24829
|
+
const costMax = costs.length ? costs.reduce((a, b) => b > a ? b : a) : null;
|
|
24658
24830
|
const fitP95 = percentile2(fits, 95);
|
|
24659
24831
|
const fitMax = fits.length ? Math.max(...fits) : null;
|
|
24660
24832
|
const exceedsStandCap = results.some((r) => r.fitExceedsCap === true);
|
|
24661
24833
|
const cutOffSamples = results.filter((r) => r.error === PROVIDER_TOKEN_LIMIT && r.fitExceedsCap !== true).length;
|
|
24662
24834
|
const noRecommendationReason = exceedsStandCap ? "answer_exceeds_stand_cap" : cutOffSamples > 0 ? "answer_cut_off" : null;
|
|
24663
|
-
const
|
|
24664
|
-
const
|
|
24665
|
-
const
|
|
24835
|
+
const { headroomBps, feeBps, usdCentsPerCredit } = terms;
|
|
24836
|
+
const missingTerms = headroomBps === null ? "headroom" : feeBps === null || usdCentsPerCredit === null ? "credit_terms" : null;
|
|
24837
|
+
const full = headroomBps !== null && feeBps !== null && usdCentsPerCredit !== null ? { headroomBps, feeBps, usdCentsPerCredit } : null;
|
|
24838
|
+
const billing = feeBps !== null && usdCentsPerCredit !== null ? { feeBps, usdCentsPerCredit } : null;
|
|
24839
|
+
const costBased = full !== null && costP95 !== null ? ceilingCreditsFor(costP95, full) : null;
|
|
24840
|
+
const recommended = costBased === null || noRecommendationReason !== null ? null : Math.max(costBased, fitP95 ?? 0);
|
|
24841
|
+
const ceiling = recommended === null || costMax === null || full === null ? null : Math.max(ceilingCreditsFor(costMax, full), recommended, fitMax ?? 0);
|
|
24842
|
+
const average = billing !== null ? averageCreditsPerCall(costs, billing) : null;
|
|
24843
|
+
const estimate = ceiling === null || billing === null ? null : Math.min(ceiling, perCallEstimateCreditsFor(costs, billing) ?? 1);
|
|
24666
24844
|
return {
|
|
24845
|
+
costs,
|
|
24667
24846
|
charged,
|
|
24668
|
-
|
|
24669
|
-
|
|
24670
|
-
|
|
24847
|
+
costP50: percentilePicos(costs, 50),
|
|
24848
|
+
costP95,
|
|
24849
|
+
costMax,
|
|
24850
|
+
chargedP50: percentile2(charged, 50),
|
|
24851
|
+
chargedP95: percentile2(charged, 95),
|
|
24852
|
+
chargedMax: charged.length ? Math.max(...charged) : null,
|
|
24671
24853
|
fits,
|
|
24672
24854
|
fitP95,
|
|
24673
24855
|
fitMax,
|
|
24674
|
-
|
|
24675
|
-
|
|
24676
|
-
|
|
24677
|
-
|
|
24856
|
+
costBasedMaxCredits: costBased,
|
|
24857
|
+
recommendedMaxCredits: recommended,
|
|
24858
|
+
recommendedPerCallMaxCredits: ceiling,
|
|
24859
|
+
averageCreditsPerCall: average,
|
|
24860
|
+
recommendedPerCallEstimateCredits: estimate,
|
|
24861
|
+
lengthRaisedCeiling: recommended !== null && costBased !== null && recommended > costBased,
|
|
24678
24862
|
cutOffSamples,
|
|
24679
24863
|
exceedsStandCap,
|
|
24680
|
-
noRecommendationReason
|
|
24864
|
+
noRecommendationReason,
|
|
24865
|
+
missingTerms
|
|
24681
24866
|
};
|
|
24682
24867
|
}
|
|
24683
|
-
var LLM_LENGTH_RAISED_LINE = "The
|
|
24684
|
-
var LLM_EXCEEDS_STAND_CAP_LINE = "No
|
|
24685
|
-
function benchCutOffLine(cutOff,
|
|
24686
|
-
return `No
|
|
24868
|
+
var LLM_LENGTH_RAISED_LINE = "The ceiling also decides how long the answer may be: at the cost-based ceiling the answer would be cut off, so declare this one.";
|
|
24869
|
+
var LLM_EXCEEDS_STAND_CAP_LINE = "No ceiling recommended: this answer is longer than one call on this stand may produce. Ask for a shorter answer \u2014 fewer fields, shorter strings, a length the prompt states \u2014 and benchmark again.";
|
|
24870
|
+
function benchCutOffLine(cutOff, maxCredits) {
|
|
24871
|
+
return `No ceiling recommended: ${cutOff} sample${cutOff === 1 ? " was" : "s were"} cut off at this ceiling (--max-credits ${maxCredits}) before the answer was finished, so its real length is unknown. Re-run with a higher --max-credits.`;
|
|
24687
24872
|
}
|
|
24688
24873
|
var ACTIVE_STATUSES = /* @__PURE__ */ new Set(["requires_confirmation", "queued", "dispatching", "awaiting_external"]);
|
|
24689
24874
|
function rowState(row) {
|
|
@@ -24694,7 +24879,22 @@ function rowState(row) {
|
|
|
24694
24879
|
function rowHoldsSlot(row) {
|
|
24695
24880
|
if (typeof row.slotHeld === "boolean") return row.slotHeld;
|
|
24696
24881
|
const state = rowState(row);
|
|
24697
|
-
return state === "active" || state === "awaiting bill" && (row.reservedCoins ?? 0) > 0;
|
|
24882
|
+
return state === "active" || state === "awaiting bill" && ((row.reservedCoins ?? 0) > 0 || (row.reservedCredits ?? 0) > 0);
|
|
24883
|
+
}
|
|
24884
|
+
function rowMoney(row) {
|
|
24885
|
+
const credits = row.currency === "credits" || row.currency === void 0 && typeof row.reservedCredits === "number" && typeof row.reservedCoins !== "number";
|
|
24886
|
+
if (credits) {
|
|
24887
|
+
return {
|
|
24888
|
+
unit: "credits",
|
|
24889
|
+
reserved: typeof row.reservedCredits === "number" ? row.reservedCredits : null,
|
|
24890
|
+
charged: typeof row.chargedCredits === "number" ? row.chargedCredits : null
|
|
24891
|
+
};
|
|
24892
|
+
}
|
|
24893
|
+
return {
|
|
24894
|
+
unit: "coin",
|
|
24895
|
+
reserved: typeof row.reservedCoins === "number" ? row.reservedCoins : null,
|
|
24896
|
+
charged: typeof row.chargedCoins === "number" ? row.chargedCoins : null
|
|
24897
|
+
};
|
|
24698
24898
|
}
|
|
24699
24899
|
function providerRefusalStatus(error) {
|
|
24700
24900
|
const m = /^provider_http_(\d{3})$/.exec(error ?? "");
|
|
@@ -24713,7 +24913,7 @@ function formatAge(iso, now = Date.now()) {
|
|
|
24713
24913
|
}
|
|
24714
24914
|
async function runBench(args) {
|
|
24715
24915
|
const { apiUrl, token, projectId, cwd, opts, log } = args;
|
|
24716
|
-
const
|
|
24916
|
+
const maxCredits = opts.maxCredits ?? opts.maxCoins;
|
|
24717
24917
|
const samples = opts.samples ?? DEFAULT_BENCH_SAMPLES;
|
|
24718
24918
|
const prompt = opts.benchPrompt.trim();
|
|
24719
24919
|
const auth = { Authorization: `Bearer ${token}`, "Content-Type": "application/json" };
|
|
@@ -24735,7 +24935,7 @@ async function runBench(args) {
|
|
|
24735
24935
|
const lane = await readRuntimeLane(apiUrl, token);
|
|
24736
24936
|
if (lane.state !== "live" || !lane.models || lane.models.length === 0) {
|
|
24737
24937
|
if (lane.state === "off") {
|
|
24738
|
-
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
24938
|
+
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCredits, opts), status: "off", error: null });
|
|
24739
24939
|
else {
|
|
24740
24940
|
log.plain(LLM_LANE_OFF_LINE);
|
|
24741
24941
|
log.dim(" Nothing was spent.");
|
|
@@ -24762,17 +24962,27 @@ async function runBench(args) {
|
|
|
24762
24962
|
);
|
|
24763
24963
|
return;
|
|
24764
24964
|
}
|
|
24765
|
-
const
|
|
24965
|
+
const walletBefore = await fetchCreditsSnapshot(apiUrl, token);
|
|
24966
|
+
if (walletBefore?.emailVerified === false) {
|
|
24967
|
+
benchFailed(opts, log, LLM_BENCH_UNVERIFIED_LINE);
|
|
24968
|
+
return;
|
|
24969
|
+
}
|
|
24970
|
+
if (walletBefore !== null && walletBefore.spendable < maxCredits) {
|
|
24971
|
+
benchFailed(opts, log, benchBalanceShortLine(walletBefore.spendable, maxCredits));
|
|
24972
|
+
return;
|
|
24973
|
+
}
|
|
24974
|
+
const balanceBefore = walletBefore?.spendable ?? null;
|
|
24766
24975
|
if (!opts.json) {
|
|
24767
24976
|
log.plain(c.bold("genex llm bench"));
|
|
24768
24977
|
log.dim(` ${apiUrl}`);
|
|
24769
24978
|
log.plain("");
|
|
24770
24979
|
log.plain(` Model ${c.cyan(modelId)}`);
|
|
24771
24980
|
log.plain(` Samples ${samples} real attempt${samples === 1 ? "" : "s"}, ${outputFormat} output`);
|
|
24772
|
-
log.plain(` Approved ${
|
|
24981
|
+
log.plain(` Approved ${maxCredits} credit${maxCredits === 1 ? "" : "s"} per attempt \u2014 at worst ${maxCredits * samples} credits for this run, from your own balance`);
|
|
24773
24982
|
log.plain(
|
|
24774
|
-
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore}
|
|
24983
|
+
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore} credits spendable`}`
|
|
24775
24984
|
);
|
|
24985
|
+
log.dim(" Billed as used: each attempt is charged its real cost plus the platform fee, rounded up to a whole credit.");
|
|
24776
24986
|
log.plain("");
|
|
24777
24987
|
}
|
|
24778
24988
|
const base = `${apiUrl}/api/runtime/development/projects/${encodeURIComponent(projectId)}/generations`;
|
|
@@ -24798,7 +25008,7 @@ async function runBench(args) {
|
|
|
24798
25008
|
const res = await call2(base, {
|
|
24799
25009
|
method: "POST",
|
|
24800
25010
|
body: JSON.stringify({
|
|
24801
|
-
|
|
25011
|
+
maxCredits,
|
|
24802
25012
|
request: {
|
|
24803
25013
|
idempotencyKey: `bench-${randomUUID2()}`,
|
|
24804
25014
|
modelId,
|
|
@@ -24828,13 +25038,14 @@ async function runBench(args) {
|
|
|
24828
25038
|
if (refusedAt !== null) providerRefused++;
|
|
24829
25039
|
if (!opts.json) {
|
|
24830
25040
|
const length = row.outputTokens !== null ? ` \xB7 ${row.outputTokens}${row.maxOutputTokens !== null ? ` of ${row.maxOutputTokens}` : ""} tokens out` : "";
|
|
25041
|
+
const charged = row.chargedCredits === null ? "\u2014" : String(row.chargedCredits);
|
|
24831
25042
|
log.plain(
|
|
24832
|
-
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${
|
|
25043
|
+
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${charged.padStart(4)} credit${row.chargedCredits === 1 ? "" : "s"} charged \xB7 cost ${usdFromPicos(row.costUsdPicos)}${length}${row.error ? ` \xB7 ${row.error}` : ""}`
|
|
24833
25044
|
);
|
|
24834
25045
|
if (refusedAt !== null) printProviderRefusal(log, refusedAt, row.providerMessage, row.status === "unknown");
|
|
24835
25046
|
if (row.error === PROVIDER_TOKEN_LIMIT) {
|
|
24836
25047
|
log.dim(
|
|
24837
|
-
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no
|
|
25048
|
+
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no ceiling makes room for it; it is not a sample." : ` The answer was cut off at this ceiling (--max-credits ${maxCredits}) before it was finished \u2014 it was charged, and it is not a sample.`
|
|
24838
25049
|
);
|
|
24839
25050
|
}
|
|
24840
25051
|
}
|
|
@@ -24842,16 +25053,19 @@ async function runBench(args) {
|
|
|
24842
25053
|
} finally {
|
|
24843
25054
|
process.removeListener("SIGINT", onSigint);
|
|
24844
25055
|
}
|
|
24845
|
-
const
|
|
24846
|
-
|
|
24847
|
-
|
|
24848
|
-
|
|
24849
|
-
|
|
24850
|
-
const
|
|
25056
|
+
const terms = {
|
|
25057
|
+
headroomBps: lane.recommendedDeclaredHeadroomBps,
|
|
25058
|
+
feeBps: lane.feeBps,
|
|
25059
|
+
usdCentsPerCredit: lane.usdCentsPerCredit
|
|
25060
|
+
};
|
|
25061
|
+
const rec = benchRecommendation(results, terms);
|
|
25062
|
+
const settledCount = rec.charged.length;
|
|
24851
25063
|
const lengthBlocked = rec.noRecommendationReason !== null;
|
|
24852
|
-
const balanceAfter = await
|
|
25064
|
+
const balanceAfter = (await fetchCreditsSnapshot(apiUrl, token))?.spendable ?? null;
|
|
25065
|
+
const picosOrNull = (v) => v === null ? null : String(v);
|
|
24853
25066
|
const record = {
|
|
24854
|
-
v:
|
|
25067
|
+
v: 2,
|
|
25068
|
+
currency: "credits",
|
|
24855
25069
|
ranAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
24856
25070
|
apiUrl,
|
|
24857
25071
|
projectId,
|
|
@@ -24860,28 +25074,34 @@ async function runBench(args) {
|
|
|
24860
25074
|
outputFormat,
|
|
24861
25075
|
samples,
|
|
24862
25076
|
completed: settledCount,
|
|
24863
|
-
|
|
24864
|
-
|
|
24865
|
-
|
|
24866
|
-
max,
|
|
24867
|
-
|
|
24868
|
-
|
|
24869
|
-
recommendedPerCallMaxCoins: ceiling,
|
|
24870
|
-
maxCoinsPerSample: maxCoins,
|
|
24871
|
-
fitCoins: rec.fits,
|
|
25077
|
+
maxCreditsPerSample: maxCredits,
|
|
25078
|
+
costUsdPicos: rec.costs.map(String),
|
|
25079
|
+
chargedCredits: rec.charged,
|
|
25080
|
+
cost: { p50: picosOrNull(rec.costP50), p95: picosOrNull(rec.costP95), max: picosOrNull(rec.costMax) },
|
|
25081
|
+
charged: { p50: rec.chargedP50, p95: rec.chargedP95, max: rec.chargedMax },
|
|
25082
|
+
fitCredits: rec.fits,
|
|
24872
25083
|
fitP95: rec.fitP95,
|
|
24873
|
-
|
|
24874
|
-
|
|
25084
|
+
fitMax: rec.fitMax,
|
|
25085
|
+
recommendedDeclaredHeadroomBps: terms.headroomBps,
|
|
25086
|
+
feeBps: terms.feeBps,
|
|
25087
|
+
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
25088
|
+
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
25089
|
+
recommendedMaxCredits: rec.recommendedMaxCredits,
|
|
25090
|
+
recommendedPerCallMaxCredits: rec.recommendedPerCallMaxCredits,
|
|
25091
|
+
averageCreditsPerCall: rec.averageCreditsPerCall,
|
|
25092
|
+
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
25093
|
+
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
24875
25094
|
cutOffSamples: rec.cutOffSamples,
|
|
24876
25095
|
noRecommendationReason: rec.noRecommendationReason
|
|
24877
25096
|
};
|
|
24878
25097
|
const savedTo = await saveBench(cwd, record);
|
|
24879
25098
|
if (opts.json) {
|
|
24880
|
-
const ok =
|
|
25099
|
+
const ok = settledCount > 0 && !lengthBlocked;
|
|
24881
25100
|
writeJsonLine({
|
|
24882
25101
|
command: "llm bench",
|
|
24883
25102
|
status: ok ? "ok" : "failed",
|
|
24884
25103
|
error: ok ? null : rec.noRecommendationReason ?? "no_sample_settled",
|
|
25104
|
+
currency: "credits",
|
|
24885
25105
|
apiUrl,
|
|
24886
25106
|
projectId,
|
|
24887
25107
|
modelId,
|
|
@@ -24890,35 +25110,54 @@ async function runBench(args) {
|
|
|
24890
25110
|
schemaPath: opts.schemaPath ?? null,
|
|
24891
25111
|
samples,
|
|
24892
25112
|
completed: settledCount,
|
|
24893
|
-
|
|
25113
|
+
maxCreditsPerSample: maxCredits,
|
|
25114
|
+
// Spendable credits before and after the run; null when unreadable.
|
|
24894
25115
|
balanceBefore,
|
|
24895
25116
|
balanceAfter,
|
|
24896
25117
|
results,
|
|
24897
25118
|
// The create-time refusal that ended the run, or null. Beside the
|
|
24898
25119
|
// samples, never among them: no attempt existed and nothing was spent.
|
|
24899
25120
|
refusal: refusal2,
|
|
24900
|
-
|
|
24901
|
-
//
|
|
25121
|
+
// Over the settled samples: the real provider cost (pico-dollar strings
|
|
25122
|
+
// and USD) and the whole credits charged.
|
|
25123
|
+
costUsdPicos: record.cost,
|
|
25124
|
+
costUsd: {
|
|
25125
|
+
p50: rec.costP50 === null ? null : Number(rec.costP50) / 1e12,
|
|
25126
|
+
p95: rec.costP95 === null ? null : Number(rec.costP95) / 1e12,
|
|
25127
|
+
max: rec.costMax === null ? null : Number(rec.costMax) / 1e12
|
|
25128
|
+
},
|
|
25129
|
+
chargedCredits: record.charged,
|
|
25130
|
+
// The answer's LENGTH, priced by the server per sample (`fitCredits`,
|
|
24902
25131
|
// headroom already on the tokens), over the settled samples.
|
|
24903
|
-
|
|
24904
|
-
|
|
24905
|
-
|
|
25132
|
+
fitCredits: { p95: rec.fitP95, max: rec.fitMax },
|
|
25133
|
+
recommendedDeclaredHeadroomBps: terms.headroomBps,
|
|
25134
|
+
feeBps: terms.feeBps,
|
|
25135
|
+
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
25136
|
+
missingTerms: rec.missingTerms,
|
|
25137
|
+
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
25138
|
+
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
24906
25139
|
cutOffSamples: rec.cutOffSamples,
|
|
24907
25140
|
exceedsStandCap: rec.exceedsStandCap,
|
|
24908
25141
|
noRecommendationReason: rec.noRecommendationReason,
|
|
24909
|
-
|
|
24910
|
-
|
|
24911
|
-
|
|
25142
|
+
recommendedMaxCredits: rec.recommendedMaxCredits,
|
|
25143
|
+
recommendedPerCallMaxCredits: rec.recommendedPerCallMaxCredits,
|
|
25144
|
+
averageCreditsPerCall: rec.averageCreditsPerCall,
|
|
25145
|
+
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
24912
25146
|
savedTo
|
|
24913
25147
|
});
|
|
24914
25148
|
if (!ok) process.exitCode = 1;
|
|
24915
25149
|
return;
|
|
24916
25150
|
}
|
|
24917
25151
|
log.plain("");
|
|
25152
|
+
const printMeasured = () => {
|
|
25153
|
+
log.plain(c.bold(" Real cost per call"));
|
|
25154
|
+
log.plain(` p50 ${usdFromPicos(rec.costP50)} p95 ${usdFromPicos(rec.costP95)} max ${usdFromPicos(rec.costMax)} (${settledCount} of ${samples} settled)`);
|
|
25155
|
+
log.plain(c.bold(" Charged credits"));
|
|
25156
|
+
log.plain(` p50 ${rec.chargedP50} p95 ${rec.chargedP95} max ${rec.chargedMax} (whole credits, rounded up per call)`);
|
|
25157
|
+
};
|
|
24918
25158
|
if (lengthBlocked) {
|
|
24919
|
-
if (
|
|
24920
|
-
|
|
24921
|
-
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
25159
|
+
if (settledCount > 0) {
|
|
25160
|
+
printMeasured();
|
|
24922
25161
|
log.plain("");
|
|
24923
25162
|
}
|
|
24924
25163
|
printRecommendation(log, record);
|
|
@@ -24926,17 +25165,16 @@ async function runBench(args) {
|
|
|
24926
25165
|
process.exitCode = 1;
|
|
24927
25166
|
return;
|
|
24928
25167
|
}
|
|
24929
|
-
if (
|
|
25168
|
+
if (settledCount === 0) {
|
|
24930
25169
|
log.error(
|
|
24931
|
-
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to
|
|
25170
|
+
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to measure from." : " No sample settled, so there is nothing to measure from."
|
|
24932
25171
|
);
|
|
24933
25172
|
process.exitCode = 1;
|
|
24934
25173
|
return;
|
|
24935
25174
|
}
|
|
24936
|
-
|
|
24937
|
-
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
25175
|
+
printMeasured();
|
|
24938
25176
|
if (balanceBefore !== null && balanceAfter !== null) {
|
|
24939
|
-
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter}
|
|
25177
|
+
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter} credits spendable`);
|
|
24940
25178
|
}
|
|
24941
25179
|
log.plain("");
|
|
24942
25180
|
printRecommendation(log, record);
|
|
@@ -24948,43 +25186,58 @@ function printRecommendation(log, record) {
|
|
|
24948
25186
|
return;
|
|
24949
25187
|
}
|
|
24950
25188
|
if (record.noRecommendationReason === "answer_cut_off") {
|
|
24951
|
-
log.warn(` ${benchCutOffLine(record.cutOffSamples
|
|
25189
|
+
log.warn(` ${benchCutOffLine(record.cutOffSamples || 1, record.maxCreditsPerSample)}`);
|
|
24952
25190
|
return;
|
|
24953
25191
|
}
|
|
24954
|
-
|
|
24955
|
-
|
|
25192
|
+
const attempts = `${record.completed} settled attempt${record.completed === 1 ? "" : "s"} on ${record.modelId}`;
|
|
25193
|
+
if (record.recommendedMaxCredits === null || record.cost.p95 === null) {
|
|
25194
|
+
log.warn(" No ceiling recommended.");
|
|
24956
25195
|
log.dim(
|
|
24957
|
-
record.recommendedDeclaredHeadroomBps === null ? " This stand served no recommended headroom, and the multiplier is the server's to set \u2014" : "
|
|
25196
|
+
record.cost.p95 === null ? " No sample settled, so there is no p95 to build a ceiling on \u2014" : record.recommendedDeclaredHeadroomBps === null ? " This stand served no recommended headroom, and the multiplier is the server's to set \u2014" : " This stand served no platform fee or credit value, and those are the server's to set \u2014"
|
|
24958
25197
|
);
|
|
24959
25198
|
log.dim(" declaring a number from this run would be a guess dressed as a measurement.");
|
|
25199
|
+
printAverage(log, record, attempts);
|
|
24960
25200
|
return;
|
|
24961
25201
|
}
|
|
24962
|
-
log.plain(c.bold(` Declare
|
|
24963
|
-
if (record.
|
|
25202
|
+
log.plain(c.bold(` Declare maxCredits: ${record.recommendedMaxCredits}`));
|
|
25203
|
+
if (record.lengthRaisedCeiling) {
|
|
24964
25204
|
log.plain(` ${LLM_LENGTH_RAISED_LINE}`);
|
|
24965
25205
|
log.dim(
|
|
24966
|
-
` = the smallest
|
|
25206
|
+
` = the smallest ceiling whose answer allowance holds the answer plus this stand's headroom (p95 ${record.fitP95 ?? "\u2014"}), above the`
|
|
24967
25207
|
);
|
|
24968
25208
|
log.dim(
|
|
24969
|
-
`
|
|
25209
|
+
` cost-based ${record.costBasedMaxCredits ?? "\u2014"} (p95 real cost ${usdFromPicos(record.cost.p95)} with the platform fee, ${record.feeBps} bps, and ${record.recommendedDeclaredHeadroomBps} bps of headroom), over ${attempts}.`
|
|
24970
25210
|
);
|
|
24971
25211
|
} else {
|
|
24972
25212
|
log.dim(
|
|
24973
|
-
` = p95
|
|
25213
|
+
` = p95 real cost (${usdFromPicos(record.cost.p95)}) with this stand's platform fee (${record.feeBps} bps) and recommended headroom (${record.recommendedDeclaredHeadroomBps} bps), in whole credits,`
|
|
24974
25214
|
);
|
|
24975
|
-
log.dim(` measured over ${
|
|
25215
|
+
log.dim(` measured over ${attempts}.`);
|
|
24976
25216
|
}
|
|
24977
|
-
log.dim(" That number is a
|
|
24978
|
-
|
|
24979
|
-
|
|
25217
|
+
log.dim(" That number is a CEILING, not a price: a call is billed its real cost as used, never more than it \u2014");
|
|
25218
|
+
log.dim(" and it also sizes how long the answer may be.");
|
|
25219
|
+
if (record.recommendedPerCallMaxCredits !== null) {
|
|
25220
|
+
log.plain(c.bold(` Grant perCallMaxCredits: ${record.recommendedPerCallMaxCredits}`));
|
|
24980
25221
|
log.dim(
|
|
24981
|
-
record.
|
|
25222
|
+
record.lengthRaisedCeiling ? ` = the same over the worst sample (${usdFromPicos(record.cost.max)}) or the longest answer's ceiling, whichever is larger, and never below the ceiling above it.` : ` = the same over the worst sample (${usdFromPicos(record.cost.max)}), and never below the ceiling above it.`
|
|
24982
25223
|
);
|
|
24983
|
-
|
|
25224
|
+
}
|
|
25225
|
+
printAverage(log, record, attempts);
|
|
25226
|
+
}
|
|
25227
|
+
function printAverage(log, record, attempts) {
|
|
25228
|
+
if (record.averageCreditsPerCall === null) return;
|
|
25229
|
+
log.plain(` Per-call estimate: about ${formatCredits(record.averageCreditsPerCall)} credits per call`);
|
|
25230
|
+
log.dim(` = the average real cost with the platform fee, no headroom, over ${attempts}.`);
|
|
25231
|
+
if (record.recommendedPerCallEstimateCredits !== null) {
|
|
25232
|
+
log.plain(c.bold(` Grant perCallEstimateCredits: ${record.recommendedPerCallEstimateCredits}`));
|
|
25233
|
+
log.dim(` = that average rounded up to a whole credit. disclosure.estimatedCreditsPerPeriod = your calls per period \xD7 ${formatCredits(record.averageCreditsPerCall)}, rounded up.`);
|
|
24984
25234
|
}
|
|
24985
25235
|
}
|
|
24986
25236
|
async function explainBenchRefusal(res, call2, base, log, opts, index, settledSoFar) {
|
|
24987
|
-
if (printedStructuredError(res))
|
|
25237
|
+
if (printedStructuredError(res)) {
|
|
25238
|
+
const printed = await res.json().catch(() => ({}));
|
|
25239
|
+
return { stop: true, refusal: { status: res.status, error: typeof printed.error === "string" ? printed.error : null, slotsHeld: null, slotLimit: null } };
|
|
25240
|
+
}
|
|
24988
25241
|
const body = await res.json().catch(() => ({}));
|
|
24989
25242
|
const refusal2 = { status: res.status, error: body.error ?? null, slotsHeld: null, slotLimit: null };
|
|
24990
25243
|
if (res.status === 429 && body.error === "generation_limit") {
|
|
@@ -24994,6 +25247,10 @@ async function explainBenchRefusal(res, call2, base, log, opts, index, settledSo
|
|
|
24994
25247
|
if (!opts.json) log.error(` Sample ${index + 1} ${benchSlotsHeldSentence(counts)}`);
|
|
24995
25248
|
return { stop: true, refusal: refusal2 };
|
|
24996
25249
|
}
|
|
25250
|
+
if (res.status === 403 && body.error === "credits_unverified") {
|
|
25251
|
+
if (!opts.json) log.error(` ${LLM_BENCH_UNVERIFIED_LINE}`);
|
|
25252
|
+
return { stop: true, refusal: refusal2 };
|
|
25253
|
+
}
|
|
24997
25254
|
if (opts.json) return { stop: res.status !== 429, refusal: refusal2 };
|
|
24998
25255
|
if (res.status === 404 && body.error === "not_found") {
|
|
24999
25256
|
log.error(` ${LLM_LANE_OFF_LINE}`);
|
|
@@ -25009,7 +25266,7 @@ async function explainBenchRefusal(res, call2, base, log, opts, index, settledSo
|
|
|
25009
25266
|
return { stop: true, refusal: refusal2 };
|
|
25010
25267
|
}
|
|
25011
25268
|
if (res.status === 402) {
|
|
25012
|
-
log.error(" Not enough
|
|
25269
|
+
log.error(" Not enough credits to start the attempt \u2014 a bench is paid from your own balance.");
|
|
25013
25270
|
return { stop: true, refusal: refusal2 };
|
|
25014
25271
|
}
|
|
25015
25272
|
log.error(
|
|
@@ -25018,7 +25275,7 @@ async function explainBenchRefusal(res, call2, base, log, opts, index, settledSo
|
|
|
25018
25275
|
return { stop: res.status >= 500 || res.status === 401 || res.status === 403, refusal: refusal2 };
|
|
25019
25276
|
}
|
|
25020
25277
|
function printProviderRefusal(log, status2, message, awaitingBill = false) {
|
|
25021
|
-
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so this call
|
|
25278
|
+
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so what this call reserved stays held until it resolves (see \`genex llm status\`); it is not a sample.`);
|
|
25022
25279
|
else log.dim(" The provider refused this call at its door \u2014 no inference ran, it cost nothing, and it is not a sample.");
|
|
25023
25280
|
if (message) log.dim(` Provider said: ${message}`);
|
|
25024
25281
|
if (status2 === 401 || status2 === 403) {
|
|
@@ -25044,19 +25301,6 @@ async function pollSettled(call2, url, timeoutSec) {
|
|
|
25044
25301
|
}
|
|
25045
25302
|
return last;
|
|
25046
25303
|
}
|
|
25047
|
-
async function readCoinBalance(apiUrl, token) {
|
|
25048
|
-
try {
|
|
25049
|
-
const res = await apiFetch(`${apiUrl}/api/coin/balance`, {
|
|
25050
|
-
headers: { Authorization: `Bearer ${token}` },
|
|
25051
|
-
signal: AbortSignal.timeout(6e3)
|
|
25052
|
-
});
|
|
25053
|
-
if (!res.ok) return null;
|
|
25054
|
-
const body = await res.json().catch(() => null);
|
|
25055
|
-
return typeof body?.spendable === "number" ? body.spendable : null;
|
|
25056
|
-
} catch {
|
|
25057
|
-
return null;
|
|
25058
|
-
}
|
|
25059
|
-
}
|
|
25060
25304
|
async function saveBench(cwd, record) {
|
|
25061
25305
|
try {
|
|
25062
25306
|
const file = path36.join(cwd, LLM_BENCH_FILE);
|
|
@@ -25072,9 +25316,10 @@ function benchFailed(opts, log, message) {
|
|
|
25072
25316
|
else log.error(message);
|
|
25073
25317
|
process.exitCode = 1;
|
|
25074
25318
|
}
|
|
25075
|
-
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
25319
|
+
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCredits, opts) {
|
|
25076
25320
|
return {
|
|
25077
25321
|
command: "llm bench",
|
|
25322
|
+
currency: "credits",
|
|
25078
25323
|
apiUrl,
|
|
25079
25324
|
projectId,
|
|
25080
25325
|
modelId: opts.modelId ?? null,
|
|
@@ -25083,21 +25328,28 @@ function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCoi
|
|
|
25083
25328
|
schemaPath: opts.schemaPath ?? null,
|
|
25084
25329
|
samples,
|
|
25085
25330
|
completed: 0,
|
|
25086
|
-
|
|
25331
|
+
maxCreditsPerSample: maxCredits,
|
|
25087
25332
|
balanceBefore: null,
|
|
25088
25333
|
balanceAfter: null,
|
|
25089
25334
|
results: [],
|
|
25090
25335
|
refusal: null,
|
|
25091
|
-
|
|
25092
|
-
|
|
25093
|
-
|
|
25094
|
-
|
|
25336
|
+
costUsdPicos: { p50: null, p95: null, max: null },
|
|
25337
|
+
costUsd: { p50: null, p95: null, max: null },
|
|
25338
|
+
chargedCredits: { p50: null, p95: null, max: null },
|
|
25339
|
+
fitCredits: { p95: null, max: null },
|
|
25340
|
+
recommendedDeclaredHeadroomBps: null,
|
|
25341
|
+
feeBps: null,
|
|
25342
|
+
usdCentsPerCredit: null,
|
|
25343
|
+
missingTerms: null,
|
|
25344
|
+
costBasedMaxCredits: null,
|
|
25345
|
+
lengthRaisedCeiling: false,
|
|
25095
25346
|
cutOffSamples: 0,
|
|
25096
25347
|
exceedsStandCap: false,
|
|
25097
25348
|
noRecommendationReason: null,
|
|
25098
|
-
|
|
25099
|
-
|
|
25100
|
-
|
|
25349
|
+
recommendedMaxCredits: null,
|
|
25350
|
+
recommendedPerCallMaxCredits: null,
|
|
25351
|
+
averageCreditsPerCall: null,
|
|
25352
|
+
recommendedPerCallEstimateCredits: null,
|
|
25101
25353
|
savedTo: null
|
|
25102
25354
|
};
|
|
25103
25355
|
}
|
|
@@ -25160,7 +25412,8 @@ async function reportStatus(args) {
|
|
|
25160
25412
|
for (const row of rows) {
|
|
25161
25413
|
const state = rowState(row);
|
|
25162
25414
|
const mark = state === "active" ? c.cyan("\u25CF") : state === "awaiting bill" ? c.yellow("\u25D0") : c.green("\u25CB");
|
|
25163
|
-
const
|
|
25415
|
+
const money2 = rowMoney(row);
|
|
25416
|
+
const reserved = money2.reserved === null ? "reserved \u2014" : `${money2.reserved} ${money2.unit} reserved`;
|
|
25164
25417
|
log.plain(
|
|
25165
25418
|
` ${mark} ${row.id ?? "?"} ${row.modelId ?? "?"} ${state.padEnd(13)} ${formatAge(row.createdAt, now).padStart(4)} old ${reserved}${row.error ? ` ${row.error}` : ""}${rowHoldsSlot(row) ? "" : c.dim(" (no slot)")}`
|
|
25166
25419
|
);
|
|
@@ -25232,10 +25485,11 @@ async function cancelCall(args) {
|
|
|
25232
25485
|
if (!opts.json) {
|
|
25233
25486
|
log.warn(` ${row.id} (${row.status ?? "?"}) \u2014 ${sentence}.`);
|
|
25234
25487
|
if (state === "awaiting bill") {
|
|
25235
|
-
log.dim(` Nothing to cancel:
|
|
25488
|
+
log.dim(` Nothing to cancel: what it reserved releases when the bill resolves, and it stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`);
|
|
25236
25489
|
if (row.providerMessage?.trim()) log.dim(` Provider said: ${row.providerMessage.trim()}`);
|
|
25237
25490
|
} else {
|
|
25238
|
-
|
|
25491
|
+
const money2 = rowMoney(row);
|
|
25492
|
+
log.dim(` ${money2.charged === null ? "charge unknown" : `${money2.charged} ${money2.unit} charged`}${row.error ? ` \xB7 ${row.error}` : ""}.`);
|
|
25239
25493
|
}
|
|
25240
25494
|
}
|
|
25241
25495
|
return done("ok", null, state === "awaiting bill" ? "awaiting_bill" : "already_final", row);
|
|
@@ -25257,78 +25511,126 @@ async function cancelCall(args) {
|
|
|
25257
25511
|
const after = await res.json().catch(() => null) ?? row;
|
|
25258
25512
|
if (!opts.json) {
|
|
25259
25513
|
log.success(` Cancelled ${row.id} (was ${row.status ?? "active"}).`);
|
|
25260
|
-
log.dim(`
|
|
25514
|
+
log.dim(` What it reserved releases once the bill settles; ${c.cyan("genex llm status")} shows it until then.`);
|
|
25261
25515
|
}
|
|
25262
25516
|
return done("ok", null, "cancelled", after);
|
|
25263
25517
|
}
|
|
25264
|
-
async function
|
|
25265
|
-
let
|
|
25518
|
+
async function readSavedBench(cwd) {
|
|
25519
|
+
let parsed;
|
|
25266
25520
|
try {
|
|
25267
|
-
|
|
25521
|
+
parsed = JSON.parse(await fs35.readFile(path36.join(cwd, LLM_BENCH_FILE), "utf8"));
|
|
25268
25522
|
} catch {
|
|
25269
|
-
|
|
25523
|
+
return null;
|
|
25270
25524
|
}
|
|
25271
|
-
if (!
|
|
25525
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null;
|
|
25526
|
+
const v = parsed;
|
|
25527
|
+
if (v.v === 2 && v.currency === "credits" && v.cost && v.charged) return { kind: "credits", record: parsed };
|
|
25528
|
+
return { kind: "coins", record: parsed };
|
|
25529
|
+
}
|
|
25530
|
+
var LLM_PRICE_COIN_TERMS_LINE = "This bench was measured under the old COIN terms, when a game declared a fixed price per call. In-game calls are now billed in the player's credits, as used, and a game declares a per-call CEILING (maxCredits) instead \u2014 a coin price is not that number. Re-run the bench for the figures to declare.";
|
|
25531
|
+
async function reportSavedPrice(cwd, opts, log) {
|
|
25532
|
+
const saved = await readSavedBench(cwd);
|
|
25533
|
+
const rerun = 'genex llm bench "<prompt>" --max-credits <n> --user-approved';
|
|
25534
|
+
const creditKeys = (record2) => ({
|
|
25535
|
+
costUsdPicos: record2?.cost ?? { p50: null, p95: null, max: null },
|
|
25536
|
+
chargedCredits: record2?.charged ?? { p50: null, p95: null, max: null },
|
|
25537
|
+
fitP95: record2?.fitP95 ?? null,
|
|
25538
|
+
fitMax: record2?.fitMax ?? null,
|
|
25539
|
+
recommendedDeclaredHeadroomBps: record2?.recommendedDeclaredHeadroomBps ?? null,
|
|
25540
|
+
feeBps: record2?.feeBps ?? null,
|
|
25541
|
+
usdCentsPerCredit: record2?.usdCentsPerCredit ?? null,
|
|
25542
|
+
costBasedMaxCredits: record2?.costBasedMaxCredits ?? null,
|
|
25543
|
+
lengthRaisedCeiling: record2?.lengthRaisedCeiling ?? false,
|
|
25544
|
+
cutOffSamples: record2?.cutOffSamples ?? 0,
|
|
25545
|
+
noRecommendationReason: record2?.noRecommendationReason ?? null,
|
|
25546
|
+
recommendedMaxCredits: record2?.recommendedMaxCredits ?? null,
|
|
25547
|
+
recommendedPerCallMaxCredits: record2?.recommendedPerCallMaxCredits ?? null,
|
|
25548
|
+
averageCreditsPerCall: record2?.averageCreditsPerCall ?? null,
|
|
25549
|
+
recommendedPerCallEstimateCredits: record2?.recommendedPerCallEstimateCredits ?? null
|
|
25550
|
+
});
|
|
25551
|
+
if (!saved) {
|
|
25272
25552
|
if (opts.json) {
|
|
25273
25553
|
writeJsonLine({
|
|
25274
25554
|
command: "llm price",
|
|
25275
25555
|
status: "failed",
|
|
25276
25556
|
error: "no_bench",
|
|
25557
|
+
currency: null,
|
|
25277
25558
|
ranAt: null,
|
|
25278
25559
|
modelId: null,
|
|
25279
25560
|
samples: null,
|
|
25280
25561
|
completed: null,
|
|
25281
|
-
|
|
25282
|
-
|
|
25283
|
-
max: null,
|
|
25284
|
-
fitP95: null,
|
|
25285
|
-
chargedBasedEstimateCoins: null,
|
|
25286
|
-
lengthRaisedPrice: false,
|
|
25287
|
-
cutOffSamples: 0,
|
|
25288
|
-
noRecommendationReason: null,
|
|
25289
|
-
recommendedDeclaredHeadroomBps: null,
|
|
25290
|
-
recommendedEstimateCoins: null,
|
|
25291
|
-
recommendedPerCallMaxCoins: null,
|
|
25562
|
+
...creditKeys(null),
|
|
25563
|
+
legacyCoinTerms: null,
|
|
25292
25564
|
savedTo: null
|
|
25293
25565
|
});
|
|
25294
25566
|
} else {
|
|
25295
25567
|
log.error("Nothing has been benchmarked in this folder yet.");
|
|
25296
|
-
log.dim(` Run ${c.cyan(
|
|
25568
|
+
log.dim(` Run ${c.cyan(rerun)} first.`);
|
|
25297
25569
|
}
|
|
25298
25570
|
process.exitCode = 1;
|
|
25299
25571
|
return;
|
|
25300
25572
|
}
|
|
25573
|
+
if (saved.kind === "coins") {
|
|
25574
|
+
const old = saved.record;
|
|
25575
|
+
const legacy = {
|
|
25576
|
+
p50: old.p50 ?? null,
|
|
25577
|
+
p95: old.p95 ?? null,
|
|
25578
|
+
max: old.max ?? null,
|
|
25579
|
+
recommendedEstimateCoins: old.recommendedEstimateCoins ?? null,
|
|
25580
|
+
recommendedPerCallMaxCoins: old.recommendedPerCallMaxCoins ?? null,
|
|
25581
|
+
maxCoinsPerSample: old.maxCoinsPerSample ?? null,
|
|
25582
|
+
noRecommendationReason: old.noRecommendationReason ?? null
|
|
25583
|
+
};
|
|
25584
|
+
if (opts.json) {
|
|
25585
|
+
writeJsonLine({
|
|
25586
|
+
command: "llm price",
|
|
25587
|
+
status: "failed",
|
|
25588
|
+
error: "coin_terms",
|
|
25589
|
+
currency: "coins",
|
|
25590
|
+
ranAt: old.ranAt ?? null,
|
|
25591
|
+
modelId: old.modelId ?? null,
|
|
25592
|
+
samples: old.samples ?? null,
|
|
25593
|
+
completed: old.completed ?? null,
|
|
25594
|
+
...creditKeys(null),
|
|
25595
|
+
legacyCoinTerms: legacy,
|
|
25596
|
+
savedTo: LLM_BENCH_FILE
|
|
25597
|
+
});
|
|
25598
|
+
} else {
|
|
25599
|
+
log.plain(c.bold("genex llm price"));
|
|
25600
|
+
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${old.ranAt ?? "at an unknown time"}${old.modelId ? ` on ${old.modelId}` : ""}`);
|
|
25601
|
+
log.plain("");
|
|
25602
|
+
log.warn(` ${LLM_PRICE_COIN_TERMS_LINE}`);
|
|
25603
|
+
const measured = legacy.p95 === null ? "no settled sample" : `charged coins p50 ${legacy.p50} \xB7 p95 ${legacy.p95} \xB7 max ${legacy.max}`;
|
|
25604
|
+
const recommended = legacy.recommendedEstimateCoins === null ? "no price recommended" : `it recommended estimateCoins ${legacy.recommendedEstimateCoins}${legacy.recommendedPerCallMaxCoins != null ? ` and perCallMaxCoins ${legacy.recommendedPerCallMaxCoins}` : ""}`;
|
|
25605
|
+
log.dim(` What it measured then: ${measured}; ${recommended}.`);
|
|
25606
|
+
log.dim(` Re-run: ${c.cyan(rerun)}`);
|
|
25607
|
+
}
|
|
25608
|
+
process.exitCode = 1;
|
|
25609
|
+
return;
|
|
25610
|
+
}
|
|
25611
|
+
const record = saved.record;
|
|
25301
25612
|
if (opts.json) {
|
|
25302
25613
|
writeJsonLine({
|
|
25303
25614
|
command: "llm price",
|
|
25304
|
-
status: record.
|
|
25305
|
-
error: record.
|
|
25615
|
+
status: record.recommendedMaxCredits === null ? "failed" : "ok",
|
|
25616
|
+
error: record.recommendedMaxCredits === null ? record.noRecommendationReason ?? "no_recommendation" : null,
|
|
25617
|
+
currency: "credits",
|
|
25306
25618
|
ranAt: record.ranAt ?? null,
|
|
25307
25619
|
modelId: record.modelId ?? null,
|
|
25308
25620
|
samples: record.samples ?? null,
|
|
25309
25621
|
completed: record.completed ?? null,
|
|
25310
|
-
|
|
25311
|
-
|
|
25312
|
-
max: record.max ?? null,
|
|
25313
|
-
// Absent from a file an older CLI wrote; null / false / 0 then, never missing.
|
|
25314
|
-
fitP95: record.fitP95 ?? null,
|
|
25315
|
-
chargedBasedEstimateCoins: record.chargedBasedEstimateCoins ?? null,
|
|
25316
|
-
lengthRaisedPrice: record.lengthRaisedPrice ?? false,
|
|
25317
|
-
cutOffSamples: record.cutOffSamples ?? 0,
|
|
25318
|
-
noRecommendationReason: record.noRecommendationReason ?? null,
|
|
25319
|
-
recommendedDeclaredHeadroomBps: record.recommendedDeclaredHeadroomBps ?? null,
|
|
25320
|
-
recommendedEstimateCoins: record.recommendedEstimateCoins ?? null,
|
|
25321
|
-
recommendedPerCallMaxCoins: record.recommendedPerCallMaxCoins ?? null,
|
|
25622
|
+
...creditKeys(record),
|
|
25623
|
+
legacyCoinTerms: null,
|
|
25322
25624
|
savedTo: LLM_BENCH_FILE
|
|
25323
25625
|
});
|
|
25324
|
-
if (record.
|
|
25626
|
+
if (record.recommendedMaxCredits === null) process.exitCode = 1;
|
|
25325
25627
|
return;
|
|
25326
25628
|
}
|
|
25327
25629
|
log.plain(c.bold("genex llm price"));
|
|
25328
25630
|
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${record.ranAt}`);
|
|
25329
25631
|
log.plain("");
|
|
25330
25632
|
printRecommendation(log, record);
|
|
25331
|
-
if (record.
|
|
25633
|
+
if (record.recommendedMaxCredits === null) process.exitCode = 1;
|
|
25332
25634
|
}
|
|
25333
25635
|
function writeJsonLine(value) {
|
|
25334
25636
|
process.stdout.write(`${JSON.stringify(value)}
|
|
@@ -25588,7 +25890,7 @@ function runtimeRow(state, lane, toolsOnly = false) {
|
|
|
25588
25890
|
return {
|
|
25589
25891
|
label: "Runtime",
|
|
25590
25892
|
value: `in-game LLM lane live \xB7 ${n} model${n === 1 ? "" : "s"}${keyRefused ? ` \xB7 ${c.yellow("provider key refused")}` : ""}`,
|
|
25591
|
-
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : '
|
|
25893
|
+
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : 'Measure a call before a game declares its per-call ceiling: `npx genex llm bench "<prompt>" --max-credits <n> --user-approved`.'
|
|
25592
25894
|
};
|
|
25593
25895
|
}
|
|
25594
25896
|
case "off":
|
|
@@ -28265,16 +28567,16 @@ ${c.bold("Usage")}
|
|
|
28265
28567
|
status | cancel. "models" says whether this
|
|
28266
28568
|
stand serves the runtime LLM lane at all, and
|
|
28267
28569
|
at what provider rates. "bench" runs the real
|
|
28268
|
-
model on YOUR OWN
|
|
28269
|
-
attempt
|
|
28570
|
+
model on YOUR OWN credits and reports what each
|
|
28571
|
+
attempt really cost; it needs --max-credits <n>
|
|
28270
28572
|
--user-approved, like any spend the player has
|
|
28271
28573
|
to agree to. "price" reprints the last run's
|
|
28272
|
-
recommended
|
|
28574
|
+
recommended maxCredits. "status" lists this
|
|
28273
28575
|
project's open calls and the slots they hold;
|
|
28274
28576
|
"cancel <id>" stops an active one. Declare a
|
|
28275
|
-
game's
|
|
28276
|
-
a
|
|
28277
|
-
|
|
28577
|
+
game's per-call ceiling from a bench, never
|
|
28578
|
+
from a guess: the player is billed as used, and
|
|
28579
|
+
the ceiling also sizes how long an answer may be.
|
|
28278
28580
|
genex player <sub> [options] Answer the in-game model requests you approve, on your
|
|
28279
28581
|
OWN Claude or ChatGPT subscription: install | run |
|
|
28280
28582
|
status | stop | uninstall. "install" signs this machine
|
|
@@ -28629,6 +28931,9 @@ ${c.bold("Options for `rename`")}
|
|
|
28629
28931
|
|
|
28630
28932
|
${c.bold("Options for `preview` / `publish`")}
|
|
28631
28933
|
--no-build Skip the build step; deploy whatever is already built.
|
|
28934
|
+
--source <dir> Push <dir> as the game's editable source instead of this
|
|
28935
|
+
folder. The project, the build and dist/ still come from
|
|
28936
|
+
here. Also on 'make-remixable'.
|
|
28632
28937
|
--no-push (publish) Skip build + push; only flip the gallery flag.
|
|
28633
28938
|
--title <title> (publish) Gallery title.
|
|
28634
28939
|
--description <text> (publish) Gallery description.
|
|
@@ -28741,7 +29046,7 @@ ${c.bold("Examples")}
|
|
|
28741
29046
|
genex shop remove sku_123
|
|
28742
29047
|
genex llm models
|
|
28743
29048
|
genex llm models --all
|
|
28744
|
-
genex llm bench "Reply with one short taunt." --samples 5 --max-
|
|
29049
|
+
genex llm bench "Reply with one short taunt." --samples 5 --max-credits 20 --user-approved
|
|
28745
29050
|
genex llm price
|
|
28746
29051
|
genex llm status
|
|
28747
29052
|
genex llm cancel <id>
|
|
@@ -28978,22 +29283,24 @@ ${c.bold("How the allowance works")}
|
|
|
28978
29283
|
|
|
28979
29284
|
Every number printed is live from your account; prices can change without a deploy.
|
|
28980
29285
|
`,
|
|
28981
|
-
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what should the game declare?
|
|
29286
|
+
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what ceiling should the game declare?
|
|
28982
29287
|
|
|
28983
29288
|
${c.bold("Usage")}
|
|
28984
29289
|
genex llm models [--all] [--json] Which models this stand serves, their provider rates per
|
|
28985
|
-
million tokens,
|
|
28986
|
-
|
|
28987
|
-
|
|
28988
|
-
|
|
28989
|
-
on this stand: it says so
|
|
28990
|
-
|
|
28991
|
-
|
|
28992
|
-
|
|
28993
|
-
|
|
28994
|
-
|
|
28995
|
-
|
|
28996
|
-
|
|
29290
|
+
million tokens, the platform fee a call is billed at, and
|
|
29291
|
+
the headroom it recommends over a benchmark. The catalog
|
|
29292
|
+
is synced from OpenRouter, so the FEATURED short list is
|
|
29293
|
+
printed by default and --all prints every row (--json
|
|
29294
|
+
always carries every row). Off on this stand: it says so
|
|
29295
|
+
and exits 0 \u2014 build the feature without a model rather
|
|
29296
|
+
than promising a 404.
|
|
29297
|
+
genex llm bench "<prompt>" --max-credits <n> --user-approved [options]
|
|
29298
|
+
Run the real model through the development lane and
|
|
29299
|
+
report what each attempt really COST and was charged.
|
|
29300
|
+
Refused without both flags, before any network call:
|
|
29301
|
+
this spends YOUR OWN CREDITS, a spend the build's asset
|
|
29302
|
+
allowance does not count. Needs a folder linked to a
|
|
29303
|
+
game you own, and a balance that covers one attempt.
|
|
28997
29304
|
genex llm price [--json] The recommendation from the last bench in this folder.
|
|
28998
29305
|
Reads a file; spends nothing.
|
|
28999
29306
|
genex llm status [--json] This project's open development calls: active, or
|
|
@@ -29010,29 +29317,32 @@ ${c.bold("Bench options")}
|
|
|
29010
29317
|
--samples <n> How many real attempts (default ${DEFAULT_BENCH_SAMPLES}). More samples, better p95.
|
|
29011
29318
|
--schema <file> A JSON file holding the output schema; implies JSON output.
|
|
29012
29319
|
--json-output Ask for JSON without a schema. --text Ask for text (the default).
|
|
29013
|
-
--max-
|
|
29320
|
+
--max-credits <n> The per-attempt ceiling YOU approved \u2014 a bench runs on your own credits,
|
|
29014
29321
|
never a player's. The run's worst case is that number times --samples,
|
|
29015
|
-
and it is printed before anything starts.
|
|
29322
|
+
and it is printed before anything starts. (--max-coins is its deprecated
|
|
29323
|
+
name, read as the same number.)
|
|
29016
29324
|
--timeout <seconds> Bounds the wait for each attempt.
|
|
29017
29325
|
--json One machine-readable object; every key is always present, null when
|
|
29018
29326
|
the CLI could not learn it.
|
|
29019
29327
|
|
|
29020
29328
|
${c.bold("Why a benchmark and not an estimate")}
|
|
29021
|
-
|
|
29022
|
-
|
|
29023
|
-
|
|
29024
|
-
|
|
29025
|
-
|
|
29026
|
-
|
|
29027
|
-
|
|
29028
|
-
|
|
29029
|
-
|
|
29329
|
+
An in-game call is paid from the PLAYER's credits and billed as used: its real provider cost
|
|
29330
|
+
plus the platform fee, never more than the per-call CEILING the game declares (\`maxCredits\`,
|
|
29331
|
+
or \`perCallMaxCredits\` on a standing budget). The ceiling is not a price, but it is still a
|
|
29332
|
+
number with two jobs: it bounds what one call may cost, and it funds how LONG the answer may
|
|
29333
|
+
be. Three layers sit between the provider's rate card and it \u2014 the provider's cost for YOUR
|
|
29334
|
+
prompt, the platform fee, and your headroom \u2014 and only a real call measures the first. The
|
|
29335
|
+
bench is billed to YOU, on the development lane; the player-funded lane is what the
|
|
29336
|
+
published game uses. The recommendation is the p95 real cost with the server's fee and
|
|
29337
|
+
headroom applied, in whole credits; a stand that serves no headroom or no fee gets no
|
|
29338
|
+
recommendation, because those multipliers are not the CLI's to invent.
|
|
29030
29339
|
|
|
29031
|
-
|
|
29032
|
-
|
|
29033
|
-
|
|
29034
|
-
|
|
29035
|
-
|
|
29340
|
+
Because the ceiling sizes each call's output allowance, the recommendation is never below
|
|
29341
|
+
the smallest ceiling that leaves room for the benchmarked answer, as the server computes it.
|
|
29342
|
+
A sample cut off at your --max-credits (provider_token_limit) is not a sample \u2014 re-run with a
|
|
29343
|
+
higher --max-credits; an answer longer than one call on this stand may produce gets no
|
|
29344
|
+
ceiling at all \u2014 ask for a shorter one. The run also prints the measured average per call,
|
|
29345
|
+
fee included, which is the honest estimate a budget's disclosure is built on.
|
|
29036
29346
|
|
|
29037
29347
|
Results land in ${LLM_BENCH_FILE} \u2014 gitignored, machine-local, re-read by \`genex llm price\`.
|
|
29038
29348
|
`,
|
|
@@ -29266,6 +29576,9 @@ function parseArgs(argv) {
|
|
|
29266
29576
|
...Object.keys(UI_NUMBER_FLAGS),
|
|
29267
29577
|
"--source-repo-url",
|
|
29268
29578
|
"--source-author",
|
|
29579
|
+
// `preview` / `publish` / `make-remixable --source <dir>`: the folder pushed as the
|
|
29580
|
+
// game's source when it is not the one the command runs in.
|
|
29581
|
+
"--source",
|
|
29269
29582
|
"--license",
|
|
29270
29583
|
"--purchase-url",
|
|
29271
29584
|
"--product-name",
|
|
@@ -29280,11 +29593,13 @@ function parseArgs(argv) {
|
|
|
29280
29593
|
"--limit",
|
|
29281
29594
|
// `genex budget --assets <credits>` — the allowance the player approved.
|
|
29282
29595
|
"--assets",
|
|
29283
|
-
// `genex llm bench` value flags. `--max-
|
|
29284
|
-
//
|
|
29596
|
+
// `genex llm bench` value flags. `--max-credits` is the same approval
|
|
29597
|
+
// shape as `--assets`, but its own gate: the asset allowance does not
|
|
29598
|
+
// count a bench. `--max-coins` is its deprecated name (same number).
|
|
29285
29599
|
"--model",
|
|
29286
29600
|
"--samples",
|
|
29287
29601
|
"--schema",
|
|
29602
|
+
"--max-credits",
|
|
29288
29603
|
"--max-coins",
|
|
29289
29604
|
// `genex player install --label <name>` — what this machine is called in
|
|
29290
29605
|
// the dashboard. Its own key rather than the generic 2nd positional,
|
|
@@ -29838,12 +30153,14 @@ function applyValueFlag(options, flag, value) {
|
|
|
29838
30153
|
options.samples = n;
|
|
29839
30154
|
break;
|
|
29840
30155
|
}
|
|
30156
|
+
case "--max-credits":
|
|
29841
30157
|
case "--max-coins": {
|
|
29842
30158
|
const n = Number(value);
|
|
29843
30159
|
if (!Number.isInteger(n) || n < 1) {
|
|
29844
|
-
throw new Error(`Invalid
|
|
30160
|
+
throw new Error(`Invalid ${flag} value: ${value} (whole credits, 1 or more)`);
|
|
29845
30161
|
}
|
|
29846
|
-
options.
|
|
30162
|
+
if (flag === "--max-credits") options.maxCredits = n;
|
|
30163
|
+
else options.maxCoins = n;
|
|
29847
30164
|
break;
|
|
29848
30165
|
}
|
|
29849
30166
|
case "--timeout": {
|
|
@@ -30026,6 +30343,9 @@ function applyValueFlag(options, flag, value) {
|
|
|
30026
30343
|
case "--src":
|
|
30027
30344
|
options.src = value;
|
|
30028
30345
|
break;
|
|
30346
|
+
case "--source":
|
|
30347
|
+
options.sourceDir = value;
|
|
30348
|
+
break;
|
|
30029
30349
|
case "--clip":
|
|
30030
30350
|
options.clipPath = value;
|
|
30031
30351
|
break;
|
|
@@ -30141,9 +30461,9 @@ async function main() {
|
|
|
30141
30461
|
await runBudget(parsed.options);
|
|
30142
30462
|
break;
|
|
30143
30463
|
// The in-game model lane: is it live here, what does one call really
|
|
30144
|
-
// cost, what should the game declare. `bench`
|
|
30145
|
-
//
|
|
30146
|
-
//
|
|
30464
|
+
// cost, what ceiling should the game declare. `bench` spends the
|
|
30465
|
+
// builder's own credits OUTSIDE the asset allowance, so it carries its
|
|
30466
|
+
// own --max-credits/--user-approved gate.
|
|
30147
30467
|
case "llm":
|
|
30148
30468
|
await runLlm(parsed.options);
|
|
30149
30469
|
break;
|