@genex-ai/cli-demo 1.36.8 → 1.36.9-dev.913
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{blender-mcp-QVEGHE5L.js → blender-mcp-WSURPZET.js} +2 -2
- package/dist/{blender-serve-66OHC4ML.js → blender-serve-EH2MOW3U.js} +1 -1
- package/dist/{chunk-4BBTPRJW.js → chunk-EABIVAGT.js} +2 -2
- package/dist/{chunk-QI7FIYBY.js → chunk-K4EHXOZK.js} +1 -1
- package/dist/index.js +575 -216
- package/package.json +1 -1
- package/templates/skills/genex-game-director/SKILL.md +1 -1
- package/templates/skills/genex-getting-started/SKILL.md +2 -2
- package/templates/skills/genex-llm-in-games/SKILL.md +209 -72
- package/templates/skills/genex-llm-in-games/references/pricing.md +126 -85
- package/templates/skills/genex-monetization/SKILL.md +1 -9
- package/templates/skills/genex-threejs-embed-auth/SKILL.md +3 -2
- package/templates/skills/genex-threejs-multiplayer/SKILL.md +48 -24
- package/templates/skills/genex-threejs-multiplayer/references/genex-netcode-feel-checklist.md +3 -2
- package/templates/skills/genex-threejs-multiplayer/references/genre-recipes.md +1 -1
- package/templates/skills/genex-tool-llm/SKILL.md +11 -9
- package/templates/skills/genex-updates/SKILL.md +1 -1
package/dist/index.js
CHANGED
|
@@ -57,7 +57,7 @@ import {
|
|
|
57
57
|
writeSecretFile,
|
|
58
58
|
writeUserToken,
|
|
59
59
|
writeWorkspace
|
|
60
|
-
} from "./chunk-
|
|
60
|
+
} from "./chunk-EABIVAGT.js";
|
|
61
61
|
import {
|
|
62
62
|
CLI_CHANNEL,
|
|
63
63
|
DEFAULT_API_URL,
|
|
@@ -78,7 +78,7 @@ import {
|
|
|
78
78
|
normalizeApiOrigin,
|
|
79
79
|
originKey,
|
|
80
80
|
resolveAgentTargets
|
|
81
|
-
} from "./chunk-
|
|
81
|
+
} from "./chunk-K4EHXOZK.js";
|
|
82
82
|
|
|
83
83
|
// src/instrument.ts
|
|
84
84
|
import * as Sentry from "@sentry/node";
|
|
@@ -876,6 +876,90 @@ async function writeGameConfigFiles(meta, log, cwd = process.cwd()) {
|
|
|
876
876
|
await writeIfAbsent(path3.join(cwd, ".env.development.local"), overrides, log);
|
|
877
877
|
}
|
|
878
878
|
}
|
|
879
|
+
var SLUG_CONFIG_FILES = ["src/genex.config.ts", "src/genex.config.js"];
|
|
880
|
+
var SLUG_ENV_FILES = [".env.production.local", ".env.production", ".env.local", ".env"];
|
|
881
|
+
var SLUG_LITERAL = /^(["'`])([a-z0-9][a-z0-9-]*)\1$/;
|
|
882
|
+
function topLevelFields(code, start) {
|
|
883
|
+
const fields = [];
|
|
884
|
+
let field = "";
|
|
885
|
+
let depth = 0;
|
|
886
|
+
for (let i = start; i < code.length; i++) {
|
|
887
|
+
const ch = code[i];
|
|
888
|
+
if (ch === "/" && code[i + 1] === "/") {
|
|
889
|
+
const end = code.indexOf("\n", i);
|
|
890
|
+
if (end === -1) return null;
|
|
891
|
+
i = end - 1;
|
|
892
|
+
continue;
|
|
893
|
+
}
|
|
894
|
+
if (ch === "/" && code[i + 1] === "*") {
|
|
895
|
+
const end = code.indexOf("*/", i + 2);
|
|
896
|
+
if (end === -1) return null;
|
|
897
|
+
i = end + 1;
|
|
898
|
+
continue;
|
|
899
|
+
}
|
|
900
|
+
if (ch === '"' || ch === "'" || ch === "`") {
|
|
901
|
+
let j = i + 1;
|
|
902
|
+
while (j < code.length && code[j] !== ch) j += code[j] === "\\" ? 2 : 1;
|
|
903
|
+
if (j >= code.length) return null;
|
|
904
|
+
field += code.slice(i, j + 1);
|
|
905
|
+
i = j;
|
|
906
|
+
continue;
|
|
907
|
+
}
|
|
908
|
+
if (ch === "(" || ch === "[" || ch === "{") depth++;
|
|
909
|
+
else if (ch === ")" || ch === "]") depth--;
|
|
910
|
+
else if (ch === "}") {
|
|
911
|
+
if (depth === 0) {
|
|
912
|
+
fields.push(field);
|
|
913
|
+
return fields;
|
|
914
|
+
}
|
|
915
|
+
depth--;
|
|
916
|
+
} else if (ch === "," && depth === 0) {
|
|
917
|
+
fields.push(field);
|
|
918
|
+
field = "";
|
|
919
|
+
continue;
|
|
920
|
+
}
|
|
921
|
+
field += ch;
|
|
922
|
+
}
|
|
923
|
+
return null;
|
|
924
|
+
}
|
|
925
|
+
function bakedConfigSlug(content) {
|
|
926
|
+
const object = /export\s+const\s+GENEX(?:\s*:[^=]+)?\s*=\s*\{/.exec(content);
|
|
927
|
+
if (!object) return null;
|
|
928
|
+
const fields = topLevelFields(content, object.index + object[0].length);
|
|
929
|
+
const value = fields?.map((f) => /^\s*(?:slug|["']slug["'])\s*:([\s\S]*)$/.exec(f)?.[1]?.trim()).find((v) => v !== void 0);
|
|
930
|
+
if (!value) return null;
|
|
931
|
+
const fallback = value.includes("??") ? value.slice(value.lastIndexOf("??") + 2).trim() : value;
|
|
932
|
+
return SLUG_LITERAL.exec(fallback)?.[2] ?? null;
|
|
933
|
+
}
|
|
934
|
+
function envFileSlug(content) {
|
|
935
|
+
let found = null;
|
|
936
|
+
for (const line of content.split(/\r?\n/)) {
|
|
937
|
+
const match = /^[ \t]*(?:export[ \t]+)?VITE_GENEX_SLUG[ \t]*=[ \t]*(.*)$/.exec(line);
|
|
938
|
+
if (!match) continue;
|
|
939
|
+
const raw = match[1].trim();
|
|
940
|
+
const quote2 = raw[0] === '"' || raw[0] === "'" ? raw[0] : null;
|
|
941
|
+
const value = quote2 ? raw.slice(1, raw.indexOf(quote2, 1) === -1 ? void 0 : raw.indexOf(quote2, 1)) : raw.replace(/[ \t]+#.*$/, "");
|
|
942
|
+
if (value.trim()) found = value.trim();
|
|
943
|
+
}
|
|
944
|
+
return found;
|
|
945
|
+
}
|
|
946
|
+
async function findForeignSlug(cwd, slug, env = process.env) {
|
|
947
|
+
const read = (rel) => fs2.readFile(path3.join(cwd, rel), "utf8").catch(() => null);
|
|
948
|
+
const judge = (file, found) => found === slug ? null : { file, found };
|
|
949
|
+
const fromShell = env.VITE_GENEX_SLUG?.trim();
|
|
950
|
+
if (fromShell) return judge("VITE_GENEX_SLUG (environment)", fromShell);
|
|
951
|
+
for (const rel of SLUG_ENV_FILES) {
|
|
952
|
+
const content = await read(rel);
|
|
953
|
+
const found = content === null ? null : envFileSlug(content);
|
|
954
|
+
if (found) return judge(rel, found);
|
|
955
|
+
}
|
|
956
|
+
for (const rel of SLUG_CONFIG_FILES) {
|
|
957
|
+
const content = await read(rel);
|
|
958
|
+
const found = content === null ? null : bakedConfigSlug(content);
|
|
959
|
+
if (found !== null) return judge(rel, found);
|
|
960
|
+
}
|
|
961
|
+
return null;
|
|
962
|
+
}
|
|
879
963
|
|
|
880
964
|
// src/lib/ssh.ts
|
|
881
965
|
import fs3 from "fs/promises";
|
|
@@ -981,7 +1065,7 @@ Important note: put soul into your creations, with many details and love. Aim to
|
|
|
981
1065
|
21. A turn ends in exactly one of two ways: on a question the player must answer before the next step can be chosen, or on a handoff \u2014 the draft link, what changed, what to try, and what is still open. Never close a turn on a promise. "I'll keep building", "while I keep working", "adding that now" are things you say and then DO before you stop; if you are stopping, say that you have stopped and what remains. While the Build plan's \`Now:\` line has open work and no answer is needed from the player, the turn is not over: preview, hand off, and start the next milestone in the same turn, until the requested outcome is reached or the platform's own budget ends the session. If a stop was forced on you anyway, the first line of your next turn is the promise you left and where it stands.
|
|
982
1066
|
22. **What the player sees must read as the thing it is**, in this game's own style: a building as that building, a person as a person, a prop as that prop, a surface as its material \u2014 and a coloured box, a capsule, or a flat grey block standing in for one of them is never its finished version. Generation is the default route to that bar wherever code will not honestly reach it: the player's character, whatever the request names, whatever the player walks up to, enters, or interacts with, and the music and sound underneath \u2014 reach for the belt on your own judgment, without stopping to ask; a status line saying what you queued is enough, and \`--no-wait\` keeps you building while it lands. Procedural code is a first-class engine for what is structural, repeated, distant, or parametric \u2014 terrain, sky, fences, paving, modular kits, filler \u2014 and for anything else only when the result meets the same bar and you have looked at it in a capture before its row says \`landed\`. Start with one reusable implementation per distinct required gameplay or visual role. Background populations default to reusable procedural bodies and motion or compatible existing rigged assets. Catalog clips need a compatible skeleton and may be paid. Reserve paid custom bodies for the player and characters examined or interacted with closely, while honoring explicit user requests. Add variants only for an unmet requirement; unspent allowance is not unfinished work. Allowance questions use timeoutPolicy no-consent: silence does not raise the allowance. Record role, reuse and completion criteria in the existing Assets table. Decide per object, not per category, and write the route on each Assets row in \`DESIGN.md\` \u2014 the paid flow for generated pieces, the procedural flow with its local path for code-built ones. An Assets table with no generation in it is a decision, not a default: record it as \`Generation: none \u2014 <why code alone reaches the bar here>\` or generate.
|
|
983
1067
|
23. **The screenshots you take to look at your own work go in \`.genex/scratch/\`.** Captures, render comparisons, before/after strips, traces and metrics dumps are how you SEE what you built \u2014 they are not part of what you built, and \`genex preview\` pushes the whole folder to the game's source, every time, forever. \`.genex/scratch/\` already exists for this and is already ignored, so there is nothing to set up: write the capture as \`.genex/scratch/arena-before.png\` and read it back from there. Never put them in a folder of your own at the top level: \`progress/\`, \`reports/\`, \`shots/\` and their kind are pushed like source and become permanent weight in the game and in every remix of it \u2014 one real game reached 11 GB and 10,332 committed screenshots exactly this way. If you find such a folder already there, add it to \`.gitignore\`; never delete the player's files (law 18). One file in \`.genex/scratch/\` IS read, by the platform: \`.genex/scratch/cover.png\` is the game's cover \u2014 the frame you would put on its poster: real gameplay at its signature moment, well lit, no menus, popups or busy HUD, landscape 16:9, and never any text (the page sets the title beside it). The next \`genex preview\` makes it the cover; save a better moment over it whenever one comes along, and if your browser tool can set the cover itself, use that instead. Before the first publish, and whenever the game's look changes, load \`$genex-cover\`: it chooses, stages, lights and judges that frame.
|
|
984
|
-
24. **Benchmark before declaring an in-game LLM
|
|
1068
|
+
24. **Benchmark before declaring an in-game LLM ceiling.** If the game calls a language model while the player plays \u2014 an NPC answering in its own words, a quest written for this save, a judge reading what the player typed \u2014 the PLAYER pays for it from their Genex credits, billed as used, and the game declares no price, only a per-call CEILING: \`maxCredits\` on \`generate()\`, \`perCallMaxCredits\` on a standing budget. That ceiling bounds what one call may cost AND funds how long its answer may be, so never pick it by judgement or from a vendor's rate card. Run the real prompt on your own credits \u2014 \`npx genex llm bench "<the prompt>" --schema <file> --samples 3 --max-credits <n> --user-approved\` \u2014 and declare the figures it prints, recorded in \`DESIGN.md\` with its model and date; re-benchmark whenever the prompt, the schema or the model changes, because a prompt edit changes the ceiling. Check the lane first with \`npx genex llm models\`: where it is off the routes 404, so build the feature behind a graceful unavailable state and say so in the handoff. A repeated call is a standing budget the player approves once (\`requestSpendGrant()\`, then \`generate({ grantId })\`), never a popup per turn, and its disclosed call rate and credits per period are computed from the game's own loop and the bench's measured average. The coin spellings (\`estimateCoins\`, \`perCallMaxCoins\`) are deprecated \u2014 write the credit names. Model output is data the game validates against its own expectation \u2014 never executed, and never authority over coin, credits, items, entitlements or rewards. Load \`$genex-llm-in-games\` before writing any of it.
|
|
985
1069
|
${CONTRACT_END}
|
|
986
1070
|
`;
|
|
987
1071
|
var TOOLS_CONTRACT_HEAD = `# Genex Tools (always in effect in this folder)
|
|
@@ -995,7 +1079,7 @@ var TOOLS_CONTRACT_OFFER = `5. **When the game is built and playable \u2014 neve
|
|
|
995
1079
|
var TOOLS_CONTRACT_HOSTED_LAWS = `5. **This game is hosted on Genex now, and publishing is glue \u2014 never a rebuild.** The game ships exactly as the user built it: \`initEmbed()\` first in the boot code (the \`genex-threejs-embed-auth\` card has the call; the renderer draws before any \`await waitForPlayer()\`), a static build into \`dist/\` with relative asset paths, then \`npx genex preview\`. Never restructure, reformat or "improve" the game to ship it, never scaffold a new app around it, and never start a design document or a build plan for it \u2014 the user's own process is theirs. Load the \`genex-tool-publish\` card before the first preview; it owns the vocabulary, the links and the limits.
|
|
996
1080
|
6. **Ship first, then report.** \`preview\` prints a preflight \u2014 phone memory, a missing volume slider, an asset nobody wired, a viewport line. It never blocks a deploy, so push the build as it is, hand over the link, THEN relay each preflight line to the user in one plain sentence with an offer to fix it, and fix one only on their yes. The preflight is a report for the user, not a to-do list for you.
|
|
997
1081
|
7. **The link is the game's page, and there are two versions.** After every preview give the user \`<dashboard>/draft/<slug>\` \u2014 \`dashboardOrigins[0]\` and \`slug\` from \`.genex/project.json\` \u2014 never localhost, a file path or the bare play origin. \`preview\` updates the draft and never touches what players are on; the first release is \`npx genex publish\` (it lists the game); after that "publish it", "update it" and "yes" all mean \`npx genex promote\` \u2014 the exact draft build, no rebuild. Ask once per round of work, in one line, and keep working while you wait.`;
|
|
998
|
-
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with Genex
|
|
1082
|
+
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with their Genex credits or their own Claude/ChatGPT subscription, and approves it on a Genex sheet; your game just calls \`generate()\`. Want it that way?`;
|
|
999
1083
|
function toolsContractLlmLaw(n, hosted) {
|
|
1000
1084
|
const onYes = hosted ? "On a yes, load `$genex-tool-llm` and then `$genex-llm-in-games`, which owns the build." : "On a yes, load `$genex-tool-llm`, which owns the rest: it checks the lane with `npx genex llm models` first, then runs `npx genex init --convert` (a hosted game is a static browser build \u2014 a local server holding a key can never ship).";
|
|
1001
1085
|
return `${n}. **A model running while people PLAY is a platform feature \u2014 never something you wire onto the user's own meter.** When a request implies one (NPCs that talk or decide in their own words, content written from what the player types, a prompt box in the game, "let the player pick a model"), recognise it, say in ONE line: "${TOOLS_CONTRACT_LLM_OFFER}" \u2014 then ASK and wait. ${onYes} On a no, build the authored version instead. Either way, never ship a game that calls a model on a key or an account of the user's own: every visitor would spend their money with nobody approving it, and the credential is readable in the bundle. Player-funded generation is text and JSON only \u2014 3D, images, video and audio stay the asset lanes above, on the user's meter.`;
|
|
@@ -2630,6 +2714,26 @@ async function detectEmbedSdkVersion(cwd = process.cwd()) {
|
|
|
2630
2714
|
return null;
|
|
2631
2715
|
}
|
|
2632
2716
|
}
|
|
2717
|
+
var MIN_MULTIPLAYER_SDK = { major: 0, minor: 17 };
|
|
2718
|
+
async function outdatedMultiplayerSdk(cwd = process.cwd()) {
|
|
2719
|
+
const readJson = async (rel) => {
|
|
2720
|
+
try {
|
|
2721
|
+
return JSON.parse(await fs13.readFile(path14.join(cwd, rel), "utf8"));
|
|
2722
|
+
} catch {
|
|
2723
|
+
return null;
|
|
2724
|
+
}
|
|
2725
|
+
};
|
|
2726
|
+
const pkg = await readJson("package.json");
|
|
2727
|
+
const range = pkg?.dependencies?.["@genex-ai/multiplayer"] ?? pkg?.devDependencies?.["@genex-ai/multiplayer"];
|
|
2728
|
+
if (!range) return null;
|
|
2729
|
+
const installed = (await readJson("node_modules/@genex-ai/multiplayer/package.json"))?.version;
|
|
2730
|
+
const version = typeof installed === "string" ? installed : range;
|
|
2731
|
+
const match = /^[\^~>=v\s]*(\d+)\.(\d+)/.exec(version);
|
|
2732
|
+
if (!match || /[|<]|\s-\s/.test(version)) return null;
|
|
2733
|
+
const [major, minor] = [Number(match[1]), Number(match[2])];
|
|
2734
|
+
const old = major < MIN_MULTIPLAYER_SDK.major || major === MIN_MULTIPLAYER_SDK.major && minor < MIN_MULTIPLAYER_SDK.minor;
|
|
2735
|
+
return old ? version : null;
|
|
2736
|
+
}
|
|
2633
2737
|
async function detectMultiplayer(cwd = process.cwd()) {
|
|
2634
2738
|
try {
|
|
2635
2739
|
const raw = await fs13.readFile(path14.join(cwd, "package.json"), "utf8");
|
|
@@ -5690,6 +5794,23 @@ async function deployGame(ctx, opts, log) {
|
|
|
5690
5794
|
return false;
|
|
5691
5795
|
}
|
|
5692
5796
|
}
|
|
5797
|
+
const expectedSlug = ctx.slug ?? meta?.slug;
|
|
5798
|
+
if (expectedSlug) {
|
|
5799
|
+
const foreign = await findForeignSlug(cwd, expectedSlug);
|
|
5800
|
+
if (foreign) {
|
|
5801
|
+
log.error(
|
|
5802
|
+
`${foreign.file} names the game "${foreign.found}", but this folder is the Genex game "${expectedSlug}". The build would sign players into "${foreign.found}". Set it to "${expectedSlug}" and deploy again. Nothing was deployed.`
|
|
5803
|
+
);
|
|
5804
|
+
return false;
|
|
5805
|
+
}
|
|
5806
|
+
}
|
|
5807
|
+
const oldSdk = await outdatedMultiplayerSdk(cwd);
|
|
5808
|
+
if (oldSdk) {
|
|
5809
|
+
log.error(
|
|
5810
|
+
`This game uses @genex-ai/multiplayer ${oldSdk}, which Genex's multiplayer server no longer accepts. Run \`npm i @genex-ai/multiplayer@^0.17.0\` and deploy again. Nothing was deployed.`
|
|
5811
|
+
);
|
|
5812
|
+
return false;
|
|
5813
|
+
}
|
|
5693
5814
|
if (!opts.noBuild && await hasBuildScript(cwd)) {
|
|
5694
5815
|
log.step("Building the production bundle\u2026");
|
|
5695
5816
|
const built = await run("npm", ["run", "build"]);
|
|
@@ -6813,6 +6934,12 @@ async function runPreview(opts) {
|
|
|
6813
6934
|
log.plain("");
|
|
6814
6935
|
log.plain(nudge);
|
|
6815
6936
|
}
|
|
6937
|
+
if (detections.multiplayer) {
|
|
6938
|
+
log.plain("");
|
|
6939
|
+
log.plain(
|
|
6940
|
+
"Multiplayer on this preview runs in rooms of its own, apart from the live game. Test it as two different players: open the preview-- address printed above in a normal and an incognito window, or send it to a friend. The game's page is yours only."
|
|
6941
|
+
);
|
|
6942
|
+
}
|
|
6816
6943
|
}
|
|
6817
6944
|
|
|
6818
6945
|
// src/commands/remix.ts
|
|
@@ -23609,7 +23736,7 @@ async function runBlender(opts) {
|
|
|
23609
23736
|
return 1;
|
|
23610
23737
|
}
|
|
23611
23738
|
if (sub === "serve") {
|
|
23612
|
-
const { serveLocalBlender } = await import("./blender-serve-
|
|
23739
|
+
const { serveLocalBlender } = await import("./blender-serve-EH2MOW3U.js");
|
|
23613
23740
|
const port = Number(process.env.GENEX_BLENDER_PORT ?? 8088);
|
|
23614
23741
|
log.step(`Starting a local Blender service on port ${port}`);
|
|
23615
23742
|
log.plain(
|
|
@@ -23618,7 +23745,7 @@ async function runBlender(opts) {
|
|
|
23618
23745
|
return serveLocalBlender({ port, log });
|
|
23619
23746
|
}
|
|
23620
23747
|
if (sub === "mcp") {
|
|
23621
|
-
const { runBlenderMcp } = await import("./blender-mcp-
|
|
23748
|
+
const { runBlenderMcp } = await import("./blender-mcp-WSURPZET.js");
|
|
23622
23749
|
return runBlenderMcp();
|
|
23623
23750
|
}
|
|
23624
23751
|
if (sub === "seat") {
|
|
@@ -24126,7 +24253,12 @@ import fs35 from "fs/promises";
|
|
|
24126
24253
|
import path36 from "path";
|
|
24127
24254
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
24128
24255
|
var SUBS3 = ["models", "bench", "price", "status", "cancel"];
|
|
24129
|
-
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN
|
|
24256
|
+
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN credits \u2014 your balance, not a player's, and a spend this build's asset allowance does not count. Re-run with --max-credits <n> --user-approved only after the person at the keyboard agreed to the number.";
|
|
24257
|
+
var LLM_MAX_COINS_DEPRECATED = "--max-coins is deprecated: it is read as --max-credits, the same number \u2014 a bench spends credits now.";
|
|
24258
|
+
function benchBalanceShortLine(spendable, maxCredits) {
|
|
24259
|
+
return `Your spendable credits (${spendable}) cannot cover one attempt at --max-credits ${maxCredits} \u2014 nothing was sent, nothing was spent. Add credits, or re-run with a lower --max-credits once the person at the keyboard agrees to it.`;
|
|
24260
|
+
}
|
|
24261
|
+
var LLM_BENCH_UNVERIFIED_LINE = "Credits pay only for a verified email, and this account's is not verified yet (credits_unverified) \u2014 nothing was sent, nothing was spent. Verify it, then re-run.";
|
|
24130
24262
|
var LLM_LANE_OFF_LINE = "The runtime LLM lane is off on this stand \u2014 in-game generate() answers 404 here, and nothing can be benchmarked.";
|
|
24131
24263
|
var LLM_TOOLS_CONVERT_HINT = [
|
|
24132
24264
|
"In-game model calls need a hosted game. This folder is a Genex Tools workspace, so nothing here",
|
|
@@ -24139,7 +24271,7 @@ function serverSlotCounts(body) {
|
|
|
24139
24271
|
return { held: body.slotsHeld, limit: body.slotLimit };
|
|
24140
24272
|
}
|
|
24141
24273
|
var PENDING_SLOT_RELEASE_MINUTES = 10;
|
|
24142
|
-
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so
|
|
24274
|
+
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so what it reserved stays held until the bill resolves; such a row stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`;
|
|
24143
24275
|
var LLM_STATUS_NO_COUNT_LINE = "This stand does not report how many slots are held across your projects, so no count is shown.";
|
|
24144
24276
|
var LLM_CANCEL_AWAITING_BILL = "this call already stopped; its bill is awaiting the provider";
|
|
24145
24277
|
var LLM_CANCEL_ALREADY_FINAL = "this call is already final";
|
|
@@ -24153,11 +24285,33 @@ var DEFAULT_SAMPLE_TIMEOUT_SEC = 180;
|
|
|
24153
24285
|
function parseProviderKey(value) {
|
|
24154
24286
|
return value === "ok" || value === "rejected" || value === "unknown" ? value : null;
|
|
24155
24287
|
}
|
|
24288
|
+
function bpsOrNull(value) {
|
|
24289
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
24290
|
+
}
|
|
24291
|
+
function standingCreditGrantOrNull(value) {
|
|
24292
|
+
if (!value || typeof value !== "object") return null;
|
|
24293
|
+
const v = value;
|
|
24294
|
+
const whole = (x) => typeof x === "number" && Number.isSafeInteger(x) && x >= 0;
|
|
24295
|
+
if (!whole(v.minCredits) || !whole(v.maxCredits) || !whole(v.stepCredits) || !whole(v.defaultCredits)) return null;
|
|
24296
|
+
return {
|
|
24297
|
+
minCredits: v.minCredits,
|
|
24298
|
+
maxCredits: v.maxCredits,
|
|
24299
|
+
stepCredits: v.stepCredits,
|
|
24300
|
+
defaultCredits: v.defaultCredits,
|
|
24301
|
+
...whole(v.minUsdCents) ? { minUsdCents: v.minUsdCents } : {},
|
|
24302
|
+
...whole(v.maxUsdCents) ? { maxUsdCents: v.maxUsdCents } : {},
|
|
24303
|
+
...whole(v.stepUsdCents) ? { stepUsdCents: v.stepUsdCents } : {}
|
|
24304
|
+
};
|
|
24305
|
+
}
|
|
24156
24306
|
async function readRuntimeLane(apiUrl, token) {
|
|
24157
24307
|
const empty = (state) => ({
|
|
24158
24308
|
state,
|
|
24159
24309
|
models: null,
|
|
24160
24310
|
recommendedDeclaredHeadroomBps: null,
|
|
24311
|
+
currency: null,
|
|
24312
|
+
feeBps: null,
|
|
24313
|
+
usdCentsPerCredit: null,
|
|
24314
|
+
standingCreditGrant: null,
|
|
24161
24315
|
externalProviders: null,
|
|
24162
24316
|
total: null,
|
|
24163
24317
|
featuredCount: null,
|
|
@@ -24176,7 +24330,11 @@ async function readRuntimeLane(apiUrl, token) {
|
|
|
24176
24330
|
return {
|
|
24177
24331
|
state: "live",
|
|
24178
24332
|
models: body.models,
|
|
24179
|
-
recommendedDeclaredHeadroomBps:
|
|
24333
|
+
recommendedDeclaredHeadroomBps: bpsOrNull(body.recommendedDeclaredHeadroomBps),
|
|
24334
|
+
currency: body.currency === "credits" || body.currency === "coins" ? body.currency : null,
|
|
24335
|
+
feeBps: bpsOrNull(body.feeBps),
|
|
24336
|
+
usdCentsPerCredit: typeof body.usdCentsPerCredit === "number" && Number.isSafeInteger(body.usdCentsPerCredit) && body.usdCentsPerCredit >= 1 ? body.usdCentsPerCredit : null,
|
|
24337
|
+
standingCreditGrant: standingCreditGrantOrNull(body.standingCreditGrant),
|
|
24180
24338
|
externalProviders: Array.isArray(body.externalProviders) ? body.externalProviders : null,
|
|
24181
24339
|
total: typeof body.total === "number" && Number.isFinite(body.total) ? body.total : null,
|
|
24182
24340
|
featuredCount: typeof body.featuredCount === "number" && Number.isFinite(body.featuredCount) ? body.featuredCount : null,
|
|
@@ -24192,18 +24350,45 @@ function percentile2(values, p) {
|
|
|
24192
24350
|
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
24193
24351
|
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
24194
24352
|
}
|
|
24195
|
-
function
|
|
24196
|
-
|
|
24353
|
+
function percentilePicos(values, p) {
|
|
24354
|
+
if (values.length === 0) return null;
|
|
24355
|
+
const sorted = [...values].sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
24356
|
+
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
24357
|
+
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
24197
24358
|
}
|
|
24198
|
-
|
|
24199
|
-
|
|
24359
|
+
var PICOS_PER_CENT = 10000000000n;
|
|
24360
|
+
function ceilingCreditsFor(costPicos, terms) {
|
|
24361
|
+
if (costPicos <= 0n) return 1;
|
|
24362
|
+
const num = costPicos * BigInt(1e4 + terms.feeBps) * BigInt(1e4 + terms.headroomBps);
|
|
24363
|
+
const den = 100000000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
24364
|
+
return Math.max(1, Number((num + den - 1n) / den));
|
|
24365
|
+
}
|
|
24366
|
+
function averageCreditsPerCall(costsPicos, terms) {
|
|
24367
|
+
if (costsPicos.length === 0) return null;
|
|
24368
|
+
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
24369
|
+
return Number(sum * BigInt(1e4 + terms.feeBps)) / Number(BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT);
|
|
24370
|
+
}
|
|
24371
|
+
function perCallEstimateCreditsFor(costsPicos, terms) {
|
|
24372
|
+
if (costsPicos.length === 0) return null;
|
|
24373
|
+
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
24374
|
+
const num = sum * BigInt(1e4 + terms.feeBps);
|
|
24375
|
+
const den = BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
24376
|
+
return Math.max(1, Number((num + den - 1n) / den));
|
|
24377
|
+
}
|
|
24378
|
+
function formatCredits(value) {
|
|
24379
|
+
if (!Number.isFinite(value)) return "\u2014";
|
|
24380
|
+
if (value >= 100) return String(Math.round(value));
|
|
24381
|
+
return String(Number(value.toPrecision(value < 1 ? 2 : 3)));
|
|
24382
|
+
}
|
|
24383
|
+
function usdFromPicos(picos) {
|
|
24384
|
+
if (picos === null || picos === void 0) return "unknown";
|
|
24385
|
+
const value = typeof picos === "bigint" ? picos : /^\d+$/.test(picos) ? BigInt(picos) : null;
|
|
24386
|
+
if (value === null) return "unknown";
|
|
24387
|
+
return `$${(Number(value) / 1e12).toFixed(6)}`;
|
|
24200
24388
|
}
|
|
24201
24389
|
function usdPerMillion(value) {
|
|
24202
24390
|
return `$${value.toFixed(value < 1 ? 4 : 2)}`;
|
|
24203
24391
|
}
|
|
24204
|
-
function usd(value) {
|
|
24205
|
-
return value === null ? "unknown" : `$${value.toFixed(6)}`;
|
|
24206
|
-
}
|
|
24207
24392
|
async function runLlm(opts = {}) {
|
|
24208
24393
|
const log = createLogger({ quiet: opts.quiet || opts.json });
|
|
24209
24394
|
const cwd = opts.cwd ?? process.cwd();
|
|
@@ -24229,16 +24414,23 @@ async function runLlm(opts = {}) {
|
|
|
24229
24414
|
return;
|
|
24230
24415
|
}
|
|
24231
24416
|
if (sub === "bench") {
|
|
24232
|
-
if (opts.maxCoins
|
|
24417
|
+
if (opts.maxCoins !== void 0) process.stderr.write(`${c.yellow("!")} ${LLM_MAX_COINS_DEPRECATED}
|
|
24418
|
+
`);
|
|
24419
|
+
if (opts.maxCoins !== void 0 && opts.maxCredits !== void 0 && opts.maxCoins !== opts.maxCredits) {
|
|
24420
|
+
fail5("llm bench", `Pass --max-credits alone \u2014 --max-coins is its deprecated name, and the two disagree (${opts.maxCredits} vs ${opts.maxCoins}).`);
|
|
24421
|
+
return;
|
|
24422
|
+
}
|
|
24423
|
+
const approvedCredits = opts.maxCredits ?? opts.maxCoins;
|
|
24424
|
+
if (approvedCredits === void 0 || opts.userApproved !== true) {
|
|
24233
24425
|
fail5("llm bench", LLM_BENCH_APPROVAL_REQUIRED);
|
|
24234
24426
|
return;
|
|
24235
24427
|
}
|
|
24236
|
-
if (!Number.isInteger(
|
|
24237
|
-
fail5("llm bench", `--max-
|
|
24428
|
+
if (!Number.isInteger(approvedCredits) || approvedCredits < 1) {
|
|
24429
|
+
fail5("llm bench", `--max-credits takes a whole number of credits, 1 or more (got ${String(approvedCredits)}).`);
|
|
24238
24430
|
return;
|
|
24239
24431
|
}
|
|
24240
24432
|
if (!opts.benchPrompt?.trim()) {
|
|
24241
|
-
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-
|
|
24433
|
+
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-credits <n> --user-approved.');
|
|
24242
24434
|
return;
|
|
24243
24435
|
}
|
|
24244
24436
|
if (opts.jsonOutput && opts.textOutput) {
|
|
@@ -24330,6 +24522,10 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24330
24522
|
models: null,
|
|
24331
24523
|
externalProviders: null,
|
|
24332
24524
|
recommendedDeclaredHeadroomBps: null,
|
|
24525
|
+
currency: null,
|
|
24526
|
+
feeBps: null,
|
|
24527
|
+
usdCentsPerCredit: null,
|
|
24528
|
+
standingCreditGrant: null,
|
|
24333
24529
|
total: null,
|
|
24334
24530
|
featuredCount: null,
|
|
24335
24531
|
// The stand was never asked, so the verdict is unknown — but the key is
|
|
@@ -24355,6 +24551,12 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24355
24551
|
models: lane.models,
|
|
24356
24552
|
externalProviders: lane.externalProviders,
|
|
24357
24553
|
recommendedDeclaredHeadroomBps: lane.recommendedDeclaredHeadroomBps,
|
|
24554
|
+
// The credit terms a call is billed on, the server's — null on a stand
|
|
24555
|
+
// that predates credit billing, never a number the CLI filled in.
|
|
24556
|
+
currency: lane.currency,
|
|
24557
|
+
feeBps: lane.feeBps,
|
|
24558
|
+
usdCentsPerCredit: lane.usdCentsPerCredit,
|
|
24559
|
+
standingCreditGrant: lane.standingCreditGrant,
|
|
24358
24560
|
total: lane.total ?? lane.models?.length ?? null,
|
|
24359
24561
|
featuredCount: lane.featuredCount ?? (lane.models ? lane.models.filter((m) => m.featured).length : null),
|
|
24360
24562
|
// `null` when the lane is not live or the server predates the probe.
|
|
@@ -24401,8 +24603,9 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24401
24603
|
);
|
|
24402
24604
|
for (const m of shown) {
|
|
24403
24605
|
const plan = m.personalPlan ? ` \xB7 personal plan: ${m.personalPlan}` : "";
|
|
24606
|
+
const kind = m.kind === "typesafe" ? " \xB7 judge (classifier): judge() under a budget, not generate() or bench" : "";
|
|
24404
24607
|
log.plain(
|
|
24405
|
-
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}`
|
|
24608
|
+
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}${kind}`
|
|
24406
24609
|
);
|
|
24407
24610
|
}
|
|
24408
24611
|
if (hiddenCount > 0) {
|
|
@@ -24418,74 +24621,104 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24418
24621
|
}
|
|
24419
24622
|
log.plain("");
|
|
24420
24623
|
if (lane.recommendedDeclaredHeadroomBps === null) {
|
|
24421
|
-
log.dim(" This stand serves no recommended headroom, so no
|
|
24624
|
+
log.dim(" This stand serves no recommended headroom, so no ceiling can be recommended from a bench.");
|
|
24422
24625
|
} else {
|
|
24423
|
-
log.dim(` Recommended headroom over benchmarked
|
|
24626
|
+
log.dim(` Recommended headroom over a benchmarked call's cost on this stand: ${lane.recommendedDeclaredHeadroomBps} bps.`);
|
|
24627
|
+
}
|
|
24628
|
+
if (lane.feeBps === null) {
|
|
24629
|
+
log.dim(" This stand serves no platform fee \u2014 it predates credit billing, so no ceiling can be recommended from a bench.");
|
|
24630
|
+
} else {
|
|
24631
|
+
log.dim(` In-game calls are billed in the player's credits, as used: the provider's cost plus this stand's platform fee (${lane.feeBps} bps).`);
|
|
24424
24632
|
}
|
|
24425
24633
|
if (toolsOnly) {
|
|
24426
24634
|
printConvertHint();
|
|
24427
24635
|
return;
|
|
24428
24636
|
}
|
|
24429
|
-
log.dim(" Those per-million rates are the PROVIDER's; what
|
|
24430
|
-
log.dim(`
|
|
24637
|
+
log.dim(" Those per-million rates are the PROVIDER's list; what one call of YOUR prompt really costs,");
|
|
24638
|
+
log.dim(` and the per-call ceiling to declare, only a real call measures: ${c.cyan('genex llm bench "<prompt>" --max-credits <n> --user-approved')}`);
|
|
24431
24639
|
}
|
|
24432
24640
|
function wholeOrNull(value) {
|
|
24433
24641
|
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
24434
24642
|
}
|
|
24643
|
+
function costPicosOf(usage) {
|
|
24644
|
+
if (typeof usage?.costUsdPicos === "string" && /^\d+$/.test(usage.costUsdPicos)) return usage.costUsdPicos;
|
|
24645
|
+
if (typeof usage?.costUsd === "number" && Number.isFinite(usage.costUsd) && usage.costUsd >= 0) {
|
|
24646
|
+
return String(BigInt(Math.round(usage.costUsd * 1e12)));
|
|
24647
|
+
}
|
|
24648
|
+
return null;
|
|
24649
|
+
}
|
|
24435
24650
|
function sampleFrom(id, settled) {
|
|
24436
24651
|
const usage = settled?.usage;
|
|
24652
|
+
const picos = costPicosOf(usage);
|
|
24437
24653
|
return {
|
|
24438
24654
|
id,
|
|
24439
24655
|
status: settled?.status ?? null,
|
|
24440
24656
|
billingStatus: settled?.billingStatus ?? null,
|
|
24441
|
-
|
|
24442
|
-
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd : null,
|
|
24657
|
+
chargedCredits: wholeOrNull(settled?.chargedCredits),
|
|
24658
|
+
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd : picos !== null ? Number(picos) / 1e12 : null,
|
|
24659
|
+
costUsdPicos: picos,
|
|
24443
24660
|
error: settled?.error ?? (settled ? null : "timed_out"),
|
|
24444
24661
|
providerMessage: typeof settled?.providerMessage === "string" && settled.providerMessage.trim() ? settled.providerMessage.trim() : null,
|
|
24445
24662
|
inputTokens: wholeOrNull(usage?.inputTokens),
|
|
24446
24663
|
outputTokens: wholeOrNull(usage?.outputTokens),
|
|
24447
24664
|
maxOutputTokens: wholeOrNull(usage?.maxOutputTokens),
|
|
24448
|
-
|
|
24665
|
+
fitCredits: typeof usage?.fitCredits === "number" && Number.isSafeInteger(usage.fitCredits) && usage.fitCredits >= 1 ? usage.fitCredits : null,
|
|
24449
24666
|
fitExceedsCap: typeof usage?.fitExceedsCap === "boolean" ? usage.fitExceedsCap : null,
|
|
24450
24667
|
truncated: typeof usage?.truncated === "boolean" ? usage.truncated : settled?.error === "provider_token_limit" ? true : null
|
|
24451
24668
|
};
|
|
24452
24669
|
}
|
|
24453
24670
|
var PROVIDER_TOKEN_LIMIT = "provider_token_limit";
|
|
24454
|
-
function benchRecommendation(results,
|
|
24455
|
-
const settled = results.filter(
|
|
24456
|
-
|
|
24457
|
-
|
|
24458
|
-
const
|
|
24459
|
-
const
|
|
24671
|
+
function benchRecommendation(results, terms) {
|
|
24672
|
+
const settled = results.filter(
|
|
24673
|
+
(r) => r.status === "succeeded" && r.billingStatus === "final" && r.chargedCredits !== null && r.costUsdPicos !== null
|
|
24674
|
+
);
|
|
24675
|
+
const costs = settled.map((r) => BigInt(r.costUsdPicos));
|
|
24676
|
+
const charged = settled.map((r) => r.chargedCredits);
|
|
24677
|
+
const fits = settled.map((r) => r.fitCredits).filter((v) => typeof v === "number");
|
|
24678
|
+
const costP95 = percentilePicos(costs, 95);
|
|
24679
|
+
const costMax = costs.length ? costs.reduce((a, b) => b > a ? b : a) : null;
|
|
24460
24680
|
const fitP95 = percentile2(fits, 95);
|
|
24461
24681
|
const fitMax = fits.length ? Math.max(...fits) : null;
|
|
24462
24682
|
const exceedsStandCap = results.some((r) => r.fitExceedsCap === true);
|
|
24463
24683
|
const cutOffSamples = results.filter((r) => r.error === PROVIDER_TOKEN_LIMIT && r.fitExceedsCap !== true).length;
|
|
24464
24684
|
const noRecommendationReason = exceedsStandCap ? "answer_exceeds_stand_cap" : cutOffSamples > 0 ? "answer_cut_off" : null;
|
|
24465
|
-
const
|
|
24466
|
-
const
|
|
24467
|
-
const
|
|
24685
|
+
const { headroomBps, feeBps, usdCentsPerCredit } = terms;
|
|
24686
|
+
const missingTerms = headroomBps === null ? "headroom" : feeBps === null || usdCentsPerCredit === null ? "credit_terms" : null;
|
|
24687
|
+
const full = headroomBps !== null && feeBps !== null && usdCentsPerCredit !== null ? { headroomBps, feeBps, usdCentsPerCredit } : null;
|
|
24688
|
+
const billing = feeBps !== null && usdCentsPerCredit !== null ? { feeBps, usdCentsPerCredit } : null;
|
|
24689
|
+
const costBased = full !== null && costP95 !== null ? ceilingCreditsFor(costP95, full) : null;
|
|
24690
|
+
const recommended = costBased === null || noRecommendationReason !== null ? null : Math.max(costBased, fitP95 ?? 0);
|
|
24691
|
+
const ceiling = recommended === null || costMax === null || full === null ? null : Math.max(ceilingCreditsFor(costMax, full), recommended, fitMax ?? 0);
|
|
24692
|
+
const average = billing !== null ? averageCreditsPerCall(costs, billing) : null;
|
|
24693
|
+
const estimate = ceiling === null || billing === null ? null : Math.min(ceiling, perCallEstimateCreditsFor(costs, billing) ?? 1);
|
|
24468
24694
|
return {
|
|
24695
|
+
costs,
|
|
24469
24696
|
charged,
|
|
24470
|
-
|
|
24471
|
-
|
|
24472
|
-
|
|
24697
|
+
costP50: percentilePicos(costs, 50),
|
|
24698
|
+
costP95,
|
|
24699
|
+
costMax,
|
|
24700
|
+
chargedP50: percentile2(charged, 50),
|
|
24701
|
+
chargedP95: percentile2(charged, 95),
|
|
24702
|
+
chargedMax: charged.length ? Math.max(...charged) : null,
|
|
24473
24703
|
fits,
|
|
24474
24704
|
fitP95,
|
|
24475
24705
|
fitMax,
|
|
24476
|
-
|
|
24477
|
-
|
|
24478
|
-
|
|
24479
|
-
|
|
24706
|
+
costBasedMaxCredits: costBased,
|
|
24707
|
+
recommendedMaxCredits: recommended,
|
|
24708
|
+
recommendedPerCallMaxCredits: ceiling,
|
|
24709
|
+
averageCreditsPerCall: average,
|
|
24710
|
+
recommendedPerCallEstimateCredits: estimate,
|
|
24711
|
+
lengthRaisedCeiling: recommended !== null && costBased !== null && recommended > costBased,
|
|
24480
24712
|
cutOffSamples,
|
|
24481
24713
|
exceedsStandCap,
|
|
24482
|
-
noRecommendationReason
|
|
24714
|
+
noRecommendationReason,
|
|
24715
|
+
missingTerms
|
|
24483
24716
|
};
|
|
24484
24717
|
}
|
|
24485
|
-
var LLM_LENGTH_RAISED_LINE = "The
|
|
24486
|
-
var LLM_EXCEEDS_STAND_CAP_LINE = "No
|
|
24487
|
-
function benchCutOffLine(cutOff,
|
|
24488
|
-
return `No
|
|
24718
|
+
var LLM_LENGTH_RAISED_LINE = "The ceiling also decides how long the answer may be: at the cost-based ceiling the answer would be cut off, so declare this one.";
|
|
24719
|
+
var LLM_EXCEEDS_STAND_CAP_LINE = "No ceiling recommended: this answer is longer than one call on this stand may produce. Ask for a shorter answer \u2014 fewer fields, shorter strings, a length the prompt states \u2014 and benchmark again.";
|
|
24720
|
+
function benchCutOffLine(cutOff, maxCredits) {
|
|
24721
|
+
return `No ceiling recommended: ${cutOff} sample${cutOff === 1 ? " was" : "s were"} cut off at this ceiling (--max-credits ${maxCredits}) before the answer was finished, so its real length is unknown. Re-run with a higher --max-credits.`;
|
|
24489
24722
|
}
|
|
24490
24723
|
var ACTIVE_STATUSES = /* @__PURE__ */ new Set(["requires_confirmation", "queued", "dispatching", "awaiting_external"]);
|
|
24491
24724
|
function rowState(row) {
|
|
@@ -24496,7 +24729,22 @@ function rowState(row) {
|
|
|
24496
24729
|
function rowHoldsSlot(row) {
|
|
24497
24730
|
if (typeof row.slotHeld === "boolean") return row.slotHeld;
|
|
24498
24731
|
const state = rowState(row);
|
|
24499
|
-
return state === "active" || state === "awaiting bill" && (row.reservedCoins ?? 0) > 0;
|
|
24732
|
+
return state === "active" || state === "awaiting bill" && ((row.reservedCoins ?? 0) > 0 || (row.reservedCredits ?? 0) > 0);
|
|
24733
|
+
}
|
|
24734
|
+
function rowMoney(row) {
|
|
24735
|
+
const credits = row.currency === "credits" || row.currency === void 0 && typeof row.reservedCredits === "number" && typeof row.reservedCoins !== "number";
|
|
24736
|
+
if (credits) {
|
|
24737
|
+
return {
|
|
24738
|
+
unit: "credits",
|
|
24739
|
+
reserved: typeof row.reservedCredits === "number" ? row.reservedCredits : null,
|
|
24740
|
+
charged: typeof row.chargedCredits === "number" ? row.chargedCredits : null
|
|
24741
|
+
};
|
|
24742
|
+
}
|
|
24743
|
+
return {
|
|
24744
|
+
unit: "coin",
|
|
24745
|
+
reserved: typeof row.reservedCoins === "number" ? row.reservedCoins : null,
|
|
24746
|
+
charged: typeof row.chargedCoins === "number" ? row.chargedCoins : null
|
|
24747
|
+
};
|
|
24500
24748
|
}
|
|
24501
24749
|
function providerRefusalStatus(error) {
|
|
24502
24750
|
const m = /^provider_http_(\d{3})$/.exec(error ?? "");
|
|
@@ -24515,7 +24763,7 @@ function formatAge(iso, now = Date.now()) {
|
|
|
24515
24763
|
}
|
|
24516
24764
|
async function runBench(args) {
|
|
24517
24765
|
const { apiUrl, token, projectId, cwd, opts, log } = args;
|
|
24518
|
-
const
|
|
24766
|
+
const maxCredits = opts.maxCredits ?? opts.maxCoins;
|
|
24519
24767
|
const samples = opts.samples ?? DEFAULT_BENCH_SAMPLES;
|
|
24520
24768
|
const prompt = opts.benchPrompt.trim();
|
|
24521
24769
|
const auth = { Authorization: `Bearer ${token}`, "Content-Type": "application/json" };
|
|
@@ -24537,7 +24785,7 @@ async function runBench(args) {
|
|
|
24537
24785
|
const lane = await readRuntimeLane(apiUrl, token);
|
|
24538
24786
|
if (lane.state !== "live" || !lane.models || lane.models.length === 0) {
|
|
24539
24787
|
if (lane.state === "off") {
|
|
24540
|
-
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
24788
|
+
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCredits, opts), status: "off", error: null });
|
|
24541
24789
|
else {
|
|
24542
24790
|
log.plain(LLM_LANE_OFF_LINE);
|
|
24543
24791
|
log.dim(" Nothing was spent.");
|
|
@@ -24564,17 +24812,27 @@ async function runBench(args) {
|
|
|
24564
24812
|
);
|
|
24565
24813
|
return;
|
|
24566
24814
|
}
|
|
24567
|
-
const
|
|
24815
|
+
const walletBefore = await fetchCreditsSnapshot(apiUrl, token);
|
|
24816
|
+
if (walletBefore?.emailVerified === false) {
|
|
24817
|
+
benchFailed(opts, log, LLM_BENCH_UNVERIFIED_LINE);
|
|
24818
|
+
return;
|
|
24819
|
+
}
|
|
24820
|
+
if (walletBefore !== null && walletBefore.spendable < maxCredits) {
|
|
24821
|
+
benchFailed(opts, log, benchBalanceShortLine(walletBefore.spendable, maxCredits));
|
|
24822
|
+
return;
|
|
24823
|
+
}
|
|
24824
|
+
const balanceBefore = walletBefore?.spendable ?? null;
|
|
24568
24825
|
if (!opts.json) {
|
|
24569
24826
|
log.plain(c.bold("genex llm bench"));
|
|
24570
24827
|
log.dim(` ${apiUrl}`);
|
|
24571
24828
|
log.plain("");
|
|
24572
24829
|
log.plain(` Model ${c.cyan(modelId)}`);
|
|
24573
24830
|
log.plain(` Samples ${samples} real attempt${samples === 1 ? "" : "s"}, ${outputFormat} output`);
|
|
24574
|
-
log.plain(` Approved ${
|
|
24831
|
+
log.plain(` Approved ${maxCredits} credit${maxCredits === 1 ? "" : "s"} per attempt \u2014 at worst ${maxCredits * samples} credits for this run, from your own balance`);
|
|
24575
24832
|
log.plain(
|
|
24576
|
-
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore}
|
|
24833
|
+
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore} credits spendable`}`
|
|
24577
24834
|
);
|
|
24835
|
+
log.dim(" Billed as used: each attempt is charged its real cost plus the platform fee, rounded up to a whole credit.");
|
|
24578
24836
|
log.plain("");
|
|
24579
24837
|
}
|
|
24580
24838
|
const base = `${apiUrl}/api/runtime/development/projects/${encodeURIComponent(projectId)}/generations`;
|
|
@@ -24600,7 +24858,7 @@ async function runBench(args) {
|
|
|
24600
24858
|
const res = await call(base, {
|
|
24601
24859
|
method: "POST",
|
|
24602
24860
|
body: JSON.stringify({
|
|
24603
|
-
|
|
24861
|
+
maxCredits,
|
|
24604
24862
|
request: {
|
|
24605
24863
|
idempotencyKey: `bench-${randomUUID2()}`,
|
|
24606
24864
|
modelId,
|
|
@@ -24630,13 +24888,14 @@ async function runBench(args) {
|
|
|
24630
24888
|
if (refusedAt !== null) providerRefused++;
|
|
24631
24889
|
if (!opts.json) {
|
|
24632
24890
|
const length = row.outputTokens !== null ? ` \xB7 ${row.outputTokens}${row.maxOutputTokens !== null ? ` of ${row.maxOutputTokens}` : ""} tokens out` : "";
|
|
24891
|
+
const charged = row.chargedCredits === null ? "\u2014" : String(row.chargedCredits);
|
|
24633
24892
|
log.plain(
|
|
24634
|
-
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${
|
|
24893
|
+
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${charged.padStart(4)} credit${row.chargedCredits === 1 ? "" : "s"} charged \xB7 cost ${usdFromPicos(row.costUsdPicos)}${length}${row.error ? ` \xB7 ${row.error}` : ""}`
|
|
24635
24894
|
);
|
|
24636
24895
|
if (refusedAt !== null) printProviderRefusal(log, refusedAt, row.providerMessage, row.status === "unknown");
|
|
24637
24896
|
if (row.error === PROVIDER_TOKEN_LIMIT) {
|
|
24638
24897
|
log.dim(
|
|
24639
|
-
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no
|
|
24898
|
+
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no ceiling makes room for it; it is not a sample." : ` The answer was cut off at this ceiling (--max-credits ${maxCredits}) before it was finished \u2014 it was charged, and it is not a sample.`
|
|
24640
24899
|
);
|
|
24641
24900
|
}
|
|
24642
24901
|
}
|
|
@@ -24644,16 +24903,19 @@ async function runBench(args) {
|
|
|
24644
24903
|
} finally {
|
|
24645
24904
|
process.removeListener("SIGINT", onSigint);
|
|
24646
24905
|
}
|
|
24647
|
-
const
|
|
24648
|
-
|
|
24649
|
-
|
|
24650
|
-
|
|
24651
|
-
|
|
24652
|
-
const
|
|
24906
|
+
const terms = {
|
|
24907
|
+
headroomBps: lane.recommendedDeclaredHeadroomBps,
|
|
24908
|
+
feeBps: lane.feeBps,
|
|
24909
|
+
usdCentsPerCredit: lane.usdCentsPerCredit
|
|
24910
|
+
};
|
|
24911
|
+
const rec = benchRecommendation(results, terms);
|
|
24912
|
+
const settledCount = rec.charged.length;
|
|
24653
24913
|
const lengthBlocked = rec.noRecommendationReason !== null;
|
|
24654
|
-
const balanceAfter = await
|
|
24914
|
+
const balanceAfter = (await fetchCreditsSnapshot(apiUrl, token))?.spendable ?? null;
|
|
24915
|
+
const picosOrNull = (v) => v === null ? null : String(v);
|
|
24655
24916
|
const record = {
|
|
24656
|
-
v:
|
|
24917
|
+
v: 2,
|
|
24918
|
+
currency: "credits",
|
|
24657
24919
|
ranAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
24658
24920
|
apiUrl,
|
|
24659
24921
|
projectId,
|
|
@@ -24662,28 +24924,34 @@ async function runBench(args) {
|
|
|
24662
24924
|
outputFormat,
|
|
24663
24925
|
samples,
|
|
24664
24926
|
completed: settledCount,
|
|
24665
|
-
|
|
24666
|
-
|
|
24667
|
-
|
|
24668
|
-
max,
|
|
24669
|
-
|
|
24670
|
-
|
|
24671
|
-
recommendedPerCallMaxCoins: ceiling,
|
|
24672
|
-
maxCoinsPerSample: maxCoins,
|
|
24673
|
-
fitCoins: rec.fits,
|
|
24927
|
+
maxCreditsPerSample: maxCredits,
|
|
24928
|
+
costUsdPicos: rec.costs.map(String),
|
|
24929
|
+
chargedCredits: rec.charged,
|
|
24930
|
+
cost: { p50: picosOrNull(rec.costP50), p95: picosOrNull(rec.costP95), max: picosOrNull(rec.costMax) },
|
|
24931
|
+
charged: { p50: rec.chargedP50, p95: rec.chargedP95, max: rec.chargedMax },
|
|
24932
|
+
fitCredits: rec.fits,
|
|
24674
24933
|
fitP95: rec.fitP95,
|
|
24675
|
-
|
|
24676
|
-
|
|
24934
|
+
fitMax: rec.fitMax,
|
|
24935
|
+
recommendedDeclaredHeadroomBps: terms.headroomBps,
|
|
24936
|
+
feeBps: terms.feeBps,
|
|
24937
|
+
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
24938
|
+
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
24939
|
+
recommendedMaxCredits: rec.recommendedMaxCredits,
|
|
24940
|
+
recommendedPerCallMaxCredits: rec.recommendedPerCallMaxCredits,
|
|
24941
|
+
averageCreditsPerCall: rec.averageCreditsPerCall,
|
|
24942
|
+
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
24943
|
+
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
24677
24944
|
cutOffSamples: rec.cutOffSamples,
|
|
24678
24945
|
noRecommendationReason: rec.noRecommendationReason
|
|
24679
24946
|
};
|
|
24680
24947
|
const savedTo = await saveBench(cwd, record);
|
|
24681
24948
|
if (opts.json) {
|
|
24682
|
-
const ok =
|
|
24949
|
+
const ok = settledCount > 0 && !lengthBlocked;
|
|
24683
24950
|
writeJsonLine({
|
|
24684
24951
|
command: "llm bench",
|
|
24685
24952
|
status: ok ? "ok" : "failed",
|
|
24686
24953
|
error: ok ? null : rec.noRecommendationReason ?? "no_sample_settled",
|
|
24954
|
+
currency: "credits",
|
|
24687
24955
|
apiUrl,
|
|
24688
24956
|
projectId,
|
|
24689
24957
|
modelId,
|
|
@@ -24692,35 +24960,54 @@ async function runBench(args) {
|
|
|
24692
24960
|
schemaPath: opts.schemaPath ?? null,
|
|
24693
24961
|
samples,
|
|
24694
24962
|
completed: settledCount,
|
|
24695
|
-
|
|
24963
|
+
maxCreditsPerSample: maxCredits,
|
|
24964
|
+
// Spendable credits before and after the run; null when unreadable.
|
|
24696
24965
|
balanceBefore,
|
|
24697
24966
|
balanceAfter,
|
|
24698
24967
|
results,
|
|
24699
24968
|
// The create-time refusal that ended the run, or null. Beside the
|
|
24700
24969
|
// samples, never among them: no attempt existed and nothing was spent.
|
|
24701
24970
|
refusal: refusal2,
|
|
24702
|
-
|
|
24703
|
-
//
|
|
24971
|
+
// Over the settled samples: the real provider cost (pico-dollar strings
|
|
24972
|
+
// and USD) and the whole credits charged.
|
|
24973
|
+
costUsdPicos: record.cost,
|
|
24974
|
+
costUsd: {
|
|
24975
|
+
p50: rec.costP50 === null ? null : Number(rec.costP50) / 1e12,
|
|
24976
|
+
p95: rec.costP95 === null ? null : Number(rec.costP95) / 1e12,
|
|
24977
|
+
max: rec.costMax === null ? null : Number(rec.costMax) / 1e12
|
|
24978
|
+
},
|
|
24979
|
+
chargedCredits: record.charged,
|
|
24980
|
+
// The answer's LENGTH, priced by the server per sample (`fitCredits`,
|
|
24704
24981
|
// headroom already on the tokens), over the settled samples.
|
|
24705
|
-
|
|
24706
|
-
|
|
24707
|
-
|
|
24982
|
+
fitCredits: { p95: rec.fitP95, max: rec.fitMax },
|
|
24983
|
+
recommendedDeclaredHeadroomBps: terms.headroomBps,
|
|
24984
|
+
feeBps: terms.feeBps,
|
|
24985
|
+
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
24986
|
+
missingTerms: rec.missingTerms,
|
|
24987
|
+
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
24988
|
+
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
24708
24989
|
cutOffSamples: rec.cutOffSamples,
|
|
24709
24990
|
exceedsStandCap: rec.exceedsStandCap,
|
|
24710
24991
|
noRecommendationReason: rec.noRecommendationReason,
|
|
24711
|
-
|
|
24712
|
-
|
|
24713
|
-
|
|
24992
|
+
recommendedMaxCredits: rec.recommendedMaxCredits,
|
|
24993
|
+
recommendedPerCallMaxCredits: rec.recommendedPerCallMaxCredits,
|
|
24994
|
+
averageCreditsPerCall: rec.averageCreditsPerCall,
|
|
24995
|
+
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
24714
24996
|
savedTo
|
|
24715
24997
|
});
|
|
24716
24998
|
if (!ok) process.exitCode = 1;
|
|
24717
24999
|
return;
|
|
24718
25000
|
}
|
|
24719
25001
|
log.plain("");
|
|
25002
|
+
const printMeasured = () => {
|
|
25003
|
+
log.plain(c.bold(" Real cost per call"));
|
|
25004
|
+
log.plain(` p50 ${usdFromPicos(rec.costP50)} p95 ${usdFromPicos(rec.costP95)} max ${usdFromPicos(rec.costMax)} (${settledCount} of ${samples} settled)`);
|
|
25005
|
+
log.plain(c.bold(" Charged credits"));
|
|
25006
|
+
log.plain(` p50 ${rec.chargedP50} p95 ${rec.chargedP95} max ${rec.chargedMax} (whole credits, rounded up per call)`);
|
|
25007
|
+
};
|
|
24720
25008
|
if (lengthBlocked) {
|
|
24721
|
-
if (
|
|
24722
|
-
|
|
24723
|
-
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
25009
|
+
if (settledCount > 0) {
|
|
25010
|
+
printMeasured();
|
|
24724
25011
|
log.plain("");
|
|
24725
25012
|
}
|
|
24726
25013
|
printRecommendation(log, record);
|
|
@@ -24728,17 +25015,16 @@ async function runBench(args) {
|
|
|
24728
25015
|
process.exitCode = 1;
|
|
24729
25016
|
return;
|
|
24730
25017
|
}
|
|
24731
|
-
if (
|
|
25018
|
+
if (settledCount === 0) {
|
|
24732
25019
|
log.error(
|
|
24733
|
-
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to
|
|
25020
|
+
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to measure from." : " No sample settled, so there is nothing to measure from."
|
|
24734
25021
|
);
|
|
24735
25022
|
process.exitCode = 1;
|
|
24736
25023
|
return;
|
|
24737
25024
|
}
|
|
24738
|
-
|
|
24739
|
-
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
25025
|
+
printMeasured();
|
|
24740
25026
|
if (balanceBefore !== null && balanceAfter !== null) {
|
|
24741
|
-
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter}
|
|
25027
|
+
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter} credits spendable`);
|
|
24742
25028
|
}
|
|
24743
25029
|
log.plain("");
|
|
24744
25030
|
printRecommendation(log, record);
|
|
@@ -24750,43 +25036,58 @@ function printRecommendation(log, record) {
|
|
|
24750
25036
|
return;
|
|
24751
25037
|
}
|
|
24752
25038
|
if (record.noRecommendationReason === "answer_cut_off") {
|
|
24753
|
-
log.warn(` ${benchCutOffLine(record.cutOffSamples
|
|
25039
|
+
log.warn(` ${benchCutOffLine(record.cutOffSamples || 1, record.maxCreditsPerSample)}`);
|
|
24754
25040
|
return;
|
|
24755
25041
|
}
|
|
24756
|
-
|
|
24757
|
-
|
|
25042
|
+
const attempts = `${record.completed} settled attempt${record.completed === 1 ? "" : "s"} on ${record.modelId}`;
|
|
25043
|
+
if (record.recommendedMaxCredits === null || record.cost.p95 === null) {
|
|
25044
|
+
log.warn(" No ceiling recommended.");
|
|
24758
25045
|
log.dim(
|
|
24759
|
-
record.recommendedDeclaredHeadroomBps === null ? " This stand served no recommended headroom, and the multiplier is the server's to set \u2014" : "
|
|
25046
|
+
record.cost.p95 === null ? " No sample settled, so there is no p95 to build a ceiling on \u2014" : record.recommendedDeclaredHeadroomBps === null ? " This stand served no recommended headroom, and the multiplier is the server's to set \u2014" : " This stand served no platform fee or credit value, and those are the server's to set \u2014"
|
|
24760
25047
|
);
|
|
24761
25048
|
log.dim(" declaring a number from this run would be a guess dressed as a measurement.");
|
|
25049
|
+
printAverage(log, record, attempts);
|
|
24762
25050
|
return;
|
|
24763
25051
|
}
|
|
24764
|
-
log.plain(c.bold(` Declare
|
|
24765
|
-
if (record.
|
|
25052
|
+
log.plain(c.bold(` Declare maxCredits: ${record.recommendedMaxCredits}`));
|
|
25053
|
+
if (record.lengthRaisedCeiling) {
|
|
24766
25054
|
log.plain(` ${LLM_LENGTH_RAISED_LINE}`);
|
|
24767
25055
|
log.dim(
|
|
24768
|
-
` = the smallest
|
|
25056
|
+
` = the smallest ceiling whose answer allowance holds the answer plus this stand's headroom (p95 ${record.fitP95 ?? "\u2014"}), above the`
|
|
24769
25057
|
);
|
|
24770
25058
|
log.dim(
|
|
24771
|
-
`
|
|
25059
|
+
` cost-based ${record.costBasedMaxCredits ?? "\u2014"} (p95 real cost ${usdFromPicos(record.cost.p95)} with the platform fee, ${record.feeBps} bps, and ${record.recommendedDeclaredHeadroomBps} bps of headroom), over ${attempts}.`
|
|
24772
25060
|
);
|
|
24773
25061
|
} else {
|
|
24774
25062
|
log.dim(
|
|
24775
|
-
` = p95
|
|
25063
|
+
` = p95 real cost (${usdFromPicos(record.cost.p95)}) with this stand's platform fee (${record.feeBps} bps) and recommended headroom (${record.recommendedDeclaredHeadroomBps} bps), in whole credits,`
|
|
24776
25064
|
);
|
|
24777
|
-
log.dim(` measured over ${
|
|
25065
|
+
log.dim(` measured over ${attempts}.`);
|
|
24778
25066
|
}
|
|
24779
|
-
log.dim(" That number is a
|
|
24780
|
-
|
|
24781
|
-
|
|
25067
|
+
log.dim(" That number is a CEILING, not a price: a call is billed its real cost as used, never more than it \u2014");
|
|
25068
|
+
log.dim(" and it also sizes how long the answer may be.");
|
|
25069
|
+
if (record.recommendedPerCallMaxCredits !== null) {
|
|
25070
|
+
log.plain(c.bold(` Grant perCallMaxCredits: ${record.recommendedPerCallMaxCredits}`));
|
|
24782
25071
|
log.dim(
|
|
24783
|
-
record.
|
|
25072
|
+
record.lengthRaisedCeiling ? ` = the same over the worst sample (${usdFromPicos(record.cost.max)}) or the longest answer's ceiling, whichever is larger, and never below the ceiling above it.` : ` = the same over the worst sample (${usdFromPicos(record.cost.max)}), and never below the ceiling above it.`
|
|
24784
25073
|
);
|
|
24785
|
-
|
|
25074
|
+
}
|
|
25075
|
+
printAverage(log, record, attempts);
|
|
25076
|
+
}
|
|
25077
|
+
function printAverage(log, record, attempts) {
|
|
25078
|
+
if (record.averageCreditsPerCall === null) return;
|
|
25079
|
+
log.plain(` Per-call estimate: about ${formatCredits(record.averageCreditsPerCall)} credits per call`);
|
|
25080
|
+
log.dim(` = the average real cost with the platform fee, no headroom, over ${attempts}.`);
|
|
25081
|
+
if (record.recommendedPerCallEstimateCredits !== null) {
|
|
25082
|
+
log.plain(c.bold(` Grant perCallEstimateCredits: ${record.recommendedPerCallEstimateCredits}`));
|
|
25083
|
+
log.dim(` = that average rounded up to a whole credit. disclosure.estimatedCreditsPerPeriod = your calls per period \xD7 ${formatCredits(record.averageCreditsPerCall)}, rounded up.`);
|
|
24786
25084
|
}
|
|
24787
25085
|
}
|
|
24788
25086
|
async function explainBenchRefusal(res, call, base, log, opts, index, settledSoFar) {
|
|
24789
|
-
if (printedStructuredError(res))
|
|
25087
|
+
if (printedStructuredError(res)) {
|
|
25088
|
+
const printed = await res.json().catch(() => ({}));
|
|
25089
|
+
return { stop: true, refusal: { status: res.status, error: typeof printed.error === "string" ? printed.error : null, slotsHeld: null, slotLimit: null } };
|
|
25090
|
+
}
|
|
24790
25091
|
const body = await res.json().catch(() => ({}));
|
|
24791
25092
|
const refusal2 = { status: res.status, error: body.error ?? null, slotsHeld: null, slotLimit: null };
|
|
24792
25093
|
if (res.status === 429 && body.error === "generation_limit") {
|
|
@@ -24796,6 +25097,10 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24796
25097
|
if (!opts.json) log.error(` Sample ${index + 1} ${benchSlotsHeldSentence(counts)}`);
|
|
24797
25098
|
return { stop: true, refusal: refusal2 };
|
|
24798
25099
|
}
|
|
25100
|
+
if (res.status === 403 && body.error === "credits_unverified") {
|
|
25101
|
+
if (!opts.json) log.error(` ${LLM_BENCH_UNVERIFIED_LINE}`);
|
|
25102
|
+
return { stop: true, refusal: refusal2 };
|
|
25103
|
+
}
|
|
24799
25104
|
if (opts.json) return { stop: res.status !== 429, refusal: refusal2 };
|
|
24800
25105
|
if (res.status === 404 && body.error === "not_found") {
|
|
24801
25106
|
log.error(` ${LLM_LANE_OFF_LINE}`);
|
|
@@ -24811,7 +25116,7 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24811
25116
|
return { stop: true, refusal: refusal2 };
|
|
24812
25117
|
}
|
|
24813
25118
|
if (res.status === 402) {
|
|
24814
|
-
log.error(" Not enough
|
|
25119
|
+
log.error(" Not enough credits to start the attempt \u2014 a bench is paid from your own balance.");
|
|
24815
25120
|
return { stop: true, refusal: refusal2 };
|
|
24816
25121
|
}
|
|
24817
25122
|
log.error(
|
|
@@ -24820,7 +25125,7 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24820
25125
|
return { stop: res.status >= 500 || res.status === 401 || res.status === 403, refusal: refusal2 };
|
|
24821
25126
|
}
|
|
24822
25127
|
function printProviderRefusal(log, status2, message, awaitingBill = false) {
|
|
24823
|
-
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so this call
|
|
25128
|
+
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so what this call reserved stays held until it resolves (see \`genex llm status\`); it is not a sample.`);
|
|
24824
25129
|
else log.dim(" The provider refused this call at its door \u2014 no inference ran, it cost nothing, and it is not a sample.");
|
|
24825
25130
|
if (message) log.dim(` Provider said: ${message}`);
|
|
24826
25131
|
if (status2 === 401 || status2 === 403) {
|
|
@@ -24846,19 +25151,6 @@ async function pollSettled(call, url, timeoutSec) {
|
|
|
24846
25151
|
}
|
|
24847
25152
|
return last;
|
|
24848
25153
|
}
|
|
24849
|
-
async function readCoinBalance(apiUrl, token) {
|
|
24850
|
-
try {
|
|
24851
|
-
const res = await apiFetch(`${apiUrl}/api/coin/balance`, {
|
|
24852
|
-
headers: { Authorization: `Bearer ${token}` },
|
|
24853
|
-
signal: AbortSignal.timeout(6e3)
|
|
24854
|
-
});
|
|
24855
|
-
if (!res.ok) return null;
|
|
24856
|
-
const body = await res.json().catch(() => null);
|
|
24857
|
-
return typeof body?.spendable === "number" ? body.spendable : null;
|
|
24858
|
-
} catch {
|
|
24859
|
-
return null;
|
|
24860
|
-
}
|
|
24861
|
-
}
|
|
24862
25154
|
async function saveBench(cwd, record) {
|
|
24863
25155
|
try {
|
|
24864
25156
|
const file = path36.join(cwd, LLM_BENCH_FILE);
|
|
@@ -24874,9 +25166,10 @@ function benchFailed(opts, log, message) {
|
|
|
24874
25166
|
else log.error(message);
|
|
24875
25167
|
process.exitCode = 1;
|
|
24876
25168
|
}
|
|
24877
|
-
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
25169
|
+
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCredits, opts) {
|
|
24878
25170
|
return {
|
|
24879
25171
|
command: "llm bench",
|
|
25172
|
+
currency: "credits",
|
|
24880
25173
|
apiUrl,
|
|
24881
25174
|
projectId,
|
|
24882
25175
|
modelId: opts.modelId ?? null,
|
|
@@ -24885,21 +25178,28 @@ function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCoi
|
|
|
24885
25178
|
schemaPath: opts.schemaPath ?? null,
|
|
24886
25179
|
samples,
|
|
24887
25180
|
completed: 0,
|
|
24888
|
-
|
|
25181
|
+
maxCreditsPerSample: maxCredits,
|
|
24889
25182
|
balanceBefore: null,
|
|
24890
25183
|
balanceAfter: null,
|
|
24891
25184
|
results: [],
|
|
24892
25185
|
refusal: null,
|
|
24893
|
-
|
|
24894
|
-
|
|
24895
|
-
|
|
24896
|
-
|
|
25186
|
+
costUsdPicos: { p50: null, p95: null, max: null },
|
|
25187
|
+
costUsd: { p50: null, p95: null, max: null },
|
|
25188
|
+
chargedCredits: { p50: null, p95: null, max: null },
|
|
25189
|
+
fitCredits: { p95: null, max: null },
|
|
25190
|
+
recommendedDeclaredHeadroomBps: null,
|
|
25191
|
+
feeBps: null,
|
|
25192
|
+
usdCentsPerCredit: null,
|
|
25193
|
+
missingTerms: null,
|
|
25194
|
+
costBasedMaxCredits: null,
|
|
25195
|
+
lengthRaisedCeiling: false,
|
|
24897
25196
|
cutOffSamples: 0,
|
|
24898
25197
|
exceedsStandCap: false,
|
|
24899
25198
|
noRecommendationReason: null,
|
|
24900
|
-
|
|
24901
|
-
|
|
24902
|
-
|
|
25199
|
+
recommendedMaxCredits: null,
|
|
25200
|
+
recommendedPerCallMaxCredits: null,
|
|
25201
|
+
averageCreditsPerCall: null,
|
|
25202
|
+
recommendedPerCallEstimateCredits: null,
|
|
24903
25203
|
savedTo: null
|
|
24904
25204
|
};
|
|
24905
25205
|
}
|
|
@@ -24962,7 +25262,8 @@ async function reportStatus(args) {
|
|
|
24962
25262
|
for (const row of rows) {
|
|
24963
25263
|
const state = rowState(row);
|
|
24964
25264
|
const mark = state === "active" ? c.cyan("\u25CF") : state === "awaiting bill" ? c.yellow("\u25D0") : c.green("\u25CB");
|
|
24965
|
-
const
|
|
25265
|
+
const money2 = rowMoney(row);
|
|
25266
|
+
const reserved = money2.reserved === null ? "reserved \u2014" : `${money2.reserved} ${money2.unit} reserved`;
|
|
24966
25267
|
log.plain(
|
|
24967
25268
|
` ${mark} ${row.id ?? "?"} ${row.modelId ?? "?"} ${state.padEnd(13)} ${formatAge(row.createdAt, now).padStart(4)} old ${reserved}${row.error ? ` ${row.error}` : ""}${rowHoldsSlot(row) ? "" : c.dim(" (no slot)")}`
|
|
24968
25269
|
);
|
|
@@ -25034,10 +25335,11 @@ async function cancelCall(args) {
|
|
|
25034
25335
|
if (!opts.json) {
|
|
25035
25336
|
log.warn(` ${row.id} (${row.status ?? "?"}) \u2014 ${sentence}.`);
|
|
25036
25337
|
if (state === "awaiting bill") {
|
|
25037
|
-
log.dim(` Nothing to cancel:
|
|
25338
|
+
log.dim(` Nothing to cancel: what it reserved releases when the bill resolves, and it stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`);
|
|
25038
25339
|
if (row.providerMessage?.trim()) log.dim(` Provider said: ${row.providerMessage.trim()}`);
|
|
25039
25340
|
} else {
|
|
25040
|
-
|
|
25341
|
+
const money2 = rowMoney(row);
|
|
25342
|
+
log.dim(` ${money2.charged === null ? "charge unknown" : `${money2.charged} ${money2.unit} charged`}${row.error ? ` \xB7 ${row.error}` : ""}.`);
|
|
25041
25343
|
}
|
|
25042
25344
|
}
|
|
25043
25345
|
return done("ok", null, state === "awaiting bill" ? "awaiting_bill" : "already_final", row);
|
|
@@ -25059,78 +25361,126 @@ async function cancelCall(args) {
|
|
|
25059
25361
|
const after = await res.json().catch(() => null) ?? row;
|
|
25060
25362
|
if (!opts.json) {
|
|
25061
25363
|
log.success(` Cancelled ${row.id} (was ${row.status ?? "active"}).`);
|
|
25062
|
-
log.dim(`
|
|
25364
|
+
log.dim(` What it reserved releases once the bill settles; ${c.cyan("genex llm status")} shows it until then.`);
|
|
25063
25365
|
}
|
|
25064
25366
|
return done("ok", null, "cancelled", after);
|
|
25065
25367
|
}
|
|
25066
|
-
async function
|
|
25067
|
-
let
|
|
25368
|
+
async function readSavedBench(cwd) {
|
|
25369
|
+
let parsed;
|
|
25068
25370
|
try {
|
|
25069
|
-
|
|
25371
|
+
parsed = JSON.parse(await fs35.readFile(path36.join(cwd, LLM_BENCH_FILE), "utf8"));
|
|
25070
25372
|
} catch {
|
|
25071
|
-
|
|
25373
|
+
return null;
|
|
25072
25374
|
}
|
|
25073
|
-
if (!
|
|
25375
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null;
|
|
25376
|
+
const v = parsed;
|
|
25377
|
+
if (v.v === 2 && v.currency === "credits" && v.cost && v.charged) return { kind: "credits", record: parsed };
|
|
25378
|
+
return { kind: "coins", record: parsed };
|
|
25379
|
+
}
|
|
25380
|
+
var LLM_PRICE_COIN_TERMS_LINE = "This bench was measured under the old COIN terms, when a game declared a fixed price per call. In-game calls are now billed in the player's credits, as used, and a game declares a per-call CEILING (maxCredits) instead \u2014 a coin price is not that number. Re-run the bench for the figures to declare.";
|
|
25381
|
+
async function reportSavedPrice(cwd, opts, log) {
|
|
25382
|
+
const saved = await readSavedBench(cwd);
|
|
25383
|
+
const rerun = 'genex llm bench "<prompt>" --max-credits <n> --user-approved';
|
|
25384
|
+
const creditKeys = (record2) => ({
|
|
25385
|
+
costUsdPicos: record2?.cost ?? { p50: null, p95: null, max: null },
|
|
25386
|
+
chargedCredits: record2?.charged ?? { p50: null, p95: null, max: null },
|
|
25387
|
+
fitP95: record2?.fitP95 ?? null,
|
|
25388
|
+
fitMax: record2?.fitMax ?? null,
|
|
25389
|
+
recommendedDeclaredHeadroomBps: record2?.recommendedDeclaredHeadroomBps ?? null,
|
|
25390
|
+
feeBps: record2?.feeBps ?? null,
|
|
25391
|
+
usdCentsPerCredit: record2?.usdCentsPerCredit ?? null,
|
|
25392
|
+
costBasedMaxCredits: record2?.costBasedMaxCredits ?? null,
|
|
25393
|
+
lengthRaisedCeiling: record2?.lengthRaisedCeiling ?? false,
|
|
25394
|
+
cutOffSamples: record2?.cutOffSamples ?? 0,
|
|
25395
|
+
noRecommendationReason: record2?.noRecommendationReason ?? null,
|
|
25396
|
+
recommendedMaxCredits: record2?.recommendedMaxCredits ?? null,
|
|
25397
|
+
recommendedPerCallMaxCredits: record2?.recommendedPerCallMaxCredits ?? null,
|
|
25398
|
+
averageCreditsPerCall: record2?.averageCreditsPerCall ?? null,
|
|
25399
|
+
recommendedPerCallEstimateCredits: record2?.recommendedPerCallEstimateCredits ?? null
|
|
25400
|
+
});
|
|
25401
|
+
if (!saved) {
|
|
25074
25402
|
if (opts.json) {
|
|
25075
25403
|
writeJsonLine({
|
|
25076
25404
|
command: "llm price",
|
|
25077
25405
|
status: "failed",
|
|
25078
25406
|
error: "no_bench",
|
|
25407
|
+
currency: null,
|
|
25079
25408
|
ranAt: null,
|
|
25080
25409
|
modelId: null,
|
|
25081
25410
|
samples: null,
|
|
25082
25411
|
completed: null,
|
|
25083
|
-
|
|
25084
|
-
|
|
25085
|
-
max: null,
|
|
25086
|
-
fitP95: null,
|
|
25087
|
-
chargedBasedEstimateCoins: null,
|
|
25088
|
-
lengthRaisedPrice: false,
|
|
25089
|
-
cutOffSamples: 0,
|
|
25090
|
-
noRecommendationReason: null,
|
|
25091
|
-
recommendedDeclaredHeadroomBps: null,
|
|
25092
|
-
recommendedEstimateCoins: null,
|
|
25093
|
-
recommendedPerCallMaxCoins: null,
|
|
25412
|
+
...creditKeys(null),
|
|
25413
|
+
legacyCoinTerms: null,
|
|
25094
25414
|
savedTo: null
|
|
25095
25415
|
});
|
|
25096
25416
|
} else {
|
|
25097
25417
|
log.error("Nothing has been benchmarked in this folder yet.");
|
|
25098
|
-
log.dim(` Run ${c.cyan(
|
|
25418
|
+
log.dim(` Run ${c.cyan(rerun)} first.`);
|
|
25419
|
+
}
|
|
25420
|
+
process.exitCode = 1;
|
|
25421
|
+
return;
|
|
25422
|
+
}
|
|
25423
|
+
if (saved.kind === "coins") {
|
|
25424
|
+
const old = saved.record;
|
|
25425
|
+
const legacy = {
|
|
25426
|
+
p50: old.p50 ?? null,
|
|
25427
|
+
p95: old.p95 ?? null,
|
|
25428
|
+
max: old.max ?? null,
|
|
25429
|
+
recommendedEstimateCoins: old.recommendedEstimateCoins ?? null,
|
|
25430
|
+
recommendedPerCallMaxCoins: old.recommendedPerCallMaxCoins ?? null,
|
|
25431
|
+
maxCoinsPerSample: old.maxCoinsPerSample ?? null,
|
|
25432
|
+
noRecommendationReason: old.noRecommendationReason ?? null
|
|
25433
|
+
};
|
|
25434
|
+
if (opts.json) {
|
|
25435
|
+
writeJsonLine({
|
|
25436
|
+
command: "llm price",
|
|
25437
|
+
status: "failed",
|
|
25438
|
+
error: "coin_terms",
|
|
25439
|
+
currency: "coins",
|
|
25440
|
+
ranAt: old.ranAt ?? null,
|
|
25441
|
+
modelId: old.modelId ?? null,
|
|
25442
|
+
samples: old.samples ?? null,
|
|
25443
|
+
completed: old.completed ?? null,
|
|
25444
|
+
...creditKeys(null),
|
|
25445
|
+
legacyCoinTerms: legacy,
|
|
25446
|
+
savedTo: LLM_BENCH_FILE
|
|
25447
|
+
});
|
|
25448
|
+
} else {
|
|
25449
|
+
log.plain(c.bold("genex llm price"));
|
|
25450
|
+
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${old.ranAt ?? "at an unknown time"}${old.modelId ? ` on ${old.modelId}` : ""}`);
|
|
25451
|
+
log.plain("");
|
|
25452
|
+
log.warn(` ${LLM_PRICE_COIN_TERMS_LINE}`);
|
|
25453
|
+
const measured = legacy.p95 === null ? "no settled sample" : `charged coins p50 ${legacy.p50} \xB7 p95 ${legacy.p95} \xB7 max ${legacy.max}`;
|
|
25454
|
+
const recommended = legacy.recommendedEstimateCoins === null ? "no price recommended" : `it recommended estimateCoins ${legacy.recommendedEstimateCoins}${legacy.recommendedPerCallMaxCoins != null ? ` and perCallMaxCoins ${legacy.recommendedPerCallMaxCoins}` : ""}`;
|
|
25455
|
+
log.dim(` What it measured then: ${measured}; ${recommended}.`);
|
|
25456
|
+
log.dim(` Re-run: ${c.cyan(rerun)}`);
|
|
25099
25457
|
}
|
|
25100
25458
|
process.exitCode = 1;
|
|
25101
25459
|
return;
|
|
25102
25460
|
}
|
|
25461
|
+
const record = saved.record;
|
|
25103
25462
|
if (opts.json) {
|
|
25104
25463
|
writeJsonLine({
|
|
25105
25464
|
command: "llm price",
|
|
25106
|
-
status: record.
|
|
25107
|
-
error: record.
|
|
25465
|
+
status: record.recommendedMaxCredits === null ? "failed" : "ok",
|
|
25466
|
+
error: record.recommendedMaxCredits === null ? record.noRecommendationReason ?? "no_recommendation" : null,
|
|
25467
|
+
currency: "credits",
|
|
25108
25468
|
ranAt: record.ranAt ?? null,
|
|
25109
25469
|
modelId: record.modelId ?? null,
|
|
25110
25470
|
samples: record.samples ?? null,
|
|
25111
25471
|
completed: record.completed ?? null,
|
|
25112
|
-
|
|
25113
|
-
|
|
25114
|
-
max: record.max ?? null,
|
|
25115
|
-
// Absent from a file an older CLI wrote; null / false / 0 then, never missing.
|
|
25116
|
-
fitP95: record.fitP95 ?? null,
|
|
25117
|
-
chargedBasedEstimateCoins: record.chargedBasedEstimateCoins ?? null,
|
|
25118
|
-
lengthRaisedPrice: record.lengthRaisedPrice ?? false,
|
|
25119
|
-
cutOffSamples: record.cutOffSamples ?? 0,
|
|
25120
|
-
noRecommendationReason: record.noRecommendationReason ?? null,
|
|
25121
|
-
recommendedDeclaredHeadroomBps: record.recommendedDeclaredHeadroomBps ?? null,
|
|
25122
|
-
recommendedEstimateCoins: record.recommendedEstimateCoins ?? null,
|
|
25123
|
-
recommendedPerCallMaxCoins: record.recommendedPerCallMaxCoins ?? null,
|
|
25472
|
+
...creditKeys(record),
|
|
25473
|
+
legacyCoinTerms: null,
|
|
25124
25474
|
savedTo: LLM_BENCH_FILE
|
|
25125
25475
|
});
|
|
25126
|
-
if (record.
|
|
25476
|
+
if (record.recommendedMaxCredits === null) process.exitCode = 1;
|
|
25127
25477
|
return;
|
|
25128
25478
|
}
|
|
25129
25479
|
log.plain(c.bold("genex llm price"));
|
|
25130
25480
|
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${record.ranAt}`);
|
|
25131
25481
|
log.plain("");
|
|
25132
25482
|
printRecommendation(log, record);
|
|
25133
|
-
if (record.
|
|
25483
|
+
if (record.recommendedMaxCredits === null) process.exitCode = 1;
|
|
25134
25484
|
}
|
|
25135
25485
|
function writeJsonLine(value) {
|
|
25136
25486
|
process.stdout.write(`${JSON.stringify(value)}
|
|
@@ -25390,7 +25740,7 @@ function runtimeRow(state, lane, toolsOnly = false) {
|
|
|
25390
25740
|
return {
|
|
25391
25741
|
label: "Runtime",
|
|
25392
25742
|
value: `in-game LLM lane live \xB7 ${n} model${n === 1 ? "" : "s"}${keyRefused ? ` \xB7 ${c.yellow("provider key refused")}` : ""}`,
|
|
25393
|
-
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : '
|
|
25743
|
+
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : 'Measure a call before a game declares its per-call ceiling: `npx genex llm bench "<prompt>" --max-credits <n> --user-approved`.'
|
|
25394
25744
|
};
|
|
25395
25745
|
}
|
|
25396
25746
|
case "off":
|
|
@@ -27346,16 +27696,16 @@ ${c.bold("Usage")}
|
|
|
27346
27696
|
status | cancel. "models" says whether this
|
|
27347
27697
|
stand serves the runtime LLM lane at all, and
|
|
27348
27698
|
at what provider rates. "bench" runs the real
|
|
27349
|
-
model on YOUR OWN
|
|
27350
|
-
attempt
|
|
27699
|
+
model on YOUR OWN credits and reports what each
|
|
27700
|
+
attempt really cost; it needs --max-credits <n>
|
|
27351
27701
|
--user-approved, like any spend the player has
|
|
27352
27702
|
to agree to. "price" reprints the last run's
|
|
27353
|
-
recommended
|
|
27703
|
+
recommended maxCredits. "status" lists this
|
|
27354
27704
|
project's open calls and the slots they hold;
|
|
27355
27705
|
"cancel <id>" stops an active one. Declare a
|
|
27356
|
-
game's
|
|
27357
|
-
a
|
|
27358
|
-
|
|
27706
|
+
game's per-call ceiling from a bench, never
|
|
27707
|
+
from a guess: the player is billed as used, and
|
|
27708
|
+
the ceiling also sizes how long an answer may be.
|
|
27359
27709
|
genex player <sub> [options] Answer the in-game model requests you approve, on your
|
|
27360
27710
|
OWN Claude or ChatGPT subscription: install | run |
|
|
27361
27711
|
status | stop | uninstall. "install" signs this machine
|
|
@@ -27817,7 +28167,7 @@ ${c.bold("Examples")}
|
|
|
27817
28167
|
genex shop remove sku_123
|
|
27818
28168
|
genex llm models
|
|
27819
28169
|
genex llm models --all
|
|
27820
|
-
genex llm bench "Reply with one short taunt." --samples 5 --max-
|
|
28170
|
+
genex llm bench "Reply with one short taunt." --samples 5 --max-credits 20 --user-approved
|
|
27821
28171
|
genex llm price
|
|
27822
28172
|
genex llm status
|
|
27823
28173
|
genex llm cancel <id>
|
|
@@ -28054,22 +28404,24 @@ ${c.bold("How the allowance works")}
|
|
|
28054
28404
|
|
|
28055
28405
|
Every number printed is live from your account; prices can change without a deploy.
|
|
28056
28406
|
`,
|
|
28057
|
-
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what should the game declare?
|
|
28407
|
+
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what ceiling should the game declare?
|
|
28058
28408
|
|
|
28059
28409
|
${c.bold("Usage")}
|
|
28060
28410
|
genex llm models [--all] [--json] Which models this stand serves, their provider rates per
|
|
28061
|
-
million tokens,
|
|
28062
|
-
|
|
28063
|
-
|
|
28064
|
-
|
|
28065
|
-
on this stand: it says so
|
|
28066
|
-
|
|
28067
|
-
|
|
28068
|
-
|
|
28069
|
-
|
|
28070
|
-
|
|
28071
|
-
|
|
28072
|
-
|
|
28411
|
+
million tokens, the platform fee a call is billed at, and
|
|
28412
|
+
the headroom it recommends over a benchmark. The catalog
|
|
28413
|
+
is synced from OpenRouter, so the FEATURED short list is
|
|
28414
|
+
printed by default and --all prints every row (--json
|
|
28415
|
+
always carries every row). Off on this stand: it says so
|
|
28416
|
+
and exits 0 \u2014 build the feature without a model rather
|
|
28417
|
+
than promising a 404.
|
|
28418
|
+
genex llm bench "<prompt>" --max-credits <n> --user-approved [options]
|
|
28419
|
+
Run the real model through the development lane and
|
|
28420
|
+
report what each attempt really COST and was charged.
|
|
28421
|
+
Refused without both flags, before any network call:
|
|
28422
|
+
this spends YOUR OWN CREDITS, a spend the build's asset
|
|
28423
|
+
allowance does not count. Needs a folder linked to a
|
|
28424
|
+
game you own, and a balance that covers one attempt.
|
|
28073
28425
|
genex llm price [--json] The recommendation from the last bench in this folder.
|
|
28074
28426
|
Reads a file; spends nothing.
|
|
28075
28427
|
genex llm status [--json] This project's open development calls: active, or
|
|
@@ -28086,29 +28438,32 @@ ${c.bold("Bench options")}
|
|
|
28086
28438
|
--samples <n> How many real attempts (default ${DEFAULT_BENCH_SAMPLES}). More samples, better p95.
|
|
28087
28439
|
--schema <file> A JSON file holding the output schema; implies JSON output.
|
|
28088
28440
|
--json-output Ask for JSON without a schema. --text Ask for text (the default).
|
|
28089
|
-
--max-
|
|
28441
|
+
--max-credits <n> The per-attempt ceiling YOU approved \u2014 a bench runs on your own credits,
|
|
28090
28442
|
never a player's. The run's worst case is that number times --samples,
|
|
28091
|
-
and it is printed before anything starts.
|
|
28443
|
+
and it is printed before anything starts. (--max-coins is its deprecated
|
|
28444
|
+
name, read as the same number.)
|
|
28092
28445
|
--timeout <seconds> Bounds the wait for each attempt.
|
|
28093
28446
|
--json One machine-readable object; every key is always present, null when
|
|
28094
28447
|
the CLI could not learn it.
|
|
28095
28448
|
|
|
28096
28449
|
${c.bold("Why a benchmark and not an estimate")}
|
|
28097
|
-
|
|
28098
|
-
|
|
28099
|
-
|
|
28100
|
-
|
|
28101
|
-
|
|
28102
|
-
|
|
28103
|
-
|
|
28104
|
-
|
|
28105
|
-
|
|
28450
|
+
An in-game call is paid from the PLAYER's credits and billed as used: its real provider cost
|
|
28451
|
+
plus the platform fee, never more than the per-call CEILING the game declares (\`maxCredits\`,
|
|
28452
|
+
or \`perCallMaxCredits\` on a standing budget). The ceiling is not a price, but it is still a
|
|
28453
|
+
number with two jobs: it bounds what one call may cost, and it funds how LONG the answer may
|
|
28454
|
+
be. Three layers sit between the provider's rate card and it \u2014 the provider's cost for YOUR
|
|
28455
|
+
prompt, the platform fee, and your headroom \u2014 and only a real call measures the first. The
|
|
28456
|
+
bench is billed to YOU, on the development lane; the player-funded lane is what the
|
|
28457
|
+
published game uses. The recommendation is the p95 real cost with the server's fee and
|
|
28458
|
+
headroom applied, in whole credits; a stand that serves no headroom or no fee gets no
|
|
28459
|
+
recommendation, because those multipliers are not the CLI's to invent.
|
|
28106
28460
|
|
|
28107
|
-
|
|
28108
|
-
|
|
28109
|
-
|
|
28110
|
-
|
|
28111
|
-
|
|
28461
|
+
Because the ceiling sizes each call's output allowance, the recommendation is never below
|
|
28462
|
+
the smallest ceiling that leaves room for the benchmarked answer, as the server computes it.
|
|
28463
|
+
A sample cut off at your --max-credits (provider_token_limit) is not a sample \u2014 re-run with a
|
|
28464
|
+
higher --max-credits; an answer longer than one call on this stand may produce gets no
|
|
28465
|
+
ceiling at all \u2014 ask for a shorter one. The run also prints the measured average per call,
|
|
28466
|
+
fee included, which is the honest estimate a budget's disclosure is built on.
|
|
28112
28467
|
|
|
28113
28468
|
Results land in ${LLM_BENCH_FILE} \u2014 gitignored, machine-local, re-read by \`genex llm price\`.
|
|
28114
28469
|
`,
|
|
@@ -28323,11 +28678,13 @@ function parseArgs(argv) {
|
|
|
28323
28678
|
"--limit",
|
|
28324
28679
|
// `genex budget --assets <credits>` — the allowance the player approved.
|
|
28325
28680
|
"--assets",
|
|
28326
|
-
// `genex llm bench` value flags. `--max-
|
|
28327
|
-
//
|
|
28681
|
+
// `genex llm bench` value flags. `--max-credits` is the same approval
|
|
28682
|
+
// shape as `--assets`, but its own gate: the asset allowance does not
|
|
28683
|
+
// count a bench. `--max-coins` is its deprecated name (same number).
|
|
28328
28684
|
"--model",
|
|
28329
28685
|
"--samples",
|
|
28330
28686
|
"--schema",
|
|
28687
|
+
"--max-credits",
|
|
28331
28688
|
"--max-coins",
|
|
28332
28689
|
// `genex player install --label <name>` — what this machine is called in
|
|
28333
28690
|
// the dashboard. Its own key rather than the generic 2nd positional,
|
|
@@ -28845,12 +29202,14 @@ function applyValueFlag(options, flag, value) {
|
|
|
28845
29202
|
options.samples = n;
|
|
28846
29203
|
break;
|
|
28847
29204
|
}
|
|
29205
|
+
case "--max-credits":
|
|
28848
29206
|
case "--max-coins": {
|
|
28849
29207
|
const n = Number(value);
|
|
28850
29208
|
if (!Number.isInteger(n) || n < 1) {
|
|
28851
|
-
throw new Error(`Invalid
|
|
29209
|
+
throw new Error(`Invalid ${flag} value: ${value} (whole credits, 1 or more)`);
|
|
28852
29210
|
}
|
|
28853
|
-
options.
|
|
29211
|
+
if (flag === "--max-credits") options.maxCredits = n;
|
|
29212
|
+
else options.maxCoins = n;
|
|
28854
29213
|
break;
|
|
28855
29214
|
}
|
|
28856
29215
|
case "--timeout": {
|
|
@@ -29148,9 +29507,9 @@ async function main() {
|
|
|
29148
29507
|
await runBudget(parsed.options);
|
|
29149
29508
|
break;
|
|
29150
29509
|
// The in-game model lane: is it live here, what does one call really
|
|
29151
|
-
// cost, what should the game declare. `bench`
|
|
29152
|
-
//
|
|
29153
|
-
//
|
|
29510
|
+
// cost, what ceiling should the game declare. `bench` spends the
|
|
29511
|
+
// builder's own credits OUTSIDE the asset allowance, so it carries its
|
|
29512
|
+
// own --max-credits/--user-approved gate.
|
|
29154
29513
|
case "llm":
|
|
29155
29514
|
await runLlm(parsed.options);
|
|
29156
29515
|
break;
|