@genex-ai/cli-demo 1.36.3 → 1.36.4-dev.807
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{blender-mcp-QVEGHE5L.js → blender-mcp-WSURPZET.js} +2 -2
- package/dist/{blender-serve-66OHC4ML.js → blender-serve-EH2MOW3U.js} +1 -1
- package/dist/{chunk-4BBTPRJW.js → chunk-EABIVAGT.js} +2 -2
- package/dist/{chunk-QI7FIYBY.js → chunk-K4EHXOZK.js} +1 -1
- package/dist/index.js +452 -217
- package/package.json +1 -1
- package/templates/skills/genex-game-director/SKILL.md +1 -1
- package/templates/skills/genex-getting-started/SKILL.md +2 -2
- package/templates/skills/genex-llm-in-games/SKILL.md +209 -72
- package/templates/skills/genex-llm-in-games/references/pricing.md +126 -85
- package/templates/skills/genex-tool-llm/SKILL.md +11 -9
- package/templates/skills/genex-updates/SKILL.md +1 -1
package/dist/index.js
CHANGED
|
@@ -57,7 +57,7 @@ import {
|
|
|
57
57
|
writeSecretFile,
|
|
58
58
|
writeUserToken,
|
|
59
59
|
writeWorkspace
|
|
60
|
-
} from "./chunk-
|
|
60
|
+
} from "./chunk-EABIVAGT.js";
|
|
61
61
|
import {
|
|
62
62
|
CLI_CHANNEL,
|
|
63
63
|
DEFAULT_API_URL,
|
|
@@ -78,7 +78,7 @@ import {
|
|
|
78
78
|
normalizeApiOrigin,
|
|
79
79
|
originKey,
|
|
80
80
|
resolveAgentTargets
|
|
81
|
-
} from "./chunk-
|
|
81
|
+
} from "./chunk-K4EHXOZK.js";
|
|
82
82
|
|
|
83
83
|
// src/instrument.ts
|
|
84
84
|
import * as Sentry from "@sentry/node";
|
|
@@ -981,7 +981,7 @@ Important note: put soul into your creations, with many details and love. Aim to
|
|
|
981
981
|
21. A turn ends in exactly one of two ways: on a question the player must answer before the next step can be chosen, or on a handoff \u2014 the draft link, what changed, what to try, and what is still open. Never close a turn on a promise. "I'll keep building", "while I keep working", "adding that now" are things you say and then DO before you stop; if you are stopping, say that you have stopped and what remains. While the Build plan's \`Now:\` line has open work and no answer is needed from the player, the turn is not over: preview, hand off, and start the next milestone in the same turn, until the requested outcome is reached or the platform's own budget ends the session. If a stop was forced on you anyway, the first line of your next turn is the promise you left and where it stands.
|
|
982
982
|
22. **What the player sees must read as the thing it is**, in this game's own style: a building as that building, a person as a person, a prop as that prop, a surface as its material \u2014 and a coloured box, a capsule, or a flat grey block standing in for one of them is never its finished version. Generation is the default route to that bar wherever code will not honestly reach it: the player's character, whatever the request names, whatever the player walks up to, enters, or interacts with, and the music and sound underneath \u2014 reach for the belt on your own judgment, without stopping to ask; a status line saying what you queued is enough, and \`--no-wait\` keeps you building while it lands. Procedural code is a first-class engine for what is structural, repeated, distant, or parametric \u2014 terrain, sky, fences, paving, modular kits, filler \u2014 and for anything else only when the result meets the same bar and you have looked at it in a capture before its row says \`landed\`. Start with one reusable implementation per distinct required gameplay or visual role. Background populations default to reusable procedural bodies and motion or compatible existing rigged assets. Catalog clips need a compatible skeleton and may be paid. Reserve paid custom bodies for the player and characters examined or interacted with closely, while honoring explicit user requests. Add variants only for an unmet requirement; unspent allowance is not unfinished work. Allowance questions use timeoutPolicy no-consent: silence does not raise the allowance. Record role, reuse and completion criteria in the existing Assets table. Decide per object, not per category, and write the route on each Assets row in \`DESIGN.md\` \u2014 the paid flow for generated pieces, the procedural flow with its local path for code-built ones. An Assets table with no generation in it is a decision, not a default: record it as \`Generation: none \u2014 <why code alone reaches the bar here>\` or generate.
|
|
983
983
|
23. **The screenshots you take to look at your own work go in \`.genex/scratch/\`.** Captures, render comparisons, before/after strips, traces and metrics dumps are how you SEE what you built \u2014 they are not part of what you built, and \`genex preview\` pushes the whole folder to the game's source, every time, forever. \`.genex/scratch/\` already exists for this and is already ignored, so there is nothing to set up: write the capture as \`.genex/scratch/arena-before.png\` and read it back from there. Never put them in a folder of your own at the top level: \`progress/\`, \`reports/\`, \`shots/\` and their kind are pushed like source and become permanent weight in the game and in every remix of it \u2014 one real game reached 11 GB and 10,332 committed screenshots exactly this way. If you find such a folder already there, add it to \`.gitignore\`; never delete the player's files (law 18). One file in \`.genex/scratch/\` IS read, by the platform: \`.genex/scratch/cover.png\` is the game's cover \u2014 the frame you would put on its poster: real gameplay at its signature moment, well lit, no menus, popups or busy HUD, landscape 16:9, and never any text (the page sets the title beside it). The next \`genex preview\` makes it the cover; save a better moment over it whenever one comes along, and if your browser tool can set the cover itself, use that instead. Before the first publish, and whenever the game's look changes, load \`$genex-cover\`: it chooses, stages, lights and judges that frame.
|
|
984
|
-
24. **Benchmark before declaring an in-game LLM
|
|
984
|
+
24. **Benchmark before declaring an in-game LLM ceiling.** If the game calls a language model while the player plays \u2014 an NPC answering in its own words, a quest written for this save, a judge reading what the player typed \u2014 the PLAYER pays for it from their Genex credits, billed as used, and the game declares no price, only a per-call CEILING: \`maxCredits\` on \`generate()\`, \`perCallMaxCredits\` on a standing budget. That ceiling bounds what one call may cost AND funds how long its answer may be, so never pick it by judgement or from a vendor's rate card. Run the real prompt on your own credits \u2014 \`npx genex llm bench "<the prompt>" --schema <file> --samples 3 --max-credits <n> --user-approved\` \u2014 and declare the figures it prints, recorded in \`DESIGN.md\` with its model and date; re-benchmark whenever the prompt, the schema or the model changes, because a prompt edit changes the ceiling. Check the lane first with \`npx genex llm models\`: where it is off the routes 404, so build the feature behind a graceful unavailable state and say so in the handoff. A repeated call is a standing budget the player approves once (\`requestSpendGrant()\`, then \`generate({ grantId })\`), never a popup per turn, and its disclosed call rate and credits per period are computed from the game's own loop and the bench's measured average. The coin spellings (\`estimateCoins\`, \`perCallMaxCoins\`) are deprecated \u2014 write the credit names. Model output is data the game validates against its own expectation \u2014 never executed, and never authority over coin, credits, items, entitlements or rewards. Load \`$genex-llm-in-games\` before writing any of it.
|
|
985
985
|
${CONTRACT_END}
|
|
986
986
|
`;
|
|
987
987
|
var TOOLS_CONTRACT_HEAD = `# Genex Tools (always in effect in this folder)
|
|
@@ -995,7 +995,7 @@ var TOOLS_CONTRACT_OFFER = `5. **When the game is built and playable \u2014 neve
|
|
|
995
995
|
var TOOLS_CONTRACT_HOSTED_LAWS = `5. **This game is hosted on Genex now, and publishing is glue \u2014 never a rebuild.** The game ships exactly as the user built it: \`initEmbed()\` first in the boot code (the \`genex-threejs-embed-auth\` card has the call; the renderer draws before any \`await waitForPlayer()\`), a static build into \`dist/\` with relative asset paths, then \`npx genex preview\`. Never restructure, reformat or "improve" the game to ship it, never scaffold a new app around it, and never start a design document or a build plan for it \u2014 the user's own process is theirs. Load the \`genex-tool-publish\` card before the first preview; it owns the vocabulary, the links and the limits.
|
|
996
996
|
6. **Ship first, then report.** \`preview\` prints a preflight \u2014 phone memory, a missing volume slider, an asset nobody wired, a viewport line. It never blocks a deploy, so push the build as it is, hand over the link, THEN relay each preflight line to the user in one plain sentence with an offer to fix it, and fix one only on their yes. The preflight is a report for the user, not a to-do list for you.
|
|
997
997
|
7. **The link is the game's page, and there are two versions.** After every preview give the user \`<dashboard>/draft/<slug>\` \u2014 \`dashboardOrigins[0]\` and \`slug\` from \`.genex/project.json\` \u2014 never localhost, a file path or the bare play origin. \`preview\` updates the draft and never touches what players are on; the first release is \`npx genex publish\` (it lists the game); after that "publish it", "update it" and "yes" all mean \`npx genex promote\` \u2014 the exact draft build, no rebuild. Ask once per round of work, in one line, and keep working while you wait.`;
|
|
998
|
-
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with Genex
|
|
998
|
+
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with their Genex credits or their own Claude/ChatGPT subscription, and approves it on a Genex sheet; your game just calls \`generate()\`. Want it that way?`;
|
|
999
999
|
function toolsContractLlmLaw(n, hosted) {
|
|
1000
1000
|
const onYes = hosted ? "On a yes, load `$genex-tool-llm` and then `$genex-llm-in-games`, which owns the build." : "On a yes, load `$genex-tool-llm`, which owns the rest: it checks the lane with `npx genex llm models` first, then runs `npx genex init --convert` (a hosted game is a static browser build \u2014 a local server holding a key can never ship).";
|
|
1001
1001
|
return `${n}. **A model running while people PLAY is a platform feature \u2014 never something you wire onto the user's own meter.** When a request implies one (NPCs that talk or decide in their own words, content written from what the player types, a prompt box in the game, "let the player pick a model"), recognise it, say in ONE line: "${TOOLS_CONTRACT_LLM_OFFER}" \u2014 then ASK and wait. ${onYes} On a no, build the authored version instead. Either way, never ship a game that calls a model on a key or an account of the user's own: every visitor would spend their money with nobody approving it, and the credential is readable in the bundle. Player-funded generation is text and JSON only \u2014 3D, images, video and audio stay the asset lanes above, on the user's meter.`;
|
|
@@ -8523,7 +8523,10 @@ function describeGenerationFailure(kind, error) {
|
|
|
8523
8523
|
return {
|
|
8524
8524
|
billing: true,
|
|
8525
8525
|
lines: [
|
|
8526
|
-
|
|
8526
|
+
// "Later" is true since the API stopped returning the dead row to a
|
|
8527
|
+
// repeated guided approval (2026-10-10): the same command starts a
|
|
8528
|
+
// fresh attempt once the platform's account is paid again.
|
|
8529
|
+
`${c.red("\u2717")} The ${kind} provider is out of credit on this stand \u2014 not your prompt and not your Genex credits (auto-refunded). It is temporary: the same command, run again later, starts a fresh attempt. Do not re-generate now: fall back to procedural/VRM for this asset and tell the player in one plain line.`,
|
|
8527
8530
|
c.dim(oneLineError(error))
|
|
8528
8531
|
]
|
|
8529
8532
|
};
|
|
@@ -22943,7 +22946,7 @@ async function runBlender(opts) {
|
|
|
22943
22946
|
return 1;
|
|
22944
22947
|
}
|
|
22945
22948
|
if (sub === "serve") {
|
|
22946
|
-
const { serveLocalBlender } = await import("./blender-serve-
|
|
22949
|
+
const { serveLocalBlender } = await import("./blender-serve-EH2MOW3U.js");
|
|
22947
22950
|
const port = Number(process.env.GENEX_BLENDER_PORT ?? 8088);
|
|
22948
22951
|
log.step(`Starting a local Blender service on port ${port}`);
|
|
22949
22952
|
log.plain(
|
|
@@ -22952,7 +22955,7 @@ async function runBlender(opts) {
|
|
|
22952
22955
|
return serveLocalBlender({ port, log });
|
|
22953
22956
|
}
|
|
22954
22957
|
if (sub === "mcp") {
|
|
22955
|
-
const { runBlenderMcp } = await import("./blender-mcp-
|
|
22958
|
+
const { runBlenderMcp } = await import("./blender-mcp-WSURPZET.js");
|
|
22956
22959
|
return runBlenderMcp();
|
|
22957
22960
|
}
|
|
22958
22961
|
if (sub === "seat") {
|
|
@@ -23460,7 +23463,12 @@ import fs35 from "fs/promises";
|
|
|
23460
23463
|
import path36 from "path";
|
|
23461
23464
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
23462
23465
|
var SUBS3 = ["models", "bench", "price", "status", "cancel"];
|
|
23463
|
-
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN
|
|
23466
|
+
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN credits \u2014 your balance, not a player's, and a spend this build's asset allowance does not count. Re-run with --max-credits <n> --user-approved only after the person at the keyboard agreed to the number.";
|
|
23467
|
+
var LLM_MAX_COINS_DEPRECATED = "--max-coins is deprecated: it is read as --max-credits, the same number \u2014 a bench spends credits now.";
|
|
23468
|
+
function benchBalanceShortLine(spendable, maxCredits) {
|
|
23469
|
+
return `Your spendable credits (${spendable}) cannot cover one attempt at --max-credits ${maxCredits} \u2014 nothing was sent, nothing was spent. Add credits, or re-run with a lower --max-credits once the person at the keyboard agrees to it.`;
|
|
23470
|
+
}
|
|
23471
|
+
var LLM_BENCH_UNVERIFIED_LINE = "Credits pay only for a verified email, and this account's is not verified yet (credits_unverified) \u2014 nothing was sent, nothing was spent. Verify it, then re-run.";
|
|
23464
23472
|
var LLM_LANE_OFF_LINE = "The runtime LLM lane is off on this stand \u2014 in-game generate() answers 404 here, and nothing can be benchmarked.";
|
|
23465
23473
|
var LLM_TOOLS_CONVERT_HINT = [
|
|
23466
23474
|
"In-game model calls need a hosted game. This folder is a Genex Tools workspace, so nothing here",
|
|
@@ -23473,7 +23481,7 @@ function serverSlotCounts(body) {
|
|
|
23473
23481
|
return { held: body.slotsHeld, limit: body.slotLimit };
|
|
23474
23482
|
}
|
|
23475
23483
|
var PENDING_SLOT_RELEASE_MINUTES = 10;
|
|
23476
|
-
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so
|
|
23484
|
+
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so what it reserved stays held until the bill resolves; such a row stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`;
|
|
23477
23485
|
var LLM_STATUS_NO_COUNT_LINE = "This stand does not report how many slots are held across your projects, so no count is shown.";
|
|
23478
23486
|
var LLM_CANCEL_AWAITING_BILL = "this call already stopped; its bill is awaiting the provider";
|
|
23479
23487
|
var LLM_CANCEL_ALREADY_FINAL = "this call is already final";
|
|
@@ -23487,11 +23495,33 @@ var DEFAULT_SAMPLE_TIMEOUT_SEC = 180;
|
|
|
23487
23495
|
function parseProviderKey(value) {
|
|
23488
23496
|
return value === "ok" || value === "rejected" || value === "unknown" ? value : null;
|
|
23489
23497
|
}
|
|
23498
|
+
function bpsOrNull(value) {
|
|
23499
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
23500
|
+
}
|
|
23501
|
+
function standingCreditGrantOrNull(value) {
|
|
23502
|
+
if (!value || typeof value !== "object") return null;
|
|
23503
|
+
const v = value;
|
|
23504
|
+
const whole = (x) => typeof x === "number" && Number.isSafeInteger(x) && x >= 0;
|
|
23505
|
+
if (!whole(v.minCredits) || !whole(v.maxCredits) || !whole(v.stepCredits) || !whole(v.defaultCredits)) return null;
|
|
23506
|
+
return {
|
|
23507
|
+
minCredits: v.minCredits,
|
|
23508
|
+
maxCredits: v.maxCredits,
|
|
23509
|
+
stepCredits: v.stepCredits,
|
|
23510
|
+
defaultCredits: v.defaultCredits,
|
|
23511
|
+
...whole(v.minUsdCents) ? { minUsdCents: v.minUsdCents } : {},
|
|
23512
|
+
...whole(v.maxUsdCents) ? { maxUsdCents: v.maxUsdCents } : {},
|
|
23513
|
+
...whole(v.stepUsdCents) ? { stepUsdCents: v.stepUsdCents } : {}
|
|
23514
|
+
};
|
|
23515
|
+
}
|
|
23490
23516
|
async function readRuntimeLane(apiUrl, token) {
|
|
23491
23517
|
const empty = (state) => ({
|
|
23492
23518
|
state,
|
|
23493
23519
|
models: null,
|
|
23494
23520
|
recommendedDeclaredHeadroomBps: null,
|
|
23521
|
+
currency: null,
|
|
23522
|
+
feeBps: null,
|
|
23523
|
+
usdCentsPerCredit: null,
|
|
23524
|
+
standingCreditGrant: null,
|
|
23495
23525
|
externalProviders: null,
|
|
23496
23526
|
total: null,
|
|
23497
23527
|
featuredCount: null,
|
|
@@ -23510,7 +23540,11 @@ async function readRuntimeLane(apiUrl, token) {
|
|
|
23510
23540
|
return {
|
|
23511
23541
|
state: "live",
|
|
23512
23542
|
models: body.models,
|
|
23513
|
-
recommendedDeclaredHeadroomBps:
|
|
23543
|
+
recommendedDeclaredHeadroomBps: bpsOrNull(body.recommendedDeclaredHeadroomBps),
|
|
23544
|
+
currency: body.currency === "credits" || body.currency === "coins" ? body.currency : null,
|
|
23545
|
+
feeBps: bpsOrNull(body.feeBps),
|
|
23546
|
+
usdCentsPerCredit: typeof body.usdCentsPerCredit === "number" && Number.isSafeInteger(body.usdCentsPerCredit) && body.usdCentsPerCredit >= 1 ? body.usdCentsPerCredit : null,
|
|
23547
|
+
standingCreditGrant: standingCreditGrantOrNull(body.standingCreditGrant),
|
|
23514
23548
|
externalProviders: Array.isArray(body.externalProviders) ? body.externalProviders : null,
|
|
23515
23549
|
total: typeof body.total === "number" && Number.isFinite(body.total) ? body.total : null,
|
|
23516
23550
|
featuredCount: typeof body.featuredCount === "number" && Number.isFinite(body.featuredCount) ? body.featuredCount : null,
|
|
@@ -23526,18 +23560,45 @@ function percentile2(values, p) {
|
|
|
23526
23560
|
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
23527
23561
|
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
23528
23562
|
}
|
|
23529
|
-
function
|
|
23530
|
-
|
|
23563
|
+
function percentilePicos(values, p) {
|
|
23564
|
+
if (values.length === 0) return null;
|
|
23565
|
+
const sorted = [...values].sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
23566
|
+
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
23567
|
+
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
23531
23568
|
}
|
|
23532
|
-
|
|
23533
|
-
|
|
23569
|
+
var PICOS_PER_CENT = 10000000000n;
|
|
23570
|
+
function ceilingCreditsFor(costPicos, terms) {
|
|
23571
|
+
if (costPicos <= 0n) return 1;
|
|
23572
|
+
const num = costPicos * BigInt(1e4 + terms.feeBps) * BigInt(1e4 + terms.headroomBps);
|
|
23573
|
+
const den = 100000000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
23574
|
+
return Math.max(1, Number((num + den - 1n) / den));
|
|
23575
|
+
}
|
|
23576
|
+
function averageCreditsPerCall(costsPicos, terms) {
|
|
23577
|
+
if (costsPicos.length === 0) return null;
|
|
23578
|
+
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
23579
|
+
return Number(sum * BigInt(1e4 + terms.feeBps)) / Number(BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT);
|
|
23580
|
+
}
|
|
23581
|
+
function perCallEstimateCreditsFor(costsPicos, terms) {
|
|
23582
|
+
if (costsPicos.length === 0) return null;
|
|
23583
|
+
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
23584
|
+
const num = sum * BigInt(1e4 + terms.feeBps);
|
|
23585
|
+
const den = BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
23586
|
+
return Math.max(1, Number((num + den - 1n) / den));
|
|
23587
|
+
}
|
|
23588
|
+
function formatCredits(value) {
|
|
23589
|
+
if (!Number.isFinite(value)) return "\u2014";
|
|
23590
|
+
if (value >= 100) return String(Math.round(value));
|
|
23591
|
+
return String(Number(value.toPrecision(value < 1 ? 2 : 3)));
|
|
23592
|
+
}
|
|
23593
|
+
function usdFromPicos(picos) {
|
|
23594
|
+
if (picos === null || picos === void 0) return "unknown";
|
|
23595
|
+
const value = typeof picos === "bigint" ? picos : /^\d+$/.test(picos) ? BigInt(picos) : null;
|
|
23596
|
+
if (value === null) return "unknown";
|
|
23597
|
+
return `$${(Number(value) / 1e12).toFixed(6)}`;
|
|
23534
23598
|
}
|
|
23535
23599
|
function usdPerMillion(value) {
|
|
23536
23600
|
return `$${value.toFixed(value < 1 ? 4 : 2)}`;
|
|
23537
23601
|
}
|
|
23538
|
-
function usd(value) {
|
|
23539
|
-
return value === null ? "unknown" : `$${value.toFixed(6)}`;
|
|
23540
|
-
}
|
|
23541
23602
|
async function runLlm(opts = {}) {
|
|
23542
23603
|
const log = createLogger({ quiet: opts.quiet || opts.json });
|
|
23543
23604
|
const cwd = opts.cwd ?? process.cwd();
|
|
@@ -23563,16 +23624,23 @@ async function runLlm(opts = {}) {
|
|
|
23563
23624
|
return;
|
|
23564
23625
|
}
|
|
23565
23626
|
if (sub === "bench") {
|
|
23566
|
-
if (opts.maxCoins
|
|
23627
|
+
if (opts.maxCoins !== void 0) process.stderr.write(`${c.yellow("!")} ${LLM_MAX_COINS_DEPRECATED}
|
|
23628
|
+
`);
|
|
23629
|
+
if (opts.maxCoins !== void 0 && opts.maxCredits !== void 0 && opts.maxCoins !== opts.maxCredits) {
|
|
23630
|
+
fail5("llm bench", `Pass --max-credits alone \u2014 --max-coins is its deprecated name, and the two disagree (${opts.maxCredits} vs ${opts.maxCoins}).`);
|
|
23631
|
+
return;
|
|
23632
|
+
}
|
|
23633
|
+
const approvedCredits = opts.maxCredits ?? opts.maxCoins;
|
|
23634
|
+
if (approvedCredits === void 0 || opts.userApproved !== true) {
|
|
23567
23635
|
fail5("llm bench", LLM_BENCH_APPROVAL_REQUIRED);
|
|
23568
23636
|
return;
|
|
23569
23637
|
}
|
|
23570
|
-
if (!Number.isInteger(
|
|
23571
|
-
fail5("llm bench", `--max-
|
|
23638
|
+
if (!Number.isInteger(approvedCredits) || approvedCredits < 1) {
|
|
23639
|
+
fail5("llm bench", `--max-credits takes a whole number of credits, 1 or more (got ${String(approvedCredits)}).`);
|
|
23572
23640
|
return;
|
|
23573
23641
|
}
|
|
23574
23642
|
if (!opts.benchPrompt?.trim()) {
|
|
23575
|
-
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-
|
|
23643
|
+
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-credits <n> --user-approved.');
|
|
23576
23644
|
return;
|
|
23577
23645
|
}
|
|
23578
23646
|
if (opts.jsonOutput && opts.textOutput) {
|
|
@@ -23664,6 +23732,10 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
23664
23732
|
models: null,
|
|
23665
23733
|
externalProviders: null,
|
|
23666
23734
|
recommendedDeclaredHeadroomBps: null,
|
|
23735
|
+
currency: null,
|
|
23736
|
+
feeBps: null,
|
|
23737
|
+
usdCentsPerCredit: null,
|
|
23738
|
+
standingCreditGrant: null,
|
|
23667
23739
|
total: null,
|
|
23668
23740
|
featuredCount: null,
|
|
23669
23741
|
// The stand was never asked, so the verdict is unknown — but the key is
|
|
@@ -23689,6 +23761,12 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
23689
23761
|
models: lane.models,
|
|
23690
23762
|
externalProviders: lane.externalProviders,
|
|
23691
23763
|
recommendedDeclaredHeadroomBps: lane.recommendedDeclaredHeadroomBps,
|
|
23764
|
+
// The credit terms a call is billed on, the server's — null on a stand
|
|
23765
|
+
// that predates credit billing, never a number the CLI filled in.
|
|
23766
|
+
currency: lane.currency,
|
|
23767
|
+
feeBps: lane.feeBps,
|
|
23768
|
+
usdCentsPerCredit: lane.usdCentsPerCredit,
|
|
23769
|
+
standingCreditGrant: lane.standingCreditGrant,
|
|
23692
23770
|
total: lane.total ?? lane.models?.length ?? null,
|
|
23693
23771
|
featuredCount: lane.featuredCount ?? (lane.models ? lane.models.filter((m) => m.featured).length : null),
|
|
23694
23772
|
// `null` when the lane is not live or the server predates the probe.
|
|
@@ -23735,8 +23813,9 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
23735
23813
|
);
|
|
23736
23814
|
for (const m of shown) {
|
|
23737
23815
|
const plan = m.personalPlan ? ` \xB7 personal plan: ${m.personalPlan}` : "";
|
|
23816
|
+
const kind = m.kind === "typesafe" ? " \xB7 judge (classifier): judge() under a budget, not generate() or bench" : "";
|
|
23738
23817
|
log.plain(
|
|
23739
|
-
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}`
|
|
23818
|
+
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}${kind}`
|
|
23740
23819
|
);
|
|
23741
23820
|
}
|
|
23742
23821
|
if (hiddenCount > 0) {
|
|
@@ -23752,74 +23831,104 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
23752
23831
|
}
|
|
23753
23832
|
log.plain("");
|
|
23754
23833
|
if (lane.recommendedDeclaredHeadroomBps === null) {
|
|
23755
|
-
log.dim(" This stand serves no recommended headroom, so no
|
|
23834
|
+
log.dim(" This stand serves no recommended headroom, so no ceiling can be recommended from a bench.");
|
|
23756
23835
|
} else {
|
|
23757
|
-
log.dim(` Recommended headroom over benchmarked
|
|
23836
|
+
log.dim(` Recommended headroom over a benchmarked call's cost on this stand: ${lane.recommendedDeclaredHeadroomBps} bps.`);
|
|
23837
|
+
}
|
|
23838
|
+
if (lane.feeBps === null) {
|
|
23839
|
+
log.dim(" This stand serves no platform fee \u2014 it predates credit billing, so no ceiling can be recommended from a bench.");
|
|
23840
|
+
} else {
|
|
23841
|
+
log.dim(` In-game calls are billed in the player's credits, as used: the provider's cost plus this stand's platform fee (${lane.feeBps} bps).`);
|
|
23758
23842
|
}
|
|
23759
23843
|
if (toolsOnly) {
|
|
23760
23844
|
printConvertHint();
|
|
23761
23845
|
return;
|
|
23762
23846
|
}
|
|
23763
|
-
log.dim(" Those per-million rates are the PROVIDER's; what
|
|
23764
|
-
log.dim(`
|
|
23847
|
+
log.dim(" Those per-million rates are the PROVIDER's list; what one call of YOUR prompt really costs,");
|
|
23848
|
+
log.dim(` and the per-call ceiling to declare, only a real call measures: ${c.cyan('genex llm bench "<prompt>" --max-credits <n> --user-approved')}`);
|
|
23765
23849
|
}
|
|
23766
23850
|
function wholeOrNull(value) {
|
|
23767
23851
|
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
23768
23852
|
}
|
|
23853
|
+
function costPicosOf(usage) {
|
|
23854
|
+
if (typeof usage?.costUsdPicos === "string" && /^\d+$/.test(usage.costUsdPicos)) return usage.costUsdPicos;
|
|
23855
|
+
if (typeof usage?.costUsd === "number" && Number.isFinite(usage.costUsd) && usage.costUsd >= 0) {
|
|
23856
|
+
return String(BigInt(Math.round(usage.costUsd * 1e12)));
|
|
23857
|
+
}
|
|
23858
|
+
return null;
|
|
23859
|
+
}
|
|
23769
23860
|
function sampleFrom(id, settled) {
|
|
23770
23861
|
const usage = settled?.usage;
|
|
23862
|
+
const picos = costPicosOf(usage);
|
|
23771
23863
|
return {
|
|
23772
23864
|
id,
|
|
23773
23865
|
status: settled?.status ?? null,
|
|
23774
23866
|
billingStatus: settled?.billingStatus ?? null,
|
|
23775
|
-
|
|
23776
|
-
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd : null,
|
|
23867
|
+
chargedCredits: wholeOrNull(settled?.chargedCredits),
|
|
23868
|
+
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd : picos !== null ? Number(picos) / 1e12 : null,
|
|
23869
|
+
costUsdPicos: picos,
|
|
23777
23870
|
error: settled?.error ?? (settled ? null : "timed_out"),
|
|
23778
23871
|
providerMessage: typeof settled?.providerMessage === "string" && settled.providerMessage.trim() ? settled.providerMessage.trim() : null,
|
|
23779
23872
|
inputTokens: wholeOrNull(usage?.inputTokens),
|
|
23780
23873
|
outputTokens: wholeOrNull(usage?.outputTokens),
|
|
23781
23874
|
maxOutputTokens: wholeOrNull(usage?.maxOutputTokens),
|
|
23782
|
-
|
|
23875
|
+
fitCredits: typeof usage?.fitCredits === "number" && Number.isSafeInteger(usage.fitCredits) && usage.fitCredits >= 1 ? usage.fitCredits : null,
|
|
23783
23876
|
fitExceedsCap: typeof usage?.fitExceedsCap === "boolean" ? usage.fitExceedsCap : null,
|
|
23784
23877
|
truncated: typeof usage?.truncated === "boolean" ? usage.truncated : settled?.error === "provider_token_limit" ? true : null
|
|
23785
23878
|
};
|
|
23786
23879
|
}
|
|
23787
23880
|
var PROVIDER_TOKEN_LIMIT = "provider_token_limit";
|
|
23788
|
-
function benchRecommendation(results,
|
|
23789
|
-
const settled = results.filter(
|
|
23790
|
-
|
|
23791
|
-
|
|
23792
|
-
const
|
|
23793
|
-
const
|
|
23881
|
+
function benchRecommendation(results, terms) {
|
|
23882
|
+
const settled = results.filter(
|
|
23883
|
+
(r) => r.status === "succeeded" && r.billingStatus === "final" && r.chargedCredits !== null && r.costUsdPicos !== null
|
|
23884
|
+
);
|
|
23885
|
+
const costs = settled.map((r) => BigInt(r.costUsdPicos));
|
|
23886
|
+
const charged = settled.map((r) => r.chargedCredits);
|
|
23887
|
+
const fits = settled.map((r) => r.fitCredits).filter((v) => typeof v === "number");
|
|
23888
|
+
const costP95 = percentilePicos(costs, 95);
|
|
23889
|
+
const costMax = costs.length ? costs.reduce((a, b) => b > a ? b : a) : null;
|
|
23794
23890
|
const fitP95 = percentile2(fits, 95);
|
|
23795
23891
|
const fitMax = fits.length ? Math.max(...fits) : null;
|
|
23796
23892
|
const exceedsStandCap = results.some((r) => r.fitExceedsCap === true);
|
|
23797
23893
|
const cutOffSamples = results.filter((r) => r.error === PROVIDER_TOKEN_LIMIT && r.fitExceedsCap !== true).length;
|
|
23798
23894
|
const noRecommendationReason = exceedsStandCap ? "answer_exceeds_stand_cap" : cutOffSamples > 0 ? "answer_cut_off" : null;
|
|
23799
|
-
const
|
|
23800
|
-
const
|
|
23801
|
-
const
|
|
23895
|
+
const { headroomBps, feeBps, usdCentsPerCredit } = terms;
|
|
23896
|
+
const missingTerms = headroomBps === null ? "headroom" : feeBps === null || usdCentsPerCredit === null ? "credit_terms" : null;
|
|
23897
|
+
const full = headroomBps !== null && feeBps !== null && usdCentsPerCredit !== null ? { headroomBps, feeBps, usdCentsPerCredit } : null;
|
|
23898
|
+
const billing = feeBps !== null && usdCentsPerCredit !== null ? { feeBps, usdCentsPerCredit } : null;
|
|
23899
|
+
const costBased = full !== null && costP95 !== null ? ceilingCreditsFor(costP95, full) : null;
|
|
23900
|
+
const recommended = costBased === null || noRecommendationReason !== null ? null : Math.max(costBased, fitP95 ?? 0);
|
|
23901
|
+
const ceiling = recommended === null || costMax === null || full === null ? null : Math.max(ceilingCreditsFor(costMax, full), recommended, fitMax ?? 0);
|
|
23902
|
+
const average = billing !== null ? averageCreditsPerCall(costs, billing) : null;
|
|
23903
|
+
const estimate = ceiling === null || billing === null ? null : Math.min(ceiling, perCallEstimateCreditsFor(costs, billing) ?? 1);
|
|
23802
23904
|
return {
|
|
23905
|
+
costs,
|
|
23803
23906
|
charged,
|
|
23804
|
-
|
|
23805
|
-
|
|
23806
|
-
|
|
23907
|
+
costP50: percentilePicos(costs, 50),
|
|
23908
|
+
costP95,
|
|
23909
|
+
costMax,
|
|
23910
|
+
chargedP50: percentile2(charged, 50),
|
|
23911
|
+
chargedP95: percentile2(charged, 95),
|
|
23912
|
+
chargedMax: charged.length ? Math.max(...charged) : null,
|
|
23807
23913
|
fits,
|
|
23808
23914
|
fitP95,
|
|
23809
23915
|
fitMax,
|
|
23810
|
-
|
|
23811
|
-
|
|
23812
|
-
|
|
23813
|
-
|
|
23916
|
+
costBasedMaxCredits: costBased,
|
|
23917
|
+
recommendedMaxCredits: recommended,
|
|
23918
|
+
recommendedPerCallMaxCredits: ceiling,
|
|
23919
|
+
averageCreditsPerCall: average,
|
|
23920
|
+
recommendedPerCallEstimateCredits: estimate,
|
|
23921
|
+
lengthRaisedCeiling: recommended !== null && costBased !== null && recommended > costBased,
|
|
23814
23922
|
cutOffSamples,
|
|
23815
23923
|
exceedsStandCap,
|
|
23816
|
-
noRecommendationReason
|
|
23924
|
+
noRecommendationReason,
|
|
23925
|
+
missingTerms
|
|
23817
23926
|
};
|
|
23818
23927
|
}
|
|
23819
|
-
var LLM_LENGTH_RAISED_LINE = "The
|
|
23820
|
-
var LLM_EXCEEDS_STAND_CAP_LINE = "No
|
|
23821
|
-
function benchCutOffLine(cutOff,
|
|
23822
|
-
return `No
|
|
23928
|
+
var LLM_LENGTH_RAISED_LINE = "The ceiling also decides how long the answer may be: at the cost-based ceiling the answer would be cut off, so declare this one.";
|
|
23929
|
+
var LLM_EXCEEDS_STAND_CAP_LINE = "No ceiling recommended: this answer is longer than one call on this stand may produce. Ask for a shorter answer \u2014 fewer fields, shorter strings, a length the prompt states \u2014 and benchmark again.";
|
|
23930
|
+
function benchCutOffLine(cutOff, maxCredits) {
|
|
23931
|
+
return `No ceiling recommended: ${cutOff} sample${cutOff === 1 ? " was" : "s were"} cut off at this ceiling (--max-credits ${maxCredits}) before the answer was finished, so its real length is unknown. Re-run with a higher --max-credits.`;
|
|
23823
23932
|
}
|
|
23824
23933
|
var ACTIVE_STATUSES = /* @__PURE__ */ new Set(["requires_confirmation", "queued", "dispatching", "awaiting_external"]);
|
|
23825
23934
|
function rowState(row) {
|
|
@@ -23830,7 +23939,22 @@ function rowState(row) {
|
|
|
23830
23939
|
function rowHoldsSlot(row) {
|
|
23831
23940
|
if (typeof row.slotHeld === "boolean") return row.slotHeld;
|
|
23832
23941
|
const state = rowState(row);
|
|
23833
|
-
return state === "active" || state === "awaiting bill" && (row.reservedCoins ?? 0) > 0;
|
|
23942
|
+
return state === "active" || state === "awaiting bill" && ((row.reservedCoins ?? 0) > 0 || (row.reservedCredits ?? 0) > 0);
|
|
23943
|
+
}
|
|
23944
|
+
function rowMoney(row) {
|
|
23945
|
+
const credits = row.currency === "credits" || row.currency === void 0 && typeof row.reservedCredits === "number" && typeof row.reservedCoins !== "number";
|
|
23946
|
+
if (credits) {
|
|
23947
|
+
return {
|
|
23948
|
+
unit: "credits",
|
|
23949
|
+
reserved: typeof row.reservedCredits === "number" ? row.reservedCredits : null,
|
|
23950
|
+
charged: typeof row.chargedCredits === "number" ? row.chargedCredits : null
|
|
23951
|
+
};
|
|
23952
|
+
}
|
|
23953
|
+
return {
|
|
23954
|
+
unit: "coin",
|
|
23955
|
+
reserved: typeof row.reservedCoins === "number" ? row.reservedCoins : null,
|
|
23956
|
+
charged: typeof row.chargedCoins === "number" ? row.chargedCoins : null
|
|
23957
|
+
};
|
|
23834
23958
|
}
|
|
23835
23959
|
function providerRefusalStatus(error) {
|
|
23836
23960
|
const m = /^provider_http_(\d{3})$/.exec(error ?? "");
|
|
@@ -23849,7 +23973,7 @@ function formatAge(iso, now = Date.now()) {
|
|
|
23849
23973
|
}
|
|
23850
23974
|
async function runBench(args) {
|
|
23851
23975
|
const { apiUrl, token, projectId, cwd, opts, log } = args;
|
|
23852
|
-
const
|
|
23976
|
+
const maxCredits = opts.maxCredits ?? opts.maxCoins;
|
|
23853
23977
|
const samples = opts.samples ?? DEFAULT_BENCH_SAMPLES;
|
|
23854
23978
|
const prompt = opts.benchPrompt.trim();
|
|
23855
23979
|
const auth = { Authorization: `Bearer ${token}`, "Content-Type": "application/json" };
|
|
@@ -23871,7 +23995,7 @@ async function runBench(args) {
|
|
|
23871
23995
|
const lane = await readRuntimeLane(apiUrl, token);
|
|
23872
23996
|
if (lane.state !== "live" || !lane.models || lane.models.length === 0) {
|
|
23873
23997
|
if (lane.state === "off") {
|
|
23874
|
-
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
23998
|
+
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCredits, opts), status: "off", error: null });
|
|
23875
23999
|
else {
|
|
23876
24000
|
log.plain(LLM_LANE_OFF_LINE);
|
|
23877
24001
|
log.dim(" Nothing was spent.");
|
|
@@ -23898,17 +24022,27 @@ async function runBench(args) {
|
|
|
23898
24022
|
);
|
|
23899
24023
|
return;
|
|
23900
24024
|
}
|
|
23901
|
-
const
|
|
24025
|
+
const walletBefore = await fetchCreditsSnapshot(apiUrl, token);
|
|
24026
|
+
if (walletBefore?.emailVerified === false) {
|
|
24027
|
+
benchFailed(opts, log, LLM_BENCH_UNVERIFIED_LINE);
|
|
24028
|
+
return;
|
|
24029
|
+
}
|
|
24030
|
+
if (walletBefore !== null && walletBefore.spendable < maxCredits) {
|
|
24031
|
+
benchFailed(opts, log, benchBalanceShortLine(walletBefore.spendable, maxCredits));
|
|
24032
|
+
return;
|
|
24033
|
+
}
|
|
24034
|
+
const balanceBefore = walletBefore?.spendable ?? null;
|
|
23902
24035
|
if (!opts.json) {
|
|
23903
24036
|
log.plain(c.bold("genex llm bench"));
|
|
23904
24037
|
log.dim(` ${apiUrl}`);
|
|
23905
24038
|
log.plain("");
|
|
23906
24039
|
log.plain(` Model ${c.cyan(modelId)}`);
|
|
23907
24040
|
log.plain(` Samples ${samples} real attempt${samples === 1 ? "" : "s"}, ${outputFormat} output`);
|
|
23908
|
-
log.plain(` Approved ${
|
|
24041
|
+
log.plain(` Approved ${maxCredits} credit${maxCredits === 1 ? "" : "s"} per attempt \u2014 at worst ${maxCredits * samples} credits for this run, from your own balance`);
|
|
23909
24042
|
log.plain(
|
|
23910
|
-
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore}
|
|
24043
|
+
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore} credits spendable`}`
|
|
23911
24044
|
);
|
|
24045
|
+
log.dim(" Billed as used: each attempt is charged its real cost plus the platform fee, rounded up to a whole credit.");
|
|
23912
24046
|
log.plain("");
|
|
23913
24047
|
}
|
|
23914
24048
|
const base = `${apiUrl}/api/runtime/development/projects/${encodeURIComponent(projectId)}/generations`;
|
|
@@ -23934,7 +24068,7 @@ async function runBench(args) {
|
|
|
23934
24068
|
const res = await call(base, {
|
|
23935
24069
|
method: "POST",
|
|
23936
24070
|
body: JSON.stringify({
|
|
23937
|
-
|
|
24071
|
+
maxCredits,
|
|
23938
24072
|
request: {
|
|
23939
24073
|
idempotencyKey: `bench-${randomUUID2()}`,
|
|
23940
24074
|
modelId,
|
|
@@ -23964,13 +24098,14 @@ async function runBench(args) {
|
|
|
23964
24098
|
if (refusedAt !== null) providerRefused++;
|
|
23965
24099
|
if (!opts.json) {
|
|
23966
24100
|
const length = row.outputTokens !== null ? ` \xB7 ${row.outputTokens}${row.maxOutputTokens !== null ? ` of ${row.maxOutputTokens}` : ""} tokens out` : "";
|
|
24101
|
+
const charged = row.chargedCredits === null ? "\u2014" : String(row.chargedCredits);
|
|
23967
24102
|
log.plain(
|
|
23968
|
-
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${
|
|
24103
|
+
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${charged.padStart(4)} credit${row.chargedCredits === 1 ? "" : "s"} charged \xB7 cost ${usdFromPicos(row.costUsdPicos)}${length}${row.error ? ` \xB7 ${row.error}` : ""}`
|
|
23969
24104
|
);
|
|
23970
24105
|
if (refusedAt !== null) printProviderRefusal(log, refusedAt, row.providerMessage, row.status === "unknown");
|
|
23971
24106
|
if (row.error === PROVIDER_TOKEN_LIMIT) {
|
|
23972
24107
|
log.dim(
|
|
23973
|
-
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no
|
|
24108
|
+
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no ceiling makes room for it; it is not a sample." : ` The answer was cut off at this ceiling (--max-credits ${maxCredits}) before it was finished \u2014 it was charged, and it is not a sample.`
|
|
23974
24109
|
);
|
|
23975
24110
|
}
|
|
23976
24111
|
}
|
|
@@ -23978,16 +24113,19 @@ async function runBench(args) {
|
|
|
23978
24113
|
} finally {
|
|
23979
24114
|
process.removeListener("SIGINT", onSigint);
|
|
23980
24115
|
}
|
|
23981
|
-
const
|
|
23982
|
-
|
|
23983
|
-
|
|
23984
|
-
|
|
23985
|
-
|
|
23986
|
-
const
|
|
24116
|
+
const terms = {
|
|
24117
|
+
headroomBps: lane.recommendedDeclaredHeadroomBps,
|
|
24118
|
+
feeBps: lane.feeBps,
|
|
24119
|
+
usdCentsPerCredit: lane.usdCentsPerCredit
|
|
24120
|
+
};
|
|
24121
|
+
const rec = benchRecommendation(results, terms);
|
|
24122
|
+
const settledCount = rec.charged.length;
|
|
23987
24123
|
const lengthBlocked = rec.noRecommendationReason !== null;
|
|
23988
|
-
const balanceAfter = await
|
|
24124
|
+
const balanceAfter = (await fetchCreditsSnapshot(apiUrl, token))?.spendable ?? null;
|
|
24125
|
+
const picosOrNull = (v) => v === null ? null : String(v);
|
|
23989
24126
|
const record = {
|
|
23990
|
-
v:
|
|
24127
|
+
v: 2,
|
|
24128
|
+
currency: "credits",
|
|
23991
24129
|
ranAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
23992
24130
|
apiUrl,
|
|
23993
24131
|
projectId,
|
|
@@ -23996,28 +24134,34 @@ async function runBench(args) {
|
|
|
23996
24134
|
outputFormat,
|
|
23997
24135
|
samples,
|
|
23998
24136
|
completed: settledCount,
|
|
23999
|
-
|
|
24000
|
-
|
|
24001
|
-
|
|
24002
|
-
max,
|
|
24003
|
-
|
|
24004
|
-
|
|
24005
|
-
recommendedPerCallMaxCoins: ceiling,
|
|
24006
|
-
maxCoinsPerSample: maxCoins,
|
|
24007
|
-
fitCoins: rec.fits,
|
|
24137
|
+
maxCreditsPerSample: maxCredits,
|
|
24138
|
+
costUsdPicos: rec.costs.map(String),
|
|
24139
|
+
chargedCredits: rec.charged,
|
|
24140
|
+
cost: { p50: picosOrNull(rec.costP50), p95: picosOrNull(rec.costP95), max: picosOrNull(rec.costMax) },
|
|
24141
|
+
charged: { p50: rec.chargedP50, p95: rec.chargedP95, max: rec.chargedMax },
|
|
24142
|
+
fitCredits: rec.fits,
|
|
24008
24143
|
fitP95: rec.fitP95,
|
|
24009
|
-
|
|
24010
|
-
|
|
24144
|
+
fitMax: rec.fitMax,
|
|
24145
|
+
recommendedDeclaredHeadroomBps: terms.headroomBps,
|
|
24146
|
+
feeBps: terms.feeBps,
|
|
24147
|
+
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
24148
|
+
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
24149
|
+
recommendedMaxCredits: rec.recommendedMaxCredits,
|
|
24150
|
+
recommendedPerCallMaxCredits: rec.recommendedPerCallMaxCredits,
|
|
24151
|
+
averageCreditsPerCall: rec.averageCreditsPerCall,
|
|
24152
|
+
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
24153
|
+
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
24011
24154
|
cutOffSamples: rec.cutOffSamples,
|
|
24012
24155
|
noRecommendationReason: rec.noRecommendationReason
|
|
24013
24156
|
};
|
|
24014
24157
|
const savedTo = await saveBench(cwd, record);
|
|
24015
24158
|
if (opts.json) {
|
|
24016
|
-
const ok =
|
|
24159
|
+
const ok = settledCount > 0 && !lengthBlocked;
|
|
24017
24160
|
writeJsonLine({
|
|
24018
24161
|
command: "llm bench",
|
|
24019
24162
|
status: ok ? "ok" : "failed",
|
|
24020
24163
|
error: ok ? null : rec.noRecommendationReason ?? "no_sample_settled",
|
|
24164
|
+
currency: "credits",
|
|
24021
24165
|
apiUrl,
|
|
24022
24166
|
projectId,
|
|
24023
24167
|
modelId,
|
|
@@ -24026,35 +24170,54 @@ async function runBench(args) {
|
|
|
24026
24170
|
schemaPath: opts.schemaPath ?? null,
|
|
24027
24171
|
samples,
|
|
24028
24172
|
completed: settledCount,
|
|
24029
|
-
|
|
24173
|
+
maxCreditsPerSample: maxCredits,
|
|
24174
|
+
// Spendable credits before and after the run; null when unreadable.
|
|
24030
24175
|
balanceBefore,
|
|
24031
24176
|
balanceAfter,
|
|
24032
24177
|
results,
|
|
24033
24178
|
// The create-time refusal that ended the run, or null. Beside the
|
|
24034
24179
|
// samples, never among them: no attempt existed and nothing was spent.
|
|
24035
24180
|
refusal: refusal2,
|
|
24036
|
-
|
|
24037
|
-
//
|
|
24181
|
+
// Over the settled samples: the real provider cost (pico-dollar strings
|
|
24182
|
+
// and USD) and the whole credits charged.
|
|
24183
|
+
costUsdPicos: record.cost,
|
|
24184
|
+
costUsd: {
|
|
24185
|
+
p50: rec.costP50 === null ? null : Number(rec.costP50) / 1e12,
|
|
24186
|
+
p95: rec.costP95 === null ? null : Number(rec.costP95) / 1e12,
|
|
24187
|
+
max: rec.costMax === null ? null : Number(rec.costMax) / 1e12
|
|
24188
|
+
},
|
|
24189
|
+
chargedCredits: record.charged,
|
|
24190
|
+
// The answer's LENGTH, priced by the server per sample (`fitCredits`,
|
|
24038
24191
|
// headroom already on the tokens), over the settled samples.
|
|
24039
|
-
|
|
24040
|
-
|
|
24041
|
-
|
|
24192
|
+
fitCredits: { p95: rec.fitP95, max: rec.fitMax },
|
|
24193
|
+
recommendedDeclaredHeadroomBps: terms.headroomBps,
|
|
24194
|
+
feeBps: terms.feeBps,
|
|
24195
|
+
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
24196
|
+
missingTerms: rec.missingTerms,
|
|
24197
|
+
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
24198
|
+
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
24042
24199
|
cutOffSamples: rec.cutOffSamples,
|
|
24043
24200
|
exceedsStandCap: rec.exceedsStandCap,
|
|
24044
24201
|
noRecommendationReason: rec.noRecommendationReason,
|
|
24045
|
-
|
|
24046
|
-
|
|
24047
|
-
|
|
24202
|
+
recommendedMaxCredits: rec.recommendedMaxCredits,
|
|
24203
|
+
recommendedPerCallMaxCredits: rec.recommendedPerCallMaxCredits,
|
|
24204
|
+
averageCreditsPerCall: rec.averageCreditsPerCall,
|
|
24205
|
+
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
24048
24206
|
savedTo
|
|
24049
24207
|
});
|
|
24050
24208
|
if (!ok) process.exitCode = 1;
|
|
24051
24209
|
return;
|
|
24052
24210
|
}
|
|
24053
24211
|
log.plain("");
|
|
24212
|
+
const printMeasured = () => {
|
|
24213
|
+
log.plain(c.bold(" Real cost per call"));
|
|
24214
|
+
log.plain(` p50 ${usdFromPicos(rec.costP50)} p95 ${usdFromPicos(rec.costP95)} max ${usdFromPicos(rec.costMax)} (${settledCount} of ${samples} settled)`);
|
|
24215
|
+
log.plain(c.bold(" Charged credits"));
|
|
24216
|
+
log.plain(` p50 ${rec.chargedP50} p95 ${rec.chargedP95} max ${rec.chargedMax} (whole credits, rounded up per call)`);
|
|
24217
|
+
};
|
|
24054
24218
|
if (lengthBlocked) {
|
|
24055
|
-
if (
|
|
24056
|
-
|
|
24057
|
-
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
24219
|
+
if (settledCount > 0) {
|
|
24220
|
+
printMeasured();
|
|
24058
24221
|
log.plain("");
|
|
24059
24222
|
}
|
|
24060
24223
|
printRecommendation(log, record);
|
|
@@ -24062,17 +24225,16 @@ async function runBench(args) {
|
|
|
24062
24225
|
process.exitCode = 1;
|
|
24063
24226
|
return;
|
|
24064
24227
|
}
|
|
24065
|
-
if (
|
|
24228
|
+
if (settledCount === 0) {
|
|
24066
24229
|
log.error(
|
|
24067
|
-
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to
|
|
24230
|
+
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to measure from." : " No sample settled, so there is nothing to measure from."
|
|
24068
24231
|
);
|
|
24069
24232
|
process.exitCode = 1;
|
|
24070
24233
|
return;
|
|
24071
24234
|
}
|
|
24072
|
-
|
|
24073
|
-
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
24235
|
+
printMeasured();
|
|
24074
24236
|
if (balanceBefore !== null && balanceAfter !== null) {
|
|
24075
|
-
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter}
|
|
24237
|
+
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter} credits spendable`);
|
|
24076
24238
|
}
|
|
24077
24239
|
log.plain("");
|
|
24078
24240
|
printRecommendation(log, record);
|
|
@@ -24084,43 +24246,58 @@ function printRecommendation(log, record) {
|
|
|
24084
24246
|
return;
|
|
24085
24247
|
}
|
|
24086
24248
|
if (record.noRecommendationReason === "answer_cut_off") {
|
|
24087
|
-
log.warn(` ${benchCutOffLine(record.cutOffSamples
|
|
24249
|
+
log.warn(` ${benchCutOffLine(record.cutOffSamples || 1, record.maxCreditsPerSample)}`);
|
|
24088
24250
|
return;
|
|
24089
24251
|
}
|
|
24090
|
-
|
|
24091
|
-
|
|
24252
|
+
const attempts = `${record.completed} settled attempt${record.completed === 1 ? "" : "s"} on ${record.modelId}`;
|
|
24253
|
+
if (record.recommendedMaxCredits === null || record.cost.p95 === null) {
|
|
24254
|
+
log.warn(" No ceiling recommended.");
|
|
24092
24255
|
log.dim(
|
|
24093
|
-
record.recommendedDeclaredHeadroomBps === null ? " This stand served no recommended headroom, and the multiplier is the server's to set \u2014" : "
|
|
24256
|
+
record.cost.p95 === null ? " No sample settled, so there is no p95 to build a ceiling on \u2014" : record.recommendedDeclaredHeadroomBps === null ? " This stand served no recommended headroom, and the multiplier is the server's to set \u2014" : " This stand served no platform fee or credit value, and those are the server's to set \u2014"
|
|
24094
24257
|
);
|
|
24095
24258
|
log.dim(" declaring a number from this run would be a guess dressed as a measurement.");
|
|
24259
|
+
printAverage(log, record, attempts);
|
|
24096
24260
|
return;
|
|
24097
24261
|
}
|
|
24098
|
-
log.plain(c.bold(` Declare
|
|
24099
|
-
if (record.
|
|
24262
|
+
log.plain(c.bold(` Declare maxCredits: ${record.recommendedMaxCredits}`));
|
|
24263
|
+
if (record.lengthRaisedCeiling) {
|
|
24100
24264
|
log.plain(` ${LLM_LENGTH_RAISED_LINE}`);
|
|
24101
24265
|
log.dim(
|
|
24102
|
-
` = the smallest
|
|
24266
|
+
` = the smallest ceiling whose answer allowance holds the answer plus this stand's headroom (p95 ${record.fitP95 ?? "\u2014"}), above the`
|
|
24103
24267
|
);
|
|
24104
24268
|
log.dim(
|
|
24105
|
-
`
|
|
24269
|
+
` cost-based ${record.costBasedMaxCredits ?? "\u2014"} (p95 real cost ${usdFromPicos(record.cost.p95)} with the platform fee, ${record.feeBps} bps, and ${record.recommendedDeclaredHeadroomBps} bps of headroom), over ${attempts}.`
|
|
24106
24270
|
);
|
|
24107
24271
|
} else {
|
|
24108
24272
|
log.dim(
|
|
24109
|
-
` = p95
|
|
24273
|
+
` = p95 real cost (${usdFromPicos(record.cost.p95)}) with this stand's platform fee (${record.feeBps} bps) and recommended headroom (${record.recommendedDeclaredHeadroomBps} bps), in whole credits,`
|
|
24110
24274
|
);
|
|
24111
|
-
log.dim(` measured over ${
|
|
24275
|
+
log.dim(` measured over ${attempts}.`);
|
|
24112
24276
|
}
|
|
24113
|
-
log.dim(" That number is a
|
|
24114
|
-
|
|
24115
|
-
|
|
24277
|
+
log.dim(" That number is a CEILING, not a price: a call is billed its real cost as used, never more than it \u2014");
|
|
24278
|
+
log.dim(" and it also sizes how long the answer may be.");
|
|
24279
|
+
if (record.recommendedPerCallMaxCredits !== null) {
|
|
24280
|
+
log.plain(c.bold(` Grant perCallMaxCredits: ${record.recommendedPerCallMaxCredits}`));
|
|
24116
24281
|
log.dim(
|
|
24117
|
-
record.
|
|
24282
|
+
record.lengthRaisedCeiling ? ` = the same over the worst sample (${usdFromPicos(record.cost.max)}) or the longest answer's ceiling, whichever is larger, and never below the ceiling above it.` : ` = the same over the worst sample (${usdFromPicos(record.cost.max)}), and never below the ceiling above it.`
|
|
24118
24283
|
);
|
|
24119
|
-
|
|
24284
|
+
}
|
|
24285
|
+
printAverage(log, record, attempts);
|
|
24286
|
+
}
|
|
24287
|
+
function printAverage(log, record, attempts) {
|
|
24288
|
+
if (record.averageCreditsPerCall === null) return;
|
|
24289
|
+
log.plain(` Per-call estimate: about ${formatCredits(record.averageCreditsPerCall)} credits per call`);
|
|
24290
|
+
log.dim(` = the average real cost with the platform fee, no headroom, over ${attempts}.`);
|
|
24291
|
+
if (record.recommendedPerCallEstimateCredits !== null) {
|
|
24292
|
+
log.plain(c.bold(` Grant perCallEstimateCredits: ${record.recommendedPerCallEstimateCredits}`));
|
|
24293
|
+
log.dim(` = that average rounded up to a whole credit. disclosure.estimatedCreditsPerPeriod = your calls per period \xD7 ${formatCredits(record.averageCreditsPerCall)}, rounded up.`);
|
|
24120
24294
|
}
|
|
24121
24295
|
}
|
|
24122
24296
|
async function explainBenchRefusal(res, call, base, log, opts, index, settledSoFar) {
|
|
24123
|
-
if (printedStructuredError(res))
|
|
24297
|
+
if (printedStructuredError(res)) {
|
|
24298
|
+
const printed = await res.json().catch(() => ({}));
|
|
24299
|
+
return { stop: true, refusal: { status: res.status, error: typeof printed.error === "string" ? printed.error : null, slotsHeld: null, slotLimit: null } };
|
|
24300
|
+
}
|
|
24124
24301
|
const body = await res.json().catch(() => ({}));
|
|
24125
24302
|
const refusal2 = { status: res.status, error: body.error ?? null, slotsHeld: null, slotLimit: null };
|
|
24126
24303
|
if (res.status === 429 && body.error === "generation_limit") {
|
|
@@ -24130,6 +24307,10 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24130
24307
|
if (!opts.json) log.error(` Sample ${index + 1} ${benchSlotsHeldSentence(counts)}`);
|
|
24131
24308
|
return { stop: true, refusal: refusal2 };
|
|
24132
24309
|
}
|
|
24310
|
+
if (res.status === 403 && body.error === "credits_unverified") {
|
|
24311
|
+
if (!opts.json) log.error(` ${LLM_BENCH_UNVERIFIED_LINE}`);
|
|
24312
|
+
return { stop: true, refusal: refusal2 };
|
|
24313
|
+
}
|
|
24133
24314
|
if (opts.json) return { stop: res.status !== 429, refusal: refusal2 };
|
|
24134
24315
|
if (res.status === 404 && body.error === "not_found") {
|
|
24135
24316
|
log.error(` ${LLM_LANE_OFF_LINE}`);
|
|
@@ -24145,7 +24326,7 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24145
24326
|
return { stop: true, refusal: refusal2 };
|
|
24146
24327
|
}
|
|
24147
24328
|
if (res.status === 402) {
|
|
24148
|
-
log.error(" Not enough
|
|
24329
|
+
log.error(" Not enough credits to start the attempt \u2014 a bench is paid from your own balance.");
|
|
24149
24330
|
return { stop: true, refusal: refusal2 };
|
|
24150
24331
|
}
|
|
24151
24332
|
log.error(
|
|
@@ -24154,7 +24335,7 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24154
24335
|
return { stop: res.status >= 500 || res.status === 401 || res.status === 403, refusal: refusal2 };
|
|
24155
24336
|
}
|
|
24156
24337
|
function printProviderRefusal(log, status2, message, awaitingBill = false) {
|
|
24157
|
-
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so this call
|
|
24338
|
+
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so what this call reserved stays held until it resolves (see \`genex llm status\`); it is not a sample.`);
|
|
24158
24339
|
else log.dim(" The provider refused this call at its door \u2014 no inference ran, it cost nothing, and it is not a sample.");
|
|
24159
24340
|
if (message) log.dim(` Provider said: ${message}`);
|
|
24160
24341
|
if (status2 === 401 || status2 === 403) {
|
|
@@ -24180,19 +24361,6 @@ async function pollSettled(call, url, timeoutSec) {
|
|
|
24180
24361
|
}
|
|
24181
24362
|
return last;
|
|
24182
24363
|
}
|
|
24183
|
-
async function readCoinBalance(apiUrl, token) {
|
|
24184
|
-
try {
|
|
24185
|
-
const res = await apiFetch(`${apiUrl}/api/coin/balance`, {
|
|
24186
|
-
headers: { Authorization: `Bearer ${token}` },
|
|
24187
|
-
signal: AbortSignal.timeout(6e3)
|
|
24188
|
-
});
|
|
24189
|
-
if (!res.ok) return null;
|
|
24190
|
-
const body = await res.json().catch(() => null);
|
|
24191
|
-
return typeof body?.spendable === "number" ? body.spendable : null;
|
|
24192
|
-
} catch {
|
|
24193
|
-
return null;
|
|
24194
|
-
}
|
|
24195
|
-
}
|
|
24196
24364
|
async function saveBench(cwd, record) {
|
|
24197
24365
|
try {
|
|
24198
24366
|
const file = path36.join(cwd, LLM_BENCH_FILE);
|
|
@@ -24208,9 +24376,10 @@ function benchFailed(opts, log, message) {
|
|
|
24208
24376
|
else log.error(message);
|
|
24209
24377
|
process.exitCode = 1;
|
|
24210
24378
|
}
|
|
24211
|
-
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
24379
|
+
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCredits, opts) {
|
|
24212
24380
|
return {
|
|
24213
24381
|
command: "llm bench",
|
|
24382
|
+
currency: "credits",
|
|
24214
24383
|
apiUrl,
|
|
24215
24384
|
projectId,
|
|
24216
24385
|
modelId: opts.modelId ?? null,
|
|
@@ -24219,21 +24388,28 @@ function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCoi
|
|
|
24219
24388
|
schemaPath: opts.schemaPath ?? null,
|
|
24220
24389
|
samples,
|
|
24221
24390
|
completed: 0,
|
|
24222
|
-
|
|
24391
|
+
maxCreditsPerSample: maxCredits,
|
|
24223
24392
|
balanceBefore: null,
|
|
24224
24393
|
balanceAfter: null,
|
|
24225
24394
|
results: [],
|
|
24226
24395
|
refusal: null,
|
|
24227
|
-
|
|
24228
|
-
|
|
24229
|
-
|
|
24230
|
-
|
|
24396
|
+
costUsdPicos: { p50: null, p95: null, max: null },
|
|
24397
|
+
costUsd: { p50: null, p95: null, max: null },
|
|
24398
|
+
chargedCredits: { p50: null, p95: null, max: null },
|
|
24399
|
+
fitCredits: { p95: null, max: null },
|
|
24400
|
+
recommendedDeclaredHeadroomBps: null,
|
|
24401
|
+
feeBps: null,
|
|
24402
|
+
usdCentsPerCredit: null,
|
|
24403
|
+
missingTerms: null,
|
|
24404
|
+
costBasedMaxCredits: null,
|
|
24405
|
+
lengthRaisedCeiling: false,
|
|
24231
24406
|
cutOffSamples: 0,
|
|
24232
24407
|
exceedsStandCap: false,
|
|
24233
24408
|
noRecommendationReason: null,
|
|
24234
|
-
|
|
24235
|
-
|
|
24236
|
-
|
|
24409
|
+
recommendedMaxCredits: null,
|
|
24410
|
+
recommendedPerCallMaxCredits: null,
|
|
24411
|
+
averageCreditsPerCall: null,
|
|
24412
|
+
recommendedPerCallEstimateCredits: null,
|
|
24237
24413
|
savedTo: null
|
|
24238
24414
|
};
|
|
24239
24415
|
}
|
|
@@ -24296,7 +24472,8 @@ async function reportStatus(args) {
|
|
|
24296
24472
|
for (const row of rows) {
|
|
24297
24473
|
const state = rowState(row);
|
|
24298
24474
|
const mark = state === "active" ? c.cyan("\u25CF") : state === "awaiting bill" ? c.yellow("\u25D0") : c.green("\u25CB");
|
|
24299
|
-
const
|
|
24475
|
+
const money2 = rowMoney(row);
|
|
24476
|
+
const reserved = money2.reserved === null ? "reserved \u2014" : `${money2.reserved} ${money2.unit} reserved`;
|
|
24300
24477
|
log.plain(
|
|
24301
24478
|
` ${mark} ${row.id ?? "?"} ${row.modelId ?? "?"} ${state.padEnd(13)} ${formatAge(row.createdAt, now).padStart(4)} old ${reserved}${row.error ? ` ${row.error}` : ""}${rowHoldsSlot(row) ? "" : c.dim(" (no slot)")}`
|
|
24302
24479
|
);
|
|
@@ -24368,10 +24545,11 @@ async function cancelCall(args) {
|
|
|
24368
24545
|
if (!opts.json) {
|
|
24369
24546
|
log.warn(` ${row.id} (${row.status ?? "?"}) \u2014 ${sentence}.`);
|
|
24370
24547
|
if (state === "awaiting bill") {
|
|
24371
|
-
log.dim(` Nothing to cancel:
|
|
24548
|
+
log.dim(` Nothing to cancel: what it reserved releases when the bill resolves, and it stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`);
|
|
24372
24549
|
if (row.providerMessage?.trim()) log.dim(` Provider said: ${row.providerMessage.trim()}`);
|
|
24373
24550
|
} else {
|
|
24374
|
-
|
|
24551
|
+
const money2 = rowMoney(row);
|
|
24552
|
+
log.dim(` ${money2.charged === null ? "charge unknown" : `${money2.charged} ${money2.unit} charged`}${row.error ? ` \xB7 ${row.error}` : ""}.`);
|
|
24375
24553
|
}
|
|
24376
24554
|
}
|
|
24377
24555
|
return done("ok", null, state === "awaiting bill" ? "awaiting_bill" : "already_final", row);
|
|
@@ -24393,78 +24571,126 @@ async function cancelCall(args) {
|
|
|
24393
24571
|
const after = await res.json().catch(() => null) ?? row;
|
|
24394
24572
|
if (!opts.json) {
|
|
24395
24573
|
log.success(` Cancelled ${row.id} (was ${row.status ?? "active"}).`);
|
|
24396
|
-
log.dim(`
|
|
24574
|
+
log.dim(` What it reserved releases once the bill settles; ${c.cyan("genex llm status")} shows it until then.`);
|
|
24397
24575
|
}
|
|
24398
24576
|
return done("ok", null, "cancelled", after);
|
|
24399
24577
|
}
|
|
24400
|
-
async function
|
|
24401
|
-
let
|
|
24578
|
+
async function readSavedBench(cwd) {
|
|
24579
|
+
let parsed;
|
|
24402
24580
|
try {
|
|
24403
|
-
|
|
24581
|
+
parsed = JSON.parse(await fs35.readFile(path36.join(cwd, LLM_BENCH_FILE), "utf8"));
|
|
24404
24582
|
} catch {
|
|
24405
|
-
|
|
24583
|
+
return null;
|
|
24406
24584
|
}
|
|
24407
|
-
if (!
|
|
24585
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null;
|
|
24586
|
+
const v = parsed;
|
|
24587
|
+
if (v.v === 2 && v.currency === "credits" && v.cost && v.charged) return { kind: "credits", record: parsed };
|
|
24588
|
+
return { kind: "coins", record: parsed };
|
|
24589
|
+
}
|
|
24590
|
+
var LLM_PRICE_COIN_TERMS_LINE = "This bench was measured under the old COIN terms, when a game declared a fixed price per call. In-game calls are now billed in the player's credits, as used, and a game declares a per-call CEILING (maxCredits) instead \u2014 a coin price is not that number. Re-run the bench for the figures to declare.";
|
|
24591
|
+
async function reportSavedPrice(cwd, opts, log) {
|
|
24592
|
+
const saved = await readSavedBench(cwd);
|
|
24593
|
+
const rerun = 'genex llm bench "<prompt>" --max-credits <n> --user-approved';
|
|
24594
|
+
const creditKeys = (record2) => ({
|
|
24595
|
+
costUsdPicos: record2?.cost ?? { p50: null, p95: null, max: null },
|
|
24596
|
+
chargedCredits: record2?.charged ?? { p50: null, p95: null, max: null },
|
|
24597
|
+
fitP95: record2?.fitP95 ?? null,
|
|
24598
|
+
fitMax: record2?.fitMax ?? null,
|
|
24599
|
+
recommendedDeclaredHeadroomBps: record2?.recommendedDeclaredHeadroomBps ?? null,
|
|
24600
|
+
feeBps: record2?.feeBps ?? null,
|
|
24601
|
+
usdCentsPerCredit: record2?.usdCentsPerCredit ?? null,
|
|
24602
|
+
costBasedMaxCredits: record2?.costBasedMaxCredits ?? null,
|
|
24603
|
+
lengthRaisedCeiling: record2?.lengthRaisedCeiling ?? false,
|
|
24604
|
+
cutOffSamples: record2?.cutOffSamples ?? 0,
|
|
24605
|
+
noRecommendationReason: record2?.noRecommendationReason ?? null,
|
|
24606
|
+
recommendedMaxCredits: record2?.recommendedMaxCredits ?? null,
|
|
24607
|
+
recommendedPerCallMaxCredits: record2?.recommendedPerCallMaxCredits ?? null,
|
|
24608
|
+
averageCreditsPerCall: record2?.averageCreditsPerCall ?? null,
|
|
24609
|
+
recommendedPerCallEstimateCredits: record2?.recommendedPerCallEstimateCredits ?? null
|
|
24610
|
+
});
|
|
24611
|
+
if (!saved) {
|
|
24408
24612
|
if (opts.json) {
|
|
24409
24613
|
writeJsonLine({
|
|
24410
24614
|
command: "llm price",
|
|
24411
24615
|
status: "failed",
|
|
24412
24616
|
error: "no_bench",
|
|
24617
|
+
currency: null,
|
|
24413
24618
|
ranAt: null,
|
|
24414
24619
|
modelId: null,
|
|
24415
24620
|
samples: null,
|
|
24416
24621
|
completed: null,
|
|
24417
|
-
|
|
24418
|
-
|
|
24419
|
-
max: null,
|
|
24420
|
-
fitP95: null,
|
|
24421
|
-
chargedBasedEstimateCoins: null,
|
|
24422
|
-
lengthRaisedPrice: false,
|
|
24423
|
-
cutOffSamples: 0,
|
|
24424
|
-
noRecommendationReason: null,
|
|
24425
|
-
recommendedDeclaredHeadroomBps: null,
|
|
24426
|
-
recommendedEstimateCoins: null,
|
|
24427
|
-
recommendedPerCallMaxCoins: null,
|
|
24622
|
+
...creditKeys(null),
|
|
24623
|
+
legacyCoinTerms: null,
|
|
24428
24624
|
savedTo: null
|
|
24429
24625
|
});
|
|
24430
24626
|
} else {
|
|
24431
24627
|
log.error("Nothing has been benchmarked in this folder yet.");
|
|
24432
|
-
log.dim(` Run ${c.cyan(
|
|
24628
|
+
log.dim(` Run ${c.cyan(rerun)} first.`);
|
|
24629
|
+
}
|
|
24630
|
+
process.exitCode = 1;
|
|
24631
|
+
return;
|
|
24632
|
+
}
|
|
24633
|
+
if (saved.kind === "coins") {
|
|
24634
|
+
const old = saved.record;
|
|
24635
|
+
const legacy = {
|
|
24636
|
+
p50: old.p50 ?? null,
|
|
24637
|
+
p95: old.p95 ?? null,
|
|
24638
|
+
max: old.max ?? null,
|
|
24639
|
+
recommendedEstimateCoins: old.recommendedEstimateCoins ?? null,
|
|
24640
|
+
recommendedPerCallMaxCoins: old.recommendedPerCallMaxCoins ?? null,
|
|
24641
|
+
maxCoinsPerSample: old.maxCoinsPerSample ?? null,
|
|
24642
|
+
noRecommendationReason: old.noRecommendationReason ?? null
|
|
24643
|
+
};
|
|
24644
|
+
if (opts.json) {
|
|
24645
|
+
writeJsonLine({
|
|
24646
|
+
command: "llm price",
|
|
24647
|
+
status: "failed",
|
|
24648
|
+
error: "coin_terms",
|
|
24649
|
+
currency: "coins",
|
|
24650
|
+
ranAt: old.ranAt ?? null,
|
|
24651
|
+
modelId: old.modelId ?? null,
|
|
24652
|
+
samples: old.samples ?? null,
|
|
24653
|
+
completed: old.completed ?? null,
|
|
24654
|
+
...creditKeys(null),
|
|
24655
|
+
legacyCoinTerms: legacy,
|
|
24656
|
+
savedTo: LLM_BENCH_FILE
|
|
24657
|
+
});
|
|
24658
|
+
} else {
|
|
24659
|
+
log.plain(c.bold("genex llm price"));
|
|
24660
|
+
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${old.ranAt ?? "at an unknown time"}${old.modelId ? ` on ${old.modelId}` : ""}`);
|
|
24661
|
+
log.plain("");
|
|
24662
|
+
log.warn(` ${LLM_PRICE_COIN_TERMS_LINE}`);
|
|
24663
|
+
const measured = legacy.p95 === null ? "no settled sample" : `charged coins p50 ${legacy.p50} \xB7 p95 ${legacy.p95} \xB7 max ${legacy.max}`;
|
|
24664
|
+
const recommended = legacy.recommendedEstimateCoins === null ? "no price recommended" : `it recommended estimateCoins ${legacy.recommendedEstimateCoins}${legacy.recommendedPerCallMaxCoins != null ? ` and perCallMaxCoins ${legacy.recommendedPerCallMaxCoins}` : ""}`;
|
|
24665
|
+
log.dim(` What it measured then: ${measured}; ${recommended}.`);
|
|
24666
|
+
log.dim(` Re-run: ${c.cyan(rerun)}`);
|
|
24433
24667
|
}
|
|
24434
24668
|
process.exitCode = 1;
|
|
24435
24669
|
return;
|
|
24436
24670
|
}
|
|
24671
|
+
const record = saved.record;
|
|
24437
24672
|
if (opts.json) {
|
|
24438
24673
|
writeJsonLine({
|
|
24439
24674
|
command: "llm price",
|
|
24440
|
-
status: record.
|
|
24441
|
-
error: record.
|
|
24675
|
+
status: record.recommendedMaxCredits === null ? "failed" : "ok",
|
|
24676
|
+
error: record.recommendedMaxCredits === null ? record.noRecommendationReason ?? "no_recommendation" : null,
|
|
24677
|
+
currency: "credits",
|
|
24442
24678
|
ranAt: record.ranAt ?? null,
|
|
24443
24679
|
modelId: record.modelId ?? null,
|
|
24444
24680
|
samples: record.samples ?? null,
|
|
24445
24681
|
completed: record.completed ?? null,
|
|
24446
|
-
|
|
24447
|
-
|
|
24448
|
-
max: record.max ?? null,
|
|
24449
|
-
// Absent from a file an older CLI wrote; null / false / 0 then, never missing.
|
|
24450
|
-
fitP95: record.fitP95 ?? null,
|
|
24451
|
-
chargedBasedEstimateCoins: record.chargedBasedEstimateCoins ?? null,
|
|
24452
|
-
lengthRaisedPrice: record.lengthRaisedPrice ?? false,
|
|
24453
|
-
cutOffSamples: record.cutOffSamples ?? 0,
|
|
24454
|
-
noRecommendationReason: record.noRecommendationReason ?? null,
|
|
24455
|
-
recommendedDeclaredHeadroomBps: record.recommendedDeclaredHeadroomBps ?? null,
|
|
24456
|
-
recommendedEstimateCoins: record.recommendedEstimateCoins ?? null,
|
|
24457
|
-
recommendedPerCallMaxCoins: record.recommendedPerCallMaxCoins ?? null,
|
|
24682
|
+
...creditKeys(record),
|
|
24683
|
+
legacyCoinTerms: null,
|
|
24458
24684
|
savedTo: LLM_BENCH_FILE
|
|
24459
24685
|
});
|
|
24460
|
-
if (record.
|
|
24686
|
+
if (record.recommendedMaxCredits === null) process.exitCode = 1;
|
|
24461
24687
|
return;
|
|
24462
24688
|
}
|
|
24463
24689
|
log.plain(c.bold("genex llm price"));
|
|
24464
24690
|
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${record.ranAt}`);
|
|
24465
24691
|
log.plain("");
|
|
24466
24692
|
printRecommendation(log, record);
|
|
24467
|
-
if (record.
|
|
24693
|
+
if (record.recommendedMaxCredits === null) process.exitCode = 1;
|
|
24468
24694
|
}
|
|
24469
24695
|
function writeJsonLine(value) {
|
|
24470
24696
|
process.stdout.write(`${JSON.stringify(value)}
|
|
@@ -24724,7 +24950,7 @@ function runtimeRow(state, lane, toolsOnly = false) {
|
|
|
24724
24950
|
return {
|
|
24725
24951
|
label: "Runtime",
|
|
24726
24952
|
value: `in-game LLM lane live \xB7 ${n} model${n === 1 ? "" : "s"}${keyRefused ? ` \xB7 ${c.yellow("provider key refused")}` : ""}`,
|
|
24727
|
-
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : '
|
|
24953
|
+
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : 'Measure a call before a game declares its per-call ceiling: `npx genex llm bench "<prompt>" --max-credits <n> --user-approved`.'
|
|
24728
24954
|
};
|
|
24729
24955
|
}
|
|
24730
24956
|
case "off":
|
|
@@ -26680,16 +26906,16 @@ ${c.bold("Usage")}
|
|
|
26680
26906
|
status | cancel. "models" says whether this
|
|
26681
26907
|
stand serves the runtime LLM lane at all, and
|
|
26682
26908
|
at what provider rates. "bench" runs the real
|
|
26683
|
-
model on YOUR OWN
|
|
26684
|
-
attempt
|
|
26909
|
+
model on YOUR OWN credits and reports what each
|
|
26910
|
+
attempt really cost; it needs --max-credits <n>
|
|
26685
26911
|
--user-approved, like any spend the player has
|
|
26686
26912
|
to agree to. "price" reprints the last run's
|
|
26687
|
-
recommended
|
|
26913
|
+
recommended maxCredits. "status" lists this
|
|
26688
26914
|
project's open calls and the slots they hold;
|
|
26689
26915
|
"cancel <id>" stops an active one. Declare a
|
|
26690
|
-
game's
|
|
26691
|
-
a
|
|
26692
|
-
|
|
26916
|
+
game's per-call ceiling from a bench, never
|
|
26917
|
+
from a guess: the player is billed as used, and
|
|
26918
|
+
the ceiling also sizes how long an answer may be.
|
|
26693
26919
|
genex player <sub> [options] Answer the in-game model requests you approve, on your
|
|
26694
26920
|
OWN Claude or ChatGPT subscription: install | run |
|
|
26695
26921
|
status | stop | uninstall. "install" signs this machine
|
|
@@ -27144,7 +27370,7 @@ ${c.bold("Examples")}
|
|
|
27144
27370
|
genex shop remove sku_123
|
|
27145
27371
|
genex llm models
|
|
27146
27372
|
genex llm models --all
|
|
27147
|
-
genex llm bench "Reply with one short taunt." --samples 5 --max-
|
|
27373
|
+
genex llm bench "Reply with one short taunt." --samples 5 --max-credits 20 --user-approved
|
|
27148
27374
|
genex llm price
|
|
27149
27375
|
genex llm status
|
|
27150
27376
|
genex llm cancel <id>
|
|
@@ -27381,22 +27607,24 @@ ${c.bold("How the allowance works")}
|
|
|
27381
27607
|
|
|
27382
27608
|
Every number printed is live from your account; prices can change without a deploy.
|
|
27383
27609
|
`,
|
|
27384
|
-
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what should the game declare?
|
|
27610
|
+
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what ceiling should the game declare?
|
|
27385
27611
|
|
|
27386
27612
|
${c.bold("Usage")}
|
|
27387
27613
|
genex llm models [--all] [--json] Which models this stand serves, their provider rates per
|
|
27388
|
-
million tokens,
|
|
27389
|
-
|
|
27390
|
-
|
|
27391
|
-
|
|
27392
|
-
on this stand: it says so
|
|
27393
|
-
|
|
27394
|
-
|
|
27395
|
-
|
|
27396
|
-
|
|
27397
|
-
|
|
27398
|
-
|
|
27399
|
-
|
|
27614
|
+
million tokens, the platform fee a call is billed at, and
|
|
27615
|
+
the headroom it recommends over a benchmark. The catalog
|
|
27616
|
+
is synced from OpenRouter, so the FEATURED short list is
|
|
27617
|
+
printed by default and --all prints every row (--json
|
|
27618
|
+
always carries every row). Off on this stand: it says so
|
|
27619
|
+
and exits 0 \u2014 build the feature without a model rather
|
|
27620
|
+
than promising a 404.
|
|
27621
|
+
genex llm bench "<prompt>" --max-credits <n> --user-approved [options]
|
|
27622
|
+
Run the real model through the development lane and
|
|
27623
|
+
report what each attempt really COST and was charged.
|
|
27624
|
+
Refused without both flags, before any network call:
|
|
27625
|
+
this spends YOUR OWN CREDITS, a spend the build's asset
|
|
27626
|
+
allowance does not count. Needs a folder linked to a
|
|
27627
|
+
game you own, and a balance that covers one attempt.
|
|
27400
27628
|
genex llm price [--json] The recommendation from the last bench in this folder.
|
|
27401
27629
|
Reads a file; spends nothing.
|
|
27402
27630
|
genex llm status [--json] This project's open development calls: active, or
|
|
@@ -27413,29 +27641,32 @@ ${c.bold("Bench options")}
|
|
|
27413
27641
|
--samples <n> How many real attempts (default ${DEFAULT_BENCH_SAMPLES}). More samples, better p95.
|
|
27414
27642
|
--schema <file> A JSON file holding the output schema; implies JSON output.
|
|
27415
27643
|
--json-output Ask for JSON without a schema. --text Ask for text (the default).
|
|
27416
|
-
--max-
|
|
27644
|
+
--max-credits <n> The per-attempt ceiling YOU approved \u2014 a bench runs on your own credits,
|
|
27417
27645
|
never a player's. The run's worst case is that number times --samples,
|
|
27418
|
-
and it is printed before anything starts.
|
|
27646
|
+
and it is printed before anything starts. (--max-coins is its deprecated
|
|
27647
|
+
name, read as the same number.)
|
|
27419
27648
|
--timeout <seconds> Bounds the wait for each attempt.
|
|
27420
27649
|
--json One machine-readable object; every key is always present, null when
|
|
27421
27650
|
the CLI could not learn it.
|
|
27422
27651
|
|
|
27423
27652
|
${c.bold("Why a benchmark and not an estimate")}
|
|
27424
|
-
|
|
27425
|
-
|
|
27426
|
-
|
|
27427
|
-
|
|
27428
|
-
|
|
27429
|
-
|
|
27430
|
-
|
|
27431
|
-
|
|
27432
|
-
|
|
27653
|
+
An in-game call is paid from the PLAYER's credits and billed as used: its real provider cost
|
|
27654
|
+
plus the platform fee, never more than the per-call CEILING the game declares (\`maxCredits\`,
|
|
27655
|
+
or \`perCallMaxCredits\` on a standing budget). The ceiling is not a price, but it is still a
|
|
27656
|
+
number with two jobs: it bounds what one call may cost, and it funds how LONG the answer may
|
|
27657
|
+
be. Three layers sit between the provider's rate card and it \u2014 the provider's cost for YOUR
|
|
27658
|
+
prompt, the platform fee, and your headroom \u2014 and only a real call measures the first. The
|
|
27659
|
+
bench is billed to YOU, on the development lane; the player-funded lane is what the
|
|
27660
|
+
published game uses. The recommendation is the p95 real cost with the server's fee and
|
|
27661
|
+
headroom applied, in whole credits; a stand that serves no headroom or no fee gets no
|
|
27662
|
+
recommendation, because those multipliers are not the CLI's to invent.
|
|
27433
27663
|
|
|
27434
|
-
|
|
27435
|
-
|
|
27436
|
-
|
|
27437
|
-
|
|
27438
|
-
|
|
27664
|
+
Because the ceiling sizes each call's output allowance, the recommendation is never below
|
|
27665
|
+
the smallest ceiling that leaves room for the benchmarked answer, as the server computes it.
|
|
27666
|
+
A sample cut off at your --max-credits (provider_token_limit) is not a sample \u2014 re-run with a
|
|
27667
|
+
higher --max-credits; an answer longer than one call on this stand may produce gets no
|
|
27668
|
+
ceiling at all \u2014 ask for a shorter one. The run also prints the measured average per call,
|
|
27669
|
+
fee included, which is the honest estimate a budget's disclosure is built on.
|
|
27439
27670
|
|
|
27440
27671
|
Results land in ${LLM_BENCH_FILE} \u2014 gitignored, machine-local, re-read by \`genex llm price\`.
|
|
27441
27672
|
`,
|
|
@@ -27650,11 +27881,13 @@ function parseArgs(argv) {
|
|
|
27650
27881
|
"--limit",
|
|
27651
27882
|
// `genex budget --assets <credits>` — the allowance the player approved.
|
|
27652
27883
|
"--assets",
|
|
27653
|
-
// `genex llm bench` value flags. `--max-
|
|
27654
|
-
//
|
|
27884
|
+
// `genex llm bench` value flags. `--max-credits` is the same approval
|
|
27885
|
+
// shape as `--assets`, but its own gate: the asset allowance does not
|
|
27886
|
+
// count a bench. `--max-coins` is its deprecated name (same number).
|
|
27655
27887
|
"--model",
|
|
27656
27888
|
"--samples",
|
|
27657
27889
|
"--schema",
|
|
27890
|
+
"--max-credits",
|
|
27658
27891
|
"--max-coins",
|
|
27659
27892
|
// `genex player install --label <name>` — what this machine is called in
|
|
27660
27893
|
// the dashboard. Its own key rather than the generic 2nd positional,
|
|
@@ -28169,12 +28402,14 @@ function applyValueFlag(options, flag, value) {
|
|
|
28169
28402
|
options.samples = n;
|
|
28170
28403
|
break;
|
|
28171
28404
|
}
|
|
28405
|
+
case "--max-credits":
|
|
28172
28406
|
case "--max-coins": {
|
|
28173
28407
|
const n = Number(value);
|
|
28174
28408
|
if (!Number.isInteger(n) || n < 1) {
|
|
28175
|
-
throw new Error(`Invalid
|
|
28409
|
+
throw new Error(`Invalid ${flag} value: ${value} (whole credits, 1 or more)`);
|
|
28176
28410
|
}
|
|
28177
|
-
options.
|
|
28411
|
+
if (flag === "--max-credits") options.maxCredits = n;
|
|
28412
|
+
else options.maxCoins = n;
|
|
28178
28413
|
break;
|
|
28179
28414
|
}
|
|
28180
28415
|
case "--timeout": {
|
|
@@ -28466,9 +28701,9 @@ async function main() {
|
|
|
28466
28701
|
await runBudget(parsed.options);
|
|
28467
28702
|
break;
|
|
28468
28703
|
// The in-game model lane: is it live here, what does one call really
|
|
28469
|
-
// cost, what should the game declare. `bench`
|
|
28470
|
-
//
|
|
28471
|
-
//
|
|
28704
|
+
// cost, what ceiling should the game declare. `bench` spends the
|
|
28705
|
+
// builder's own credits OUTSIDE the asset allowance, so it carries its
|
|
28706
|
+
// own --max-credits/--user-approved gate.
|
|
28472
28707
|
case "llm":
|
|
28473
28708
|
await runLlm(parsed.options);
|
|
28474
28709
|
break;
|