@genex-ai/cli-demo 1.36.12-dev.995 → 1.36.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{blender-mcp-WSURPZET.js → blender-mcp-QVEGHE5L.js} +2 -2
- package/dist/{blender-serve-EH2MOW3U.js → blender-serve-66OHC4ML.js} +1 -1
- package/dist/{chunk-EABIVAGT.js → chunk-4BBTPRJW.js} +2 -2
- package/dist/{chunk-K4EHXOZK.js → chunk-QI7FIYBY.js} +1 -1
- package/dist/index.js +216 -448
- package/package.json +1 -1
- package/templates/skills/genex-game-director/SKILL.md +1 -1
- package/templates/skills/genex-getting-started/SKILL.md +2 -2
- package/templates/skills/genex-llm-in-games/SKILL.md +72 -209
- package/templates/skills/genex-llm-in-games/references/pricing.md +85 -126
- package/templates/skills/genex-monetization/SKILL.md +9 -1
- package/templates/skills/genex-tool-llm/SKILL.md +9 -11
- package/templates/skills/genex-updates/SKILL.md +1 -1
package/dist/index.js
CHANGED
|
@@ -57,7 +57,7 @@ import {
|
|
|
57
57
|
writeSecretFile,
|
|
58
58
|
writeUserToken,
|
|
59
59
|
writeWorkspace
|
|
60
|
-
} from "./chunk-
|
|
60
|
+
} from "./chunk-4BBTPRJW.js";
|
|
61
61
|
import {
|
|
62
62
|
CLI_CHANNEL,
|
|
63
63
|
DEFAULT_API_URL,
|
|
@@ -78,7 +78,7 @@ import {
|
|
|
78
78
|
normalizeApiOrigin,
|
|
79
79
|
originKey,
|
|
80
80
|
resolveAgentTargets
|
|
81
|
-
} from "./chunk-
|
|
81
|
+
} from "./chunk-QI7FIYBY.js";
|
|
82
82
|
|
|
83
83
|
// src/instrument.ts
|
|
84
84
|
import { setTimeout as sleep } from "timers/promises";
|
|
@@ -1107,7 +1107,7 @@ Important note: put soul into your creations, with many details and love. Aim to
|
|
|
1107
1107
|
21. A turn ends in exactly one of two ways: on a question the player must answer before the next step can be chosen, or on a handoff \u2014 the draft link, what changed, what to try, and what is still open. Never close a turn on a promise. "I'll keep building", "while I keep working", "adding that now" are things you say and then DO before you stop; if you are stopping, say that you have stopped and what remains. While the Build plan's \`Now:\` line has open work and no answer is needed from the player, the turn is not over: preview, hand off, and start the next milestone in the same turn, until the requested outcome is reached or the platform's own budget ends the session. If a stop was forced on you anyway, the first line of your next turn is the promise you left and where it stands.
|
|
1108
1108
|
22. **What the player sees must read as the thing it is**, in this game's own style: a building as that building, a person as a person, a prop as that prop, a surface as its material \u2014 and a coloured box, a capsule, or a flat grey block standing in for one of them is never its finished version. Generation is the default route to that bar wherever code will not honestly reach it: the player's character, whatever the request names, whatever the player walks up to, enters, or interacts with, and the music and sound underneath \u2014 reach for the belt on your own judgment, without stopping to ask; a status line saying what you queued is enough, and \`--no-wait\` keeps you building while it lands. Procedural code is a first-class engine for what is structural, repeated, distant, or parametric \u2014 terrain, sky, fences, paving, modular kits, filler \u2014 and for anything else only when the result meets the same bar and you have looked at it in a capture before its row says \`landed\`. Start with one reusable implementation per distinct required gameplay or visual role. Background populations default to reusable procedural bodies and motion or compatible existing rigged assets. Catalog clips need a compatible skeleton and may be paid. Reserve paid custom bodies for the player and characters examined or interacted with closely, while honoring explicit user requests. Add variants only for an unmet requirement; unspent allowance is not unfinished work. Allowance questions use timeoutPolicy no-consent: silence does not raise the allowance. Record role, reuse and completion criteria in the existing Assets table. Decide per object, not per category, and write the route on each Assets row in \`DESIGN.md\` \u2014 the paid flow for generated pieces, the procedural flow with its local path for code-built ones. An Assets table with no generation in it is a decision, not a default: record it as \`Generation: none \u2014 <why code alone reaches the bar here>\` or generate.
|
|
1109
1109
|
23. **The screenshots you take to look at your own work go in \`.genex/scratch/\`.** Captures, render comparisons, before/after strips, traces and metrics dumps are how you SEE what you built \u2014 they are not part of what you built, and \`genex preview\` pushes the whole folder to the game's source, every time, forever. \`.genex/scratch/\` already exists for this and is already ignored, so there is nothing to set up: write the capture as \`.genex/scratch/arena-before.png\` and read it back from there. Never put them in a folder of your own at the top level: \`progress/\`, \`reports/\`, \`shots/\` and their kind are pushed like source and become permanent weight in the game and in every remix of it \u2014 one real game reached 11 GB and 10,332 committed screenshots exactly this way. If you find such a folder already there, add it to \`.gitignore\`; never delete the player's files (law 18). One file in \`.genex/scratch/\` IS read, by the platform: \`.genex/scratch/cover.png\` is the game's cover \u2014 the frame you would put on its poster: real gameplay at its signature moment, well lit, no menus, popups or busy HUD, landscape 16:9, and never any text (the page sets the title beside it). The next \`genex preview\` makes it the cover; save a better moment over it whenever one comes along, and if your browser tool can set the cover itself, use that instead. Before the first publish, and whenever the game's look changes, load \`$genex-cover\`: it chooses, stages, lights and judges that frame.
|
|
1110
|
-
24. **Benchmark before declaring an in-game LLM
|
|
1110
|
+
24. **Benchmark before declaring an in-game LLM price.** If the game calls a language model while the player plays \u2014 an NPC answering in its own words, a quest written for this save, a judge reading what the player typed \u2014 the PLAYER pays for it, and \`estimateCoins\` is the FIXED price of a started attempt: charged whether it succeeds, fails, is canceled, or stops at its own budget. Never pick that number by judgement or from a vendor's rate card. Run the real prompt on your own coins \u2014 \`npx genex llm bench "<the prompt>" --schema <file> --samples 3 --max-coins <n> --user-approved\` \u2014 and declare the figure it recommends, recorded in \`DESIGN.md\` with its model and date; re-benchmark whenever the prompt, the schema or the model changes, because a prompt edit is a price change. Check the lane first with \`npx genex llm models\`: where it is off the routes 404, so build the feature behind a graceful unavailable state and say so in the handoff. A repeated call is a standing budget the player approves once (\`requestSpendGrant()\`, then \`generate({ grantId })\`), never a popup per turn, and its disclosed call rate is computed from the game's own loop. Model output is data the game validates against its own expectation \u2014 never executed, and never authority over coin, items, entitlements or rewards. Load \`$genex-llm-in-games\` before writing any of it.
|
|
1111
1111
|
${CONTRACT_END}
|
|
1112
1112
|
`;
|
|
1113
1113
|
var TOOLS_CONTRACT_HEAD = `# Genex Tools (always in effect in this folder)
|
|
@@ -1121,7 +1121,7 @@ var TOOLS_CONTRACT_OFFER = `5. **When the game is built and playable \u2014 neve
|
|
|
1121
1121
|
var TOOLS_CONTRACT_HOSTED_LAWS = `5. **This game is hosted on Genex now, and publishing is glue \u2014 never a rebuild.** The game ships exactly as the user built it: \`initEmbed()\` first in the boot code (the \`genex-threejs-embed-auth\` card has the call; the renderer draws before any \`await waitForPlayer()\`), a static build into \`dist/\` with relative asset paths, then \`npx genex preview\`. Never restructure, reformat or "improve" the game to ship it, never scaffold a new app around it, and never start a design document or a build plan for it \u2014 the user's own process is theirs. Load the \`genex-tool-publish\` card before the first preview; it owns the vocabulary, the links and the limits.
|
|
1122
1122
|
6. **Ship first, then report.** \`preview\` prints a preflight \u2014 phone memory, a missing volume slider, an asset nobody wired, a viewport line. It never blocks a deploy, so push the build as it is, hand over the link, THEN relay each preflight line to the user in one plain sentence with an offer to fix it, and fix one only on their yes. The preflight is a report for the user, not a to-do list for you.
|
|
1123
1123
|
7. **The link is the game's page, and there are two versions.** After every preview give the user \`<dashboard>/draft/<slug>\` \u2014 \`dashboardOrigins[0]\` and \`slug\` from \`.genex/project.json\` \u2014 never localhost, a file path or the bare play origin. \`preview\` updates the draft and never touches what players are on; the first release is \`npx genex publish\` (it lists the game); after that "publish it", "update it" and "yes" all mean \`npx genex promote\` \u2014 the exact draft build, no rebuild. Ask once per round of work, in one line, and keep working while you wait.`;
|
|
1124
|
-
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with
|
|
1124
|
+
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with Genex coins or their own Claude/ChatGPT subscription, and approves it on a Genex sheet; your game just calls \`generate()\`. Want it that way?`;
|
|
1125
1125
|
function toolsContractLlmLaw(n, hosted) {
|
|
1126
1126
|
const onYes = hosted ? "On a yes, load `$genex-tool-llm` and then `$genex-llm-in-games`, which owns the build." : "On a yes, load `$genex-tool-llm`, which owns the rest: it checks the lane with `npx genex llm models` first, then runs `npx genex init --convert` (a hosted game is a static browser build \u2014 a local server holding a key can never ship).";
|
|
1127
1127
|
return `${n}. **A model running while people PLAY is a platform feature \u2014 never something you wire onto the user's own meter.** When a request implies one (NPCs that talk or decide in their own words, content written from what the player types, a prompt box in the game, "let the player pick a model"), recognise it, say in ONE line: "${TOOLS_CONTRACT_LLM_OFFER}" \u2014 then ASK and wait. ${onYes} On a no, build the authored version instead. Either way, never ship a game that calls a model on a key or an account of the user's own: every visitor would spend their money with nobody approving it, and the credential is readable in the bundle. Player-funded generation is text and JSON only \u2014 3D, images, video and audio stay the asset lanes above, on the user's meter.`;
|
|
@@ -23807,7 +23807,7 @@ async function runBlender(opts) {
|
|
|
23807
23807
|
return 1;
|
|
23808
23808
|
}
|
|
23809
23809
|
if (sub === "serve") {
|
|
23810
|
-
const { serveLocalBlender } = await import("./blender-serve-
|
|
23810
|
+
const { serveLocalBlender } = await import("./blender-serve-66OHC4ML.js");
|
|
23811
23811
|
const port = Number(process.env.GENEX_BLENDER_PORT ?? 8088);
|
|
23812
23812
|
log.step(`Starting a local Blender service on port ${port}`);
|
|
23813
23813
|
log.plain(
|
|
@@ -23816,7 +23816,7 @@ async function runBlender(opts) {
|
|
|
23816
23816
|
return serveLocalBlender({ port, log });
|
|
23817
23817
|
}
|
|
23818
23818
|
if (sub === "mcp") {
|
|
23819
|
-
const { runBlenderMcp } = await import("./blender-mcp-
|
|
23819
|
+
const { runBlenderMcp } = await import("./blender-mcp-QVEGHE5L.js");
|
|
23820
23820
|
return runBlenderMcp();
|
|
23821
23821
|
}
|
|
23822
23822
|
if (sub === "seat") {
|
|
@@ -24324,12 +24324,7 @@ import fs35 from "fs/promises";
|
|
|
24324
24324
|
import path36 from "path";
|
|
24325
24325
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
24326
24326
|
var SUBS3 = ["models", "bench", "price", "status", "cancel"];
|
|
24327
|
-
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN
|
|
24328
|
-
var LLM_MAX_COINS_DEPRECATED = "--max-coins is deprecated: it is read as --max-credits, the same number \u2014 a bench spends credits now.";
|
|
24329
|
-
function benchBalanceShortLine(spendable, maxCredits) {
|
|
24330
|
-
return `Your spendable credits (${spendable}) cannot cover one attempt at --max-credits ${maxCredits} \u2014 nothing was sent, nothing was spent. Add credits, or re-run with a lower --max-credits once the person at the keyboard agrees to it.`;
|
|
24331
|
-
}
|
|
24332
|
-
var LLM_BENCH_UNVERIFIED_LINE = "Credits pay only for a verified email, and this account's is not verified yet (credits_unverified) \u2014 nothing was sent, nothing was spent. Verify it, then re-run.";
|
|
24327
|
+
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN coin \u2014 your wallet, not a player's, and one this build's asset allowance cannot see. Re-run with --max-coins <n> --user-approved only after the person at the keyboard agreed to the number.";
|
|
24333
24328
|
var LLM_LANE_OFF_LINE = "The runtime LLM lane is off on this stand \u2014 in-game generate() answers 404 here, and nothing can be benchmarked.";
|
|
24334
24329
|
var LLM_TOOLS_CONVERT_HINT = [
|
|
24335
24330
|
"In-game model calls need a hosted game. This folder is a Genex Tools workspace, so nothing here",
|
|
@@ -24342,7 +24337,7 @@ function serverSlotCounts(body) {
|
|
|
24342
24337
|
return { held: body.slotsHeld, limit: body.slotLimit };
|
|
24343
24338
|
}
|
|
24344
24339
|
var PENDING_SLOT_RELEASE_MINUTES = 10;
|
|
24345
|
-
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so
|
|
24340
|
+
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so its coin stays held until the bill resolves; such a row stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`;
|
|
24346
24341
|
var LLM_STATUS_NO_COUNT_LINE = "This stand does not report how many slots are held across your projects, so no count is shown.";
|
|
24347
24342
|
var LLM_CANCEL_AWAITING_BILL = "this call already stopped; its bill is awaiting the provider";
|
|
24348
24343
|
var LLM_CANCEL_ALREADY_FINAL = "this call is already final";
|
|
@@ -24356,33 +24351,11 @@ var DEFAULT_SAMPLE_TIMEOUT_SEC = 180;
|
|
|
24356
24351
|
function parseProviderKey(value) {
|
|
24357
24352
|
return value === "ok" || value === "rejected" || value === "unknown" ? value : null;
|
|
24358
24353
|
}
|
|
24359
|
-
function bpsOrNull(value) {
|
|
24360
|
-
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
24361
|
-
}
|
|
24362
|
-
function standingCreditGrantOrNull(value) {
|
|
24363
|
-
if (!value || typeof value !== "object") return null;
|
|
24364
|
-
const v = value;
|
|
24365
|
-
const whole = (x) => typeof x === "number" && Number.isSafeInteger(x) && x >= 0;
|
|
24366
|
-
if (!whole(v.minCredits) || !whole(v.maxCredits) || !whole(v.stepCredits) || !whole(v.defaultCredits)) return null;
|
|
24367
|
-
return {
|
|
24368
|
-
minCredits: v.minCredits,
|
|
24369
|
-
maxCredits: v.maxCredits,
|
|
24370
|
-
stepCredits: v.stepCredits,
|
|
24371
|
-
defaultCredits: v.defaultCredits,
|
|
24372
|
-
...whole(v.minUsdCents) ? { minUsdCents: v.minUsdCents } : {},
|
|
24373
|
-
...whole(v.maxUsdCents) ? { maxUsdCents: v.maxUsdCents } : {},
|
|
24374
|
-
...whole(v.stepUsdCents) ? { stepUsdCents: v.stepUsdCents } : {}
|
|
24375
|
-
};
|
|
24376
|
-
}
|
|
24377
24354
|
async function readRuntimeLane(apiUrl, token) {
|
|
24378
24355
|
const empty = (state) => ({
|
|
24379
24356
|
state,
|
|
24380
24357
|
models: null,
|
|
24381
24358
|
recommendedDeclaredHeadroomBps: null,
|
|
24382
|
-
currency: null,
|
|
24383
|
-
feeBps: null,
|
|
24384
|
-
usdCentsPerCredit: null,
|
|
24385
|
-
standingCreditGrant: null,
|
|
24386
24359
|
externalProviders: null,
|
|
24387
24360
|
total: null,
|
|
24388
24361
|
featuredCount: null,
|
|
@@ -24401,11 +24374,7 @@ async function readRuntimeLane(apiUrl, token) {
|
|
|
24401
24374
|
return {
|
|
24402
24375
|
state: "live",
|
|
24403
24376
|
models: body.models,
|
|
24404
|
-
recommendedDeclaredHeadroomBps:
|
|
24405
|
-
currency: body.currency === "credits" || body.currency === "coins" ? body.currency : null,
|
|
24406
|
-
feeBps: bpsOrNull(body.feeBps),
|
|
24407
|
-
usdCentsPerCredit: typeof body.usdCentsPerCredit === "number" && Number.isSafeInteger(body.usdCentsPerCredit) && body.usdCentsPerCredit >= 1 ? body.usdCentsPerCredit : null,
|
|
24408
|
-
standingCreditGrant: standingCreditGrantOrNull(body.standingCreditGrant),
|
|
24377
|
+
recommendedDeclaredHeadroomBps: typeof body.recommendedDeclaredHeadroomBps === "number" && Number.isFinite(body.recommendedDeclaredHeadroomBps) ? body.recommendedDeclaredHeadroomBps : null,
|
|
24409
24378
|
externalProviders: Array.isArray(body.externalProviders) ? body.externalProviders : null,
|
|
24410
24379
|
total: typeof body.total === "number" && Number.isFinite(body.total) ? body.total : null,
|
|
24411
24380
|
featuredCount: typeof body.featuredCount === "number" && Number.isFinite(body.featuredCount) ? body.featuredCount : null,
|
|
@@ -24421,45 +24390,18 @@ function percentile2(values, p) {
|
|
|
24421
24390
|
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
24422
24391
|
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
24423
24392
|
}
|
|
24424
|
-
function
|
|
24425
|
-
|
|
24426
|
-
const sorted = [...values].sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
24427
|
-
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
24428
|
-
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
24393
|
+
function recommendEstimateCoins(p95, headroomBps) {
|
|
24394
|
+
return Math.max(1, Math.ceil(p95 * (1e4 + headroomBps) / 1e4));
|
|
24429
24395
|
}
|
|
24430
|
-
|
|
24431
|
-
|
|
24432
|
-
if (costPicos <= 0n) return 1;
|
|
24433
|
-
const num = costPicos * BigInt(1e4 + terms.feeBps) * BigInt(1e4 + terms.headroomBps);
|
|
24434
|
-
const den = 100000000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
24435
|
-
return Math.max(1, Number((num + den - 1n) / den));
|
|
24436
|
-
}
|
|
24437
|
-
function averageCreditsPerCall(costsPicos, terms) {
|
|
24438
|
-
if (costsPicos.length === 0) return null;
|
|
24439
|
-
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
24440
|
-
return Number(sum * BigInt(1e4 + terms.feeBps)) / Number(BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT);
|
|
24441
|
-
}
|
|
24442
|
-
function perCallEstimateCreditsFor(costsPicos, terms) {
|
|
24443
|
-
if (costsPicos.length === 0) return null;
|
|
24444
|
-
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
24445
|
-
const num = sum * BigInt(1e4 + terms.feeBps);
|
|
24446
|
-
const den = BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
24447
|
-
return Math.max(1, Number((num + den - 1n) / den));
|
|
24448
|
-
}
|
|
24449
|
-
function formatCredits(value) {
|
|
24450
|
-
if (!Number.isFinite(value)) return "\u2014";
|
|
24451
|
-
if (value >= 100) return String(Math.round(value));
|
|
24452
|
-
return String(Number(value.toPrecision(value < 1 ? 2 : 3)));
|
|
24453
|
-
}
|
|
24454
|
-
function usdFromPicos(picos) {
|
|
24455
|
-
if (picos === null || picos === void 0) return "unknown";
|
|
24456
|
-
const value = typeof picos === "bigint" ? picos : /^\d+$/.test(picos) ? BigInt(picos) : null;
|
|
24457
|
-
if (value === null) return "unknown";
|
|
24458
|
-
return `$${(Number(value) / 1e12).toFixed(6)}`;
|
|
24396
|
+
function recommendPerCallMaxCoins(p95, max, headroomBps) {
|
|
24397
|
+
return Math.max(recommendEstimateCoins(p95, headroomBps), recommendEstimateCoins(max, headroomBps));
|
|
24459
24398
|
}
|
|
24460
24399
|
function usdPerMillion(value) {
|
|
24461
24400
|
return `$${value.toFixed(value < 1 ? 4 : 2)}`;
|
|
24462
24401
|
}
|
|
24402
|
+
function usd(value) {
|
|
24403
|
+
return value === null ? "unknown" : `$${value.toFixed(6)}`;
|
|
24404
|
+
}
|
|
24463
24405
|
async function runLlm(opts = {}) {
|
|
24464
24406
|
const log = createLogger({ quiet: opts.quiet || opts.json });
|
|
24465
24407
|
const cwd = opts.cwd ?? process.cwd();
|
|
@@ -24485,23 +24427,16 @@ async function runLlm(opts = {}) {
|
|
|
24485
24427
|
return;
|
|
24486
24428
|
}
|
|
24487
24429
|
if (sub === "bench") {
|
|
24488
|
-
if (opts.maxCoins
|
|
24489
|
-
`);
|
|
24490
|
-
if (opts.maxCoins !== void 0 && opts.maxCredits !== void 0 && opts.maxCoins !== opts.maxCredits) {
|
|
24491
|
-
fail5("llm bench", `Pass --max-credits alone \u2014 --max-coins is its deprecated name, and the two disagree (${opts.maxCredits} vs ${opts.maxCoins}).`);
|
|
24492
|
-
return;
|
|
24493
|
-
}
|
|
24494
|
-
const approvedCredits = opts.maxCredits ?? opts.maxCoins;
|
|
24495
|
-
if (approvedCredits === void 0 || opts.userApproved !== true) {
|
|
24430
|
+
if (opts.maxCoins === void 0 || opts.userApproved !== true) {
|
|
24496
24431
|
fail5("llm bench", LLM_BENCH_APPROVAL_REQUIRED);
|
|
24497
24432
|
return;
|
|
24498
24433
|
}
|
|
24499
|
-
if (!Number.isInteger(
|
|
24500
|
-
fail5("llm bench", `--max-
|
|
24434
|
+
if (!Number.isInteger(opts.maxCoins) || opts.maxCoins < 1) {
|
|
24435
|
+
fail5("llm bench", `--max-coins takes a whole number of coin, 1 or more (got ${String(opts.maxCoins)}).`);
|
|
24501
24436
|
return;
|
|
24502
24437
|
}
|
|
24503
24438
|
if (!opts.benchPrompt?.trim()) {
|
|
24504
|
-
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-
|
|
24439
|
+
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-coins <n> --user-approved.');
|
|
24505
24440
|
return;
|
|
24506
24441
|
}
|
|
24507
24442
|
if (opts.jsonOutput && opts.textOutput) {
|
|
@@ -24593,10 +24528,6 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24593
24528
|
models: null,
|
|
24594
24529
|
externalProviders: null,
|
|
24595
24530
|
recommendedDeclaredHeadroomBps: null,
|
|
24596
|
-
currency: null,
|
|
24597
|
-
feeBps: null,
|
|
24598
|
-
usdCentsPerCredit: null,
|
|
24599
|
-
standingCreditGrant: null,
|
|
24600
24531
|
total: null,
|
|
24601
24532
|
featuredCount: null,
|
|
24602
24533
|
// The stand was never asked, so the verdict is unknown — but the key is
|
|
@@ -24622,12 +24553,6 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24622
24553
|
models: lane.models,
|
|
24623
24554
|
externalProviders: lane.externalProviders,
|
|
24624
24555
|
recommendedDeclaredHeadroomBps: lane.recommendedDeclaredHeadroomBps,
|
|
24625
|
-
// The credit terms a call is billed on, the server's — null on a stand
|
|
24626
|
-
// that predates credit billing, never a number the CLI filled in.
|
|
24627
|
-
currency: lane.currency,
|
|
24628
|
-
feeBps: lane.feeBps,
|
|
24629
|
-
usdCentsPerCredit: lane.usdCentsPerCredit,
|
|
24630
|
-
standingCreditGrant: lane.standingCreditGrant,
|
|
24631
24556
|
total: lane.total ?? lane.models?.length ?? null,
|
|
24632
24557
|
featuredCount: lane.featuredCount ?? (lane.models ? lane.models.filter((m) => m.featured).length : null),
|
|
24633
24558
|
// `null` when the lane is not live or the server predates the probe.
|
|
@@ -24674,9 +24599,8 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24674
24599
|
);
|
|
24675
24600
|
for (const m of shown) {
|
|
24676
24601
|
const plan = m.personalPlan ? ` \xB7 personal plan: ${m.personalPlan}` : "";
|
|
24677
|
-
const kind = m.kind === "typesafe" ? " \xB7 judge (classifier): judge() under a budget, not generate() or bench" : "";
|
|
24678
24602
|
log.plain(
|
|
24679
|
-
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}
|
|
24603
|
+
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}`
|
|
24680
24604
|
);
|
|
24681
24605
|
}
|
|
24682
24606
|
if (hiddenCount > 0) {
|
|
@@ -24692,104 +24616,74 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24692
24616
|
}
|
|
24693
24617
|
log.plain("");
|
|
24694
24618
|
if (lane.recommendedDeclaredHeadroomBps === null) {
|
|
24695
|
-
log.dim(" This stand serves no recommended headroom, so no
|
|
24619
|
+
log.dim(" This stand serves no recommended headroom, so no price can be recommended from a bench.");
|
|
24696
24620
|
} else {
|
|
24697
|
-
log.dim(` Recommended headroom over
|
|
24698
|
-
}
|
|
24699
|
-
if (lane.feeBps === null) {
|
|
24700
|
-
log.dim(" This stand serves no platform fee \u2014 it predates credit billing, so no ceiling can be recommended from a bench.");
|
|
24701
|
-
} else {
|
|
24702
|
-
log.dim(` In-game calls are billed in the player's credits, as used: the provider's cost plus this stand's platform fee (${lane.feeBps} bps).`);
|
|
24621
|
+
log.dim(` Recommended headroom over benchmarked coins on this stand: ${lane.recommendedDeclaredHeadroomBps} bps.`);
|
|
24703
24622
|
}
|
|
24704
24623
|
if (toolsOnly) {
|
|
24705
24624
|
printConvertHint();
|
|
24706
24625
|
return;
|
|
24707
24626
|
}
|
|
24708
|
-
log.dim(" Those per-million rates are the PROVIDER's
|
|
24709
|
-
log.dim(`
|
|
24627
|
+
log.dim(" Those per-million rates are the PROVIDER's; what your game is charged is the platform");
|
|
24628
|
+
log.dim(` tariff on top, which only a real call reveals: ${c.cyan('genex llm bench "<prompt>" --max-coins <n> --user-approved')}`);
|
|
24710
24629
|
}
|
|
24711
24630
|
function wholeOrNull(value) {
|
|
24712
24631
|
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
24713
24632
|
}
|
|
24714
|
-
function costPicosOf(usage) {
|
|
24715
|
-
if (typeof usage?.costUsdPicos === "string" && /^\d+$/.test(usage.costUsdPicos)) return usage.costUsdPicos;
|
|
24716
|
-
if (typeof usage?.costUsd === "number" && Number.isFinite(usage.costUsd) && usage.costUsd >= 0) {
|
|
24717
|
-
return String(BigInt(Math.round(usage.costUsd * 1e12)));
|
|
24718
|
-
}
|
|
24719
|
-
return null;
|
|
24720
|
-
}
|
|
24721
24633
|
function sampleFrom(id, settled) {
|
|
24722
24634
|
const usage = settled?.usage;
|
|
24723
|
-
const picos = costPicosOf(usage);
|
|
24724
24635
|
return {
|
|
24725
24636
|
id,
|
|
24726
24637
|
status: settled?.status ?? null,
|
|
24727
24638
|
billingStatus: settled?.billingStatus ?? null,
|
|
24728
|
-
|
|
24729
|
-
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd :
|
|
24730
|
-
costUsdPicos: picos,
|
|
24639
|
+
chargedCoins: typeof settled?.chargedCoins === "number" ? settled.chargedCoins : null,
|
|
24640
|
+
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd : null,
|
|
24731
24641
|
error: settled?.error ?? (settled ? null : "timed_out"),
|
|
24732
24642
|
providerMessage: typeof settled?.providerMessage === "string" && settled.providerMessage.trim() ? settled.providerMessage.trim() : null,
|
|
24733
24643
|
inputTokens: wholeOrNull(usage?.inputTokens),
|
|
24734
24644
|
outputTokens: wholeOrNull(usage?.outputTokens),
|
|
24735
24645
|
maxOutputTokens: wholeOrNull(usage?.maxOutputTokens),
|
|
24736
|
-
|
|
24646
|
+
fitCoins: typeof usage?.fitCoins === "number" && Number.isSafeInteger(usage.fitCoins) && usage.fitCoins >= 1 ? usage.fitCoins : null,
|
|
24737
24647
|
fitExceedsCap: typeof usage?.fitExceedsCap === "boolean" ? usage.fitExceedsCap : null,
|
|
24738
24648
|
truncated: typeof usage?.truncated === "boolean" ? usage.truncated : settled?.error === "provider_token_limit" ? true : null
|
|
24739
24649
|
};
|
|
24740
24650
|
}
|
|
24741
24651
|
var PROVIDER_TOKEN_LIMIT = "provider_token_limit";
|
|
24742
|
-
function benchRecommendation(results,
|
|
24743
|
-
const settled = results.filter(
|
|
24744
|
-
|
|
24745
|
-
);
|
|
24746
|
-
const
|
|
24747
|
-
const
|
|
24748
|
-
const fits = settled.map((r) => r.fitCredits).filter((v) => typeof v === "number");
|
|
24749
|
-
const costP95 = percentilePicos(costs, 95);
|
|
24750
|
-
const costMax = costs.length ? costs.reduce((a, b) => b > a ? b : a) : null;
|
|
24652
|
+
function benchRecommendation(results, headroomBps) {
|
|
24653
|
+
const settled = results.filter((r) => r.status === "succeeded" && r.billingStatus === "final" && typeof r.chargedCoins === "number");
|
|
24654
|
+
const charged = settled.map((r) => r.chargedCoins);
|
|
24655
|
+
const fits = settled.map((r) => r.fitCoins).filter((v) => typeof v === "number");
|
|
24656
|
+
const p95 = percentile2(charged, 95);
|
|
24657
|
+
const max = charged.length ? Math.max(...charged) : null;
|
|
24751
24658
|
const fitP95 = percentile2(fits, 95);
|
|
24752
24659
|
const fitMax = fits.length ? Math.max(...fits) : null;
|
|
24753
24660
|
const exceedsStandCap = results.some((r) => r.fitExceedsCap === true);
|
|
24754
24661
|
const cutOffSamples = results.filter((r) => r.error === PROVIDER_TOKEN_LIMIT && r.fitExceedsCap !== true).length;
|
|
24755
24662
|
const noRecommendationReason = exceedsStandCap ? "answer_exceeds_stand_cap" : cutOffSamples > 0 ? "answer_cut_off" : null;
|
|
24756
|
-
const
|
|
24757
|
-
const
|
|
24758
|
-
const
|
|
24759
|
-
const billing = feeBps !== null && usdCentsPerCredit !== null ? { feeBps, usdCentsPerCredit } : null;
|
|
24760
|
-
const costBased = full !== null && costP95 !== null ? ceilingCreditsFor(costP95, full) : null;
|
|
24761
|
-
const recommended = costBased === null || noRecommendationReason !== null ? null : Math.max(costBased, fitP95 ?? 0);
|
|
24762
|
-
const ceiling = recommended === null || costMax === null || full === null ? null : Math.max(ceilingCreditsFor(costMax, full), recommended, fitMax ?? 0);
|
|
24763
|
-
const average = billing !== null ? averageCreditsPerCall(costs, billing) : null;
|
|
24764
|
-
const estimate = ceiling === null || billing === null ? null : Math.min(ceiling, perCallEstimateCreditsFor(costs, billing) ?? 1);
|
|
24663
|
+
const chargedBased = p95 !== null && headroomBps !== null ? recommendEstimateCoins(p95, headroomBps) : null;
|
|
24664
|
+
const recommended = chargedBased === null || noRecommendationReason !== null ? null : Math.max(chargedBased, fitP95 ?? 0);
|
|
24665
|
+
const ceiling = recommended === null || p95 === null || max === null || headroomBps === null ? null : Math.max(recommendPerCallMaxCoins(p95, max, headroomBps), recommended, fitMax ?? 0);
|
|
24765
24666
|
return {
|
|
24766
|
-
costs,
|
|
24767
24667
|
charged,
|
|
24768
|
-
|
|
24769
|
-
|
|
24770
|
-
|
|
24771
|
-
chargedP50: percentile2(charged, 50),
|
|
24772
|
-
chargedP95: percentile2(charged, 95),
|
|
24773
|
-
chargedMax: charged.length ? Math.max(...charged) : null,
|
|
24668
|
+
p50: percentile2(charged, 50),
|
|
24669
|
+
p95,
|
|
24670
|
+
max,
|
|
24774
24671
|
fits,
|
|
24775
24672
|
fitP95,
|
|
24776
24673
|
fitMax,
|
|
24777
|
-
|
|
24778
|
-
|
|
24779
|
-
|
|
24780
|
-
|
|
24781
|
-
recommendedPerCallEstimateCredits: estimate,
|
|
24782
|
-
lengthRaisedCeiling: recommended !== null && costBased !== null && recommended > costBased,
|
|
24674
|
+
chargedBasedEstimateCoins: chargedBased,
|
|
24675
|
+
recommendedEstimateCoins: recommended,
|
|
24676
|
+
recommendedPerCallMaxCoins: ceiling,
|
|
24677
|
+
lengthRaisedPrice: recommended !== null && chargedBased !== null && recommended > chargedBased,
|
|
24783
24678
|
cutOffSamples,
|
|
24784
24679
|
exceedsStandCap,
|
|
24785
|
-
noRecommendationReason
|
|
24786
|
-
missingTerms
|
|
24680
|
+
noRecommendationReason
|
|
24787
24681
|
};
|
|
24788
24682
|
}
|
|
24789
|
-
var LLM_LENGTH_RAISED_LINE = "The
|
|
24790
|
-
var LLM_EXCEEDS_STAND_CAP_LINE = "No
|
|
24791
|
-
function benchCutOffLine(cutOff,
|
|
24792
|
-
return `No
|
|
24683
|
+
var LLM_LENGTH_RAISED_LINE = "The declared price also pays for how long the answer may be: at the charged-based price the answer would be cut off, so declare this one.";
|
|
24684
|
+
var LLM_EXCEEDS_STAND_CAP_LINE = "No price recommended: this answer is longer than one call on this stand may produce. Ask for a shorter answer \u2014 fewer fields, shorter strings, a length the prompt states \u2014 and benchmark again.";
|
|
24685
|
+
function benchCutOffLine(cutOff, maxCoins) {
|
|
24686
|
+
return `No price recommended: ${cutOff} sample${cutOff === 1 ? " was" : "s were"} cut off at this ceiling (--max-coins ${maxCoins}) before the answer was finished, so its real length is unknown. Re-run with a higher --max-coins.`;
|
|
24793
24687
|
}
|
|
24794
24688
|
var ACTIVE_STATUSES = /* @__PURE__ */ new Set(["requires_confirmation", "queued", "dispatching", "awaiting_external"]);
|
|
24795
24689
|
function rowState(row) {
|
|
@@ -24800,22 +24694,7 @@ function rowState(row) {
|
|
|
24800
24694
|
function rowHoldsSlot(row) {
|
|
24801
24695
|
if (typeof row.slotHeld === "boolean") return row.slotHeld;
|
|
24802
24696
|
const state = rowState(row);
|
|
24803
|
-
return state === "active" || state === "awaiting bill" && (
|
|
24804
|
-
}
|
|
24805
|
-
function rowMoney(row) {
|
|
24806
|
-
const credits = row.currency === "credits" || row.currency === void 0 && typeof row.reservedCredits === "number" && typeof row.reservedCoins !== "number";
|
|
24807
|
-
if (credits) {
|
|
24808
|
-
return {
|
|
24809
|
-
unit: "credits",
|
|
24810
|
-
reserved: typeof row.reservedCredits === "number" ? row.reservedCredits : null,
|
|
24811
|
-
charged: typeof row.chargedCredits === "number" ? row.chargedCredits : null
|
|
24812
|
-
};
|
|
24813
|
-
}
|
|
24814
|
-
return {
|
|
24815
|
-
unit: "coin",
|
|
24816
|
-
reserved: typeof row.reservedCoins === "number" ? row.reservedCoins : null,
|
|
24817
|
-
charged: typeof row.chargedCoins === "number" ? row.chargedCoins : null
|
|
24818
|
-
};
|
|
24697
|
+
return state === "active" || state === "awaiting bill" && (row.reservedCoins ?? 0) > 0;
|
|
24819
24698
|
}
|
|
24820
24699
|
function providerRefusalStatus(error) {
|
|
24821
24700
|
const m = /^provider_http_(\d{3})$/.exec(error ?? "");
|
|
@@ -24834,7 +24713,7 @@ function formatAge(iso, now = Date.now()) {
|
|
|
24834
24713
|
}
|
|
24835
24714
|
async function runBench(args) {
|
|
24836
24715
|
const { apiUrl, token, projectId, cwd, opts, log } = args;
|
|
24837
|
-
const
|
|
24716
|
+
const maxCoins = opts.maxCoins;
|
|
24838
24717
|
const samples = opts.samples ?? DEFAULT_BENCH_SAMPLES;
|
|
24839
24718
|
const prompt = opts.benchPrompt.trim();
|
|
24840
24719
|
const auth = { Authorization: `Bearer ${token}`, "Content-Type": "application/json" };
|
|
@@ -24856,7 +24735,7 @@ async function runBench(args) {
|
|
|
24856
24735
|
const lane = await readRuntimeLane(apiUrl, token);
|
|
24857
24736
|
if (lane.state !== "live" || !lane.models || lane.models.length === 0) {
|
|
24858
24737
|
if (lane.state === "off") {
|
|
24859
|
-
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
24738
|
+
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCoins, opts), status: "off", error: null });
|
|
24860
24739
|
else {
|
|
24861
24740
|
log.plain(LLM_LANE_OFF_LINE);
|
|
24862
24741
|
log.dim(" Nothing was spent.");
|
|
@@ -24883,27 +24762,17 @@ async function runBench(args) {
|
|
|
24883
24762
|
);
|
|
24884
24763
|
return;
|
|
24885
24764
|
}
|
|
24886
|
-
const
|
|
24887
|
-
if (walletBefore?.emailVerified === false) {
|
|
24888
|
-
benchFailed(opts, log, LLM_BENCH_UNVERIFIED_LINE);
|
|
24889
|
-
return;
|
|
24890
|
-
}
|
|
24891
|
-
if (walletBefore !== null && walletBefore.spendable < maxCredits) {
|
|
24892
|
-
benchFailed(opts, log, benchBalanceShortLine(walletBefore.spendable, maxCredits));
|
|
24893
|
-
return;
|
|
24894
|
-
}
|
|
24895
|
-
const balanceBefore = walletBefore?.spendable ?? null;
|
|
24765
|
+
const balanceBefore = await readCoinBalance(apiUrl, token);
|
|
24896
24766
|
if (!opts.json) {
|
|
24897
24767
|
log.plain(c.bold("genex llm bench"));
|
|
24898
24768
|
log.dim(` ${apiUrl}`);
|
|
24899
24769
|
log.plain("");
|
|
24900
24770
|
log.plain(` Model ${c.cyan(modelId)}`);
|
|
24901
24771
|
log.plain(` Samples ${samples} real attempt${samples === 1 ? "" : "s"}, ${outputFormat} output`);
|
|
24902
|
-
log.plain(` Approved ${
|
|
24772
|
+
log.plain(` Approved ${maxCoins} coin per attempt \u2014 at worst ${maxCoins * samples} coin for this run`);
|
|
24903
24773
|
log.plain(
|
|
24904
|
-
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore}
|
|
24774
|
+
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore} coin spendable`}`
|
|
24905
24775
|
);
|
|
24906
|
-
log.dim(" Billed as used: each attempt is charged its real cost plus the platform fee, rounded up to a whole credit.");
|
|
24907
24776
|
log.plain("");
|
|
24908
24777
|
}
|
|
24909
24778
|
const base = `${apiUrl}/api/runtime/development/projects/${encodeURIComponent(projectId)}/generations`;
|
|
@@ -24929,7 +24798,7 @@ async function runBench(args) {
|
|
|
24929
24798
|
const res = await call2(base, {
|
|
24930
24799
|
method: "POST",
|
|
24931
24800
|
body: JSON.stringify({
|
|
24932
|
-
|
|
24801
|
+
maxCoins,
|
|
24933
24802
|
request: {
|
|
24934
24803
|
idempotencyKey: `bench-${randomUUID2()}`,
|
|
24935
24804
|
modelId,
|
|
@@ -24959,14 +24828,13 @@ async function runBench(args) {
|
|
|
24959
24828
|
if (refusedAt !== null) providerRefused++;
|
|
24960
24829
|
if (!opts.json) {
|
|
24961
24830
|
const length = row.outputTokens !== null ? ` \xB7 ${row.outputTokens}${row.maxOutputTokens !== null ? ` of ${row.maxOutputTokens}` : ""} tokens out` : "";
|
|
24962
|
-
const charged = row.chargedCredits === null ? "\u2014" : String(row.chargedCredits);
|
|
24963
24831
|
log.plain(
|
|
24964
|
-
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${
|
|
24832
|
+
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${String(row.chargedCoins ?? "\u2014").padStart(4)} coin charged \xB7 provider ${usd(row.costUsd)}${length}${row.error ? ` \xB7 ${row.error}` : ""}`
|
|
24965
24833
|
);
|
|
24966
24834
|
if (refusedAt !== null) printProviderRefusal(log, refusedAt, row.providerMessage, row.status === "unknown");
|
|
24967
24835
|
if (row.error === PROVIDER_TOKEN_LIMIT) {
|
|
24968
24836
|
log.dim(
|
|
24969
|
-
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no
|
|
24837
|
+
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no price makes room for it; it is not a sample." : ` The answer was cut off at this ceiling (--max-coins ${maxCoins}) before it was finished \u2014 it was charged, and it is not a sample.`
|
|
24970
24838
|
);
|
|
24971
24839
|
}
|
|
24972
24840
|
}
|
|
@@ -24974,19 +24842,16 @@ async function runBench(args) {
|
|
|
24974
24842
|
} finally {
|
|
24975
24843
|
process.removeListener("SIGINT", onSigint);
|
|
24976
24844
|
}
|
|
24977
|
-
const
|
|
24978
|
-
|
|
24979
|
-
|
|
24980
|
-
|
|
24981
|
-
|
|
24982
|
-
const
|
|
24983
|
-
const settledCount = rec.charged.length;
|
|
24845
|
+
const headroomBps = lane.recommendedDeclaredHeadroomBps;
|
|
24846
|
+
const rec = benchRecommendation(results, headroomBps);
|
|
24847
|
+
const { charged, p50, p95, max } = rec;
|
|
24848
|
+
const settledCount = charged.length;
|
|
24849
|
+
const recommended = rec.recommendedEstimateCoins;
|
|
24850
|
+
const ceiling = rec.recommendedPerCallMaxCoins;
|
|
24984
24851
|
const lengthBlocked = rec.noRecommendationReason !== null;
|
|
24985
|
-
const balanceAfter =
|
|
24986
|
-
const picosOrNull = (v) => v === null ? null : String(v);
|
|
24852
|
+
const balanceAfter = await readCoinBalance(apiUrl, token);
|
|
24987
24853
|
const record = {
|
|
24988
|
-
v:
|
|
24989
|
-
currency: "credits",
|
|
24854
|
+
v: 1,
|
|
24990
24855
|
ranAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
24991
24856
|
apiUrl,
|
|
24992
24857
|
projectId,
|
|
@@ -24995,34 +24860,28 @@ async function runBench(args) {
|
|
|
24995
24860
|
outputFormat,
|
|
24996
24861
|
samples,
|
|
24997
24862
|
completed: settledCount,
|
|
24998
|
-
|
|
24999
|
-
|
|
25000
|
-
|
|
25001
|
-
|
|
25002
|
-
|
|
25003
|
-
|
|
24863
|
+
chargedCoins: charged,
|
|
24864
|
+
p50,
|
|
24865
|
+
p95,
|
|
24866
|
+
max,
|
|
24867
|
+
recommendedDeclaredHeadroomBps: headroomBps,
|
|
24868
|
+
recommendedEstimateCoins: recommended,
|
|
24869
|
+
recommendedPerCallMaxCoins: ceiling,
|
|
24870
|
+
maxCoinsPerSample: maxCoins,
|
|
24871
|
+
fitCoins: rec.fits,
|
|
25004
24872
|
fitP95: rec.fitP95,
|
|
25005
|
-
|
|
25006
|
-
|
|
25007
|
-
feeBps: terms.feeBps,
|
|
25008
|
-
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
25009
|
-
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
25010
|
-
recommendedMaxCredits: rec.recommendedMaxCredits,
|
|
25011
|
-
recommendedPerCallMaxCredits: rec.recommendedPerCallMaxCredits,
|
|
25012
|
-
averageCreditsPerCall: rec.averageCreditsPerCall,
|
|
25013
|
-
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
25014
|
-
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
24873
|
+
chargedBasedEstimateCoins: rec.chargedBasedEstimateCoins,
|
|
24874
|
+
lengthRaisedPrice: rec.lengthRaisedPrice,
|
|
25015
24875
|
cutOffSamples: rec.cutOffSamples,
|
|
25016
24876
|
noRecommendationReason: rec.noRecommendationReason
|
|
25017
24877
|
};
|
|
25018
24878
|
const savedTo = await saveBench(cwd, record);
|
|
25019
24879
|
if (opts.json) {
|
|
25020
|
-
const ok =
|
|
24880
|
+
const ok = charged.length > 0 && !lengthBlocked;
|
|
25021
24881
|
writeJsonLine({
|
|
25022
24882
|
command: "llm bench",
|
|
25023
24883
|
status: ok ? "ok" : "failed",
|
|
25024
24884
|
error: ok ? null : rec.noRecommendationReason ?? "no_sample_settled",
|
|
25025
|
-
currency: "credits",
|
|
25026
24885
|
apiUrl,
|
|
25027
24886
|
projectId,
|
|
25028
24887
|
modelId,
|
|
@@ -25031,54 +24890,35 @@ async function runBench(args) {
|
|
|
25031
24890
|
schemaPath: opts.schemaPath ?? null,
|
|
25032
24891
|
samples,
|
|
25033
24892
|
completed: settledCount,
|
|
25034
|
-
|
|
25035
|
-
// Spendable credits before and after the run; null when unreadable.
|
|
24893
|
+
maxCoinsPerSample: maxCoins,
|
|
25036
24894
|
balanceBefore,
|
|
25037
24895
|
balanceAfter,
|
|
25038
24896
|
results,
|
|
25039
24897
|
// The create-time refusal that ended the run, or null. Beside the
|
|
25040
24898
|
// samples, never among them: no attempt existed and nothing was spent.
|
|
25041
24899
|
refusal: refusal2,
|
|
25042
|
-
|
|
25043
|
-
//
|
|
25044
|
-
costUsdPicos: record.cost,
|
|
25045
|
-
costUsd: {
|
|
25046
|
-
p50: rec.costP50 === null ? null : Number(rec.costP50) / 1e12,
|
|
25047
|
-
p95: rec.costP95 === null ? null : Number(rec.costP95) / 1e12,
|
|
25048
|
-
max: rec.costMax === null ? null : Number(rec.costMax) / 1e12
|
|
25049
|
-
},
|
|
25050
|
-
chargedCredits: record.charged,
|
|
25051
|
-
// The answer's LENGTH, priced by the server per sample (`fitCredits`,
|
|
24900
|
+
chargedCoins: { p50, p95, max },
|
|
24901
|
+
// The answer's LENGTH, priced by the server per sample (`fitCoins`,
|
|
25052
24902
|
// headroom already on the tokens), over the settled samples.
|
|
25053
|
-
|
|
25054
|
-
|
|
25055
|
-
|
|
25056
|
-
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
25057
|
-
missingTerms: rec.missingTerms,
|
|
25058
|
-
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
25059
|
-
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
24903
|
+
fitCoins: { p95: rec.fitP95, max: rec.fitMax },
|
|
24904
|
+
chargedBasedEstimateCoins: rec.chargedBasedEstimateCoins,
|
|
24905
|
+
lengthRaisedPrice: rec.lengthRaisedPrice,
|
|
25060
24906
|
cutOffSamples: rec.cutOffSamples,
|
|
25061
24907
|
exceedsStandCap: rec.exceedsStandCap,
|
|
25062
24908
|
noRecommendationReason: rec.noRecommendationReason,
|
|
25063
|
-
|
|
25064
|
-
|
|
25065
|
-
|
|
25066
|
-
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
24909
|
+
recommendedDeclaredHeadroomBps: headroomBps,
|
|
24910
|
+
recommendedEstimateCoins: recommended,
|
|
24911
|
+
recommendedPerCallMaxCoins: ceiling,
|
|
25067
24912
|
savedTo
|
|
25068
24913
|
});
|
|
25069
24914
|
if (!ok) process.exitCode = 1;
|
|
25070
24915
|
return;
|
|
25071
24916
|
}
|
|
25072
24917
|
log.plain("");
|
|
25073
|
-
const printMeasured = () => {
|
|
25074
|
-
log.plain(c.bold(" Real cost per call"));
|
|
25075
|
-
log.plain(` p50 ${usdFromPicos(rec.costP50)} p95 ${usdFromPicos(rec.costP95)} max ${usdFromPicos(rec.costMax)} (${settledCount} of ${samples} settled)`);
|
|
25076
|
-
log.plain(c.bold(" Charged credits"));
|
|
25077
|
-
log.plain(` p50 ${rec.chargedP50} p95 ${rec.chargedP95} max ${rec.chargedMax} (whole credits, rounded up per call)`);
|
|
25078
|
-
};
|
|
25079
24918
|
if (lengthBlocked) {
|
|
25080
|
-
if (
|
|
25081
|
-
|
|
24919
|
+
if (charged.length > 0) {
|
|
24920
|
+
log.plain(c.bold(" Charged coins"));
|
|
24921
|
+
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
25082
24922
|
log.plain("");
|
|
25083
24923
|
}
|
|
25084
24924
|
printRecommendation(log, record);
|
|
@@ -25086,16 +24926,17 @@ async function runBench(args) {
|
|
|
25086
24926
|
process.exitCode = 1;
|
|
25087
24927
|
return;
|
|
25088
24928
|
}
|
|
25089
|
-
if (
|
|
24929
|
+
if (charged.length === 0) {
|
|
25090
24930
|
log.error(
|
|
25091
|
-
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to
|
|
24931
|
+
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to price from." : " No sample settled, so there is nothing to price from."
|
|
25092
24932
|
);
|
|
25093
24933
|
process.exitCode = 1;
|
|
25094
24934
|
return;
|
|
25095
24935
|
}
|
|
25096
|
-
|
|
24936
|
+
log.plain(c.bold(" Charged coins"));
|
|
24937
|
+
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
25097
24938
|
if (balanceBefore !== null && balanceAfter !== null) {
|
|
25098
|
-
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter}
|
|
24939
|
+
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter} coin spendable`);
|
|
25099
24940
|
}
|
|
25100
24941
|
log.plain("");
|
|
25101
24942
|
printRecommendation(log, record);
|
|
@@ -25107,58 +24948,43 @@ function printRecommendation(log, record) {
|
|
|
25107
24948
|
return;
|
|
25108
24949
|
}
|
|
25109
24950
|
if (record.noRecommendationReason === "answer_cut_off") {
|
|
25110
|
-
log.warn(` ${benchCutOffLine(record.cutOffSamples
|
|
24951
|
+
log.warn(` ${benchCutOffLine(record.cutOffSamples ?? 1, record.maxCoinsPerSample ?? 0)}`);
|
|
25111
24952
|
return;
|
|
25112
24953
|
}
|
|
25113
|
-
|
|
25114
|
-
|
|
25115
|
-
log.warn(" No ceiling recommended.");
|
|
24954
|
+
if (record.recommendedEstimateCoins === null || record.p95 === null) {
|
|
24955
|
+
log.warn(" No price recommended.");
|
|
25116
24956
|
log.dim(
|
|
25117
|
-
record.
|
|
24957
|
+
record.recommendedDeclaredHeadroomBps === null ? " This stand served no recommended headroom, and the multiplier is the server's to set \u2014" : " No sample settled, so there is no p95 to build a price on \u2014"
|
|
25118
24958
|
);
|
|
25119
24959
|
log.dim(" declaring a number from this run would be a guess dressed as a measurement.");
|
|
25120
|
-
printAverage(log, record, attempts);
|
|
25121
24960
|
return;
|
|
25122
24961
|
}
|
|
25123
|
-
log.plain(c.bold(` Declare
|
|
25124
|
-
if (record.
|
|
24962
|
+
log.plain(c.bold(` Declare estimateCoins: ${record.recommendedEstimateCoins}`));
|
|
24963
|
+
if (record.lengthRaisedPrice) {
|
|
25125
24964
|
log.plain(` ${LLM_LENGTH_RAISED_LINE}`);
|
|
25126
24965
|
log.dim(
|
|
25127
|
-
` = the smallest
|
|
24966
|
+
` = the smallest price whose answer allowance holds the answer plus this stand's headroom (p95 ${record.fitP95 ?? "\u2014"}), above the`
|
|
25128
24967
|
);
|
|
25129
24968
|
log.dim(
|
|
25130
|
-
`
|
|
24969
|
+
` charged-based ${record.chargedBasedEstimateCoins ?? "\u2014"} (p95 of charged coins ${record.p95} plus ${record.recommendedDeclaredHeadroomBps} bps), over ${record.completed} settled attempt${record.completed === 1 ? "" : "s"} on ${record.modelId}.`
|
|
25131
24970
|
);
|
|
25132
24971
|
} else {
|
|
25133
24972
|
log.dim(
|
|
25134
|
-
` = p95
|
|
24973
|
+
` = p95 of charged coins (${record.p95}) plus this stand's recommended headroom (${record.recommendedDeclaredHeadroomBps} bps),`
|
|
25135
24974
|
);
|
|
25136
|
-
log.dim(` measured over ${
|
|
24975
|
+
log.dim(` measured over ${record.completed} settled attempt${record.completed === 1 ? "" : "s"} on ${record.modelId}.`);
|
|
25137
24976
|
}
|
|
25138
|
-
log.dim(" That number is a
|
|
25139
|
-
|
|
25140
|
-
|
|
25141
|
-
log.plain(c.bold(` Grant perCallMaxCredits: ${record.recommendedPerCallMaxCredits}`));
|
|
24977
|
+
log.dim(" That number is a PRICE: a started attempt is charged in full, including one that fails.");
|
|
24978
|
+
if (record.recommendedPerCallMaxCoins != null) {
|
|
24979
|
+
log.plain(c.bold(` Grant perCallMaxCoins: ${record.recommendedPerCallMaxCoins}`));
|
|
25142
24980
|
log.dim(
|
|
25143
|
-
record.
|
|
24981
|
+
record.lengthRaisedPrice ? ` = the same headroom over the worst sample (${record.max}) or the longest answer's price, whichever is larger, and never below the price above it.` : ` = the same headroom over the worst sample (${record.max}), and never below the price above it.`
|
|
25144
24982
|
);
|
|
25145
|
-
|
|
25146
|
-
printAverage(log, record, attempts);
|
|
25147
|
-
}
|
|
25148
|
-
function printAverage(log, record, attempts) {
|
|
25149
|
-
if (record.averageCreditsPerCall === null) return;
|
|
25150
|
-
log.plain(` Per-call estimate: about ${formatCredits(record.averageCreditsPerCall)} credits per call`);
|
|
25151
|
-
log.dim(` = the average real cost with the platform fee, no headroom, over ${attempts}.`);
|
|
25152
|
-
if (record.recommendedPerCallEstimateCredits !== null) {
|
|
25153
|
-
log.plain(c.bold(` Grant perCallEstimateCredits: ${record.recommendedPerCallEstimateCredits}`));
|
|
25154
|
-
log.dim(` = that average rounded up to a whole credit. disclosure.estimatedCreditsPerPeriod = your calls per period \xD7 ${formatCredits(record.averageCreditsPerCall)}, rounded up.`);
|
|
24983
|
+
log.dim(" The ceiling is the price's room to be wrong; declaring one under the price is refused.");
|
|
25155
24984
|
}
|
|
25156
24985
|
}
|
|
25157
24986
|
async function explainBenchRefusal(res, call2, base, log, opts, index, settledSoFar) {
|
|
25158
|
-
if (printedStructuredError(res)) {
|
|
25159
|
-
const printed = await res.json().catch(() => ({}));
|
|
25160
|
-
return { stop: true, refusal: { status: res.status, error: typeof printed.error === "string" ? printed.error : null, slotsHeld: null, slotLimit: null } };
|
|
25161
|
-
}
|
|
24987
|
+
if (printedStructuredError(res)) return { stop: true, refusal: { status: res.status, error: null, slotsHeld: null, slotLimit: null } };
|
|
25162
24988
|
const body = await res.json().catch(() => ({}));
|
|
25163
24989
|
const refusal2 = { status: res.status, error: body.error ?? null, slotsHeld: null, slotLimit: null };
|
|
25164
24990
|
if (res.status === 429 && body.error === "generation_limit") {
|
|
@@ -25168,10 +24994,6 @@ async function explainBenchRefusal(res, call2, base, log, opts, index, settledSo
|
|
|
25168
24994
|
if (!opts.json) log.error(` Sample ${index + 1} ${benchSlotsHeldSentence(counts)}`);
|
|
25169
24995
|
return { stop: true, refusal: refusal2 };
|
|
25170
24996
|
}
|
|
25171
|
-
if (res.status === 403 && body.error === "credits_unverified") {
|
|
25172
|
-
if (!opts.json) log.error(` ${LLM_BENCH_UNVERIFIED_LINE}`);
|
|
25173
|
-
return { stop: true, refusal: refusal2 };
|
|
25174
|
-
}
|
|
25175
24997
|
if (opts.json) return { stop: res.status !== 429, refusal: refusal2 };
|
|
25176
24998
|
if (res.status === 404 && body.error === "not_found") {
|
|
25177
24999
|
log.error(` ${LLM_LANE_OFF_LINE}`);
|
|
@@ -25187,7 +25009,7 @@ async function explainBenchRefusal(res, call2, base, log, opts, index, settledSo
|
|
|
25187
25009
|
return { stop: true, refusal: refusal2 };
|
|
25188
25010
|
}
|
|
25189
25011
|
if (res.status === 402) {
|
|
25190
|
-
log.error(" Not enough
|
|
25012
|
+
log.error(" Not enough coin to start the attempt.");
|
|
25191
25013
|
return { stop: true, refusal: refusal2 };
|
|
25192
25014
|
}
|
|
25193
25015
|
log.error(
|
|
@@ -25196,7 +25018,7 @@ async function explainBenchRefusal(res, call2, base, log, opts, index, settledSo
|
|
|
25196
25018
|
return { stop: res.status >= 500 || res.status === 401 || res.status === 403, refusal: refusal2 };
|
|
25197
25019
|
}
|
|
25198
25020
|
function printProviderRefusal(log, status2, message, awaitingBill = false) {
|
|
25199
|
-
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so
|
|
25021
|
+
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so this call's coin stays held until it resolves (see \`genex llm status\`); it is not a sample.`);
|
|
25200
25022
|
else log.dim(" The provider refused this call at its door \u2014 no inference ran, it cost nothing, and it is not a sample.");
|
|
25201
25023
|
if (message) log.dim(` Provider said: ${message}`);
|
|
25202
25024
|
if (status2 === 401 || status2 === 403) {
|
|
@@ -25222,6 +25044,19 @@ async function pollSettled(call2, url, timeoutSec) {
|
|
|
25222
25044
|
}
|
|
25223
25045
|
return last;
|
|
25224
25046
|
}
|
|
25047
|
+
async function readCoinBalance(apiUrl, token) {
|
|
25048
|
+
try {
|
|
25049
|
+
const res = await apiFetch(`${apiUrl}/api/coin/balance`, {
|
|
25050
|
+
headers: { Authorization: `Bearer ${token}` },
|
|
25051
|
+
signal: AbortSignal.timeout(6e3)
|
|
25052
|
+
});
|
|
25053
|
+
if (!res.ok) return null;
|
|
25054
|
+
const body = await res.json().catch(() => null);
|
|
25055
|
+
return typeof body?.spendable === "number" ? body.spendable : null;
|
|
25056
|
+
} catch {
|
|
25057
|
+
return null;
|
|
25058
|
+
}
|
|
25059
|
+
}
|
|
25225
25060
|
async function saveBench(cwd, record) {
|
|
25226
25061
|
try {
|
|
25227
25062
|
const file = path36.join(cwd, LLM_BENCH_FILE);
|
|
@@ -25237,10 +25072,9 @@ function benchFailed(opts, log, message) {
|
|
|
25237
25072
|
else log.error(message);
|
|
25238
25073
|
process.exitCode = 1;
|
|
25239
25074
|
}
|
|
25240
|
-
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
25075
|
+
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCoins, opts) {
|
|
25241
25076
|
return {
|
|
25242
25077
|
command: "llm bench",
|
|
25243
|
-
currency: "credits",
|
|
25244
25078
|
apiUrl,
|
|
25245
25079
|
projectId,
|
|
25246
25080
|
modelId: opts.modelId ?? null,
|
|
@@ -25249,28 +25083,21 @@ function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCre
|
|
|
25249
25083
|
schemaPath: opts.schemaPath ?? null,
|
|
25250
25084
|
samples,
|
|
25251
25085
|
completed: 0,
|
|
25252
|
-
|
|
25086
|
+
maxCoinsPerSample: maxCoins,
|
|
25253
25087
|
balanceBefore: null,
|
|
25254
25088
|
balanceAfter: null,
|
|
25255
25089
|
results: [],
|
|
25256
25090
|
refusal: null,
|
|
25257
|
-
|
|
25258
|
-
|
|
25259
|
-
|
|
25260
|
-
|
|
25261
|
-
recommendedDeclaredHeadroomBps: null,
|
|
25262
|
-
feeBps: null,
|
|
25263
|
-
usdCentsPerCredit: null,
|
|
25264
|
-
missingTerms: null,
|
|
25265
|
-
costBasedMaxCredits: null,
|
|
25266
|
-
lengthRaisedCeiling: false,
|
|
25091
|
+
chargedCoins: { p50: null, p95: null, max: null },
|
|
25092
|
+
fitCoins: { p95: null, max: null },
|
|
25093
|
+
chargedBasedEstimateCoins: null,
|
|
25094
|
+
lengthRaisedPrice: false,
|
|
25267
25095
|
cutOffSamples: 0,
|
|
25268
25096
|
exceedsStandCap: false,
|
|
25269
25097
|
noRecommendationReason: null,
|
|
25270
|
-
|
|
25271
|
-
|
|
25272
|
-
|
|
25273
|
-
recommendedPerCallEstimateCredits: null,
|
|
25098
|
+
recommendedDeclaredHeadroomBps: null,
|
|
25099
|
+
recommendedEstimateCoins: null,
|
|
25100
|
+
recommendedPerCallMaxCoins: null,
|
|
25274
25101
|
savedTo: null
|
|
25275
25102
|
};
|
|
25276
25103
|
}
|
|
@@ -25333,8 +25160,7 @@ async function reportStatus(args) {
|
|
|
25333
25160
|
for (const row of rows) {
|
|
25334
25161
|
const state = rowState(row);
|
|
25335
25162
|
const mark = state === "active" ? c.cyan("\u25CF") : state === "awaiting bill" ? c.yellow("\u25D0") : c.green("\u25CB");
|
|
25336
|
-
const
|
|
25337
|
-
const reserved = money2.reserved === null ? "reserved \u2014" : `${money2.reserved} ${money2.unit} reserved`;
|
|
25163
|
+
const reserved = typeof row.reservedCoins === "number" ? `${row.reservedCoins} coin reserved` : "reserved \u2014";
|
|
25338
25164
|
log.plain(
|
|
25339
25165
|
` ${mark} ${row.id ?? "?"} ${row.modelId ?? "?"} ${state.padEnd(13)} ${formatAge(row.createdAt, now).padStart(4)} old ${reserved}${row.error ? ` ${row.error}` : ""}${rowHoldsSlot(row) ? "" : c.dim(" (no slot)")}`
|
|
25340
25166
|
);
|
|
@@ -25406,11 +25232,10 @@ async function cancelCall(args) {
|
|
|
25406
25232
|
if (!opts.json) {
|
|
25407
25233
|
log.warn(` ${row.id} (${row.status ?? "?"}) \u2014 ${sentence}.`);
|
|
25408
25234
|
if (state === "awaiting bill") {
|
|
25409
|
-
log.dim(` Nothing to cancel:
|
|
25235
|
+
log.dim(` Nothing to cancel: its coin releases when the bill resolves, and it stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`);
|
|
25410
25236
|
if (row.providerMessage?.trim()) log.dim(` Provider said: ${row.providerMessage.trim()}`);
|
|
25411
25237
|
} else {
|
|
25412
|
-
|
|
25413
|
-
log.dim(` ${money2.charged === null ? "charge unknown" : `${money2.charged} ${money2.unit} charged`}${row.error ? ` \xB7 ${row.error}` : ""}.`);
|
|
25238
|
+
log.dim(` ${typeof row.chargedCoins === "number" ? `${row.chargedCoins} coin charged` : "charge unknown"}${row.error ? ` \xB7 ${row.error}` : ""}.`);
|
|
25414
25239
|
}
|
|
25415
25240
|
}
|
|
25416
25241
|
return done("ok", null, state === "awaiting bill" ? "awaiting_bill" : "already_final", row);
|
|
@@ -25432,126 +25257,78 @@ async function cancelCall(args) {
|
|
|
25432
25257
|
const after = await res.json().catch(() => null) ?? row;
|
|
25433
25258
|
if (!opts.json) {
|
|
25434
25259
|
log.success(` Cancelled ${row.id} (was ${row.status ?? "active"}).`);
|
|
25435
|
-
log.dim(`
|
|
25260
|
+
log.dim(` Its coin releases once the bill settles; ${c.cyan("genex llm status")} shows it until then.`);
|
|
25436
25261
|
}
|
|
25437
25262
|
return done("ok", null, "cancelled", after);
|
|
25438
25263
|
}
|
|
25439
|
-
async function
|
|
25440
|
-
let
|
|
25264
|
+
async function reportSavedPrice(cwd, opts, log) {
|
|
25265
|
+
let record = null;
|
|
25441
25266
|
try {
|
|
25442
|
-
|
|
25267
|
+
record = JSON.parse(await fs35.readFile(path36.join(cwd, LLM_BENCH_FILE), "utf8"));
|
|
25443
25268
|
} catch {
|
|
25444
|
-
|
|
25269
|
+
record = null;
|
|
25445
25270
|
}
|
|
25446
|
-
if (!
|
|
25447
|
-
const v = parsed;
|
|
25448
|
-
if (v.v === 2 && v.currency === "credits" && v.cost && v.charged) return { kind: "credits", record: parsed };
|
|
25449
|
-
return { kind: "coins", record: parsed };
|
|
25450
|
-
}
|
|
25451
|
-
var LLM_PRICE_COIN_TERMS_LINE = "This bench was measured under the old COIN terms, when a game declared a fixed price per call. In-game calls are now billed in the player's credits, as used, and a game declares a per-call CEILING (maxCredits) instead \u2014 a coin price is not that number. Re-run the bench for the figures to declare.";
|
|
25452
|
-
async function reportSavedPrice(cwd, opts, log) {
|
|
25453
|
-
const saved = await readSavedBench(cwd);
|
|
25454
|
-
const rerun = 'genex llm bench "<prompt>" --max-credits <n> --user-approved';
|
|
25455
|
-
const creditKeys = (record2) => ({
|
|
25456
|
-
costUsdPicos: record2?.cost ?? { p50: null, p95: null, max: null },
|
|
25457
|
-
chargedCredits: record2?.charged ?? { p50: null, p95: null, max: null },
|
|
25458
|
-
fitP95: record2?.fitP95 ?? null,
|
|
25459
|
-
fitMax: record2?.fitMax ?? null,
|
|
25460
|
-
recommendedDeclaredHeadroomBps: record2?.recommendedDeclaredHeadroomBps ?? null,
|
|
25461
|
-
feeBps: record2?.feeBps ?? null,
|
|
25462
|
-
usdCentsPerCredit: record2?.usdCentsPerCredit ?? null,
|
|
25463
|
-
costBasedMaxCredits: record2?.costBasedMaxCredits ?? null,
|
|
25464
|
-
lengthRaisedCeiling: record2?.lengthRaisedCeiling ?? false,
|
|
25465
|
-
cutOffSamples: record2?.cutOffSamples ?? 0,
|
|
25466
|
-
noRecommendationReason: record2?.noRecommendationReason ?? null,
|
|
25467
|
-
recommendedMaxCredits: record2?.recommendedMaxCredits ?? null,
|
|
25468
|
-
recommendedPerCallMaxCredits: record2?.recommendedPerCallMaxCredits ?? null,
|
|
25469
|
-
averageCreditsPerCall: record2?.averageCreditsPerCall ?? null,
|
|
25470
|
-
recommendedPerCallEstimateCredits: record2?.recommendedPerCallEstimateCredits ?? null
|
|
25471
|
-
});
|
|
25472
|
-
if (!saved) {
|
|
25271
|
+
if (!record) {
|
|
25473
25272
|
if (opts.json) {
|
|
25474
25273
|
writeJsonLine({
|
|
25475
25274
|
command: "llm price",
|
|
25476
25275
|
status: "failed",
|
|
25477
25276
|
error: "no_bench",
|
|
25478
|
-
currency: null,
|
|
25479
25277
|
ranAt: null,
|
|
25480
25278
|
modelId: null,
|
|
25481
25279
|
samples: null,
|
|
25482
25280
|
completed: null,
|
|
25483
|
-
|
|
25484
|
-
|
|
25281
|
+
p50: null,
|
|
25282
|
+
p95: null,
|
|
25283
|
+
max: null,
|
|
25284
|
+
fitP95: null,
|
|
25285
|
+
chargedBasedEstimateCoins: null,
|
|
25286
|
+
lengthRaisedPrice: false,
|
|
25287
|
+
cutOffSamples: 0,
|
|
25288
|
+
noRecommendationReason: null,
|
|
25289
|
+
recommendedDeclaredHeadroomBps: null,
|
|
25290
|
+
recommendedEstimateCoins: null,
|
|
25291
|
+
recommendedPerCallMaxCoins: null,
|
|
25485
25292
|
savedTo: null
|
|
25486
25293
|
});
|
|
25487
25294
|
} else {
|
|
25488
25295
|
log.error("Nothing has been benchmarked in this folder yet.");
|
|
25489
|
-
log.dim(` Run ${c.cyan(
|
|
25490
|
-
}
|
|
25491
|
-
process.exitCode = 1;
|
|
25492
|
-
return;
|
|
25493
|
-
}
|
|
25494
|
-
if (saved.kind === "coins") {
|
|
25495
|
-
const old = saved.record;
|
|
25496
|
-
const legacy = {
|
|
25497
|
-
p50: old.p50 ?? null,
|
|
25498
|
-
p95: old.p95 ?? null,
|
|
25499
|
-
max: old.max ?? null,
|
|
25500
|
-
recommendedEstimateCoins: old.recommendedEstimateCoins ?? null,
|
|
25501
|
-
recommendedPerCallMaxCoins: old.recommendedPerCallMaxCoins ?? null,
|
|
25502
|
-
maxCoinsPerSample: old.maxCoinsPerSample ?? null,
|
|
25503
|
-
noRecommendationReason: old.noRecommendationReason ?? null
|
|
25504
|
-
};
|
|
25505
|
-
if (opts.json) {
|
|
25506
|
-
writeJsonLine({
|
|
25507
|
-
command: "llm price",
|
|
25508
|
-
status: "failed",
|
|
25509
|
-
error: "coin_terms",
|
|
25510
|
-
currency: "coins",
|
|
25511
|
-
ranAt: old.ranAt ?? null,
|
|
25512
|
-
modelId: old.modelId ?? null,
|
|
25513
|
-
samples: old.samples ?? null,
|
|
25514
|
-
completed: old.completed ?? null,
|
|
25515
|
-
...creditKeys(null),
|
|
25516
|
-
legacyCoinTerms: legacy,
|
|
25517
|
-
savedTo: LLM_BENCH_FILE
|
|
25518
|
-
});
|
|
25519
|
-
} else {
|
|
25520
|
-
log.plain(c.bold("genex llm price"));
|
|
25521
|
-
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${old.ranAt ?? "at an unknown time"}${old.modelId ? ` on ${old.modelId}` : ""}`);
|
|
25522
|
-
log.plain("");
|
|
25523
|
-
log.warn(` ${LLM_PRICE_COIN_TERMS_LINE}`);
|
|
25524
|
-
const measured = legacy.p95 === null ? "no settled sample" : `charged coins p50 ${legacy.p50} \xB7 p95 ${legacy.p95} \xB7 max ${legacy.max}`;
|
|
25525
|
-
const recommended = legacy.recommendedEstimateCoins === null ? "no price recommended" : `it recommended estimateCoins ${legacy.recommendedEstimateCoins}${legacy.recommendedPerCallMaxCoins != null ? ` and perCallMaxCoins ${legacy.recommendedPerCallMaxCoins}` : ""}`;
|
|
25526
|
-
log.dim(` What it measured then: ${measured}; ${recommended}.`);
|
|
25527
|
-
log.dim(` Re-run: ${c.cyan(rerun)}`);
|
|
25296
|
+
log.dim(` Run ${c.cyan('genex llm bench "<prompt>" --max-coins <n> --user-approved')} first.`);
|
|
25528
25297
|
}
|
|
25529
25298
|
process.exitCode = 1;
|
|
25530
25299
|
return;
|
|
25531
25300
|
}
|
|
25532
|
-
const record = saved.record;
|
|
25533
25301
|
if (opts.json) {
|
|
25534
25302
|
writeJsonLine({
|
|
25535
25303
|
command: "llm price",
|
|
25536
|
-
status: record.
|
|
25537
|
-
error: record.
|
|
25538
|
-
currency: "credits",
|
|
25304
|
+
status: record.recommendedEstimateCoins === null ? "failed" : "ok",
|
|
25305
|
+
error: record.recommendedEstimateCoins === null ? record.noRecommendationReason ?? "no_recommendation" : null,
|
|
25539
25306
|
ranAt: record.ranAt ?? null,
|
|
25540
25307
|
modelId: record.modelId ?? null,
|
|
25541
25308
|
samples: record.samples ?? null,
|
|
25542
25309
|
completed: record.completed ?? null,
|
|
25543
|
-
|
|
25544
|
-
|
|
25310
|
+
p50: record.p50 ?? null,
|
|
25311
|
+
p95: record.p95 ?? null,
|
|
25312
|
+
max: record.max ?? null,
|
|
25313
|
+
// Absent from a file an older CLI wrote; null / false / 0 then, never missing.
|
|
25314
|
+
fitP95: record.fitP95 ?? null,
|
|
25315
|
+
chargedBasedEstimateCoins: record.chargedBasedEstimateCoins ?? null,
|
|
25316
|
+
lengthRaisedPrice: record.lengthRaisedPrice ?? false,
|
|
25317
|
+
cutOffSamples: record.cutOffSamples ?? 0,
|
|
25318
|
+
noRecommendationReason: record.noRecommendationReason ?? null,
|
|
25319
|
+
recommendedDeclaredHeadroomBps: record.recommendedDeclaredHeadroomBps ?? null,
|
|
25320
|
+
recommendedEstimateCoins: record.recommendedEstimateCoins ?? null,
|
|
25321
|
+
recommendedPerCallMaxCoins: record.recommendedPerCallMaxCoins ?? null,
|
|
25545
25322
|
savedTo: LLM_BENCH_FILE
|
|
25546
25323
|
});
|
|
25547
|
-
if (record.
|
|
25324
|
+
if (record.recommendedEstimateCoins === null) process.exitCode = 1;
|
|
25548
25325
|
return;
|
|
25549
25326
|
}
|
|
25550
25327
|
log.plain(c.bold("genex llm price"));
|
|
25551
25328
|
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${record.ranAt}`);
|
|
25552
25329
|
log.plain("");
|
|
25553
25330
|
printRecommendation(log, record);
|
|
25554
|
-
if (record.
|
|
25331
|
+
if (record.recommendedEstimateCoins === null) process.exitCode = 1;
|
|
25555
25332
|
}
|
|
25556
25333
|
function writeJsonLine(value) {
|
|
25557
25334
|
process.stdout.write(`${JSON.stringify(value)}
|
|
@@ -25811,7 +25588,7 @@ function runtimeRow(state, lane, toolsOnly = false) {
|
|
|
25811
25588
|
return {
|
|
25812
25589
|
label: "Runtime",
|
|
25813
25590
|
value: `in-game LLM lane live \xB7 ${n} model${n === 1 ? "" : "s"}${keyRefused ? ` \xB7 ${c.yellow("provider key refused")}` : ""}`,
|
|
25814
|
-
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : '
|
|
25591
|
+
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : 'Price a call before a game declares one: `npx genex llm bench "<prompt>" --max-coins <n> --user-approved`.'
|
|
25815
25592
|
};
|
|
25816
25593
|
}
|
|
25817
25594
|
case "off":
|
|
@@ -28488,16 +28265,16 @@ ${c.bold("Usage")}
|
|
|
28488
28265
|
status | cancel. "models" says whether this
|
|
28489
28266
|
stand serves the runtime LLM lane at all, and
|
|
28490
28267
|
at what provider rates. "bench" runs the real
|
|
28491
|
-
model on YOUR OWN
|
|
28492
|
-
attempt
|
|
28268
|
+
model on YOUR OWN coin and reports what each
|
|
28269
|
+
attempt was charged; it needs --max-coins <n>
|
|
28493
28270
|
--user-approved, like any spend the player has
|
|
28494
28271
|
to agree to. "price" reprints the last run's
|
|
28495
|
-
recommended
|
|
28272
|
+
recommended estimateCoins. "status" lists this
|
|
28496
28273
|
project's open calls and the slots they hold;
|
|
28497
28274
|
"cancel <id>" stops an active one. Declare a
|
|
28498
|
-
game's
|
|
28499
|
-
|
|
28500
|
-
|
|
28275
|
+
game's price from a bench, never from a guess:
|
|
28276
|
+
a started attempt is charged in full, including
|
|
28277
|
+
one that fails.
|
|
28501
28278
|
genex player <sub> [options] Answer the in-game model requests you approve, on your
|
|
28502
28279
|
OWN Claude or ChatGPT subscription: install | run |
|
|
28503
28280
|
status | stop | uninstall. "install" signs this machine
|
|
@@ -28964,7 +28741,7 @@ ${c.bold("Examples")}
|
|
|
28964
28741
|
genex shop remove sku_123
|
|
28965
28742
|
genex llm models
|
|
28966
28743
|
genex llm models --all
|
|
28967
|
-
genex llm bench "Reply with one short taunt." --samples 5 --max-
|
|
28744
|
+
genex llm bench "Reply with one short taunt." --samples 5 --max-coins 20 --user-approved
|
|
28968
28745
|
genex llm price
|
|
28969
28746
|
genex llm status
|
|
28970
28747
|
genex llm cancel <id>
|
|
@@ -29201,24 +28978,22 @@ ${c.bold("How the allowance works")}
|
|
|
29201
28978
|
|
|
29202
28979
|
Every number printed is live from your account; prices can change without a deploy.
|
|
29203
28980
|
`,
|
|
29204
|
-
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what
|
|
28981
|
+
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what should the game declare?
|
|
29205
28982
|
|
|
29206
28983
|
${c.bold("Usage")}
|
|
29207
28984
|
genex llm models [--all] [--json] Which models this stand serves, their provider rates per
|
|
29208
|
-
million tokens, the
|
|
29209
|
-
|
|
29210
|
-
|
|
29211
|
-
|
|
29212
|
-
|
|
29213
|
-
|
|
29214
|
-
|
|
29215
|
-
|
|
29216
|
-
|
|
29217
|
-
|
|
29218
|
-
|
|
29219
|
-
|
|
29220
|
-
allowance does not count. Needs a folder linked to a
|
|
29221
|
-
game you own, and a balance that covers one attempt.
|
|
28985
|
+
million tokens, and the headroom it recommends over a
|
|
28986
|
+
benchmark. The catalog is synced from OpenRouter, so the
|
|
28987
|
+
FEATURED short list is printed by default and --all
|
|
28988
|
+
prints every row (--json always carries every row). Off
|
|
28989
|
+
on this stand: it says so and exits 0 \u2014 build the feature
|
|
28990
|
+
without a model rather than promising a 404.
|
|
28991
|
+
genex llm bench "<prompt>" --max-coins <n> --user-approved [options]
|
|
28992
|
+
Run the real model <n> times through the development
|
|
28993
|
+
lane and report what each attempt was CHARGED. Refused
|
|
28994
|
+
without both flags, before any network call: this spends
|
|
28995
|
+
YOUR OWN COIN, a wallet the build's asset allowance
|
|
28996
|
+
cannot see. Needs a folder linked to a game you own.
|
|
29222
28997
|
genex llm price [--json] The recommendation from the last bench in this folder.
|
|
29223
28998
|
Reads a file; spends nothing.
|
|
29224
28999
|
genex llm status [--json] This project's open development calls: active, or
|
|
@@ -29235,32 +29010,29 @@ ${c.bold("Bench options")}
|
|
|
29235
29010
|
--samples <n> How many real attempts (default ${DEFAULT_BENCH_SAMPLES}). More samples, better p95.
|
|
29236
29011
|
--schema <file> A JSON file holding the output schema; implies JSON output.
|
|
29237
29012
|
--json-output Ask for JSON without a schema. --text Ask for text (the default).
|
|
29238
|
-
--max-
|
|
29013
|
+
--max-coins <n> The per-attempt ceiling YOU approved \u2014 a bench runs on your own coin,
|
|
29239
29014
|
never a player's. The run's worst case is that number times --samples,
|
|
29240
|
-
and it is printed before anything starts.
|
|
29241
|
-
name, read as the same number.)
|
|
29015
|
+
and it is printed before anything starts.
|
|
29242
29016
|
--timeout <seconds> Bounds the wait for each attempt.
|
|
29243
29017
|
--json One machine-readable object; every key is always present, null when
|
|
29244
29018
|
the CLI could not learn it.
|
|
29245
29019
|
|
|
29246
29020
|
${c.bold("Why a benchmark and not an estimate")}
|
|
29247
|
-
|
|
29248
|
-
|
|
29249
|
-
|
|
29250
|
-
|
|
29251
|
-
|
|
29252
|
-
|
|
29253
|
-
|
|
29254
|
-
|
|
29255
|
-
|
|
29256
|
-
recommendation, because those multipliers are not the CLI's to invent.
|
|
29021
|
+
\`estimateCoins\` is a PRICE, not a guess: declaring it charges it, in full, even when the
|
|
29022
|
+
attempt fails, is cancelled, or stops at its budget. And three layers sit between the
|
|
29023
|
+
provider's rate card and that number \u2014 the provider's cost, the platform's tariff (already
|
|
29024
|
+
inside every charged coin), and your headroom. The bench itself is billed to YOU, on the
|
|
29025
|
+
development lane; the player-funded lane is what the published game uses. Adding a margin
|
|
29026
|
+
to the provider's rate silently drops the tariff and under-prices every call, so the
|
|
29027
|
+
recommendation is built from CHARGED COINS and the headroom comes from the server. A stand
|
|
29028
|
+
that serves no headroom gets no recommendation, because the multiplier is not the CLI's to
|
|
29029
|
+
invent.
|
|
29257
29030
|
|
|
29258
|
-
|
|
29259
|
-
|
|
29260
|
-
A sample cut off at your --max-
|
|
29261
|
-
|
|
29262
|
-
|
|
29263
|
-
fee included, which is the honest estimate a budget's disclosure is built on.
|
|
29031
|
+
The declared price also decides how LONG the answer may be: it sizes each call's output
|
|
29032
|
+
allowance. So the recommendation is never below the smallest price that leaves room for
|
|
29033
|
+
the benchmarked answer, as the server computes it. A sample cut off at your --max-coins
|
|
29034
|
+
(provider_token_limit) is not a sample \u2014 re-run with a higher --max-coins; an answer longer
|
|
29035
|
+
than one call on this stand may produce gets no price at all \u2014 ask for a shorter one.
|
|
29264
29036
|
|
|
29265
29037
|
Results land in ${LLM_BENCH_FILE} \u2014 gitignored, machine-local, re-read by \`genex llm price\`.
|
|
29266
29038
|
`,
|
|
@@ -29508,13 +29280,11 @@ function parseArgs(argv) {
|
|
|
29508
29280
|
"--limit",
|
|
29509
29281
|
// `genex budget --assets <credits>` — the allowance the player approved.
|
|
29510
29282
|
"--assets",
|
|
29511
|
-
// `genex llm bench` value flags. `--max-
|
|
29512
|
-
// shape as `--assets
|
|
29513
|
-
// count a bench. `--max-coins` is its deprecated name (same number).
|
|
29283
|
+
// `genex llm bench` value flags. `--max-coins` is the COIN half of the
|
|
29284
|
+
// same approval shape as `--assets` (a different economy, its own gate).
|
|
29514
29285
|
"--model",
|
|
29515
29286
|
"--samples",
|
|
29516
29287
|
"--schema",
|
|
29517
|
-
"--max-credits",
|
|
29518
29288
|
"--max-coins",
|
|
29519
29289
|
// `genex player install --label <name>` — what this machine is called in
|
|
29520
29290
|
// the dashboard. Its own key rather than the generic 2nd positional,
|
|
@@ -30068,14 +29838,12 @@ function applyValueFlag(options, flag, value) {
|
|
|
30068
29838
|
options.samples = n;
|
|
30069
29839
|
break;
|
|
30070
29840
|
}
|
|
30071
|
-
case "--max-credits":
|
|
30072
29841
|
case "--max-coins": {
|
|
30073
29842
|
const n = Number(value);
|
|
30074
29843
|
if (!Number.isInteger(n) || n < 1) {
|
|
30075
|
-
throw new Error(`Invalid
|
|
29844
|
+
throw new Error(`Invalid --max-coins value: ${value} (whole coin, 1 or more)`);
|
|
30076
29845
|
}
|
|
30077
|
-
|
|
30078
|
-
else options.maxCoins = n;
|
|
29846
|
+
options.maxCoins = n;
|
|
30079
29847
|
break;
|
|
30080
29848
|
}
|
|
30081
29849
|
case "--timeout": {
|
|
@@ -30373,9 +30141,9 @@ async function main() {
|
|
|
30373
30141
|
await runBudget(parsed.options);
|
|
30374
30142
|
break;
|
|
30375
30143
|
// The in-game model lane: is it live here, what does one call really
|
|
30376
|
-
// cost, what
|
|
30377
|
-
//
|
|
30378
|
-
//
|
|
30144
|
+
// cost, what should the game declare. `bench` is the only CLI command
|
|
30145
|
+
// that spends COIN, so it carries its own --max-coins/--user-approved
|
|
30146
|
+
// gate — the credits allowance cannot see that wallet.
|
|
30379
30147
|
case "llm":
|
|
30380
30148
|
await runLlm(parsed.options);
|
|
30381
30149
|
break;
|