@genex-ai/cli-demo 1.36.13-dev.1018 → 1.36.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{blender-mcp-WSURPZET.js → blender-mcp-QVEGHE5L.js} +2 -2
- package/dist/{blender-serve-EH2MOW3U.js → blender-serve-66OHC4ML.js} +1 -1
- package/dist/{chunk-EABIVAGT.js → chunk-4BBTPRJW.js} +2 -2
- package/dist/{chunk-K4EHXOZK.js → chunk-QI7FIYBY.js} +1 -1
- package/dist/index.js +216 -448
- package/package.json +1 -1
- package/templates/skills/genex-game-director/SKILL.md +1 -1
- package/templates/skills/genex-getting-started/SKILL.md +2 -2
- package/templates/skills/genex-llm-in-games/SKILL.md +72 -209
- package/templates/skills/genex-llm-in-games/references/pricing.md +85 -126
- package/templates/skills/genex-monetization/SKILL.md +9 -1
- package/templates/skills/genex-tool-llm/SKILL.md +9 -11
- package/templates/skills/genex-updates/SKILL.md +1 -1
package/dist/index.js
CHANGED
|
@@ -57,7 +57,7 @@ import {
|
|
|
57
57
|
writeSecretFile,
|
|
58
58
|
writeUserToken,
|
|
59
59
|
writeWorkspace
|
|
60
|
-
} from "./chunk-
|
|
60
|
+
} from "./chunk-4BBTPRJW.js";
|
|
61
61
|
import {
|
|
62
62
|
CLI_CHANNEL,
|
|
63
63
|
DEFAULT_API_URL,
|
|
@@ -78,7 +78,7 @@ import {
|
|
|
78
78
|
normalizeApiOrigin,
|
|
79
79
|
originKey,
|
|
80
80
|
resolveAgentTargets
|
|
81
|
-
} from "./chunk-
|
|
81
|
+
} from "./chunk-QI7FIYBY.js";
|
|
82
82
|
|
|
83
83
|
// src/instrument.ts
|
|
84
84
|
import { setTimeout as sleep } from "timers/promises";
|
|
@@ -1134,7 +1134,7 @@ Important note: put soul into your creations, with many details and love. Aim to
|
|
|
1134
1134
|
21. A turn ends in exactly one of two ways: on a question the player must answer before the next step can be chosen, or on a handoff \u2014 the draft link, what changed, what to try, and what is still open. Never close a turn on a promise. "I'll keep building", "while I keep working", "adding that now" are things you say and then DO before you stop; if you are stopping, say that you have stopped and what remains. While the Build plan's \`Now:\` line has open work and no answer is needed from the player, the turn is not over: preview, hand off, and start the next milestone in the same turn, until the requested outcome is reached or the platform's own budget ends the session. If a stop was forced on you anyway, the first line of your next turn is the promise you left and where it stands.
|
|
1135
1135
|
22. **What the player sees must read as the thing it is**, in this game's own style: a building as that building, a person as a person, a prop as that prop, a surface as its material \u2014 and a coloured box, a capsule, or a flat grey block standing in for one of them is never its finished version. Generation is the default route to that bar wherever code will not honestly reach it: the player's character, whatever the request names, whatever the player walks up to, enters, or interacts with, and the music and sound underneath \u2014 reach for the belt on your own judgment, without stopping to ask; a status line saying what you queued is enough, and \`--no-wait\` keeps you building while it lands. Procedural code is a first-class engine for what is structural, repeated, distant, or parametric \u2014 terrain, sky, fences, paving, modular kits, filler \u2014 and for anything else only when the result meets the same bar and you have looked at it in a capture before its row says \`landed\`. Start with one reusable implementation per distinct required gameplay or visual role. Background populations default to reusable procedural bodies and motion or compatible existing rigged assets. Catalog clips need a compatible skeleton and may be paid. Reserve paid custom bodies for the player and characters examined or interacted with closely, while honoring explicit user requests. Add variants only for an unmet requirement; unspent allowance is not unfinished work. Allowance questions use timeoutPolicy no-consent: silence does not raise the allowance. Record role, reuse and completion criteria in the existing Assets table. Decide per object, not per category, and write the route on each Assets row in \`DESIGN.md\` \u2014 the paid flow for generated pieces, the procedural flow with its local path for code-built ones. An Assets table with no generation in it is a decision, not a default: record it as \`Generation: none \u2014 <why code alone reaches the bar here>\` or generate.
|
|
1136
1136
|
23. **The screenshots you take to look at your own work go in \`.genex/scratch/\`.** Captures, render comparisons, before/after strips, traces and metrics dumps are how you SEE what you built \u2014 they are not part of what you built, and \`genex preview\` pushes the whole folder to the game's source, every time, forever. \`.genex/scratch/\` already exists for this and is already ignored, so there is nothing to set up: write the capture as \`.genex/scratch/arena-before.png\` and read it back from there. Never put them in a folder of your own at the top level: \`progress/\`, \`reports/\`, \`shots/\` and their kind are pushed like source and become permanent weight in the game and in every remix of it \u2014 one real game reached 11 GB and 10,332 committed screenshots exactly this way. If you find such a folder already there, add it to \`.gitignore\`; never delete the player's files (law 18). One file in \`.genex/scratch/\` IS read, by the platform: \`.genex/scratch/cover.png\` is the game's cover \u2014 the frame you would put on its poster: real gameplay at its signature moment, well lit, no menus, popups or busy HUD, landscape 16:9, and never any text (the page sets the title beside it). The next \`genex preview\` makes it the cover; save a better moment over it whenever one comes along, and if your browser tool can set the cover itself, use that instead. Before the first publish, and whenever the game's look changes, load \`$genex-cover\`: it chooses, stages, lights and judges that frame.
|
|
1137
|
-
24. **Benchmark before declaring an in-game LLM
|
|
1137
|
+
24. **Benchmark before declaring an in-game LLM price.** If the game calls a language model while the player plays \u2014 an NPC answering in its own words, a quest written for this save, a judge reading what the player typed \u2014 the PLAYER pays for it, and \`estimateCoins\` is the FIXED price of a started attempt: charged whether it succeeds, fails, is canceled, or stops at its own budget. Never pick that number by judgement or from a vendor's rate card. Run the real prompt on your own coins \u2014 \`npx genex llm bench "<the prompt>" --schema <file> --samples 3 --max-coins <n> --user-approved\` \u2014 and declare the figure it recommends, recorded in \`DESIGN.md\` with its model and date; re-benchmark whenever the prompt, the schema or the model changes, because a prompt edit is a price change. Check the lane first with \`npx genex llm models\`: where it is off the routes 404, so build the feature behind a graceful unavailable state and say so in the handoff. A repeated call is a standing budget the player approves once (\`requestSpendGrant()\`, then \`generate({ grantId })\`), never a popup per turn, and its disclosed call rate is computed from the game's own loop. Model output is data the game validates against its own expectation \u2014 never executed, and never authority over coin, items, entitlements or rewards. Load \`$genex-llm-in-games\` before writing any of it.
|
|
1138
1138
|
${CONTRACT_END}
|
|
1139
1139
|
`;
|
|
1140
1140
|
var TOOLS_CONTRACT_HEAD = `# Genex Tools (always in effect in this folder)
|
|
@@ -1148,7 +1148,7 @@ var TOOLS_CONTRACT_OFFER = `5. **When the game is built and playable \u2014 neve
|
|
|
1148
1148
|
var TOOLS_CONTRACT_HOSTED_LAWS = `5. **This game is hosted on Genex now, and publishing is glue \u2014 never a rebuild.** The game ships exactly as the user built it: \`initEmbed()\` first in the boot code (the \`genex-threejs-embed-auth\` card has the call; the renderer draws before any \`await waitForPlayer()\`), a static build into \`dist/\` with relative asset paths, then \`npx genex preview\`. Never restructure, reformat or "improve" the game to ship it, never scaffold a new app around it, and never start a design document or a build plan for it \u2014 the user's own process is theirs. Load the \`genex-tool-publish\` card before the first preview; it owns the vocabulary, the links and the limits.
|
|
1149
1149
|
6. **Ship first, then report.** \`preview\` prints a preflight \u2014 phone memory, a missing volume slider, an asset nobody wired, a viewport line. It never blocks a deploy, so push the build as it is, hand over the link, THEN relay each preflight line to the user in one plain sentence with an offer to fix it, and fix one only on their yes. The preflight is a report for the user, not a to-do list for you.
|
|
1150
1150
|
7. **The link is the game's page, and there are two versions.** After every preview give the user \`<dashboard>/draft/<slug>\` \u2014 \`dashboardOrigins[0]\` and \`slug\` from \`.genex/project.json\` \u2014 never localhost, a file path or the bare play origin. \`preview\` updates the draft and never touches what players are on; the first release is \`npx genex publish\` (it lists the game); after that "publish it", "update it" and "yes" all mean \`npx genex promote\` \u2014 the exact draft build, no rebuild. Ask once per round of work, in one line, and keep working while you wait.`;
|
|
1151
|
-
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with
|
|
1151
|
+
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with Genex coins or their own Claude/ChatGPT subscription, and approves it on a Genex sheet; your game just calls \`generate()\`. Want it that way?`;
|
|
1152
1152
|
function toolsContractLlmLaw(n, hosted) {
|
|
1153
1153
|
const onYes = hosted ? "On a yes, load `$genex-tool-llm` and then `$genex-llm-in-games`, which owns the build." : "On a yes, load `$genex-tool-llm`, which owns the rest: it checks the lane with `npx genex llm models` first, then runs `npx genex init --convert` (a hosted game is a static browser build \u2014 a local server holding a key can never ship).";
|
|
1154
1154
|
return `${n}. **A model running while people PLAY is a platform feature \u2014 never something you wire onto the user's own meter.** When a request implies one (NPCs that talk or decide in their own words, content written from what the player types, a prompt box in the game, "let the player pick a model"), recognise it, say in ONE line: "${TOOLS_CONTRACT_LLM_OFFER}" \u2014 then ASK and wait. ${onYes} On a no, build the authored version instead. Either way, never ship a game that calls a model on a key or an account of the user's own: every visitor would spend their money with nobody approving it, and the credential is readable in the bundle. Player-funded generation is text and JSON only \u2014 3D, images, video and audio stay the asset lanes above, on the user's meter.`;
|
|
@@ -23886,7 +23886,7 @@ async function runBlender(opts) {
|
|
|
23886
23886
|
return 1;
|
|
23887
23887
|
}
|
|
23888
23888
|
if (sub === "serve") {
|
|
23889
|
-
const { serveLocalBlender } = await import("./blender-serve-
|
|
23889
|
+
const { serveLocalBlender } = await import("./blender-serve-66OHC4ML.js");
|
|
23890
23890
|
const port = Number(process.env.GENEX_BLENDER_PORT ?? 8088);
|
|
23891
23891
|
log.step(`Starting a local Blender service on port ${port}`);
|
|
23892
23892
|
log.plain(
|
|
@@ -23895,7 +23895,7 @@ async function runBlender(opts) {
|
|
|
23895
23895
|
return serveLocalBlender({ port, log });
|
|
23896
23896
|
}
|
|
23897
23897
|
if (sub === "mcp") {
|
|
23898
|
-
const { runBlenderMcp } = await import("./blender-mcp-
|
|
23898
|
+
const { runBlenderMcp } = await import("./blender-mcp-QVEGHE5L.js");
|
|
23899
23899
|
return runBlenderMcp();
|
|
23900
23900
|
}
|
|
23901
23901
|
if (sub === "seat") {
|
|
@@ -24403,12 +24403,7 @@ import fs35 from "fs/promises";
|
|
|
24403
24403
|
import path36 from "path";
|
|
24404
24404
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
24405
24405
|
var SUBS3 = ["models", "bench", "price", "status", "cancel"];
|
|
24406
|
-
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN
|
|
24407
|
-
var LLM_MAX_COINS_DEPRECATED = "--max-coins is deprecated: it is read as --max-credits, the same number \u2014 a bench spends credits now.";
|
|
24408
|
-
function benchBalanceShortLine(spendable, maxCredits) {
|
|
24409
|
-
return `Your spendable credits (${spendable}) cannot cover one attempt at --max-credits ${maxCredits} \u2014 nothing was sent, nothing was spent. Add credits, or re-run with a lower --max-credits once the person at the keyboard agrees to it.`;
|
|
24410
|
-
}
|
|
24411
|
-
var LLM_BENCH_UNVERIFIED_LINE = "Credits pay only for a verified email, and this account's is not verified yet (credits_unverified) \u2014 nothing was sent, nothing was spent. Verify it, then re-run.";
|
|
24406
|
+
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN coin \u2014 your wallet, not a player's, and one this build's asset allowance cannot see. Re-run with --max-coins <n> --user-approved only after the person at the keyboard agreed to the number.";
|
|
24412
24407
|
var LLM_LANE_OFF_LINE = "The runtime LLM lane is off on this stand \u2014 in-game generate() answers 404 here, and nothing can be benchmarked.";
|
|
24413
24408
|
var LLM_TOOLS_CONVERT_HINT = [
|
|
24414
24409
|
"In-game model calls need a hosted game. This folder is a Genex Tools workspace, so nothing here",
|
|
@@ -24421,7 +24416,7 @@ function serverSlotCounts(body) {
|
|
|
24421
24416
|
return { held: body.slotsHeld, limit: body.slotLimit };
|
|
24422
24417
|
}
|
|
24423
24418
|
var PENDING_SLOT_RELEASE_MINUTES = 10;
|
|
24424
|
-
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so
|
|
24419
|
+
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so its coin stays held until the bill resolves; such a row stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`;
|
|
24425
24420
|
var LLM_STATUS_NO_COUNT_LINE = "This stand does not report how many slots are held across your projects, so no count is shown.";
|
|
24426
24421
|
var LLM_CANCEL_AWAITING_BILL = "this call already stopped; its bill is awaiting the provider";
|
|
24427
24422
|
var LLM_CANCEL_ALREADY_FINAL = "this call is already final";
|
|
@@ -24435,33 +24430,11 @@ var DEFAULT_SAMPLE_TIMEOUT_SEC = 180;
|
|
|
24435
24430
|
function parseProviderKey(value) {
|
|
24436
24431
|
return value === "ok" || value === "rejected" || value === "unknown" ? value : null;
|
|
24437
24432
|
}
|
|
24438
|
-
function bpsOrNull(value) {
|
|
24439
|
-
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
24440
|
-
}
|
|
24441
|
-
function standingCreditGrantOrNull(value) {
|
|
24442
|
-
if (!value || typeof value !== "object") return null;
|
|
24443
|
-
const v = value;
|
|
24444
|
-
const whole = (x) => typeof x === "number" && Number.isSafeInteger(x) && x >= 0;
|
|
24445
|
-
if (!whole(v.minCredits) || !whole(v.maxCredits) || !whole(v.stepCredits) || !whole(v.defaultCredits)) return null;
|
|
24446
|
-
return {
|
|
24447
|
-
minCredits: v.minCredits,
|
|
24448
|
-
maxCredits: v.maxCredits,
|
|
24449
|
-
stepCredits: v.stepCredits,
|
|
24450
|
-
defaultCredits: v.defaultCredits,
|
|
24451
|
-
...whole(v.minUsdCents) ? { minUsdCents: v.minUsdCents } : {},
|
|
24452
|
-
...whole(v.maxUsdCents) ? { maxUsdCents: v.maxUsdCents } : {},
|
|
24453
|
-
...whole(v.stepUsdCents) ? { stepUsdCents: v.stepUsdCents } : {}
|
|
24454
|
-
};
|
|
24455
|
-
}
|
|
24456
24433
|
async function readRuntimeLane(apiUrl, token) {
|
|
24457
24434
|
const empty = (state) => ({
|
|
24458
24435
|
state,
|
|
24459
24436
|
models: null,
|
|
24460
24437
|
recommendedDeclaredHeadroomBps: null,
|
|
24461
|
-
currency: null,
|
|
24462
|
-
feeBps: null,
|
|
24463
|
-
usdCentsPerCredit: null,
|
|
24464
|
-
standingCreditGrant: null,
|
|
24465
24438
|
externalProviders: null,
|
|
24466
24439
|
total: null,
|
|
24467
24440
|
featuredCount: null,
|
|
@@ -24480,11 +24453,7 @@ async function readRuntimeLane(apiUrl, token) {
|
|
|
24480
24453
|
return {
|
|
24481
24454
|
state: "live",
|
|
24482
24455
|
models: body.models,
|
|
24483
|
-
recommendedDeclaredHeadroomBps:
|
|
24484
|
-
currency: body.currency === "credits" || body.currency === "coins" ? body.currency : null,
|
|
24485
|
-
feeBps: bpsOrNull(body.feeBps),
|
|
24486
|
-
usdCentsPerCredit: typeof body.usdCentsPerCredit === "number" && Number.isSafeInteger(body.usdCentsPerCredit) && body.usdCentsPerCredit >= 1 ? body.usdCentsPerCredit : null,
|
|
24487
|
-
standingCreditGrant: standingCreditGrantOrNull(body.standingCreditGrant),
|
|
24456
|
+
recommendedDeclaredHeadroomBps: typeof body.recommendedDeclaredHeadroomBps === "number" && Number.isFinite(body.recommendedDeclaredHeadroomBps) ? body.recommendedDeclaredHeadroomBps : null,
|
|
24488
24457
|
externalProviders: Array.isArray(body.externalProviders) ? body.externalProviders : null,
|
|
24489
24458
|
total: typeof body.total === "number" && Number.isFinite(body.total) ? body.total : null,
|
|
24490
24459
|
featuredCount: typeof body.featuredCount === "number" && Number.isFinite(body.featuredCount) ? body.featuredCount : null,
|
|
@@ -24500,45 +24469,18 @@ function percentile2(values, p) {
|
|
|
24500
24469
|
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
24501
24470
|
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
24502
24471
|
}
|
|
24503
|
-
function
|
|
24504
|
-
|
|
24505
|
-
const sorted = [...values].sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
24506
|
-
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
24507
|
-
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
24472
|
+
function recommendEstimateCoins(p95, headroomBps) {
|
|
24473
|
+
return Math.max(1, Math.ceil(p95 * (1e4 + headroomBps) / 1e4));
|
|
24508
24474
|
}
|
|
24509
|
-
|
|
24510
|
-
|
|
24511
|
-
if (costPicos <= 0n) return 1;
|
|
24512
|
-
const num = costPicos * BigInt(1e4 + terms.feeBps) * BigInt(1e4 + terms.headroomBps);
|
|
24513
|
-
const den = 100000000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
24514
|
-
return Math.max(1, Number((num + den - 1n) / den));
|
|
24515
|
-
}
|
|
24516
|
-
function averageCreditsPerCall(costsPicos, terms) {
|
|
24517
|
-
if (costsPicos.length === 0) return null;
|
|
24518
|
-
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
24519
|
-
return Number(sum * BigInt(1e4 + terms.feeBps)) / Number(BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT);
|
|
24520
|
-
}
|
|
24521
|
-
function perCallEstimateCreditsFor(costsPicos, terms) {
|
|
24522
|
-
if (costsPicos.length === 0) return null;
|
|
24523
|
-
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
24524
|
-
const num = sum * BigInt(1e4 + terms.feeBps);
|
|
24525
|
-
const den = BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
24526
|
-
return Math.max(1, Number((num + den - 1n) / den));
|
|
24527
|
-
}
|
|
24528
|
-
function formatCredits(value) {
|
|
24529
|
-
if (!Number.isFinite(value)) return "\u2014";
|
|
24530
|
-
if (value >= 100) return String(Math.round(value));
|
|
24531
|
-
return String(Number(value.toPrecision(value < 1 ? 2 : 3)));
|
|
24532
|
-
}
|
|
24533
|
-
function usdFromPicos(picos) {
|
|
24534
|
-
if (picos === null || picos === void 0) return "unknown";
|
|
24535
|
-
const value = typeof picos === "bigint" ? picos : /^\d+$/.test(picos) ? BigInt(picos) : null;
|
|
24536
|
-
if (value === null) return "unknown";
|
|
24537
|
-
return `$${(Number(value) / 1e12).toFixed(6)}`;
|
|
24475
|
+
function recommendPerCallMaxCoins(p95, max, headroomBps) {
|
|
24476
|
+
return Math.max(recommendEstimateCoins(p95, headroomBps), recommendEstimateCoins(max, headroomBps));
|
|
24538
24477
|
}
|
|
24539
24478
|
function usdPerMillion(value) {
|
|
24540
24479
|
return `$${value.toFixed(value < 1 ? 4 : 2)}`;
|
|
24541
24480
|
}
|
|
24481
|
+
function usd(value) {
|
|
24482
|
+
return value === null ? "unknown" : `$${value.toFixed(6)}`;
|
|
24483
|
+
}
|
|
24542
24484
|
async function runLlm(opts = {}) {
|
|
24543
24485
|
const log = createLogger({ quiet: opts.quiet || opts.json });
|
|
24544
24486
|
const cwd = opts.cwd ?? process.cwd();
|
|
@@ -24564,23 +24506,16 @@ async function runLlm(opts = {}) {
|
|
|
24564
24506
|
return;
|
|
24565
24507
|
}
|
|
24566
24508
|
if (sub === "bench") {
|
|
24567
|
-
if (opts.maxCoins
|
|
24568
|
-
`);
|
|
24569
|
-
if (opts.maxCoins !== void 0 && opts.maxCredits !== void 0 && opts.maxCoins !== opts.maxCredits) {
|
|
24570
|
-
fail5("llm bench", `Pass --max-credits alone \u2014 --max-coins is its deprecated name, and the two disagree (${opts.maxCredits} vs ${opts.maxCoins}).`);
|
|
24571
|
-
return;
|
|
24572
|
-
}
|
|
24573
|
-
const approvedCredits = opts.maxCredits ?? opts.maxCoins;
|
|
24574
|
-
if (approvedCredits === void 0 || opts.userApproved !== true) {
|
|
24509
|
+
if (opts.maxCoins === void 0 || opts.userApproved !== true) {
|
|
24575
24510
|
fail5("llm bench", LLM_BENCH_APPROVAL_REQUIRED);
|
|
24576
24511
|
return;
|
|
24577
24512
|
}
|
|
24578
|
-
if (!Number.isInteger(
|
|
24579
|
-
fail5("llm bench", `--max-
|
|
24513
|
+
if (!Number.isInteger(opts.maxCoins) || opts.maxCoins < 1) {
|
|
24514
|
+
fail5("llm bench", `--max-coins takes a whole number of coin, 1 or more (got ${String(opts.maxCoins)}).`);
|
|
24580
24515
|
return;
|
|
24581
24516
|
}
|
|
24582
24517
|
if (!opts.benchPrompt?.trim()) {
|
|
24583
|
-
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-
|
|
24518
|
+
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-coins <n> --user-approved.');
|
|
24584
24519
|
return;
|
|
24585
24520
|
}
|
|
24586
24521
|
if (opts.jsonOutput && opts.textOutput) {
|
|
@@ -24672,10 +24607,6 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24672
24607
|
models: null,
|
|
24673
24608
|
externalProviders: null,
|
|
24674
24609
|
recommendedDeclaredHeadroomBps: null,
|
|
24675
|
-
currency: null,
|
|
24676
|
-
feeBps: null,
|
|
24677
|
-
usdCentsPerCredit: null,
|
|
24678
|
-
standingCreditGrant: null,
|
|
24679
24610
|
total: null,
|
|
24680
24611
|
featuredCount: null,
|
|
24681
24612
|
// The stand was never asked, so the verdict is unknown — but the key is
|
|
@@ -24701,12 +24632,6 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24701
24632
|
models: lane.models,
|
|
24702
24633
|
externalProviders: lane.externalProviders,
|
|
24703
24634
|
recommendedDeclaredHeadroomBps: lane.recommendedDeclaredHeadroomBps,
|
|
24704
|
-
// The credit terms a call is billed on, the server's — null on a stand
|
|
24705
|
-
// that predates credit billing, never a number the CLI filled in.
|
|
24706
|
-
currency: lane.currency,
|
|
24707
|
-
feeBps: lane.feeBps,
|
|
24708
|
-
usdCentsPerCredit: lane.usdCentsPerCredit,
|
|
24709
|
-
standingCreditGrant: lane.standingCreditGrant,
|
|
24710
24635
|
total: lane.total ?? lane.models?.length ?? null,
|
|
24711
24636
|
featuredCount: lane.featuredCount ?? (lane.models ? lane.models.filter((m) => m.featured).length : null),
|
|
24712
24637
|
// `null` when the lane is not live or the server predates the probe.
|
|
@@ -24753,9 +24678,8 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24753
24678
|
);
|
|
24754
24679
|
for (const m of shown) {
|
|
24755
24680
|
const plan = m.personalPlan ? ` \xB7 personal plan: ${m.personalPlan}` : "";
|
|
24756
|
-
const kind = m.kind === "typesafe" ? " \xB7 judge (classifier): judge() under a budget, not generate() or bench" : "";
|
|
24757
24681
|
log.plain(
|
|
24758
|
-
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}
|
|
24682
|
+
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}`
|
|
24759
24683
|
);
|
|
24760
24684
|
}
|
|
24761
24685
|
if (hiddenCount > 0) {
|
|
@@ -24771,104 +24695,74 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24771
24695
|
}
|
|
24772
24696
|
log.plain("");
|
|
24773
24697
|
if (lane.recommendedDeclaredHeadroomBps === null) {
|
|
24774
|
-
log.dim(" This stand serves no recommended headroom, so no
|
|
24698
|
+
log.dim(" This stand serves no recommended headroom, so no price can be recommended from a bench.");
|
|
24775
24699
|
} else {
|
|
24776
|
-
log.dim(` Recommended headroom over
|
|
24777
|
-
}
|
|
24778
|
-
if (lane.feeBps === null) {
|
|
24779
|
-
log.dim(" This stand serves no platform fee \u2014 it predates credit billing, so no ceiling can be recommended from a bench.");
|
|
24780
|
-
} else {
|
|
24781
|
-
log.dim(` In-game calls are billed in the player's credits, as used: the provider's cost plus this stand's platform fee (${lane.feeBps} bps).`);
|
|
24700
|
+
log.dim(` Recommended headroom over benchmarked coins on this stand: ${lane.recommendedDeclaredHeadroomBps} bps.`);
|
|
24782
24701
|
}
|
|
24783
24702
|
if (toolsOnly) {
|
|
24784
24703
|
printConvertHint();
|
|
24785
24704
|
return;
|
|
24786
24705
|
}
|
|
24787
|
-
log.dim(" Those per-million rates are the PROVIDER's
|
|
24788
|
-
log.dim(`
|
|
24706
|
+
log.dim(" Those per-million rates are the PROVIDER's; what your game is charged is the platform");
|
|
24707
|
+
log.dim(` tariff on top, which only a real call reveals: ${c.cyan('genex llm bench "<prompt>" --max-coins <n> --user-approved')}`);
|
|
24789
24708
|
}
|
|
24790
24709
|
function wholeOrNull(value) {
|
|
24791
24710
|
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
24792
24711
|
}
|
|
24793
|
-
function costPicosOf(usage) {
|
|
24794
|
-
if (typeof usage?.costUsdPicos === "string" && /^\d+$/.test(usage.costUsdPicos)) return usage.costUsdPicos;
|
|
24795
|
-
if (typeof usage?.costUsd === "number" && Number.isFinite(usage.costUsd) && usage.costUsd >= 0) {
|
|
24796
|
-
return String(BigInt(Math.round(usage.costUsd * 1e12)));
|
|
24797
|
-
}
|
|
24798
|
-
return null;
|
|
24799
|
-
}
|
|
24800
24712
|
function sampleFrom(id, settled) {
|
|
24801
24713
|
const usage = settled?.usage;
|
|
24802
|
-
const picos = costPicosOf(usage);
|
|
24803
24714
|
return {
|
|
24804
24715
|
id,
|
|
24805
24716
|
status: settled?.status ?? null,
|
|
24806
24717
|
billingStatus: settled?.billingStatus ?? null,
|
|
24807
|
-
|
|
24808
|
-
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd :
|
|
24809
|
-
costUsdPicos: picos,
|
|
24718
|
+
chargedCoins: typeof settled?.chargedCoins === "number" ? settled.chargedCoins : null,
|
|
24719
|
+
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd : null,
|
|
24810
24720
|
error: settled?.error ?? (settled ? null : "timed_out"),
|
|
24811
24721
|
providerMessage: typeof settled?.providerMessage === "string" && settled.providerMessage.trim() ? settled.providerMessage.trim() : null,
|
|
24812
24722
|
inputTokens: wholeOrNull(usage?.inputTokens),
|
|
24813
24723
|
outputTokens: wholeOrNull(usage?.outputTokens),
|
|
24814
24724
|
maxOutputTokens: wholeOrNull(usage?.maxOutputTokens),
|
|
24815
|
-
|
|
24725
|
+
fitCoins: typeof usage?.fitCoins === "number" && Number.isSafeInteger(usage.fitCoins) && usage.fitCoins >= 1 ? usage.fitCoins : null,
|
|
24816
24726
|
fitExceedsCap: typeof usage?.fitExceedsCap === "boolean" ? usage.fitExceedsCap : null,
|
|
24817
24727
|
truncated: typeof usage?.truncated === "boolean" ? usage.truncated : settled?.error === "provider_token_limit" ? true : null
|
|
24818
24728
|
};
|
|
24819
24729
|
}
|
|
24820
24730
|
var PROVIDER_TOKEN_LIMIT = "provider_token_limit";
|
|
24821
|
-
function benchRecommendation(results,
|
|
24822
|
-
const settled = results.filter(
|
|
24823
|
-
|
|
24824
|
-
);
|
|
24825
|
-
const
|
|
24826
|
-
const
|
|
24827
|
-
const fits = settled.map((r) => r.fitCredits).filter((v) => typeof v === "number");
|
|
24828
|
-
const costP95 = percentilePicos(costs, 95);
|
|
24829
|
-
const costMax = costs.length ? costs.reduce((a, b) => b > a ? b : a) : null;
|
|
24731
|
+
function benchRecommendation(results, headroomBps) {
|
|
24732
|
+
const settled = results.filter((r) => r.status === "succeeded" && r.billingStatus === "final" && typeof r.chargedCoins === "number");
|
|
24733
|
+
const charged = settled.map((r) => r.chargedCoins);
|
|
24734
|
+
const fits = settled.map((r) => r.fitCoins).filter((v) => typeof v === "number");
|
|
24735
|
+
const p95 = percentile2(charged, 95);
|
|
24736
|
+
const max = charged.length ? Math.max(...charged) : null;
|
|
24830
24737
|
const fitP95 = percentile2(fits, 95);
|
|
24831
24738
|
const fitMax = fits.length ? Math.max(...fits) : null;
|
|
24832
24739
|
const exceedsStandCap = results.some((r) => r.fitExceedsCap === true);
|
|
24833
24740
|
const cutOffSamples = results.filter((r) => r.error === PROVIDER_TOKEN_LIMIT && r.fitExceedsCap !== true).length;
|
|
24834
24741
|
const noRecommendationReason = exceedsStandCap ? "answer_exceeds_stand_cap" : cutOffSamples > 0 ? "answer_cut_off" : null;
|
|
24835
|
-
const
|
|
24836
|
-
const
|
|
24837
|
-
const
|
|
24838
|
-
const billing = feeBps !== null && usdCentsPerCredit !== null ? { feeBps, usdCentsPerCredit } : null;
|
|
24839
|
-
const costBased = full !== null && costP95 !== null ? ceilingCreditsFor(costP95, full) : null;
|
|
24840
|
-
const recommended = costBased === null || noRecommendationReason !== null ? null : Math.max(costBased, fitP95 ?? 0);
|
|
24841
|
-
const ceiling = recommended === null || costMax === null || full === null ? null : Math.max(ceilingCreditsFor(costMax, full), recommended, fitMax ?? 0);
|
|
24842
|
-
const average = billing !== null ? averageCreditsPerCall(costs, billing) : null;
|
|
24843
|
-
const estimate = ceiling === null || billing === null ? null : Math.min(ceiling, perCallEstimateCreditsFor(costs, billing) ?? 1);
|
|
24742
|
+
const chargedBased = p95 !== null && headroomBps !== null ? recommendEstimateCoins(p95, headroomBps) : null;
|
|
24743
|
+
const recommended = chargedBased === null || noRecommendationReason !== null ? null : Math.max(chargedBased, fitP95 ?? 0);
|
|
24744
|
+
const ceiling = recommended === null || p95 === null || max === null || headroomBps === null ? null : Math.max(recommendPerCallMaxCoins(p95, max, headroomBps), recommended, fitMax ?? 0);
|
|
24844
24745
|
return {
|
|
24845
|
-
costs,
|
|
24846
24746
|
charged,
|
|
24847
|
-
|
|
24848
|
-
|
|
24849
|
-
|
|
24850
|
-
chargedP50: percentile2(charged, 50),
|
|
24851
|
-
chargedP95: percentile2(charged, 95),
|
|
24852
|
-
chargedMax: charged.length ? Math.max(...charged) : null,
|
|
24747
|
+
p50: percentile2(charged, 50),
|
|
24748
|
+
p95,
|
|
24749
|
+
max,
|
|
24853
24750
|
fits,
|
|
24854
24751
|
fitP95,
|
|
24855
24752
|
fitMax,
|
|
24856
|
-
|
|
24857
|
-
|
|
24858
|
-
|
|
24859
|
-
|
|
24860
|
-
recommendedPerCallEstimateCredits: estimate,
|
|
24861
|
-
lengthRaisedCeiling: recommended !== null && costBased !== null && recommended > costBased,
|
|
24753
|
+
chargedBasedEstimateCoins: chargedBased,
|
|
24754
|
+
recommendedEstimateCoins: recommended,
|
|
24755
|
+
recommendedPerCallMaxCoins: ceiling,
|
|
24756
|
+
lengthRaisedPrice: recommended !== null && chargedBased !== null && recommended > chargedBased,
|
|
24862
24757
|
cutOffSamples,
|
|
24863
24758
|
exceedsStandCap,
|
|
24864
|
-
noRecommendationReason
|
|
24865
|
-
missingTerms
|
|
24759
|
+
noRecommendationReason
|
|
24866
24760
|
};
|
|
24867
24761
|
}
|
|
24868
|
-
var LLM_LENGTH_RAISED_LINE = "The
|
|
24869
|
-
var LLM_EXCEEDS_STAND_CAP_LINE = "No
|
|
24870
|
-
function benchCutOffLine(cutOff,
|
|
24871
|
-
return `No
|
|
24762
|
+
var LLM_LENGTH_RAISED_LINE = "The declared price also pays for how long the answer may be: at the charged-based price the answer would be cut off, so declare this one.";
|
|
24763
|
+
var LLM_EXCEEDS_STAND_CAP_LINE = "No price recommended: this answer is longer than one call on this stand may produce. Ask for a shorter answer \u2014 fewer fields, shorter strings, a length the prompt states \u2014 and benchmark again.";
|
|
24764
|
+
function benchCutOffLine(cutOff, maxCoins) {
|
|
24765
|
+
return `No price recommended: ${cutOff} sample${cutOff === 1 ? " was" : "s were"} cut off at this ceiling (--max-coins ${maxCoins}) before the answer was finished, so its real length is unknown. Re-run with a higher --max-coins.`;
|
|
24872
24766
|
}
|
|
24873
24767
|
var ACTIVE_STATUSES = /* @__PURE__ */ new Set(["requires_confirmation", "queued", "dispatching", "awaiting_external"]);
|
|
24874
24768
|
function rowState(row) {
|
|
@@ -24879,22 +24773,7 @@ function rowState(row) {
|
|
|
24879
24773
|
function rowHoldsSlot(row) {
|
|
24880
24774
|
if (typeof row.slotHeld === "boolean") return row.slotHeld;
|
|
24881
24775
|
const state = rowState(row);
|
|
24882
|
-
return state === "active" || state === "awaiting bill" && (
|
|
24883
|
-
}
|
|
24884
|
-
function rowMoney(row) {
|
|
24885
|
-
const credits = row.currency === "credits" || row.currency === void 0 && typeof row.reservedCredits === "number" && typeof row.reservedCoins !== "number";
|
|
24886
|
-
if (credits) {
|
|
24887
|
-
return {
|
|
24888
|
-
unit: "credits",
|
|
24889
|
-
reserved: typeof row.reservedCredits === "number" ? row.reservedCredits : null,
|
|
24890
|
-
charged: typeof row.chargedCredits === "number" ? row.chargedCredits : null
|
|
24891
|
-
};
|
|
24892
|
-
}
|
|
24893
|
-
return {
|
|
24894
|
-
unit: "coin",
|
|
24895
|
-
reserved: typeof row.reservedCoins === "number" ? row.reservedCoins : null,
|
|
24896
|
-
charged: typeof row.chargedCoins === "number" ? row.chargedCoins : null
|
|
24897
|
-
};
|
|
24776
|
+
return state === "active" || state === "awaiting bill" && (row.reservedCoins ?? 0) > 0;
|
|
24898
24777
|
}
|
|
24899
24778
|
function providerRefusalStatus(error) {
|
|
24900
24779
|
const m = /^provider_http_(\d{3})$/.exec(error ?? "");
|
|
@@ -24913,7 +24792,7 @@ function formatAge(iso, now = Date.now()) {
|
|
|
24913
24792
|
}
|
|
24914
24793
|
async function runBench(args) {
|
|
24915
24794
|
const { apiUrl, token, projectId, cwd, opts, log } = args;
|
|
24916
|
-
const
|
|
24795
|
+
const maxCoins = opts.maxCoins;
|
|
24917
24796
|
const samples = opts.samples ?? DEFAULT_BENCH_SAMPLES;
|
|
24918
24797
|
const prompt = opts.benchPrompt.trim();
|
|
24919
24798
|
const auth = { Authorization: `Bearer ${token}`, "Content-Type": "application/json" };
|
|
@@ -24935,7 +24814,7 @@ async function runBench(args) {
|
|
|
24935
24814
|
const lane = await readRuntimeLane(apiUrl, token);
|
|
24936
24815
|
if (lane.state !== "live" || !lane.models || lane.models.length === 0) {
|
|
24937
24816
|
if (lane.state === "off") {
|
|
24938
|
-
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
24817
|
+
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCoins, opts), status: "off", error: null });
|
|
24939
24818
|
else {
|
|
24940
24819
|
log.plain(LLM_LANE_OFF_LINE);
|
|
24941
24820
|
log.dim(" Nothing was spent.");
|
|
@@ -24962,27 +24841,17 @@ async function runBench(args) {
|
|
|
24962
24841
|
);
|
|
24963
24842
|
return;
|
|
24964
24843
|
}
|
|
24965
|
-
const
|
|
24966
|
-
if (walletBefore?.emailVerified === false) {
|
|
24967
|
-
benchFailed(opts, log, LLM_BENCH_UNVERIFIED_LINE);
|
|
24968
|
-
return;
|
|
24969
|
-
}
|
|
24970
|
-
if (walletBefore !== null && walletBefore.spendable < maxCredits) {
|
|
24971
|
-
benchFailed(opts, log, benchBalanceShortLine(walletBefore.spendable, maxCredits));
|
|
24972
|
-
return;
|
|
24973
|
-
}
|
|
24974
|
-
const balanceBefore = walletBefore?.spendable ?? null;
|
|
24844
|
+
const balanceBefore = await readCoinBalance(apiUrl, token);
|
|
24975
24845
|
if (!opts.json) {
|
|
24976
24846
|
log.plain(c.bold("genex llm bench"));
|
|
24977
24847
|
log.dim(` ${apiUrl}`);
|
|
24978
24848
|
log.plain("");
|
|
24979
24849
|
log.plain(` Model ${c.cyan(modelId)}`);
|
|
24980
24850
|
log.plain(` Samples ${samples} real attempt${samples === 1 ? "" : "s"}, ${outputFormat} output`);
|
|
24981
|
-
log.plain(` Approved ${
|
|
24851
|
+
log.plain(` Approved ${maxCoins} coin per attempt \u2014 at worst ${maxCoins * samples} coin for this run`);
|
|
24982
24852
|
log.plain(
|
|
24983
|
-
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore}
|
|
24853
|
+
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore} coin spendable`}`
|
|
24984
24854
|
);
|
|
24985
|
-
log.dim(" Billed as used: each attempt is charged its real cost plus the platform fee, rounded up to a whole credit.");
|
|
24986
24855
|
log.plain("");
|
|
24987
24856
|
}
|
|
24988
24857
|
const base = `${apiUrl}/api/runtime/development/projects/${encodeURIComponent(projectId)}/generations`;
|
|
@@ -25008,7 +24877,7 @@ async function runBench(args) {
|
|
|
25008
24877
|
const res = await call2(base, {
|
|
25009
24878
|
method: "POST",
|
|
25010
24879
|
body: JSON.stringify({
|
|
25011
|
-
|
|
24880
|
+
maxCoins,
|
|
25012
24881
|
request: {
|
|
25013
24882
|
idempotencyKey: `bench-${randomUUID2()}`,
|
|
25014
24883
|
modelId,
|
|
@@ -25038,14 +24907,13 @@ async function runBench(args) {
|
|
|
25038
24907
|
if (refusedAt !== null) providerRefused++;
|
|
25039
24908
|
if (!opts.json) {
|
|
25040
24909
|
const length = row.outputTokens !== null ? ` \xB7 ${row.outputTokens}${row.maxOutputTokens !== null ? ` of ${row.maxOutputTokens}` : ""} tokens out` : "";
|
|
25041
|
-
const charged = row.chargedCredits === null ? "\u2014" : String(row.chargedCredits);
|
|
25042
24910
|
log.plain(
|
|
25043
|
-
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${
|
|
24911
|
+
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${String(row.chargedCoins ?? "\u2014").padStart(4)} coin charged \xB7 provider ${usd(row.costUsd)}${length}${row.error ? ` \xB7 ${row.error}` : ""}`
|
|
25044
24912
|
);
|
|
25045
24913
|
if (refusedAt !== null) printProviderRefusal(log, refusedAt, row.providerMessage, row.status === "unknown");
|
|
25046
24914
|
if (row.error === PROVIDER_TOKEN_LIMIT) {
|
|
25047
24915
|
log.dim(
|
|
25048
|
-
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no
|
|
24916
|
+
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no price makes room for it; it is not a sample." : ` The answer was cut off at this ceiling (--max-coins ${maxCoins}) before it was finished \u2014 it was charged, and it is not a sample.`
|
|
25049
24917
|
);
|
|
25050
24918
|
}
|
|
25051
24919
|
}
|
|
@@ -25053,19 +24921,16 @@ async function runBench(args) {
|
|
|
25053
24921
|
} finally {
|
|
25054
24922
|
process.removeListener("SIGINT", onSigint);
|
|
25055
24923
|
}
|
|
25056
|
-
const
|
|
25057
|
-
|
|
25058
|
-
|
|
25059
|
-
|
|
25060
|
-
|
|
25061
|
-
const
|
|
25062
|
-
const settledCount = rec.charged.length;
|
|
24924
|
+
const headroomBps = lane.recommendedDeclaredHeadroomBps;
|
|
24925
|
+
const rec = benchRecommendation(results, headroomBps);
|
|
24926
|
+
const { charged, p50, p95, max } = rec;
|
|
24927
|
+
const settledCount = charged.length;
|
|
24928
|
+
const recommended = rec.recommendedEstimateCoins;
|
|
24929
|
+
const ceiling = rec.recommendedPerCallMaxCoins;
|
|
25063
24930
|
const lengthBlocked = rec.noRecommendationReason !== null;
|
|
25064
|
-
const balanceAfter =
|
|
25065
|
-
const picosOrNull = (v) => v === null ? null : String(v);
|
|
24931
|
+
const balanceAfter = await readCoinBalance(apiUrl, token);
|
|
25066
24932
|
const record = {
|
|
25067
|
-
v:
|
|
25068
|
-
currency: "credits",
|
|
24933
|
+
v: 1,
|
|
25069
24934
|
ranAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
25070
24935
|
apiUrl,
|
|
25071
24936
|
projectId,
|
|
@@ -25074,34 +24939,28 @@ async function runBench(args) {
|
|
|
25074
24939
|
outputFormat,
|
|
25075
24940
|
samples,
|
|
25076
24941
|
completed: settledCount,
|
|
25077
|
-
|
|
25078
|
-
|
|
25079
|
-
|
|
25080
|
-
|
|
25081
|
-
|
|
25082
|
-
|
|
24942
|
+
chargedCoins: charged,
|
|
24943
|
+
p50,
|
|
24944
|
+
p95,
|
|
24945
|
+
max,
|
|
24946
|
+
recommendedDeclaredHeadroomBps: headroomBps,
|
|
24947
|
+
recommendedEstimateCoins: recommended,
|
|
24948
|
+
recommendedPerCallMaxCoins: ceiling,
|
|
24949
|
+
maxCoinsPerSample: maxCoins,
|
|
24950
|
+
fitCoins: rec.fits,
|
|
25083
24951
|
fitP95: rec.fitP95,
|
|
25084
|
-
|
|
25085
|
-
|
|
25086
|
-
feeBps: terms.feeBps,
|
|
25087
|
-
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
25088
|
-
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
25089
|
-
recommendedMaxCredits: rec.recommendedMaxCredits,
|
|
25090
|
-
recommendedPerCallMaxCredits: rec.recommendedPerCallMaxCredits,
|
|
25091
|
-
averageCreditsPerCall: rec.averageCreditsPerCall,
|
|
25092
|
-
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
25093
|
-
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
24952
|
+
chargedBasedEstimateCoins: rec.chargedBasedEstimateCoins,
|
|
24953
|
+
lengthRaisedPrice: rec.lengthRaisedPrice,
|
|
25094
24954
|
cutOffSamples: rec.cutOffSamples,
|
|
25095
24955
|
noRecommendationReason: rec.noRecommendationReason
|
|
25096
24956
|
};
|
|
25097
24957
|
const savedTo = await saveBench(cwd, record);
|
|
25098
24958
|
if (opts.json) {
|
|
25099
|
-
const ok =
|
|
24959
|
+
const ok = charged.length > 0 && !lengthBlocked;
|
|
25100
24960
|
writeJsonLine({
|
|
25101
24961
|
command: "llm bench",
|
|
25102
24962
|
status: ok ? "ok" : "failed",
|
|
25103
24963
|
error: ok ? null : rec.noRecommendationReason ?? "no_sample_settled",
|
|
25104
|
-
currency: "credits",
|
|
25105
24964
|
apiUrl,
|
|
25106
24965
|
projectId,
|
|
25107
24966
|
modelId,
|
|
@@ -25110,54 +24969,35 @@ async function runBench(args) {
|
|
|
25110
24969
|
schemaPath: opts.schemaPath ?? null,
|
|
25111
24970
|
samples,
|
|
25112
24971
|
completed: settledCount,
|
|
25113
|
-
|
|
25114
|
-
// Spendable credits before and after the run; null when unreadable.
|
|
24972
|
+
maxCoinsPerSample: maxCoins,
|
|
25115
24973
|
balanceBefore,
|
|
25116
24974
|
balanceAfter,
|
|
25117
24975
|
results,
|
|
25118
24976
|
// The create-time refusal that ended the run, or null. Beside the
|
|
25119
24977
|
// samples, never among them: no attempt existed and nothing was spent.
|
|
25120
24978
|
refusal: refusal2,
|
|
25121
|
-
|
|
25122
|
-
//
|
|
25123
|
-
costUsdPicos: record.cost,
|
|
25124
|
-
costUsd: {
|
|
25125
|
-
p50: rec.costP50 === null ? null : Number(rec.costP50) / 1e12,
|
|
25126
|
-
p95: rec.costP95 === null ? null : Number(rec.costP95) / 1e12,
|
|
25127
|
-
max: rec.costMax === null ? null : Number(rec.costMax) / 1e12
|
|
25128
|
-
},
|
|
25129
|
-
chargedCredits: record.charged,
|
|
25130
|
-
// The answer's LENGTH, priced by the server per sample (`fitCredits`,
|
|
24979
|
+
chargedCoins: { p50, p95, max },
|
|
24980
|
+
// The answer's LENGTH, priced by the server per sample (`fitCoins`,
|
|
25131
24981
|
// headroom already on the tokens), over the settled samples.
|
|
25132
|
-
|
|
25133
|
-
|
|
25134
|
-
|
|
25135
|
-
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
25136
|
-
missingTerms: rec.missingTerms,
|
|
25137
|
-
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
25138
|
-
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
24982
|
+
fitCoins: { p95: rec.fitP95, max: rec.fitMax },
|
|
24983
|
+
chargedBasedEstimateCoins: rec.chargedBasedEstimateCoins,
|
|
24984
|
+
lengthRaisedPrice: rec.lengthRaisedPrice,
|
|
25139
24985
|
cutOffSamples: rec.cutOffSamples,
|
|
25140
24986
|
exceedsStandCap: rec.exceedsStandCap,
|
|
25141
24987
|
noRecommendationReason: rec.noRecommendationReason,
|
|
25142
|
-
|
|
25143
|
-
|
|
25144
|
-
|
|
25145
|
-
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
24988
|
+
recommendedDeclaredHeadroomBps: headroomBps,
|
|
24989
|
+
recommendedEstimateCoins: recommended,
|
|
24990
|
+
recommendedPerCallMaxCoins: ceiling,
|
|
25146
24991
|
savedTo
|
|
25147
24992
|
});
|
|
25148
24993
|
if (!ok) process.exitCode = 1;
|
|
25149
24994
|
return;
|
|
25150
24995
|
}
|
|
25151
24996
|
log.plain("");
|
|
25152
|
-
const printMeasured = () => {
|
|
25153
|
-
log.plain(c.bold(" Real cost per call"));
|
|
25154
|
-
log.plain(` p50 ${usdFromPicos(rec.costP50)} p95 ${usdFromPicos(rec.costP95)} max ${usdFromPicos(rec.costMax)} (${settledCount} of ${samples} settled)`);
|
|
25155
|
-
log.plain(c.bold(" Charged credits"));
|
|
25156
|
-
log.plain(` p50 ${rec.chargedP50} p95 ${rec.chargedP95} max ${rec.chargedMax} (whole credits, rounded up per call)`);
|
|
25157
|
-
};
|
|
25158
24997
|
if (lengthBlocked) {
|
|
25159
|
-
if (
|
|
25160
|
-
|
|
24998
|
+
if (charged.length > 0) {
|
|
24999
|
+
log.plain(c.bold(" Charged coins"));
|
|
25000
|
+
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
25161
25001
|
log.plain("");
|
|
25162
25002
|
}
|
|
25163
25003
|
printRecommendation(log, record);
|
|
@@ -25165,16 +25005,17 @@ async function runBench(args) {
|
|
|
25165
25005
|
process.exitCode = 1;
|
|
25166
25006
|
return;
|
|
25167
25007
|
}
|
|
25168
|
-
if (
|
|
25008
|
+
if (charged.length === 0) {
|
|
25169
25009
|
log.error(
|
|
25170
|
-
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to
|
|
25010
|
+
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to price from." : " No sample settled, so there is nothing to price from."
|
|
25171
25011
|
);
|
|
25172
25012
|
process.exitCode = 1;
|
|
25173
25013
|
return;
|
|
25174
25014
|
}
|
|
25175
|
-
|
|
25015
|
+
log.plain(c.bold(" Charged coins"));
|
|
25016
|
+
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
25176
25017
|
if (balanceBefore !== null && balanceAfter !== null) {
|
|
25177
|
-
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter}
|
|
25018
|
+
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter} coin spendable`);
|
|
25178
25019
|
}
|
|
25179
25020
|
log.plain("");
|
|
25180
25021
|
printRecommendation(log, record);
|
|
@@ -25186,58 +25027,43 @@ function printRecommendation(log, record) {
|
|
|
25186
25027
|
return;
|
|
25187
25028
|
}
|
|
25188
25029
|
if (record.noRecommendationReason === "answer_cut_off") {
|
|
25189
|
-
log.warn(` ${benchCutOffLine(record.cutOffSamples
|
|
25030
|
+
log.warn(` ${benchCutOffLine(record.cutOffSamples ?? 1, record.maxCoinsPerSample ?? 0)}`);
|
|
25190
25031
|
return;
|
|
25191
25032
|
}
|
|
25192
|
-
|
|
25193
|
-
|
|
25194
|
-
log.warn(" No ceiling recommended.");
|
|
25033
|
+
if (record.recommendedEstimateCoins === null || record.p95 === null) {
|
|
25034
|
+
log.warn(" No price recommended.");
|
|
25195
25035
|
log.dim(
|
|
25196
|
-
record.
|
|
25036
|
+
record.recommendedDeclaredHeadroomBps === null ? " This stand served no recommended headroom, and the multiplier is the server's to set \u2014" : " No sample settled, so there is no p95 to build a price on \u2014"
|
|
25197
25037
|
);
|
|
25198
25038
|
log.dim(" declaring a number from this run would be a guess dressed as a measurement.");
|
|
25199
|
-
printAverage(log, record, attempts);
|
|
25200
25039
|
return;
|
|
25201
25040
|
}
|
|
25202
|
-
log.plain(c.bold(` Declare
|
|
25203
|
-
if (record.
|
|
25041
|
+
log.plain(c.bold(` Declare estimateCoins: ${record.recommendedEstimateCoins}`));
|
|
25042
|
+
if (record.lengthRaisedPrice) {
|
|
25204
25043
|
log.plain(` ${LLM_LENGTH_RAISED_LINE}`);
|
|
25205
25044
|
log.dim(
|
|
25206
|
-
` = the smallest
|
|
25045
|
+
` = the smallest price whose answer allowance holds the answer plus this stand's headroom (p95 ${record.fitP95 ?? "\u2014"}), above the`
|
|
25207
25046
|
);
|
|
25208
25047
|
log.dim(
|
|
25209
|
-
`
|
|
25048
|
+
` charged-based ${record.chargedBasedEstimateCoins ?? "\u2014"} (p95 of charged coins ${record.p95} plus ${record.recommendedDeclaredHeadroomBps} bps), over ${record.completed} settled attempt${record.completed === 1 ? "" : "s"} on ${record.modelId}.`
|
|
25210
25049
|
);
|
|
25211
25050
|
} else {
|
|
25212
25051
|
log.dim(
|
|
25213
|
-
` = p95
|
|
25052
|
+
` = p95 of charged coins (${record.p95}) plus this stand's recommended headroom (${record.recommendedDeclaredHeadroomBps} bps),`
|
|
25214
25053
|
);
|
|
25215
|
-
log.dim(` measured over ${
|
|
25054
|
+
log.dim(` measured over ${record.completed} settled attempt${record.completed === 1 ? "" : "s"} on ${record.modelId}.`);
|
|
25216
25055
|
}
|
|
25217
|
-
log.dim(" That number is a
|
|
25218
|
-
|
|
25219
|
-
|
|
25220
|
-
log.plain(c.bold(` Grant perCallMaxCredits: ${record.recommendedPerCallMaxCredits}`));
|
|
25056
|
+
log.dim(" That number is a PRICE: a started attempt is charged in full, including one that fails.");
|
|
25057
|
+
if (record.recommendedPerCallMaxCoins != null) {
|
|
25058
|
+
log.plain(c.bold(` Grant perCallMaxCoins: ${record.recommendedPerCallMaxCoins}`));
|
|
25221
25059
|
log.dim(
|
|
25222
|
-
record.
|
|
25060
|
+
record.lengthRaisedPrice ? ` = the same headroom over the worst sample (${record.max}) or the longest answer's price, whichever is larger, and never below the price above it.` : ` = the same headroom over the worst sample (${record.max}), and never below the price above it.`
|
|
25223
25061
|
);
|
|
25224
|
-
|
|
25225
|
-
printAverage(log, record, attempts);
|
|
25226
|
-
}
|
|
25227
|
-
function printAverage(log, record, attempts) {
|
|
25228
|
-
if (record.averageCreditsPerCall === null) return;
|
|
25229
|
-
log.plain(` Per-call estimate: about ${formatCredits(record.averageCreditsPerCall)} credits per call`);
|
|
25230
|
-
log.dim(` = the average real cost with the platform fee, no headroom, over ${attempts}.`);
|
|
25231
|
-
if (record.recommendedPerCallEstimateCredits !== null) {
|
|
25232
|
-
log.plain(c.bold(` Grant perCallEstimateCredits: ${record.recommendedPerCallEstimateCredits}`));
|
|
25233
|
-
log.dim(` = that average rounded up to a whole credit. disclosure.estimatedCreditsPerPeriod = your calls per period \xD7 ${formatCredits(record.averageCreditsPerCall)}, rounded up.`);
|
|
25062
|
+
log.dim(" The ceiling is the price's room to be wrong; declaring one under the price is refused.");
|
|
25234
25063
|
}
|
|
25235
25064
|
}
|
|
25236
25065
|
async function explainBenchRefusal(res, call2, base, log, opts, index, settledSoFar) {
|
|
25237
|
-
if (printedStructuredError(res)) {
|
|
25238
|
-
const printed = await res.json().catch(() => ({}));
|
|
25239
|
-
return { stop: true, refusal: { status: res.status, error: typeof printed.error === "string" ? printed.error : null, slotsHeld: null, slotLimit: null } };
|
|
25240
|
-
}
|
|
25066
|
+
if (printedStructuredError(res)) return { stop: true, refusal: { status: res.status, error: null, slotsHeld: null, slotLimit: null } };
|
|
25241
25067
|
const body = await res.json().catch(() => ({}));
|
|
25242
25068
|
const refusal2 = { status: res.status, error: body.error ?? null, slotsHeld: null, slotLimit: null };
|
|
25243
25069
|
if (res.status === 429 && body.error === "generation_limit") {
|
|
@@ -25247,10 +25073,6 @@ async function explainBenchRefusal(res, call2, base, log, opts, index, settledSo
|
|
|
25247
25073
|
if (!opts.json) log.error(` Sample ${index + 1} ${benchSlotsHeldSentence(counts)}`);
|
|
25248
25074
|
return { stop: true, refusal: refusal2 };
|
|
25249
25075
|
}
|
|
25250
|
-
if (res.status === 403 && body.error === "credits_unverified") {
|
|
25251
|
-
if (!opts.json) log.error(` ${LLM_BENCH_UNVERIFIED_LINE}`);
|
|
25252
|
-
return { stop: true, refusal: refusal2 };
|
|
25253
|
-
}
|
|
25254
25076
|
if (opts.json) return { stop: res.status !== 429, refusal: refusal2 };
|
|
25255
25077
|
if (res.status === 404 && body.error === "not_found") {
|
|
25256
25078
|
log.error(` ${LLM_LANE_OFF_LINE}`);
|
|
@@ -25266,7 +25088,7 @@ async function explainBenchRefusal(res, call2, base, log, opts, index, settledSo
|
|
|
25266
25088
|
return { stop: true, refusal: refusal2 };
|
|
25267
25089
|
}
|
|
25268
25090
|
if (res.status === 402) {
|
|
25269
|
-
log.error(" Not enough
|
|
25091
|
+
log.error(" Not enough coin to start the attempt.");
|
|
25270
25092
|
return { stop: true, refusal: refusal2 };
|
|
25271
25093
|
}
|
|
25272
25094
|
log.error(
|
|
@@ -25275,7 +25097,7 @@ async function explainBenchRefusal(res, call2, base, log, opts, index, settledSo
|
|
|
25275
25097
|
return { stop: res.status >= 500 || res.status === 401 || res.status === 403, refusal: refusal2 };
|
|
25276
25098
|
}
|
|
25277
25099
|
function printProviderRefusal(log, status2, message, awaitingBill = false) {
|
|
25278
|
-
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so
|
|
25100
|
+
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so this call's coin stays held until it resolves (see \`genex llm status\`); it is not a sample.`);
|
|
25279
25101
|
else log.dim(" The provider refused this call at its door \u2014 no inference ran, it cost nothing, and it is not a sample.");
|
|
25280
25102
|
if (message) log.dim(` Provider said: ${message}`);
|
|
25281
25103
|
if (status2 === 401 || status2 === 403) {
|
|
@@ -25301,6 +25123,19 @@ async function pollSettled(call2, url, timeoutSec) {
|
|
|
25301
25123
|
}
|
|
25302
25124
|
return last;
|
|
25303
25125
|
}
|
|
25126
|
+
async function readCoinBalance(apiUrl, token) {
|
|
25127
|
+
try {
|
|
25128
|
+
const res = await apiFetch(`${apiUrl}/api/coin/balance`, {
|
|
25129
|
+
headers: { Authorization: `Bearer ${token}` },
|
|
25130
|
+
signal: AbortSignal.timeout(6e3)
|
|
25131
|
+
});
|
|
25132
|
+
if (!res.ok) return null;
|
|
25133
|
+
const body = await res.json().catch(() => null);
|
|
25134
|
+
return typeof body?.spendable === "number" ? body.spendable : null;
|
|
25135
|
+
} catch {
|
|
25136
|
+
return null;
|
|
25137
|
+
}
|
|
25138
|
+
}
|
|
25304
25139
|
async function saveBench(cwd, record) {
|
|
25305
25140
|
try {
|
|
25306
25141
|
const file = path36.join(cwd, LLM_BENCH_FILE);
|
|
@@ -25316,10 +25151,9 @@ function benchFailed(opts, log, message) {
|
|
|
25316
25151
|
else log.error(message);
|
|
25317
25152
|
process.exitCode = 1;
|
|
25318
25153
|
}
|
|
25319
|
-
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
25154
|
+
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCoins, opts) {
|
|
25320
25155
|
return {
|
|
25321
25156
|
command: "llm bench",
|
|
25322
|
-
currency: "credits",
|
|
25323
25157
|
apiUrl,
|
|
25324
25158
|
projectId,
|
|
25325
25159
|
modelId: opts.modelId ?? null,
|
|
@@ -25328,28 +25162,21 @@ function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCre
|
|
|
25328
25162
|
schemaPath: opts.schemaPath ?? null,
|
|
25329
25163
|
samples,
|
|
25330
25164
|
completed: 0,
|
|
25331
|
-
|
|
25165
|
+
maxCoinsPerSample: maxCoins,
|
|
25332
25166
|
balanceBefore: null,
|
|
25333
25167
|
balanceAfter: null,
|
|
25334
25168
|
results: [],
|
|
25335
25169
|
refusal: null,
|
|
25336
|
-
|
|
25337
|
-
|
|
25338
|
-
|
|
25339
|
-
|
|
25340
|
-
recommendedDeclaredHeadroomBps: null,
|
|
25341
|
-
feeBps: null,
|
|
25342
|
-
usdCentsPerCredit: null,
|
|
25343
|
-
missingTerms: null,
|
|
25344
|
-
costBasedMaxCredits: null,
|
|
25345
|
-
lengthRaisedCeiling: false,
|
|
25170
|
+
chargedCoins: { p50: null, p95: null, max: null },
|
|
25171
|
+
fitCoins: { p95: null, max: null },
|
|
25172
|
+
chargedBasedEstimateCoins: null,
|
|
25173
|
+
lengthRaisedPrice: false,
|
|
25346
25174
|
cutOffSamples: 0,
|
|
25347
25175
|
exceedsStandCap: false,
|
|
25348
25176
|
noRecommendationReason: null,
|
|
25349
|
-
|
|
25350
|
-
|
|
25351
|
-
|
|
25352
|
-
recommendedPerCallEstimateCredits: null,
|
|
25177
|
+
recommendedDeclaredHeadroomBps: null,
|
|
25178
|
+
recommendedEstimateCoins: null,
|
|
25179
|
+
recommendedPerCallMaxCoins: null,
|
|
25353
25180
|
savedTo: null
|
|
25354
25181
|
};
|
|
25355
25182
|
}
|
|
@@ -25412,8 +25239,7 @@ async function reportStatus(args) {
|
|
|
25412
25239
|
for (const row of rows) {
|
|
25413
25240
|
const state = rowState(row);
|
|
25414
25241
|
const mark = state === "active" ? c.cyan("\u25CF") : state === "awaiting bill" ? c.yellow("\u25D0") : c.green("\u25CB");
|
|
25415
|
-
const
|
|
25416
|
-
const reserved = money2.reserved === null ? "reserved \u2014" : `${money2.reserved} ${money2.unit} reserved`;
|
|
25242
|
+
const reserved = typeof row.reservedCoins === "number" ? `${row.reservedCoins} coin reserved` : "reserved \u2014";
|
|
25417
25243
|
log.plain(
|
|
25418
25244
|
` ${mark} ${row.id ?? "?"} ${row.modelId ?? "?"} ${state.padEnd(13)} ${formatAge(row.createdAt, now).padStart(4)} old ${reserved}${row.error ? ` ${row.error}` : ""}${rowHoldsSlot(row) ? "" : c.dim(" (no slot)")}`
|
|
25419
25245
|
);
|
|
@@ -25485,11 +25311,10 @@ async function cancelCall(args) {
|
|
|
25485
25311
|
if (!opts.json) {
|
|
25486
25312
|
log.warn(` ${row.id} (${row.status ?? "?"}) \u2014 ${sentence}.`);
|
|
25487
25313
|
if (state === "awaiting bill") {
|
|
25488
|
-
log.dim(` Nothing to cancel:
|
|
25314
|
+
log.dim(` Nothing to cancel: its coin releases when the bill resolves, and it stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`);
|
|
25489
25315
|
if (row.providerMessage?.trim()) log.dim(` Provider said: ${row.providerMessage.trim()}`);
|
|
25490
25316
|
} else {
|
|
25491
|
-
|
|
25492
|
-
log.dim(` ${money2.charged === null ? "charge unknown" : `${money2.charged} ${money2.unit} charged`}${row.error ? ` \xB7 ${row.error}` : ""}.`);
|
|
25317
|
+
log.dim(` ${typeof row.chargedCoins === "number" ? `${row.chargedCoins} coin charged` : "charge unknown"}${row.error ? ` \xB7 ${row.error}` : ""}.`);
|
|
25493
25318
|
}
|
|
25494
25319
|
}
|
|
25495
25320
|
return done("ok", null, state === "awaiting bill" ? "awaiting_bill" : "already_final", row);
|
|
@@ -25511,126 +25336,78 @@ async function cancelCall(args) {
|
|
|
25511
25336
|
const after = await res.json().catch(() => null) ?? row;
|
|
25512
25337
|
if (!opts.json) {
|
|
25513
25338
|
log.success(` Cancelled ${row.id} (was ${row.status ?? "active"}).`);
|
|
25514
|
-
log.dim(`
|
|
25339
|
+
log.dim(` Its coin releases once the bill settles; ${c.cyan("genex llm status")} shows it until then.`);
|
|
25515
25340
|
}
|
|
25516
25341
|
return done("ok", null, "cancelled", after);
|
|
25517
25342
|
}
|
|
25518
|
-
async function
|
|
25519
|
-
let
|
|
25343
|
+
async function reportSavedPrice(cwd, opts, log) {
|
|
25344
|
+
let record = null;
|
|
25520
25345
|
try {
|
|
25521
|
-
|
|
25346
|
+
record = JSON.parse(await fs35.readFile(path36.join(cwd, LLM_BENCH_FILE), "utf8"));
|
|
25522
25347
|
} catch {
|
|
25523
|
-
|
|
25348
|
+
record = null;
|
|
25524
25349
|
}
|
|
25525
|
-
if (!
|
|
25526
|
-
const v = parsed;
|
|
25527
|
-
if (v.v === 2 && v.currency === "credits" && v.cost && v.charged) return { kind: "credits", record: parsed };
|
|
25528
|
-
return { kind: "coins", record: parsed };
|
|
25529
|
-
}
|
|
25530
|
-
var LLM_PRICE_COIN_TERMS_LINE = "This bench was measured under the old COIN terms, when a game declared a fixed price per call. In-game calls are now billed in the player's credits, as used, and a game declares a per-call CEILING (maxCredits) instead \u2014 a coin price is not that number. Re-run the bench for the figures to declare.";
|
|
25531
|
-
async function reportSavedPrice(cwd, opts, log) {
|
|
25532
|
-
const saved = await readSavedBench(cwd);
|
|
25533
|
-
const rerun = 'genex llm bench "<prompt>" --max-credits <n> --user-approved';
|
|
25534
|
-
const creditKeys = (record2) => ({
|
|
25535
|
-
costUsdPicos: record2?.cost ?? { p50: null, p95: null, max: null },
|
|
25536
|
-
chargedCredits: record2?.charged ?? { p50: null, p95: null, max: null },
|
|
25537
|
-
fitP95: record2?.fitP95 ?? null,
|
|
25538
|
-
fitMax: record2?.fitMax ?? null,
|
|
25539
|
-
recommendedDeclaredHeadroomBps: record2?.recommendedDeclaredHeadroomBps ?? null,
|
|
25540
|
-
feeBps: record2?.feeBps ?? null,
|
|
25541
|
-
usdCentsPerCredit: record2?.usdCentsPerCredit ?? null,
|
|
25542
|
-
costBasedMaxCredits: record2?.costBasedMaxCredits ?? null,
|
|
25543
|
-
lengthRaisedCeiling: record2?.lengthRaisedCeiling ?? false,
|
|
25544
|
-
cutOffSamples: record2?.cutOffSamples ?? 0,
|
|
25545
|
-
noRecommendationReason: record2?.noRecommendationReason ?? null,
|
|
25546
|
-
recommendedMaxCredits: record2?.recommendedMaxCredits ?? null,
|
|
25547
|
-
recommendedPerCallMaxCredits: record2?.recommendedPerCallMaxCredits ?? null,
|
|
25548
|
-
averageCreditsPerCall: record2?.averageCreditsPerCall ?? null,
|
|
25549
|
-
recommendedPerCallEstimateCredits: record2?.recommendedPerCallEstimateCredits ?? null
|
|
25550
|
-
});
|
|
25551
|
-
if (!saved) {
|
|
25350
|
+
if (!record) {
|
|
25552
25351
|
if (opts.json) {
|
|
25553
25352
|
writeJsonLine({
|
|
25554
25353
|
command: "llm price",
|
|
25555
25354
|
status: "failed",
|
|
25556
25355
|
error: "no_bench",
|
|
25557
|
-
currency: null,
|
|
25558
25356
|
ranAt: null,
|
|
25559
25357
|
modelId: null,
|
|
25560
25358
|
samples: null,
|
|
25561
25359
|
completed: null,
|
|
25562
|
-
|
|
25563
|
-
|
|
25360
|
+
p50: null,
|
|
25361
|
+
p95: null,
|
|
25362
|
+
max: null,
|
|
25363
|
+
fitP95: null,
|
|
25364
|
+
chargedBasedEstimateCoins: null,
|
|
25365
|
+
lengthRaisedPrice: false,
|
|
25366
|
+
cutOffSamples: 0,
|
|
25367
|
+
noRecommendationReason: null,
|
|
25368
|
+
recommendedDeclaredHeadroomBps: null,
|
|
25369
|
+
recommendedEstimateCoins: null,
|
|
25370
|
+
recommendedPerCallMaxCoins: null,
|
|
25564
25371
|
savedTo: null
|
|
25565
25372
|
});
|
|
25566
25373
|
} else {
|
|
25567
25374
|
log.error("Nothing has been benchmarked in this folder yet.");
|
|
25568
|
-
log.dim(` Run ${c.cyan(
|
|
25569
|
-
}
|
|
25570
|
-
process.exitCode = 1;
|
|
25571
|
-
return;
|
|
25572
|
-
}
|
|
25573
|
-
if (saved.kind === "coins") {
|
|
25574
|
-
const old = saved.record;
|
|
25575
|
-
const legacy = {
|
|
25576
|
-
p50: old.p50 ?? null,
|
|
25577
|
-
p95: old.p95 ?? null,
|
|
25578
|
-
max: old.max ?? null,
|
|
25579
|
-
recommendedEstimateCoins: old.recommendedEstimateCoins ?? null,
|
|
25580
|
-
recommendedPerCallMaxCoins: old.recommendedPerCallMaxCoins ?? null,
|
|
25581
|
-
maxCoinsPerSample: old.maxCoinsPerSample ?? null,
|
|
25582
|
-
noRecommendationReason: old.noRecommendationReason ?? null
|
|
25583
|
-
};
|
|
25584
|
-
if (opts.json) {
|
|
25585
|
-
writeJsonLine({
|
|
25586
|
-
command: "llm price",
|
|
25587
|
-
status: "failed",
|
|
25588
|
-
error: "coin_terms",
|
|
25589
|
-
currency: "coins",
|
|
25590
|
-
ranAt: old.ranAt ?? null,
|
|
25591
|
-
modelId: old.modelId ?? null,
|
|
25592
|
-
samples: old.samples ?? null,
|
|
25593
|
-
completed: old.completed ?? null,
|
|
25594
|
-
...creditKeys(null),
|
|
25595
|
-
legacyCoinTerms: legacy,
|
|
25596
|
-
savedTo: LLM_BENCH_FILE
|
|
25597
|
-
});
|
|
25598
|
-
} else {
|
|
25599
|
-
log.plain(c.bold("genex llm price"));
|
|
25600
|
-
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${old.ranAt ?? "at an unknown time"}${old.modelId ? ` on ${old.modelId}` : ""}`);
|
|
25601
|
-
log.plain("");
|
|
25602
|
-
log.warn(` ${LLM_PRICE_COIN_TERMS_LINE}`);
|
|
25603
|
-
const measured = legacy.p95 === null ? "no settled sample" : `charged coins p50 ${legacy.p50} \xB7 p95 ${legacy.p95} \xB7 max ${legacy.max}`;
|
|
25604
|
-
const recommended = legacy.recommendedEstimateCoins === null ? "no price recommended" : `it recommended estimateCoins ${legacy.recommendedEstimateCoins}${legacy.recommendedPerCallMaxCoins != null ? ` and perCallMaxCoins ${legacy.recommendedPerCallMaxCoins}` : ""}`;
|
|
25605
|
-
log.dim(` What it measured then: ${measured}; ${recommended}.`);
|
|
25606
|
-
log.dim(` Re-run: ${c.cyan(rerun)}`);
|
|
25375
|
+
log.dim(` Run ${c.cyan('genex llm bench "<prompt>" --max-coins <n> --user-approved')} first.`);
|
|
25607
25376
|
}
|
|
25608
25377
|
process.exitCode = 1;
|
|
25609
25378
|
return;
|
|
25610
25379
|
}
|
|
25611
|
-
const record = saved.record;
|
|
25612
25380
|
if (opts.json) {
|
|
25613
25381
|
writeJsonLine({
|
|
25614
25382
|
command: "llm price",
|
|
25615
|
-
status: record.
|
|
25616
|
-
error: record.
|
|
25617
|
-
currency: "credits",
|
|
25383
|
+
status: record.recommendedEstimateCoins === null ? "failed" : "ok",
|
|
25384
|
+
error: record.recommendedEstimateCoins === null ? record.noRecommendationReason ?? "no_recommendation" : null,
|
|
25618
25385
|
ranAt: record.ranAt ?? null,
|
|
25619
25386
|
modelId: record.modelId ?? null,
|
|
25620
25387
|
samples: record.samples ?? null,
|
|
25621
25388
|
completed: record.completed ?? null,
|
|
25622
|
-
|
|
25623
|
-
|
|
25389
|
+
p50: record.p50 ?? null,
|
|
25390
|
+
p95: record.p95 ?? null,
|
|
25391
|
+
max: record.max ?? null,
|
|
25392
|
+
// Absent from a file an older CLI wrote; null / false / 0 then, never missing.
|
|
25393
|
+
fitP95: record.fitP95 ?? null,
|
|
25394
|
+
chargedBasedEstimateCoins: record.chargedBasedEstimateCoins ?? null,
|
|
25395
|
+
lengthRaisedPrice: record.lengthRaisedPrice ?? false,
|
|
25396
|
+
cutOffSamples: record.cutOffSamples ?? 0,
|
|
25397
|
+
noRecommendationReason: record.noRecommendationReason ?? null,
|
|
25398
|
+
recommendedDeclaredHeadroomBps: record.recommendedDeclaredHeadroomBps ?? null,
|
|
25399
|
+
recommendedEstimateCoins: record.recommendedEstimateCoins ?? null,
|
|
25400
|
+
recommendedPerCallMaxCoins: record.recommendedPerCallMaxCoins ?? null,
|
|
25624
25401
|
savedTo: LLM_BENCH_FILE
|
|
25625
25402
|
});
|
|
25626
|
-
if (record.
|
|
25403
|
+
if (record.recommendedEstimateCoins === null) process.exitCode = 1;
|
|
25627
25404
|
return;
|
|
25628
25405
|
}
|
|
25629
25406
|
log.plain(c.bold("genex llm price"));
|
|
25630
25407
|
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${record.ranAt}`);
|
|
25631
25408
|
log.plain("");
|
|
25632
25409
|
printRecommendation(log, record);
|
|
25633
|
-
if (record.
|
|
25410
|
+
if (record.recommendedEstimateCoins === null) process.exitCode = 1;
|
|
25634
25411
|
}
|
|
25635
25412
|
function writeJsonLine(value) {
|
|
25636
25413
|
process.stdout.write(`${JSON.stringify(value)}
|
|
@@ -25890,7 +25667,7 @@ function runtimeRow(state, lane, toolsOnly = false) {
|
|
|
25890
25667
|
return {
|
|
25891
25668
|
label: "Runtime",
|
|
25892
25669
|
value: `in-game LLM lane live \xB7 ${n} model${n === 1 ? "" : "s"}${keyRefused ? ` \xB7 ${c.yellow("provider key refused")}` : ""}`,
|
|
25893
|
-
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : '
|
|
25670
|
+
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : 'Price a call before a game declares one: `npx genex llm bench "<prompt>" --max-coins <n> --user-approved`.'
|
|
25894
25671
|
};
|
|
25895
25672
|
}
|
|
25896
25673
|
case "off":
|
|
@@ -28567,16 +28344,16 @@ ${c.bold("Usage")}
|
|
|
28567
28344
|
status | cancel. "models" says whether this
|
|
28568
28345
|
stand serves the runtime LLM lane at all, and
|
|
28569
28346
|
at what provider rates. "bench" runs the real
|
|
28570
|
-
model on YOUR OWN
|
|
28571
|
-
attempt
|
|
28347
|
+
model on YOUR OWN coin and reports what each
|
|
28348
|
+
attempt was charged; it needs --max-coins <n>
|
|
28572
28349
|
--user-approved, like any spend the player has
|
|
28573
28350
|
to agree to. "price" reprints the last run's
|
|
28574
|
-
recommended
|
|
28351
|
+
recommended estimateCoins. "status" lists this
|
|
28575
28352
|
project's open calls and the slots they hold;
|
|
28576
28353
|
"cancel <id>" stops an active one. Declare a
|
|
28577
|
-
game's
|
|
28578
|
-
|
|
28579
|
-
|
|
28354
|
+
game's price from a bench, never from a guess:
|
|
28355
|
+
a started attempt is charged in full, including
|
|
28356
|
+
one that fails.
|
|
28580
28357
|
genex player <sub> [options] Answer the in-game model requests you approve, on your
|
|
28581
28358
|
OWN Claude or ChatGPT subscription: install | run |
|
|
28582
28359
|
status | stop | uninstall. "install" signs this machine
|
|
@@ -29046,7 +28823,7 @@ ${c.bold("Examples")}
|
|
|
29046
28823
|
genex shop remove sku_123
|
|
29047
28824
|
genex llm models
|
|
29048
28825
|
genex llm models --all
|
|
29049
|
-
genex llm bench "Reply with one short taunt." --samples 5 --max-
|
|
28826
|
+
genex llm bench "Reply with one short taunt." --samples 5 --max-coins 20 --user-approved
|
|
29050
28827
|
genex llm price
|
|
29051
28828
|
genex llm status
|
|
29052
28829
|
genex llm cancel <id>
|
|
@@ -29283,24 +29060,22 @@ ${c.bold("How the allowance works")}
|
|
|
29283
29060
|
|
|
29284
29061
|
Every number printed is live from your account; prices can change without a deploy.
|
|
29285
29062
|
`,
|
|
29286
|
-
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what
|
|
29063
|
+
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what should the game declare?
|
|
29287
29064
|
|
|
29288
29065
|
${c.bold("Usage")}
|
|
29289
29066
|
genex llm models [--all] [--json] Which models this stand serves, their provider rates per
|
|
29290
|
-
million tokens, the
|
|
29291
|
-
|
|
29292
|
-
|
|
29293
|
-
|
|
29294
|
-
|
|
29295
|
-
|
|
29296
|
-
|
|
29297
|
-
|
|
29298
|
-
|
|
29299
|
-
|
|
29300
|
-
|
|
29301
|
-
|
|
29302
|
-
allowance does not count. Needs a folder linked to a
|
|
29303
|
-
game you own, and a balance that covers one attempt.
|
|
29067
|
+
million tokens, and the headroom it recommends over a
|
|
29068
|
+
benchmark. The catalog is synced from OpenRouter, so the
|
|
29069
|
+
FEATURED short list is printed by default and --all
|
|
29070
|
+
prints every row (--json always carries every row). Off
|
|
29071
|
+
on this stand: it says so and exits 0 \u2014 build the feature
|
|
29072
|
+
without a model rather than promising a 404.
|
|
29073
|
+
genex llm bench "<prompt>" --max-coins <n> --user-approved [options]
|
|
29074
|
+
Run the real model <n> times through the development
|
|
29075
|
+
lane and report what each attempt was CHARGED. Refused
|
|
29076
|
+
without both flags, before any network call: this spends
|
|
29077
|
+
YOUR OWN COIN, a wallet the build's asset allowance
|
|
29078
|
+
cannot see. Needs a folder linked to a game you own.
|
|
29304
29079
|
genex llm price [--json] The recommendation from the last bench in this folder.
|
|
29305
29080
|
Reads a file; spends nothing.
|
|
29306
29081
|
genex llm status [--json] This project's open development calls: active, or
|
|
@@ -29317,32 +29092,29 @@ ${c.bold("Bench options")}
|
|
|
29317
29092
|
--samples <n> How many real attempts (default ${DEFAULT_BENCH_SAMPLES}). More samples, better p95.
|
|
29318
29093
|
--schema <file> A JSON file holding the output schema; implies JSON output.
|
|
29319
29094
|
--json-output Ask for JSON without a schema. --text Ask for text (the default).
|
|
29320
|
-
--max-
|
|
29095
|
+
--max-coins <n> The per-attempt ceiling YOU approved \u2014 a bench runs on your own coin,
|
|
29321
29096
|
never a player's. The run's worst case is that number times --samples,
|
|
29322
|
-
and it is printed before anything starts.
|
|
29323
|
-
name, read as the same number.)
|
|
29097
|
+
and it is printed before anything starts.
|
|
29324
29098
|
--timeout <seconds> Bounds the wait for each attempt.
|
|
29325
29099
|
--json One machine-readable object; every key is always present, null when
|
|
29326
29100
|
the CLI could not learn it.
|
|
29327
29101
|
|
|
29328
29102
|
${c.bold("Why a benchmark and not an estimate")}
|
|
29329
|
-
|
|
29330
|
-
|
|
29331
|
-
|
|
29332
|
-
|
|
29333
|
-
|
|
29334
|
-
|
|
29335
|
-
|
|
29336
|
-
|
|
29337
|
-
|
|
29338
|
-
recommendation, because those multipliers are not the CLI's to invent.
|
|
29103
|
+
\`estimateCoins\` is a PRICE, not a guess: declaring it charges it, in full, even when the
|
|
29104
|
+
attempt fails, is cancelled, or stops at its budget. And three layers sit between the
|
|
29105
|
+
provider's rate card and that number \u2014 the provider's cost, the platform's tariff (already
|
|
29106
|
+
inside every charged coin), and your headroom. The bench itself is billed to YOU, on the
|
|
29107
|
+
development lane; the player-funded lane is what the published game uses. Adding a margin
|
|
29108
|
+
to the provider's rate silently drops the tariff and under-prices every call, so the
|
|
29109
|
+
recommendation is built from CHARGED COINS and the headroom comes from the server. A stand
|
|
29110
|
+
that serves no headroom gets no recommendation, because the multiplier is not the CLI's to
|
|
29111
|
+
invent.
|
|
29339
29112
|
|
|
29340
|
-
|
|
29341
|
-
|
|
29342
|
-
A sample cut off at your --max-
|
|
29343
|
-
|
|
29344
|
-
|
|
29345
|
-
fee included, which is the honest estimate a budget's disclosure is built on.
|
|
29113
|
+
The declared price also decides how LONG the answer may be: it sizes each call's output
|
|
29114
|
+
allowance. So the recommendation is never below the smallest price that leaves room for
|
|
29115
|
+
the benchmarked answer, as the server computes it. A sample cut off at your --max-coins
|
|
29116
|
+
(provider_token_limit) is not a sample \u2014 re-run with a higher --max-coins; an answer longer
|
|
29117
|
+
than one call on this stand may produce gets no price at all \u2014 ask for a shorter one.
|
|
29346
29118
|
|
|
29347
29119
|
Results land in ${LLM_BENCH_FILE} \u2014 gitignored, machine-local, re-read by \`genex llm price\`.
|
|
29348
29120
|
`,
|
|
@@ -29593,13 +29365,11 @@ function parseArgs(argv) {
|
|
|
29593
29365
|
"--limit",
|
|
29594
29366
|
// `genex budget --assets <credits>` — the allowance the player approved.
|
|
29595
29367
|
"--assets",
|
|
29596
|
-
// `genex llm bench` value flags. `--max-
|
|
29597
|
-
// shape as `--assets
|
|
29598
|
-
// count a bench. `--max-coins` is its deprecated name (same number).
|
|
29368
|
+
// `genex llm bench` value flags. `--max-coins` is the COIN half of the
|
|
29369
|
+
// same approval shape as `--assets` (a different economy, its own gate).
|
|
29599
29370
|
"--model",
|
|
29600
29371
|
"--samples",
|
|
29601
29372
|
"--schema",
|
|
29602
|
-
"--max-credits",
|
|
29603
29373
|
"--max-coins",
|
|
29604
29374
|
// `genex player install --label <name>` — what this machine is called in
|
|
29605
29375
|
// the dashboard. Its own key rather than the generic 2nd positional,
|
|
@@ -30153,14 +29923,12 @@ function applyValueFlag(options, flag, value) {
|
|
|
30153
29923
|
options.samples = n;
|
|
30154
29924
|
break;
|
|
30155
29925
|
}
|
|
30156
|
-
case "--max-credits":
|
|
30157
29926
|
case "--max-coins": {
|
|
30158
29927
|
const n = Number(value);
|
|
30159
29928
|
if (!Number.isInteger(n) || n < 1) {
|
|
30160
|
-
throw new Error(`Invalid
|
|
29929
|
+
throw new Error(`Invalid --max-coins value: ${value} (whole coin, 1 or more)`);
|
|
30161
29930
|
}
|
|
30162
|
-
|
|
30163
|
-
else options.maxCoins = n;
|
|
29931
|
+
options.maxCoins = n;
|
|
30164
29932
|
break;
|
|
30165
29933
|
}
|
|
30166
29934
|
case "--timeout": {
|
|
@@ -30461,9 +30229,9 @@ async function main() {
|
|
|
30461
30229
|
await runBudget(parsed.options);
|
|
30462
30230
|
break;
|
|
30463
30231
|
// The in-game model lane: is it live here, what does one call really
|
|
30464
|
-
// cost, what
|
|
30465
|
-
//
|
|
30466
|
-
//
|
|
30232
|
+
// cost, what should the game declare. `bench` is the only CLI command
|
|
30233
|
+
// that spends COIN, so it carries its own --max-coins/--user-approved
|
|
30234
|
+
// gate — the credits allowance cannot see that wallet.
|
|
30467
30235
|
case "llm":
|
|
30468
30236
|
await runLlm(parsed.options);
|
|
30469
30237
|
break;
|