@genex-ai/cli-demo 1.36.9-dev.913 → 1.36.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{blender-mcp-WSURPZET.js → blender-mcp-QVEGHE5L.js} +2 -2
- package/dist/{blender-serve-EH2MOW3U.js → blender-serve-66OHC4ML.js} +1 -1
- package/dist/{chunk-EABIVAGT.js → chunk-4BBTPRJW.js} +2 -2
- package/dist/{chunk-K4EHXOZK.js → chunk-QI7FIYBY.js} +1 -1
- package/dist/index.js +216 -448
- package/package.json +1 -1
- package/templates/skills/genex-game-director/SKILL.md +1 -1
- package/templates/skills/genex-getting-started/SKILL.md +2 -2
- package/templates/skills/genex-llm-in-games/SKILL.md +72 -209
- package/templates/skills/genex-llm-in-games/references/pricing.md +85 -126
- package/templates/skills/genex-monetization/SKILL.md +9 -1
- package/templates/skills/genex-tool-llm/SKILL.md +9 -11
- package/templates/skills/genex-updates/SKILL.md +1 -1
package/dist/index.js
CHANGED
|
@@ -57,7 +57,7 @@ import {
|
|
|
57
57
|
writeSecretFile,
|
|
58
58
|
writeUserToken,
|
|
59
59
|
writeWorkspace
|
|
60
|
-
} from "./chunk-
|
|
60
|
+
} from "./chunk-4BBTPRJW.js";
|
|
61
61
|
import {
|
|
62
62
|
CLI_CHANNEL,
|
|
63
63
|
DEFAULT_API_URL,
|
|
@@ -78,7 +78,7 @@ import {
|
|
|
78
78
|
normalizeApiOrigin,
|
|
79
79
|
originKey,
|
|
80
80
|
resolveAgentTargets
|
|
81
|
-
} from "./chunk-
|
|
81
|
+
} from "./chunk-QI7FIYBY.js";
|
|
82
82
|
|
|
83
83
|
// src/instrument.ts
|
|
84
84
|
import * as Sentry from "@sentry/node";
|
|
@@ -1065,7 +1065,7 @@ Important note: put soul into your creations, with many details and love. Aim to
|
|
|
1065
1065
|
21. A turn ends in exactly one of two ways: on a question the player must answer before the next step can be chosen, or on a handoff \u2014 the draft link, what changed, what to try, and what is still open. Never close a turn on a promise. "I'll keep building", "while I keep working", "adding that now" are things you say and then DO before you stop; if you are stopping, say that you have stopped and what remains. While the Build plan's \`Now:\` line has open work and no answer is needed from the player, the turn is not over: preview, hand off, and start the next milestone in the same turn, until the requested outcome is reached or the platform's own budget ends the session. If a stop was forced on you anyway, the first line of your next turn is the promise you left and where it stands.
|
|
1066
1066
|
22. **What the player sees must read as the thing it is**, in this game's own style: a building as that building, a person as a person, a prop as that prop, a surface as its material \u2014 and a coloured box, a capsule, or a flat grey block standing in for one of them is never its finished version. Generation is the default route to that bar wherever code will not honestly reach it: the player's character, whatever the request names, whatever the player walks up to, enters, or interacts with, and the music and sound underneath \u2014 reach for the belt on your own judgment, without stopping to ask; a status line saying what you queued is enough, and \`--no-wait\` keeps you building while it lands. Procedural code is a first-class engine for what is structural, repeated, distant, or parametric \u2014 terrain, sky, fences, paving, modular kits, filler \u2014 and for anything else only when the result meets the same bar and you have looked at it in a capture before its row says \`landed\`. Start with one reusable implementation per distinct required gameplay or visual role. Background populations default to reusable procedural bodies and motion or compatible existing rigged assets. Catalog clips need a compatible skeleton and may be paid. Reserve paid custom bodies for the player and characters examined or interacted with closely, while honoring explicit user requests. Add variants only for an unmet requirement; unspent allowance is not unfinished work. Allowance questions use timeoutPolicy no-consent: silence does not raise the allowance. Record role, reuse and completion criteria in the existing Assets table. Decide per object, not per category, and write the route on each Assets row in \`DESIGN.md\` \u2014 the paid flow for generated pieces, the procedural flow with its local path for code-built ones. An Assets table with no generation in it is a decision, not a default: record it as \`Generation: none \u2014 <why code alone reaches the bar here>\` or generate.
|
|
1067
1067
|
23. **The screenshots you take to look at your own work go in \`.genex/scratch/\`.** Captures, render comparisons, before/after strips, traces and metrics dumps are how you SEE what you built \u2014 they are not part of what you built, and \`genex preview\` pushes the whole folder to the game's source, every time, forever. \`.genex/scratch/\` already exists for this and is already ignored, so there is nothing to set up: write the capture as \`.genex/scratch/arena-before.png\` and read it back from there. Never put them in a folder of your own at the top level: \`progress/\`, \`reports/\`, \`shots/\` and their kind are pushed like source and become permanent weight in the game and in every remix of it \u2014 one real game reached 11 GB and 10,332 committed screenshots exactly this way. If you find such a folder already there, add it to \`.gitignore\`; never delete the player's files (law 18). One file in \`.genex/scratch/\` IS read, by the platform: \`.genex/scratch/cover.png\` is the game's cover \u2014 the frame you would put on its poster: real gameplay at its signature moment, well lit, no menus, popups or busy HUD, landscape 16:9, and never any text (the page sets the title beside it). The next \`genex preview\` makes it the cover; save a better moment over it whenever one comes along, and if your browser tool can set the cover itself, use that instead. Before the first publish, and whenever the game's look changes, load \`$genex-cover\`: it chooses, stages, lights and judges that frame.
|
|
1068
|
-
24. **Benchmark before declaring an in-game LLM
|
|
1068
|
+
24. **Benchmark before declaring an in-game LLM price.** If the game calls a language model while the player plays \u2014 an NPC answering in its own words, a quest written for this save, a judge reading what the player typed \u2014 the PLAYER pays for it, and \`estimateCoins\` is the FIXED price of a started attempt: charged whether it succeeds, fails, is canceled, or stops at its own budget. Never pick that number by judgement or from a vendor's rate card. Run the real prompt on your own coins \u2014 \`npx genex llm bench "<the prompt>" --schema <file> --samples 3 --max-coins <n> --user-approved\` \u2014 and declare the figure it recommends, recorded in \`DESIGN.md\` with its model and date; re-benchmark whenever the prompt, the schema or the model changes, because a prompt edit is a price change. Check the lane first with \`npx genex llm models\`: where it is off the routes 404, so build the feature behind a graceful unavailable state and say so in the handoff. A repeated call is a standing budget the player approves once (\`requestSpendGrant()\`, then \`generate({ grantId })\`), never a popup per turn, and its disclosed call rate is computed from the game's own loop. Model output is data the game validates against its own expectation \u2014 never executed, and never authority over coin, items, entitlements or rewards. Load \`$genex-llm-in-games\` before writing any of it.
|
|
1069
1069
|
${CONTRACT_END}
|
|
1070
1070
|
`;
|
|
1071
1071
|
var TOOLS_CONTRACT_HEAD = `# Genex Tools (always in effect in this folder)
|
|
@@ -1079,7 +1079,7 @@ var TOOLS_CONTRACT_OFFER = `5. **When the game is built and playable \u2014 neve
|
|
|
1079
1079
|
var TOOLS_CONTRACT_HOSTED_LAWS = `5. **This game is hosted on Genex now, and publishing is glue \u2014 never a rebuild.** The game ships exactly as the user built it: \`initEmbed()\` first in the boot code (the \`genex-threejs-embed-auth\` card has the call; the renderer draws before any \`await waitForPlayer()\`), a static build into \`dist/\` with relative asset paths, then \`npx genex preview\`. Never restructure, reformat or "improve" the game to ship it, never scaffold a new app around it, and never start a design document or a build plan for it \u2014 the user's own process is theirs. Load the \`genex-tool-publish\` card before the first preview; it owns the vocabulary, the links and the limits.
|
|
1080
1080
|
6. **Ship first, then report.** \`preview\` prints a preflight \u2014 phone memory, a missing volume slider, an asset nobody wired, a viewport line. It never blocks a deploy, so push the build as it is, hand over the link, THEN relay each preflight line to the user in one plain sentence with an offer to fix it, and fix one only on their yes. The preflight is a report for the user, not a to-do list for you.
|
|
1081
1081
|
7. **The link is the game's page, and there are two versions.** After every preview give the user \`<dashboard>/draft/<slug>\` \u2014 \`dashboardOrigins[0]\` and \`slug\` from \`.genex/project.json\` \u2014 never localhost, a file path or the bare play origin. \`preview\` updates the draft and never touches what players are on; the first release is \`npx genex publish\` (it lists the game); after that "publish it", "update it" and "yes" all mean \`npx genex promote\` \u2014 the exact draft build, no rebuild. Ask once per round of work, in one line, and keep working while you wait.`;
|
|
1082
|
-
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with
|
|
1082
|
+
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with Genex coins or their own Claude/ChatGPT subscription, and approves it on a Genex sheet; your game just calls \`generate()\`. Want it that way?`;
|
|
1083
1083
|
function toolsContractLlmLaw(n, hosted) {
|
|
1084
1084
|
const onYes = hosted ? "On a yes, load `$genex-tool-llm` and then `$genex-llm-in-games`, which owns the build." : "On a yes, load `$genex-tool-llm`, which owns the rest: it checks the lane with `npx genex llm models` first, then runs `npx genex init --convert` (a hosted game is a static browser build \u2014 a local server holding a key can never ship).";
|
|
1085
1085
|
return `${n}. **A model running while people PLAY is a platform feature \u2014 never something you wire onto the user's own meter.** When a request implies one (NPCs that talk or decide in their own words, content written from what the player types, a prompt box in the game, "let the player pick a model"), recognise it, say in ONE line: "${TOOLS_CONTRACT_LLM_OFFER}" \u2014 then ASK and wait. ${onYes} On a no, build the authored version instead. Either way, never ship a game that calls a model on a key or an account of the user's own: every visitor would spend their money with nobody approving it, and the credential is readable in the bundle. Player-funded generation is text and JSON only \u2014 3D, images, video and audio stay the asset lanes above, on the user's meter.`;
|
|
@@ -23736,7 +23736,7 @@ async function runBlender(opts) {
|
|
|
23736
23736
|
return 1;
|
|
23737
23737
|
}
|
|
23738
23738
|
if (sub === "serve") {
|
|
23739
|
-
const { serveLocalBlender } = await import("./blender-serve-
|
|
23739
|
+
const { serveLocalBlender } = await import("./blender-serve-66OHC4ML.js");
|
|
23740
23740
|
const port = Number(process.env.GENEX_BLENDER_PORT ?? 8088);
|
|
23741
23741
|
log.step(`Starting a local Blender service on port ${port}`);
|
|
23742
23742
|
log.plain(
|
|
@@ -23745,7 +23745,7 @@ async function runBlender(opts) {
|
|
|
23745
23745
|
return serveLocalBlender({ port, log });
|
|
23746
23746
|
}
|
|
23747
23747
|
if (sub === "mcp") {
|
|
23748
|
-
const { runBlenderMcp } = await import("./blender-mcp-
|
|
23748
|
+
const { runBlenderMcp } = await import("./blender-mcp-QVEGHE5L.js");
|
|
23749
23749
|
return runBlenderMcp();
|
|
23750
23750
|
}
|
|
23751
23751
|
if (sub === "seat") {
|
|
@@ -24253,12 +24253,7 @@ import fs35 from "fs/promises";
|
|
|
24253
24253
|
import path36 from "path";
|
|
24254
24254
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
24255
24255
|
var SUBS3 = ["models", "bench", "price", "status", "cancel"];
|
|
24256
|
-
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN
|
|
24257
|
-
var LLM_MAX_COINS_DEPRECATED = "--max-coins is deprecated: it is read as --max-credits, the same number \u2014 a bench spends credits now.";
|
|
24258
|
-
function benchBalanceShortLine(spendable, maxCredits) {
|
|
24259
|
-
return `Your spendable credits (${spendable}) cannot cover one attempt at --max-credits ${maxCredits} \u2014 nothing was sent, nothing was spent. Add credits, or re-run with a lower --max-credits once the person at the keyboard agrees to it.`;
|
|
24260
|
-
}
|
|
24261
|
-
var LLM_BENCH_UNVERIFIED_LINE = "Credits pay only for a verified email, and this account's is not verified yet (credits_unverified) \u2014 nothing was sent, nothing was spent. Verify it, then re-run.";
|
|
24256
|
+
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN coin \u2014 your wallet, not a player's, and one this build's asset allowance cannot see. Re-run with --max-coins <n> --user-approved only after the person at the keyboard agreed to the number.";
|
|
24262
24257
|
var LLM_LANE_OFF_LINE = "The runtime LLM lane is off on this stand \u2014 in-game generate() answers 404 here, and nothing can be benchmarked.";
|
|
24263
24258
|
var LLM_TOOLS_CONVERT_HINT = [
|
|
24264
24259
|
"In-game model calls need a hosted game. This folder is a Genex Tools workspace, so nothing here",
|
|
@@ -24271,7 +24266,7 @@ function serverSlotCounts(body) {
|
|
|
24271
24266
|
return { held: body.slotsHeld, limit: body.slotLimit };
|
|
24272
24267
|
}
|
|
24273
24268
|
var PENDING_SLOT_RELEASE_MINUTES = 10;
|
|
24274
|
-
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so
|
|
24269
|
+
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so its coin stays held until the bill resolves; such a row stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`;
|
|
24275
24270
|
var LLM_STATUS_NO_COUNT_LINE = "This stand does not report how many slots are held across your projects, so no count is shown.";
|
|
24276
24271
|
var LLM_CANCEL_AWAITING_BILL = "this call already stopped; its bill is awaiting the provider";
|
|
24277
24272
|
var LLM_CANCEL_ALREADY_FINAL = "this call is already final";
|
|
@@ -24285,33 +24280,11 @@ var DEFAULT_SAMPLE_TIMEOUT_SEC = 180;
|
|
|
24285
24280
|
function parseProviderKey(value) {
|
|
24286
24281
|
return value === "ok" || value === "rejected" || value === "unknown" ? value : null;
|
|
24287
24282
|
}
|
|
24288
|
-
function bpsOrNull(value) {
|
|
24289
|
-
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
24290
|
-
}
|
|
24291
|
-
function standingCreditGrantOrNull(value) {
|
|
24292
|
-
if (!value || typeof value !== "object") return null;
|
|
24293
|
-
const v = value;
|
|
24294
|
-
const whole = (x) => typeof x === "number" && Number.isSafeInteger(x) && x >= 0;
|
|
24295
|
-
if (!whole(v.minCredits) || !whole(v.maxCredits) || !whole(v.stepCredits) || !whole(v.defaultCredits)) return null;
|
|
24296
|
-
return {
|
|
24297
|
-
minCredits: v.minCredits,
|
|
24298
|
-
maxCredits: v.maxCredits,
|
|
24299
|
-
stepCredits: v.stepCredits,
|
|
24300
|
-
defaultCredits: v.defaultCredits,
|
|
24301
|
-
...whole(v.minUsdCents) ? { minUsdCents: v.minUsdCents } : {},
|
|
24302
|
-
...whole(v.maxUsdCents) ? { maxUsdCents: v.maxUsdCents } : {},
|
|
24303
|
-
...whole(v.stepUsdCents) ? { stepUsdCents: v.stepUsdCents } : {}
|
|
24304
|
-
};
|
|
24305
|
-
}
|
|
24306
24283
|
async function readRuntimeLane(apiUrl, token) {
|
|
24307
24284
|
const empty = (state) => ({
|
|
24308
24285
|
state,
|
|
24309
24286
|
models: null,
|
|
24310
24287
|
recommendedDeclaredHeadroomBps: null,
|
|
24311
|
-
currency: null,
|
|
24312
|
-
feeBps: null,
|
|
24313
|
-
usdCentsPerCredit: null,
|
|
24314
|
-
standingCreditGrant: null,
|
|
24315
24288
|
externalProviders: null,
|
|
24316
24289
|
total: null,
|
|
24317
24290
|
featuredCount: null,
|
|
@@ -24330,11 +24303,7 @@ async function readRuntimeLane(apiUrl, token) {
|
|
|
24330
24303
|
return {
|
|
24331
24304
|
state: "live",
|
|
24332
24305
|
models: body.models,
|
|
24333
|
-
recommendedDeclaredHeadroomBps:
|
|
24334
|
-
currency: body.currency === "credits" || body.currency === "coins" ? body.currency : null,
|
|
24335
|
-
feeBps: bpsOrNull(body.feeBps),
|
|
24336
|
-
usdCentsPerCredit: typeof body.usdCentsPerCredit === "number" && Number.isSafeInteger(body.usdCentsPerCredit) && body.usdCentsPerCredit >= 1 ? body.usdCentsPerCredit : null,
|
|
24337
|
-
standingCreditGrant: standingCreditGrantOrNull(body.standingCreditGrant),
|
|
24306
|
+
recommendedDeclaredHeadroomBps: typeof body.recommendedDeclaredHeadroomBps === "number" && Number.isFinite(body.recommendedDeclaredHeadroomBps) ? body.recommendedDeclaredHeadroomBps : null,
|
|
24338
24307
|
externalProviders: Array.isArray(body.externalProviders) ? body.externalProviders : null,
|
|
24339
24308
|
total: typeof body.total === "number" && Number.isFinite(body.total) ? body.total : null,
|
|
24340
24309
|
featuredCount: typeof body.featuredCount === "number" && Number.isFinite(body.featuredCount) ? body.featuredCount : null,
|
|
@@ -24350,45 +24319,18 @@ function percentile2(values, p) {
|
|
|
24350
24319
|
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
24351
24320
|
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
24352
24321
|
}
|
|
24353
|
-
function
|
|
24354
|
-
|
|
24355
|
-
const sorted = [...values].sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
24356
|
-
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
24357
|
-
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
24322
|
+
function recommendEstimateCoins(p95, headroomBps) {
|
|
24323
|
+
return Math.max(1, Math.ceil(p95 * (1e4 + headroomBps) / 1e4));
|
|
24358
24324
|
}
|
|
24359
|
-
|
|
24360
|
-
|
|
24361
|
-
if (costPicos <= 0n) return 1;
|
|
24362
|
-
const num = costPicos * BigInt(1e4 + terms.feeBps) * BigInt(1e4 + terms.headroomBps);
|
|
24363
|
-
const den = 100000000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
24364
|
-
return Math.max(1, Number((num + den - 1n) / den));
|
|
24365
|
-
}
|
|
24366
|
-
function averageCreditsPerCall(costsPicos, terms) {
|
|
24367
|
-
if (costsPicos.length === 0) return null;
|
|
24368
|
-
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
24369
|
-
return Number(sum * BigInt(1e4 + terms.feeBps)) / Number(BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT);
|
|
24370
|
-
}
|
|
24371
|
-
function perCallEstimateCreditsFor(costsPicos, terms) {
|
|
24372
|
-
if (costsPicos.length === 0) return null;
|
|
24373
|
-
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
24374
|
-
const num = sum * BigInt(1e4 + terms.feeBps);
|
|
24375
|
-
const den = BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
24376
|
-
return Math.max(1, Number((num + den - 1n) / den));
|
|
24377
|
-
}
|
|
24378
|
-
function formatCredits(value) {
|
|
24379
|
-
if (!Number.isFinite(value)) return "\u2014";
|
|
24380
|
-
if (value >= 100) return String(Math.round(value));
|
|
24381
|
-
return String(Number(value.toPrecision(value < 1 ? 2 : 3)));
|
|
24382
|
-
}
|
|
24383
|
-
function usdFromPicos(picos) {
|
|
24384
|
-
if (picos === null || picos === void 0) return "unknown";
|
|
24385
|
-
const value = typeof picos === "bigint" ? picos : /^\d+$/.test(picos) ? BigInt(picos) : null;
|
|
24386
|
-
if (value === null) return "unknown";
|
|
24387
|
-
return `$${(Number(value) / 1e12).toFixed(6)}`;
|
|
24325
|
+
function recommendPerCallMaxCoins(p95, max, headroomBps) {
|
|
24326
|
+
return Math.max(recommendEstimateCoins(p95, headroomBps), recommendEstimateCoins(max, headroomBps));
|
|
24388
24327
|
}
|
|
24389
24328
|
function usdPerMillion(value) {
|
|
24390
24329
|
return `$${value.toFixed(value < 1 ? 4 : 2)}`;
|
|
24391
24330
|
}
|
|
24331
|
+
function usd(value) {
|
|
24332
|
+
return value === null ? "unknown" : `$${value.toFixed(6)}`;
|
|
24333
|
+
}
|
|
24392
24334
|
async function runLlm(opts = {}) {
|
|
24393
24335
|
const log = createLogger({ quiet: opts.quiet || opts.json });
|
|
24394
24336
|
const cwd = opts.cwd ?? process.cwd();
|
|
@@ -24414,23 +24356,16 @@ async function runLlm(opts = {}) {
|
|
|
24414
24356
|
return;
|
|
24415
24357
|
}
|
|
24416
24358
|
if (sub === "bench") {
|
|
24417
|
-
if (opts.maxCoins
|
|
24418
|
-
`);
|
|
24419
|
-
if (opts.maxCoins !== void 0 && opts.maxCredits !== void 0 && opts.maxCoins !== opts.maxCredits) {
|
|
24420
|
-
fail5("llm bench", `Pass --max-credits alone \u2014 --max-coins is its deprecated name, and the two disagree (${opts.maxCredits} vs ${opts.maxCoins}).`);
|
|
24421
|
-
return;
|
|
24422
|
-
}
|
|
24423
|
-
const approvedCredits = opts.maxCredits ?? opts.maxCoins;
|
|
24424
|
-
if (approvedCredits === void 0 || opts.userApproved !== true) {
|
|
24359
|
+
if (opts.maxCoins === void 0 || opts.userApproved !== true) {
|
|
24425
24360
|
fail5("llm bench", LLM_BENCH_APPROVAL_REQUIRED);
|
|
24426
24361
|
return;
|
|
24427
24362
|
}
|
|
24428
|
-
if (!Number.isInteger(
|
|
24429
|
-
fail5("llm bench", `--max-
|
|
24363
|
+
if (!Number.isInteger(opts.maxCoins) || opts.maxCoins < 1) {
|
|
24364
|
+
fail5("llm bench", `--max-coins takes a whole number of coin, 1 or more (got ${String(opts.maxCoins)}).`);
|
|
24430
24365
|
return;
|
|
24431
24366
|
}
|
|
24432
24367
|
if (!opts.benchPrompt?.trim()) {
|
|
24433
|
-
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-
|
|
24368
|
+
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-coins <n> --user-approved.');
|
|
24434
24369
|
return;
|
|
24435
24370
|
}
|
|
24436
24371
|
if (opts.jsonOutput && opts.textOutput) {
|
|
@@ -24522,10 +24457,6 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24522
24457
|
models: null,
|
|
24523
24458
|
externalProviders: null,
|
|
24524
24459
|
recommendedDeclaredHeadroomBps: null,
|
|
24525
|
-
currency: null,
|
|
24526
|
-
feeBps: null,
|
|
24527
|
-
usdCentsPerCredit: null,
|
|
24528
|
-
standingCreditGrant: null,
|
|
24529
24460
|
total: null,
|
|
24530
24461
|
featuredCount: null,
|
|
24531
24462
|
// The stand was never asked, so the verdict is unknown — but the key is
|
|
@@ -24551,12 +24482,6 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24551
24482
|
models: lane.models,
|
|
24552
24483
|
externalProviders: lane.externalProviders,
|
|
24553
24484
|
recommendedDeclaredHeadroomBps: lane.recommendedDeclaredHeadroomBps,
|
|
24554
|
-
// The credit terms a call is billed on, the server's — null on a stand
|
|
24555
|
-
// that predates credit billing, never a number the CLI filled in.
|
|
24556
|
-
currency: lane.currency,
|
|
24557
|
-
feeBps: lane.feeBps,
|
|
24558
|
-
usdCentsPerCredit: lane.usdCentsPerCredit,
|
|
24559
|
-
standingCreditGrant: lane.standingCreditGrant,
|
|
24560
24485
|
total: lane.total ?? lane.models?.length ?? null,
|
|
24561
24486
|
featuredCount: lane.featuredCount ?? (lane.models ? lane.models.filter((m) => m.featured).length : null),
|
|
24562
24487
|
// `null` when the lane is not live or the server predates the probe.
|
|
@@ -24603,9 +24528,8 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24603
24528
|
);
|
|
24604
24529
|
for (const m of shown) {
|
|
24605
24530
|
const plan = m.personalPlan ? ` \xB7 personal plan: ${m.personalPlan}` : "";
|
|
24606
|
-
const kind = m.kind === "typesafe" ? " \xB7 judge (classifier): judge() under a budget, not generate() or bench" : "";
|
|
24607
24531
|
log.plain(
|
|
24608
|
-
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}
|
|
24532
|
+
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}`
|
|
24609
24533
|
);
|
|
24610
24534
|
}
|
|
24611
24535
|
if (hiddenCount > 0) {
|
|
@@ -24621,104 +24545,74 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24621
24545
|
}
|
|
24622
24546
|
log.plain("");
|
|
24623
24547
|
if (lane.recommendedDeclaredHeadroomBps === null) {
|
|
24624
|
-
log.dim(" This stand serves no recommended headroom, so no
|
|
24548
|
+
log.dim(" This stand serves no recommended headroom, so no price can be recommended from a bench.");
|
|
24625
24549
|
} else {
|
|
24626
|
-
log.dim(` Recommended headroom over
|
|
24627
|
-
}
|
|
24628
|
-
if (lane.feeBps === null) {
|
|
24629
|
-
log.dim(" This stand serves no platform fee \u2014 it predates credit billing, so no ceiling can be recommended from a bench.");
|
|
24630
|
-
} else {
|
|
24631
|
-
log.dim(` In-game calls are billed in the player's credits, as used: the provider's cost plus this stand's platform fee (${lane.feeBps} bps).`);
|
|
24550
|
+
log.dim(` Recommended headroom over benchmarked coins on this stand: ${lane.recommendedDeclaredHeadroomBps} bps.`);
|
|
24632
24551
|
}
|
|
24633
24552
|
if (toolsOnly) {
|
|
24634
24553
|
printConvertHint();
|
|
24635
24554
|
return;
|
|
24636
24555
|
}
|
|
24637
|
-
log.dim(" Those per-million rates are the PROVIDER's
|
|
24638
|
-
log.dim(`
|
|
24556
|
+
log.dim(" Those per-million rates are the PROVIDER's; what your game is charged is the platform");
|
|
24557
|
+
log.dim(` tariff on top, which only a real call reveals: ${c.cyan('genex llm bench "<prompt>" --max-coins <n> --user-approved')}`);
|
|
24639
24558
|
}
|
|
24640
24559
|
function wholeOrNull(value) {
|
|
24641
24560
|
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
24642
24561
|
}
|
|
24643
|
-
function costPicosOf(usage) {
|
|
24644
|
-
if (typeof usage?.costUsdPicos === "string" && /^\d+$/.test(usage.costUsdPicos)) return usage.costUsdPicos;
|
|
24645
|
-
if (typeof usage?.costUsd === "number" && Number.isFinite(usage.costUsd) && usage.costUsd >= 0) {
|
|
24646
|
-
return String(BigInt(Math.round(usage.costUsd * 1e12)));
|
|
24647
|
-
}
|
|
24648
|
-
return null;
|
|
24649
|
-
}
|
|
24650
24562
|
function sampleFrom(id, settled) {
|
|
24651
24563
|
const usage = settled?.usage;
|
|
24652
|
-
const picos = costPicosOf(usage);
|
|
24653
24564
|
return {
|
|
24654
24565
|
id,
|
|
24655
24566
|
status: settled?.status ?? null,
|
|
24656
24567
|
billingStatus: settled?.billingStatus ?? null,
|
|
24657
|
-
|
|
24658
|
-
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd :
|
|
24659
|
-
costUsdPicos: picos,
|
|
24568
|
+
chargedCoins: typeof settled?.chargedCoins === "number" ? settled.chargedCoins : null,
|
|
24569
|
+
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd : null,
|
|
24660
24570
|
error: settled?.error ?? (settled ? null : "timed_out"),
|
|
24661
24571
|
providerMessage: typeof settled?.providerMessage === "string" && settled.providerMessage.trim() ? settled.providerMessage.trim() : null,
|
|
24662
24572
|
inputTokens: wholeOrNull(usage?.inputTokens),
|
|
24663
24573
|
outputTokens: wholeOrNull(usage?.outputTokens),
|
|
24664
24574
|
maxOutputTokens: wholeOrNull(usage?.maxOutputTokens),
|
|
24665
|
-
|
|
24575
|
+
fitCoins: typeof usage?.fitCoins === "number" && Number.isSafeInteger(usage.fitCoins) && usage.fitCoins >= 1 ? usage.fitCoins : null,
|
|
24666
24576
|
fitExceedsCap: typeof usage?.fitExceedsCap === "boolean" ? usage.fitExceedsCap : null,
|
|
24667
24577
|
truncated: typeof usage?.truncated === "boolean" ? usage.truncated : settled?.error === "provider_token_limit" ? true : null
|
|
24668
24578
|
};
|
|
24669
24579
|
}
|
|
24670
24580
|
var PROVIDER_TOKEN_LIMIT = "provider_token_limit";
|
|
24671
|
-
function benchRecommendation(results,
|
|
24672
|
-
const settled = results.filter(
|
|
24673
|
-
|
|
24674
|
-
);
|
|
24675
|
-
const
|
|
24676
|
-
const
|
|
24677
|
-
const fits = settled.map((r) => r.fitCredits).filter((v) => typeof v === "number");
|
|
24678
|
-
const costP95 = percentilePicos(costs, 95);
|
|
24679
|
-
const costMax = costs.length ? costs.reduce((a, b) => b > a ? b : a) : null;
|
|
24581
|
+
function benchRecommendation(results, headroomBps) {
|
|
24582
|
+
const settled = results.filter((r) => r.status === "succeeded" && r.billingStatus === "final" && typeof r.chargedCoins === "number");
|
|
24583
|
+
const charged = settled.map((r) => r.chargedCoins);
|
|
24584
|
+
const fits = settled.map((r) => r.fitCoins).filter((v) => typeof v === "number");
|
|
24585
|
+
const p95 = percentile2(charged, 95);
|
|
24586
|
+
const max = charged.length ? Math.max(...charged) : null;
|
|
24680
24587
|
const fitP95 = percentile2(fits, 95);
|
|
24681
24588
|
const fitMax = fits.length ? Math.max(...fits) : null;
|
|
24682
24589
|
const exceedsStandCap = results.some((r) => r.fitExceedsCap === true);
|
|
24683
24590
|
const cutOffSamples = results.filter((r) => r.error === PROVIDER_TOKEN_LIMIT && r.fitExceedsCap !== true).length;
|
|
24684
24591
|
const noRecommendationReason = exceedsStandCap ? "answer_exceeds_stand_cap" : cutOffSamples > 0 ? "answer_cut_off" : null;
|
|
24685
|
-
const
|
|
24686
|
-
const
|
|
24687
|
-
const
|
|
24688
|
-
const billing = feeBps !== null && usdCentsPerCredit !== null ? { feeBps, usdCentsPerCredit } : null;
|
|
24689
|
-
const costBased = full !== null && costP95 !== null ? ceilingCreditsFor(costP95, full) : null;
|
|
24690
|
-
const recommended = costBased === null || noRecommendationReason !== null ? null : Math.max(costBased, fitP95 ?? 0);
|
|
24691
|
-
const ceiling = recommended === null || costMax === null || full === null ? null : Math.max(ceilingCreditsFor(costMax, full), recommended, fitMax ?? 0);
|
|
24692
|
-
const average = billing !== null ? averageCreditsPerCall(costs, billing) : null;
|
|
24693
|
-
const estimate = ceiling === null || billing === null ? null : Math.min(ceiling, perCallEstimateCreditsFor(costs, billing) ?? 1);
|
|
24592
|
+
const chargedBased = p95 !== null && headroomBps !== null ? recommendEstimateCoins(p95, headroomBps) : null;
|
|
24593
|
+
const recommended = chargedBased === null || noRecommendationReason !== null ? null : Math.max(chargedBased, fitP95 ?? 0);
|
|
24594
|
+
const ceiling = recommended === null || p95 === null || max === null || headroomBps === null ? null : Math.max(recommendPerCallMaxCoins(p95, max, headroomBps), recommended, fitMax ?? 0);
|
|
24694
24595
|
return {
|
|
24695
|
-
costs,
|
|
24696
24596
|
charged,
|
|
24697
|
-
|
|
24698
|
-
|
|
24699
|
-
|
|
24700
|
-
chargedP50: percentile2(charged, 50),
|
|
24701
|
-
chargedP95: percentile2(charged, 95),
|
|
24702
|
-
chargedMax: charged.length ? Math.max(...charged) : null,
|
|
24597
|
+
p50: percentile2(charged, 50),
|
|
24598
|
+
p95,
|
|
24599
|
+
max,
|
|
24703
24600
|
fits,
|
|
24704
24601
|
fitP95,
|
|
24705
24602
|
fitMax,
|
|
24706
|
-
|
|
24707
|
-
|
|
24708
|
-
|
|
24709
|
-
|
|
24710
|
-
recommendedPerCallEstimateCredits: estimate,
|
|
24711
|
-
lengthRaisedCeiling: recommended !== null && costBased !== null && recommended > costBased,
|
|
24603
|
+
chargedBasedEstimateCoins: chargedBased,
|
|
24604
|
+
recommendedEstimateCoins: recommended,
|
|
24605
|
+
recommendedPerCallMaxCoins: ceiling,
|
|
24606
|
+
lengthRaisedPrice: recommended !== null && chargedBased !== null && recommended > chargedBased,
|
|
24712
24607
|
cutOffSamples,
|
|
24713
24608
|
exceedsStandCap,
|
|
24714
|
-
noRecommendationReason
|
|
24715
|
-
missingTerms
|
|
24609
|
+
noRecommendationReason
|
|
24716
24610
|
};
|
|
24717
24611
|
}
|
|
24718
|
-
var LLM_LENGTH_RAISED_LINE = "The
|
|
24719
|
-
var LLM_EXCEEDS_STAND_CAP_LINE = "No
|
|
24720
|
-
function benchCutOffLine(cutOff,
|
|
24721
|
-
return `No
|
|
24612
|
+
var LLM_LENGTH_RAISED_LINE = "The declared price also pays for how long the answer may be: at the charged-based price the answer would be cut off, so declare this one.";
|
|
24613
|
+
var LLM_EXCEEDS_STAND_CAP_LINE = "No price recommended: this answer is longer than one call on this stand may produce. Ask for a shorter answer \u2014 fewer fields, shorter strings, a length the prompt states \u2014 and benchmark again.";
|
|
24614
|
+
function benchCutOffLine(cutOff, maxCoins) {
|
|
24615
|
+
return `No price recommended: ${cutOff} sample${cutOff === 1 ? " was" : "s were"} cut off at this ceiling (--max-coins ${maxCoins}) before the answer was finished, so its real length is unknown. Re-run with a higher --max-coins.`;
|
|
24722
24616
|
}
|
|
24723
24617
|
var ACTIVE_STATUSES = /* @__PURE__ */ new Set(["requires_confirmation", "queued", "dispatching", "awaiting_external"]);
|
|
24724
24618
|
function rowState(row) {
|
|
@@ -24729,22 +24623,7 @@ function rowState(row) {
|
|
|
24729
24623
|
function rowHoldsSlot(row) {
|
|
24730
24624
|
if (typeof row.slotHeld === "boolean") return row.slotHeld;
|
|
24731
24625
|
const state = rowState(row);
|
|
24732
|
-
return state === "active" || state === "awaiting bill" && (
|
|
24733
|
-
}
|
|
24734
|
-
function rowMoney(row) {
|
|
24735
|
-
const credits = row.currency === "credits" || row.currency === void 0 && typeof row.reservedCredits === "number" && typeof row.reservedCoins !== "number";
|
|
24736
|
-
if (credits) {
|
|
24737
|
-
return {
|
|
24738
|
-
unit: "credits",
|
|
24739
|
-
reserved: typeof row.reservedCredits === "number" ? row.reservedCredits : null,
|
|
24740
|
-
charged: typeof row.chargedCredits === "number" ? row.chargedCredits : null
|
|
24741
|
-
};
|
|
24742
|
-
}
|
|
24743
|
-
return {
|
|
24744
|
-
unit: "coin",
|
|
24745
|
-
reserved: typeof row.reservedCoins === "number" ? row.reservedCoins : null,
|
|
24746
|
-
charged: typeof row.chargedCoins === "number" ? row.chargedCoins : null
|
|
24747
|
-
};
|
|
24626
|
+
return state === "active" || state === "awaiting bill" && (row.reservedCoins ?? 0) > 0;
|
|
24748
24627
|
}
|
|
24749
24628
|
function providerRefusalStatus(error) {
|
|
24750
24629
|
const m = /^provider_http_(\d{3})$/.exec(error ?? "");
|
|
@@ -24763,7 +24642,7 @@ function formatAge(iso, now = Date.now()) {
|
|
|
24763
24642
|
}
|
|
24764
24643
|
async function runBench(args) {
|
|
24765
24644
|
const { apiUrl, token, projectId, cwd, opts, log } = args;
|
|
24766
|
-
const
|
|
24645
|
+
const maxCoins = opts.maxCoins;
|
|
24767
24646
|
const samples = opts.samples ?? DEFAULT_BENCH_SAMPLES;
|
|
24768
24647
|
const prompt = opts.benchPrompt.trim();
|
|
24769
24648
|
const auth = { Authorization: `Bearer ${token}`, "Content-Type": "application/json" };
|
|
@@ -24785,7 +24664,7 @@ async function runBench(args) {
|
|
|
24785
24664
|
const lane = await readRuntimeLane(apiUrl, token);
|
|
24786
24665
|
if (lane.state !== "live" || !lane.models || lane.models.length === 0) {
|
|
24787
24666
|
if (lane.state === "off") {
|
|
24788
|
-
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
24667
|
+
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCoins, opts), status: "off", error: null });
|
|
24789
24668
|
else {
|
|
24790
24669
|
log.plain(LLM_LANE_OFF_LINE);
|
|
24791
24670
|
log.dim(" Nothing was spent.");
|
|
@@ -24812,27 +24691,17 @@ async function runBench(args) {
|
|
|
24812
24691
|
);
|
|
24813
24692
|
return;
|
|
24814
24693
|
}
|
|
24815
|
-
const
|
|
24816
|
-
if (walletBefore?.emailVerified === false) {
|
|
24817
|
-
benchFailed(opts, log, LLM_BENCH_UNVERIFIED_LINE);
|
|
24818
|
-
return;
|
|
24819
|
-
}
|
|
24820
|
-
if (walletBefore !== null && walletBefore.spendable < maxCredits) {
|
|
24821
|
-
benchFailed(opts, log, benchBalanceShortLine(walletBefore.spendable, maxCredits));
|
|
24822
|
-
return;
|
|
24823
|
-
}
|
|
24824
|
-
const balanceBefore = walletBefore?.spendable ?? null;
|
|
24694
|
+
const balanceBefore = await readCoinBalance(apiUrl, token);
|
|
24825
24695
|
if (!opts.json) {
|
|
24826
24696
|
log.plain(c.bold("genex llm bench"));
|
|
24827
24697
|
log.dim(` ${apiUrl}`);
|
|
24828
24698
|
log.plain("");
|
|
24829
24699
|
log.plain(` Model ${c.cyan(modelId)}`);
|
|
24830
24700
|
log.plain(` Samples ${samples} real attempt${samples === 1 ? "" : "s"}, ${outputFormat} output`);
|
|
24831
|
-
log.plain(` Approved ${
|
|
24701
|
+
log.plain(` Approved ${maxCoins} coin per attempt \u2014 at worst ${maxCoins * samples} coin for this run`);
|
|
24832
24702
|
log.plain(
|
|
24833
|
-
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore}
|
|
24703
|
+
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore} coin spendable`}`
|
|
24834
24704
|
);
|
|
24835
|
-
log.dim(" Billed as used: each attempt is charged its real cost plus the platform fee, rounded up to a whole credit.");
|
|
24836
24705
|
log.plain("");
|
|
24837
24706
|
}
|
|
24838
24707
|
const base = `${apiUrl}/api/runtime/development/projects/${encodeURIComponent(projectId)}/generations`;
|
|
@@ -24858,7 +24727,7 @@ async function runBench(args) {
|
|
|
24858
24727
|
const res = await call(base, {
|
|
24859
24728
|
method: "POST",
|
|
24860
24729
|
body: JSON.stringify({
|
|
24861
|
-
|
|
24730
|
+
maxCoins,
|
|
24862
24731
|
request: {
|
|
24863
24732
|
idempotencyKey: `bench-${randomUUID2()}`,
|
|
24864
24733
|
modelId,
|
|
@@ -24888,14 +24757,13 @@ async function runBench(args) {
|
|
|
24888
24757
|
if (refusedAt !== null) providerRefused++;
|
|
24889
24758
|
if (!opts.json) {
|
|
24890
24759
|
const length = row.outputTokens !== null ? ` \xB7 ${row.outputTokens}${row.maxOutputTokens !== null ? ` of ${row.maxOutputTokens}` : ""} tokens out` : "";
|
|
24891
|
-
const charged = row.chargedCredits === null ? "\u2014" : String(row.chargedCredits);
|
|
24892
24760
|
log.plain(
|
|
24893
|
-
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${
|
|
24761
|
+
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${String(row.chargedCoins ?? "\u2014").padStart(4)} coin charged \xB7 provider ${usd(row.costUsd)}${length}${row.error ? ` \xB7 ${row.error}` : ""}`
|
|
24894
24762
|
);
|
|
24895
24763
|
if (refusedAt !== null) printProviderRefusal(log, refusedAt, row.providerMessage, row.status === "unknown");
|
|
24896
24764
|
if (row.error === PROVIDER_TOKEN_LIMIT) {
|
|
24897
24765
|
log.dim(
|
|
24898
|
-
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no
|
|
24766
|
+
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no price makes room for it; it is not a sample." : ` The answer was cut off at this ceiling (--max-coins ${maxCoins}) before it was finished \u2014 it was charged, and it is not a sample.`
|
|
24899
24767
|
);
|
|
24900
24768
|
}
|
|
24901
24769
|
}
|
|
@@ -24903,19 +24771,16 @@ async function runBench(args) {
|
|
|
24903
24771
|
} finally {
|
|
24904
24772
|
process.removeListener("SIGINT", onSigint);
|
|
24905
24773
|
}
|
|
24906
|
-
const
|
|
24907
|
-
|
|
24908
|
-
|
|
24909
|
-
|
|
24910
|
-
|
|
24911
|
-
const
|
|
24912
|
-
const settledCount = rec.charged.length;
|
|
24774
|
+
const headroomBps = lane.recommendedDeclaredHeadroomBps;
|
|
24775
|
+
const rec = benchRecommendation(results, headroomBps);
|
|
24776
|
+
const { charged, p50, p95, max } = rec;
|
|
24777
|
+
const settledCount = charged.length;
|
|
24778
|
+
const recommended = rec.recommendedEstimateCoins;
|
|
24779
|
+
const ceiling = rec.recommendedPerCallMaxCoins;
|
|
24913
24780
|
const lengthBlocked = rec.noRecommendationReason !== null;
|
|
24914
|
-
const balanceAfter =
|
|
24915
|
-
const picosOrNull = (v) => v === null ? null : String(v);
|
|
24781
|
+
const balanceAfter = await readCoinBalance(apiUrl, token);
|
|
24916
24782
|
const record = {
|
|
24917
|
-
v:
|
|
24918
|
-
currency: "credits",
|
|
24783
|
+
v: 1,
|
|
24919
24784
|
ranAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
24920
24785
|
apiUrl,
|
|
24921
24786
|
projectId,
|
|
@@ -24924,34 +24789,28 @@ async function runBench(args) {
|
|
|
24924
24789
|
outputFormat,
|
|
24925
24790
|
samples,
|
|
24926
24791
|
completed: settledCount,
|
|
24927
|
-
|
|
24928
|
-
|
|
24929
|
-
|
|
24930
|
-
|
|
24931
|
-
|
|
24932
|
-
|
|
24792
|
+
chargedCoins: charged,
|
|
24793
|
+
p50,
|
|
24794
|
+
p95,
|
|
24795
|
+
max,
|
|
24796
|
+
recommendedDeclaredHeadroomBps: headroomBps,
|
|
24797
|
+
recommendedEstimateCoins: recommended,
|
|
24798
|
+
recommendedPerCallMaxCoins: ceiling,
|
|
24799
|
+
maxCoinsPerSample: maxCoins,
|
|
24800
|
+
fitCoins: rec.fits,
|
|
24933
24801
|
fitP95: rec.fitP95,
|
|
24934
|
-
|
|
24935
|
-
|
|
24936
|
-
feeBps: terms.feeBps,
|
|
24937
|
-
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
24938
|
-
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
24939
|
-
recommendedMaxCredits: rec.recommendedMaxCredits,
|
|
24940
|
-
recommendedPerCallMaxCredits: rec.recommendedPerCallMaxCredits,
|
|
24941
|
-
averageCreditsPerCall: rec.averageCreditsPerCall,
|
|
24942
|
-
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
24943
|
-
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
24802
|
+
chargedBasedEstimateCoins: rec.chargedBasedEstimateCoins,
|
|
24803
|
+
lengthRaisedPrice: rec.lengthRaisedPrice,
|
|
24944
24804
|
cutOffSamples: rec.cutOffSamples,
|
|
24945
24805
|
noRecommendationReason: rec.noRecommendationReason
|
|
24946
24806
|
};
|
|
24947
24807
|
const savedTo = await saveBench(cwd, record);
|
|
24948
24808
|
if (opts.json) {
|
|
24949
|
-
const ok =
|
|
24809
|
+
const ok = charged.length > 0 && !lengthBlocked;
|
|
24950
24810
|
writeJsonLine({
|
|
24951
24811
|
command: "llm bench",
|
|
24952
24812
|
status: ok ? "ok" : "failed",
|
|
24953
24813
|
error: ok ? null : rec.noRecommendationReason ?? "no_sample_settled",
|
|
24954
|
-
currency: "credits",
|
|
24955
24814
|
apiUrl,
|
|
24956
24815
|
projectId,
|
|
24957
24816
|
modelId,
|
|
@@ -24960,54 +24819,35 @@ async function runBench(args) {
|
|
|
24960
24819
|
schemaPath: opts.schemaPath ?? null,
|
|
24961
24820
|
samples,
|
|
24962
24821
|
completed: settledCount,
|
|
24963
|
-
|
|
24964
|
-
// Spendable credits before and after the run; null when unreadable.
|
|
24822
|
+
maxCoinsPerSample: maxCoins,
|
|
24965
24823
|
balanceBefore,
|
|
24966
24824
|
balanceAfter,
|
|
24967
24825
|
results,
|
|
24968
24826
|
// The create-time refusal that ended the run, or null. Beside the
|
|
24969
24827
|
// samples, never among them: no attempt existed and nothing was spent.
|
|
24970
24828
|
refusal: refusal2,
|
|
24971
|
-
|
|
24972
|
-
//
|
|
24973
|
-
costUsdPicos: record.cost,
|
|
24974
|
-
costUsd: {
|
|
24975
|
-
p50: rec.costP50 === null ? null : Number(rec.costP50) / 1e12,
|
|
24976
|
-
p95: rec.costP95 === null ? null : Number(rec.costP95) / 1e12,
|
|
24977
|
-
max: rec.costMax === null ? null : Number(rec.costMax) / 1e12
|
|
24978
|
-
},
|
|
24979
|
-
chargedCredits: record.charged,
|
|
24980
|
-
// The answer's LENGTH, priced by the server per sample (`fitCredits`,
|
|
24829
|
+
chargedCoins: { p50, p95, max },
|
|
24830
|
+
// The answer's LENGTH, priced by the server per sample (`fitCoins`,
|
|
24981
24831
|
// headroom already on the tokens), over the settled samples.
|
|
24982
|
-
|
|
24983
|
-
|
|
24984
|
-
|
|
24985
|
-
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
24986
|
-
missingTerms: rec.missingTerms,
|
|
24987
|
-
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
24988
|
-
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
24832
|
+
fitCoins: { p95: rec.fitP95, max: rec.fitMax },
|
|
24833
|
+
chargedBasedEstimateCoins: rec.chargedBasedEstimateCoins,
|
|
24834
|
+
lengthRaisedPrice: rec.lengthRaisedPrice,
|
|
24989
24835
|
cutOffSamples: rec.cutOffSamples,
|
|
24990
24836
|
exceedsStandCap: rec.exceedsStandCap,
|
|
24991
24837
|
noRecommendationReason: rec.noRecommendationReason,
|
|
24992
|
-
|
|
24993
|
-
|
|
24994
|
-
|
|
24995
|
-
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
24838
|
+
recommendedDeclaredHeadroomBps: headroomBps,
|
|
24839
|
+
recommendedEstimateCoins: recommended,
|
|
24840
|
+
recommendedPerCallMaxCoins: ceiling,
|
|
24996
24841
|
savedTo
|
|
24997
24842
|
});
|
|
24998
24843
|
if (!ok) process.exitCode = 1;
|
|
24999
24844
|
return;
|
|
25000
24845
|
}
|
|
25001
24846
|
log.plain("");
|
|
25002
|
-
const printMeasured = () => {
|
|
25003
|
-
log.plain(c.bold(" Real cost per call"));
|
|
25004
|
-
log.plain(` p50 ${usdFromPicos(rec.costP50)} p95 ${usdFromPicos(rec.costP95)} max ${usdFromPicos(rec.costMax)} (${settledCount} of ${samples} settled)`);
|
|
25005
|
-
log.plain(c.bold(" Charged credits"));
|
|
25006
|
-
log.plain(` p50 ${rec.chargedP50} p95 ${rec.chargedP95} max ${rec.chargedMax} (whole credits, rounded up per call)`);
|
|
25007
|
-
};
|
|
25008
24847
|
if (lengthBlocked) {
|
|
25009
|
-
if (
|
|
25010
|
-
|
|
24848
|
+
if (charged.length > 0) {
|
|
24849
|
+
log.plain(c.bold(" Charged coins"));
|
|
24850
|
+
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
25011
24851
|
log.plain("");
|
|
25012
24852
|
}
|
|
25013
24853
|
printRecommendation(log, record);
|
|
@@ -25015,16 +24855,17 @@ async function runBench(args) {
|
|
|
25015
24855
|
process.exitCode = 1;
|
|
25016
24856
|
return;
|
|
25017
24857
|
}
|
|
25018
|
-
if (
|
|
24858
|
+
if (charged.length === 0) {
|
|
25019
24859
|
log.error(
|
|
25020
|
-
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to
|
|
24860
|
+
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to price from." : " No sample settled, so there is nothing to price from."
|
|
25021
24861
|
);
|
|
25022
24862
|
process.exitCode = 1;
|
|
25023
24863
|
return;
|
|
25024
24864
|
}
|
|
25025
|
-
|
|
24865
|
+
log.plain(c.bold(" Charged coins"));
|
|
24866
|
+
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
25026
24867
|
if (balanceBefore !== null && balanceAfter !== null) {
|
|
25027
|
-
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter}
|
|
24868
|
+
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter} coin spendable`);
|
|
25028
24869
|
}
|
|
25029
24870
|
log.plain("");
|
|
25030
24871
|
printRecommendation(log, record);
|
|
@@ -25036,58 +24877,43 @@ function printRecommendation(log, record) {
|
|
|
25036
24877
|
return;
|
|
25037
24878
|
}
|
|
25038
24879
|
if (record.noRecommendationReason === "answer_cut_off") {
|
|
25039
|
-
log.warn(` ${benchCutOffLine(record.cutOffSamples
|
|
24880
|
+
log.warn(` ${benchCutOffLine(record.cutOffSamples ?? 1, record.maxCoinsPerSample ?? 0)}`);
|
|
25040
24881
|
return;
|
|
25041
24882
|
}
|
|
25042
|
-
|
|
25043
|
-
|
|
25044
|
-
log.warn(" No ceiling recommended.");
|
|
24883
|
+
if (record.recommendedEstimateCoins === null || record.p95 === null) {
|
|
24884
|
+
log.warn(" No price recommended.");
|
|
25045
24885
|
log.dim(
|
|
25046
|
-
record.
|
|
24886
|
+
record.recommendedDeclaredHeadroomBps === null ? " This stand served no recommended headroom, and the multiplier is the server's to set \u2014" : " No sample settled, so there is no p95 to build a price on \u2014"
|
|
25047
24887
|
);
|
|
25048
24888
|
log.dim(" declaring a number from this run would be a guess dressed as a measurement.");
|
|
25049
|
-
printAverage(log, record, attempts);
|
|
25050
24889
|
return;
|
|
25051
24890
|
}
|
|
25052
|
-
log.plain(c.bold(` Declare
|
|
25053
|
-
if (record.
|
|
24891
|
+
log.plain(c.bold(` Declare estimateCoins: ${record.recommendedEstimateCoins}`));
|
|
24892
|
+
if (record.lengthRaisedPrice) {
|
|
25054
24893
|
log.plain(` ${LLM_LENGTH_RAISED_LINE}`);
|
|
25055
24894
|
log.dim(
|
|
25056
|
-
` = the smallest
|
|
24895
|
+
` = the smallest price whose answer allowance holds the answer plus this stand's headroom (p95 ${record.fitP95 ?? "\u2014"}), above the`
|
|
25057
24896
|
);
|
|
25058
24897
|
log.dim(
|
|
25059
|
-
`
|
|
24898
|
+
` charged-based ${record.chargedBasedEstimateCoins ?? "\u2014"} (p95 of charged coins ${record.p95} plus ${record.recommendedDeclaredHeadroomBps} bps), over ${record.completed} settled attempt${record.completed === 1 ? "" : "s"} on ${record.modelId}.`
|
|
25060
24899
|
);
|
|
25061
24900
|
} else {
|
|
25062
24901
|
log.dim(
|
|
25063
|
-
` = p95
|
|
24902
|
+
` = p95 of charged coins (${record.p95}) plus this stand's recommended headroom (${record.recommendedDeclaredHeadroomBps} bps),`
|
|
25064
24903
|
);
|
|
25065
|
-
log.dim(` measured over ${
|
|
24904
|
+
log.dim(` measured over ${record.completed} settled attempt${record.completed === 1 ? "" : "s"} on ${record.modelId}.`);
|
|
25066
24905
|
}
|
|
25067
|
-
log.dim(" That number is a
|
|
25068
|
-
|
|
25069
|
-
|
|
25070
|
-
log.plain(c.bold(` Grant perCallMaxCredits: ${record.recommendedPerCallMaxCredits}`));
|
|
24906
|
+
log.dim(" That number is a PRICE: a started attempt is charged in full, including one that fails.");
|
|
24907
|
+
if (record.recommendedPerCallMaxCoins != null) {
|
|
24908
|
+
log.plain(c.bold(` Grant perCallMaxCoins: ${record.recommendedPerCallMaxCoins}`));
|
|
25071
24909
|
log.dim(
|
|
25072
|
-
record.
|
|
24910
|
+
record.lengthRaisedPrice ? ` = the same headroom over the worst sample (${record.max}) or the longest answer's price, whichever is larger, and never below the price above it.` : ` = the same headroom over the worst sample (${record.max}), and never below the price above it.`
|
|
25073
24911
|
);
|
|
25074
|
-
|
|
25075
|
-
printAverage(log, record, attempts);
|
|
25076
|
-
}
|
|
25077
|
-
function printAverage(log, record, attempts) {
|
|
25078
|
-
if (record.averageCreditsPerCall === null) return;
|
|
25079
|
-
log.plain(` Per-call estimate: about ${formatCredits(record.averageCreditsPerCall)} credits per call`);
|
|
25080
|
-
log.dim(` = the average real cost with the platform fee, no headroom, over ${attempts}.`);
|
|
25081
|
-
if (record.recommendedPerCallEstimateCredits !== null) {
|
|
25082
|
-
log.plain(c.bold(` Grant perCallEstimateCredits: ${record.recommendedPerCallEstimateCredits}`));
|
|
25083
|
-
log.dim(` = that average rounded up to a whole credit. disclosure.estimatedCreditsPerPeriod = your calls per period \xD7 ${formatCredits(record.averageCreditsPerCall)}, rounded up.`);
|
|
24912
|
+
log.dim(" The ceiling is the price's room to be wrong; declaring one under the price is refused.");
|
|
25084
24913
|
}
|
|
25085
24914
|
}
|
|
25086
24915
|
async function explainBenchRefusal(res, call, base, log, opts, index, settledSoFar) {
|
|
25087
|
-
if (printedStructuredError(res)) {
|
|
25088
|
-
const printed = await res.json().catch(() => ({}));
|
|
25089
|
-
return { stop: true, refusal: { status: res.status, error: typeof printed.error === "string" ? printed.error : null, slotsHeld: null, slotLimit: null } };
|
|
25090
|
-
}
|
|
24916
|
+
if (printedStructuredError(res)) return { stop: true, refusal: { status: res.status, error: null, slotsHeld: null, slotLimit: null } };
|
|
25091
24917
|
const body = await res.json().catch(() => ({}));
|
|
25092
24918
|
const refusal2 = { status: res.status, error: body.error ?? null, slotsHeld: null, slotLimit: null };
|
|
25093
24919
|
if (res.status === 429 && body.error === "generation_limit") {
|
|
@@ -25097,10 +24923,6 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
25097
24923
|
if (!opts.json) log.error(` Sample ${index + 1} ${benchSlotsHeldSentence(counts)}`);
|
|
25098
24924
|
return { stop: true, refusal: refusal2 };
|
|
25099
24925
|
}
|
|
25100
|
-
if (res.status === 403 && body.error === "credits_unverified") {
|
|
25101
|
-
if (!opts.json) log.error(` ${LLM_BENCH_UNVERIFIED_LINE}`);
|
|
25102
|
-
return { stop: true, refusal: refusal2 };
|
|
25103
|
-
}
|
|
25104
24926
|
if (opts.json) return { stop: res.status !== 429, refusal: refusal2 };
|
|
25105
24927
|
if (res.status === 404 && body.error === "not_found") {
|
|
25106
24928
|
log.error(` ${LLM_LANE_OFF_LINE}`);
|
|
@@ -25116,7 +24938,7 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
25116
24938
|
return { stop: true, refusal: refusal2 };
|
|
25117
24939
|
}
|
|
25118
24940
|
if (res.status === 402) {
|
|
25119
|
-
log.error(" Not enough
|
|
24941
|
+
log.error(" Not enough coin to start the attempt.");
|
|
25120
24942
|
return { stop: true, refusal: refusal2 };
|
|
25121
24943
|
}
|
|
25122
24944
|
log.error(
|
|
@@ -25125,7 +24947,7 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
25125
24947
|
return { stop: res.status >= 500 || res.status === 401 || res.status === 403, refusal: refusal2 };
|
|
25126
24948
|
}
|
|
25127
24949
|
function printProviderRefusal(log, status2, message, awaitingBill = false) {
|
|
25128
|
-
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so
|
|
24950
|
+
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so this call's coin stays held until it resolves (see \`genex llm status\`); it is not a sample.`);
|
|
25129
24951
|
else log.dim(" The provider refused this call at its door \u2014 no inference ran, it cost nothing, and it is not a sample.");
|
|
25130
24952
|
if (message) log.dim(` Provider said: ${message}`);
|
|
25131
24953
|
if (status2 === 401 || status2 === 403) {
|
|
@@ -25151,6 +24973,19 @@ async function pollSettled(call, url, timeoutSec) {
|
|
|
25151
24973
|
}
|
|
25152
24974
|
return last;
|
|
25153
24975
|
}
|
|
24976
|
+
async function readCoinBalance(apiUrl, token) {
|
|
24977
|
+
try {
|
|
24978
|
+
const res = await apiFetch(`${apiUrl}/api/coin/balance`, {
|
|
24979
|
+
headers: { Authorization: `Bearer ${token}` },
|
|
24980
|
+
signal: AbortSignal.timeout(6e3)
|
|
24981
|
+
});
|
|
24982
|
+
if (!res.ok) return null;
|
|
24983
|
+
const body = await res.json().catch(() => null);
|
|
24984
|
+
return typeof body?.spendable === "number" ? body.spendable : null;
|
|
24985
|
+
} catch {
|
|
24986
|
+
return null;
|
|
24987
|
+
}
|
|
24988
|
+
}
|
|
25154
24989
|
async function saveBench(cwd, record) {
|
|
25155
24990
|
try {
|
|
25156
24991
|
const file = path36.join(cwd, LLM_BENCH_FILE);
|
|
@@ -25166,10 +25001,9 @@ function benchFailed(opts, log, message) {
|
|
|
25166
25001
|
else log.error(message);
|
|
25167
25002
|
process.exitCode = 1;
|
|
25168
25003
|
}
|
|
25169
|
-
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
25004
|
+
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCoins, opts) {
|
|
25170
25005
|
return {
|
|
25171
25006
|
command: "llm bench",
|
|
25172
|
-
currency: "credits",
|
|
25173
25007
|
apiUrl,
|
|
25174
25008
|
projectId,
|
|
25175
25009
|
modelId: opts.modelId ?? null,
|
|
@@ -25178,28 +25012,21 @@ function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCre
|
|
|
25178
25012
|
schemaPath: opts.schemaPath ?? null,
|
|
25179
25013
|
samples,
|
|
25180
25014
|
completed: 0,
|
|
25181
|
-
|
|
25015
|
+
maxCoinsPerSample: maxCoins,
|
|
25182
25016
|
balanceBefore: null,
|
|
25183
25017
|
balanceAfter: null,
|
|
25184
25018
|
results: [],
|
|
25185
25019
|
refusal: null,
|
|
25186
|
-
|
|
25187
|
-
|
|
25188
|
-
|
|
25189
|
-
|
|
25190
|
-
recommendedDeclaredHeadroomBps: null,
|
|
25191
|
-
feeBps: null,
|
|
25192
|
-
usdCentsPerCredit: null,
|
|
25193
|
-
missingTerms: null,
|
|
25194
|
-
costBasedMaxCredits: null,
|
|
25195
|
-
lengthRaisedCeiling: false,
|
|
25020
|
+
chargedCoins: { p50: null, p95: null, max: null },
|
|
25021
|
+
fitCoins: { p95: null, max: null },
|
|
25022
|
+
chargedBasedEstimateCoins: null,
|
|
25023
|
+
lengthRaisedPrice: false,
|
|
25196
25024
|
cutOffSamples: 0,
|
|
25197
25025
|
exceedsStandCap: false,
|
|
25198
25026
|
noRecommendationReason: null,
|
|
25199
|
-
|
|
25200
|
-
|
|
25201
|
-
|
|
25202
|
-
recommendedPerCallEstimateCredits: null,
|
|
25027
|
+
recommendedDeclaredHeadroomBps: null,
|
|
25028
|
+
recommendedEstimateCoins: null,
|
|
25029
|
+
recommendedPerCallMaxCoins: null,
|
|
25203
25030
|
savedTo: null
|
|
25204
25031
|
};
|
|
25205
25032
|
}
|
|
@@ -25262,8 +25089,7 @@ async function reportStatus(args) {
|
|
|
25262
25089
|
for (const row of rows) {
|
|
25263
25090
|
const state = rowState(row);
|
|
25264
25091
|
const mark = state === "active" ? c.cyan("\u25CF") : state === "awaiting bill" ? c.yellow("\u25D0") : c.green("\u25CB");
|
|
25265
|
-
const
|
|
25266
|
-
const reserved = money2.reserved === null ? "reserved \u2014" : `${money2.reserved} ${money2.unit} reserved`;
|
|
25092
|
+
const reserved = typeof row.reservedCoins === "number" ? `${row.reservedCoins} coin reserved` : "reserved \u2014";
|
|
25267
25093
|
log.plain(
|
|
25268
25094
|
` ${mark} ${row.id ?? "?"} ${row.modelId ?? "?"} ${state.padEnd(13)} ${formatAge(row.createdAt, now).padStart(4)} old ${reserved}${row.error ? ` ${row.error}` : ""}${rowHoldsSlot(row) ? "" : c.dim(" (no slot)")}`
|
|
25269
25095
|
);
|
|
@@ -25335,11 +25161,10 @@ async function cancelCall(args) {
|
|
|
25335
25161
|
if (!opts.json) {
|
|
25336
25162
|
log.warn(` ${row.id} (${row.status ?? "?"}) \u2014 ${sentence}.`);
|
|
25337
25163
|
if (state === "awaiting bill") {
|
|
25338
|
-
log.dim(` Nothing to cancel:
|
|
25164
|
+
log.dim(` Nothing to cancel: its coin releases when the bill resolves, and it stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`);
|
|
25339
25165
|
if (row.providerMessage?.trim()) log.dim(` Provider said: ${row.providerMessage.trim()}`);
|
|
25340
25166
|
} else {
|
|
25341
|
-
|
|
25342
|
-
log.dim(` ${money2.charged === null ? "charge unknown" : `${money2.charged} ${money2.unit} charged`}${row.error ? ` \xB7 ${row.error}` : ""}.`);
|
|
25167
|
+
log.dim(` ${typeof row.chargedCoins === "number" ? `${row.chargedCoins} coin charged` : "charge unknown"}${row.error ? ` \xB7 ${row.error}` : ""}.`);
|
|
25343
25168
|
}
|
|
25344
25169
|
}
|
|
25345
25170
|
return done("ok", null, state === "awaiting bill" ? "awaiting_bill" : "already_final", row);
|
|
@@ -25361,126 +25186,78 @@ async function cancelCall(args) {
|
|
|
25361
25186
|
const after = await res.json().catch(() => null) ?? row;
|
|
25362
25187
|
if (!opts.json) {
|
|
25363
25188
|
log.success(` Cancelled ${row.id} (was ${row.status ?? "active"}).`);
|
|
25364
|
-
log.dim(`
|
|
25189
|
+
log.dim(` Its coin releases once the bill settles; ${c.cyan("genex llm status")} shows it until then.`);
|
|
25365
25190
|
}
|
|
25366
25191
|
return done("ok", null, "cancelled", after);
|
|
25367
25192
|
}
|
|
25368
|
-
async function
|
|
25369
|
-
let
|
|
25193
|
+
async function reportSavedPrice(cwd, opts, log) {
|
|
25194
|
+
let record = null;
|
|
25370
25195
|
try {
|
|
25371
|
-
|
|
25196
|
+
record = JSON.parse(await fs35.readFile(path36.join(cwd, LLM_BENCH_FILE), "utf8"));
|
|
25372
25197
|
} catch {
|
|
25373
|
-
|
|
25198
|
+
record = null;
|
|
25374
25199
|
}
|
|
25375
|
-
if (!
|
|
25376
|
-
const v = parsed;
|
|
25377
|
-
if (v.v === 2 && v.currency === "credits" && v.cost && v.charged) return { kind: "credits", record: parsed };
|
|
25378
|
-
return { kind: "coins", record: parsed };
|
|
25379
|
-
}
|
|
25380
|
-
var LLM_PRICE_COIN_TERMS_LINE = "This bench was measured under the old COIN terms, when a game declared a fixed price per call. In-game calls are now billed in the player's credits, as used, and a game declares a per-call CEILING (maxCredits) instead \u2014 a coin price is not that number. Re-run the bench for the figures to declare.";
|
|
25381
|
-
async function reportSavedPrice(cwd, opts, log) {
|
|
25382
|
-
const saved = await readSavedBench(cwd);
|
|
25383
|
-
const rerun = 'genex llm bench "<prompt>" --max-credits <n> --user-approved';
|
|
25384
|
-
const creditKeys = (record2) => ({
|
|
25385
|
-
costUsdPicos: record2?.cost ?? { p50: null, p95: null, max: null },
|
|
25386
|
-
chargedCredits: record2?.charged ?? { p50: null, p95: null, max: null },
|
|
25387
|
-
fitP95: record2?.fitP95 ?? null,
|
|
25388
|
-
fitMax: record2?.fitMax ?? null,
|
|
25389
|
-
recommendedDeclaredHeadroomBps: record2?.recommendedDeclaredHeadroomBps ?? null,
|
|
25390
|
-
feeBps: record2?.feeBps ?? null,
|
|
25391
|
-
usdCentsPerCredit: record2?.usdCentsPerCredit ?? null,
|
|
25392
|
-
costBasedMaxCredits: record2?.costBasedMaxCredits ?? null,
|
|
25393
|
-
lengthRaisedCeiling: record2?.lengthRaisedCeiling ?? false,
|
|
25394
|
-
cutOffSamples: record2?.cutOffSamples ?? 0,
|
|
25395
|
-
noRecommendationReason: record2?.noRecommendationReason ?? null,
|
|
25396
|
-
recommendedMaxCredits: record2?.recommendedMaxCredits ?? null,
|
|
25397
|
-
recommendedPerCallMaxCredits: record2?.recommendedPerCallMaxCredits ?? null,
|
|
25398
|
-
averageCreditsPerCall: record2?.averageCreditsPerCall ?? null,
|
|
25399
|
-
recommendedPerCallEstimateCredits: record2?.recommendedPerCallEstimateCredits ?? null
|
|
25400
|
-
});
|
|
25401
|
-
if (!saved) {
|
|
25200
|
+
if (!record) {
|
|
25402
25201
|
if (opts.json) {
|
|
25403
25202
|
writeJsonLine({
|
|
25404
25203
|
command: "llm price",
|
|
25405
25204
|
status: "failed",
|
|
25406
25205
|
error: "no_bench",
|
|
25407
|
-
currency: null,
|
|
25408
25206
|
ranAt: null,
|
|
25409
25207
|
modelId: null,
|
|
25410
25208
|
samples: null,
|
|
25411
25209
|
completed: null,
|
|
25412
|
-
|
|
25413
|
-
|
|
25210
|
+
p50: null,
|
|
25211
|
+
p95: null,
|
|
25212
|
+
max: null,
|
|
25213
|
+
fitP95: null,
|
|
25214
|
+
chargedBasedEstimateCoins: null,
|
|
25215
|
+
lengthRaisedPrice: false,
|
|
25216
|
+
cutOffSamples: 0,
|
|
25217
|
+
noRecommendationReason: null,
|
|
25218
|
+
recommendedDeclaredHeadroomBps: null,
|
|
25219
|
+
recommendedEstimateCoins: null,
|
|
25220
|
+
recommendedPerCallMaxCoins: null,
|
|
25414
25221
|
savedTo: null
|
|
25415
25222
|
});
|
|
25416
25223
|
} else {
|
|
25417
25224
|
log.error("Nothing has been benchmarked in this folder yet.");
|
|
25418
|
-
log.dim(` Run ${c.cyan(
|
|
25419
|
-
}
|
|
25420
|
-
process.exitCode = 1;
|
|
25421
|
-
return;
|
|
25422
|
-
}
|
|
25423
|
-
if (saved.kind === "coins") {
|
|
25424
|
-
const old = saved.record;
|
|
25425
|
-
const legacy = {
|
|
25426
|
-
p50: old.p50 ?? null,
|
|
25427
|
-
p95: old.p95 ?? null,
|
|
25428
|
-
max: old.max ?? null,
|
|
25429
|
-
recommendedEstimateCoins: old.recommendedEstimateCoins ?? null,
|
|
25430
|
-
recommendedPerCallMaxCoins: old.recommendedPerCallMaxCoins ?? null,
|
|
25431
|
-
maxCoinsPerSample: old.maxCoinsPerSample ?? null,
|
|
25432
|
-
noRecommendationReason: old.noRecommendationReason ?? null
|
|
25433
|
-
};
|
|
25434
|
-
if (opts.json) {
|
|
25435
|
-
writeJsonLine({
|
|
25436
|
-
command: "llm price",
|
|
25437
|
-
status: "failed",
|
|
25438
|
-
error: "coin_terms",
|
|
25439
|
-
currency: "coins",
|
|
25440
|
-
ranAt: old.ranAt ?? null,
|
|
25441
|
-
modelId: old.modelId ?? null,
|
|
25442
|
-
samples: old.samples ?? null,
|
|
25443
|
-
completed: old.completed ?? null,
|
|
25444
|
-
...creditKeys(null),
|
|
25445
|
-
legacyCoinTerms: legacy,
|
|
25446
|
-
savedTo: LLM_BENCH_FILE
|
|
25447
|
-
});
|
|
25448
|
-
} else {
|
|
25449
|
-
log.plain(c.bold("genex llm price"));
|
|
25450
|
-
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${old.ranAt ?? "at an unknown time"}${old.modelId ? ` on ${old.modelId}` : ""}`);
|
|
25451
|
-
log.plain("");
|
|
25452
|
-
log.warn(` ${LLM_PRICE_COIN_TERMS_LINE}`);
|
|
25453
|
-
const measured = legacy.p95 === null ? "no settled sample" : `charged coins p50 ${legacy.p50} \xB7 p95 ${legacy.p95} \xB7 max ${legacy.max}`;
|
|
25454
|
-
const recommended = legacy.recommendedEstimateCoins === null ? "no price recommended" : `it recommended estimateCoins ${legacy.recommendedEstimateCoins}${legacy.recommendedPerCallMaxCoins != null ? ` and perCallMaxCoins ${legacy.recommendedPerCallMaxCoins}` : ""}`;
|
|
25455
|
-
log.dim(` What it measured then: ${measured}; ${recommended}.`);
|
|
25456
|
-
log.dim(` Re-run: ${c.cyan(rerun)}`);
|
|
25225
|
+
log.dim(` Run ${c.cyan('genex llm bench "<prompt>" --max-coins <n> --user-approved')} first.`);
|
|
25457
25226
|
}
|
|
25458
25227
|
process.exitCode = 1;
|
|
25459
25228
|
return;
|
|
25460
25229
|
}
|
|
25461
|
-
const record = saved.record;
|
|
25462
25230
|
if (opts.json) {
|
|
25463
25231
|
writeJsonLine({
|
|
25464
25232
|
command: "llm price",
|
|
25465
|
-
status: record.
|
|
25466
|
-
error: record.
|
|
25467
|
-
currency: "credits",
|
|
25233
|
+
status: record.recommendedEstimateCoins === null ? "failed" : "ok",
|
|
25234
|
+
error: record.recommendedEstimateCoins === null ? record.noRecommendationReason ?? "no_recommendation" : null,
|
|
25468
25235
|
ranAt: record.ranAt ?? null,
|
|
25469
25236
|
modelId: record.modelId ?? null,
|
|
25470
25237
|
samples: record.samples ?? null,
|
|
25471
25238
|
completed: record.completed ?? null,
|
|
25472
|
-
|
|
25473
|
-
|
|
25239
|
+
p50: record.p50 ?? null,
|
|
25240
|
+
p95: record.p95 ?? null,
|
|
25241
|
+
max: record.max ?? null,
|
|
25242
|
+
// Absent from a file an older CLI wrote; null / false / 0 then, never missing.
|
|
25243
|
+
fitP95: record.fitP95 ?? null,
|
|
25244
|
+
chargedBasedEstimateCoins: record.chargedBasedEstimateCoins ?? null,
|
|
25245
|
+
lengthRaisedPrice: record.lengthRaisedPrice ?? false,
|
|
25246
|
+
cutOffSamples: record.cutOffSamples ?? 0,
|
|
25247
|
+
noRecommendationReason: record.noRecommendationReason ?? null,
|
|
25248
|
+
recommendedDeclaredHeadroomBps: record.recommendedDeclaredHeadroomBps ?? null,
|
|
25249
|
+
recommendedEstimateCoins: record.recommendedEstimateCoins ?? null,
|
|
25250
|
+
recommendedPerCallMaxCoins: record.recommendedPerCallMaxCoins ?? null,
|
|
25474
25251
|
savedTo: LLM_BENCH_FILE
|
|
25475
25252
|
});
|
|
25476
|
-
if (record.
|
|
25253
|
+
if (record.recommendedEstimateCoins === null) process.exitCode = 1;
|
|
25477
25254
|
return;
|
|
25478
25255
|
}
|
|
25479
25256
|
log.plain(c.bold("genex llm price"));
|
|
25480
25257
|
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${record.ranAt}`);
|
|
25481
25258
|
log.plain("");
|
|
25482
25259
|
printRecommendation(log, record);
|
|
25483
|
-
if (record.
|
|
25260
|
+
if (record.recommendedEstimateCoins === null) process.exitCode = 1;
|
|
25484
25261
|
}
|
|
25485
25262
|
function writeJsonLine(value) {
|
|
25486
25263
|
process.stdout.write(`${JSON.stringify(value)}
|
|
@@ -25740,7 +25517,7 @@ function runtimeRow(state, lane, toolsOnly = false) {
|
|
|
25740
25517
|
return {
|
|
25741
25518
|
label: "Runtime",
|
|
25742
25519
|
value: `in-game LLM lane live \xB7 ${n} model${n === 1 ? "" : "s"}${keyRefused ? ` \xB7 ${c.yellow("provider key refused")}` : ""}`,
|
|
25743
|
-
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : '
|
|
25520
|
+
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : 'Price a call before a game declares one: `npx genex llm bench "<prompt>" --max-coins <n> --user-approved`.'
|
|
25744
25521
|
};
|
|
25745
25522
|
}
|
|
25746
25523
|
case "off":
|
|
@@ -27696,16 +27473,16 @@ ${c.bold("Usage")}
|
|
|
27696
27473
|
status | cancel. "models" says whether this
|
|
27697
27474
|
stand serves the runtime LLM lane at all, and
|
|
27698
27475
|
at what provider rates. "bench" runs the real
|
|
27699
|
-
model on YOUR OWN
|
|
27700
|
-
attempt
|
|
27476
|
+
model on YOUR OWN coin and reports what each
|
|
27477
|
+
attempt was charged; it needs --max-coins <n>
|
|
27701
27478
|
--user-approved, like any spend the player has
|
|
27702
27479
|
to agree to. "price" reprints the last run's
|
|
27703
|
-
recommended
|
|
27480
|
+
recommended estimateCoins. "status" lists this
|
|
27704
27481
|
project's open calls and the slots they hold;
|
|
27705
27482
|
"cancel <id>" stops an active one. Declare a
|
|
27706
|
-
game's
|
|
27707
|
-
|
|
27708
|
-
|
|
27483
|
+
game's price from a bench, never from a guess:
|
|
27484
|
+
a started attempt is charged in full, including
|
|
27485
|
+
one that fails.
|
|
27709
27486
|
genex player <sub> [options] Answer the in-game model requests you approve, on your
|
|
27710
27487
|
OWN Claude or ChatGPT subscription: install | run |
|
|
27711
27488
|
status | stop | uninstall. "install" signs this machine
|
|
@@ -28167,7 +27944,7 @@ ${c.bold("Examples")}
|
|
|
28167
27944
|
genex shop remove sku_123
|
|
28168
27945
|
genex llm models
|
|
28169
27946
|
genex llm models --all
|
|
28170
|
-
genex llm bench "Reply with one short taunt." --samples 5 --max-
|
|
27947
|
+
genex llm bench "Reply with one short taunt." --samples 5 --max-coins 20 --user-approved
|
|
28171
27948
|
genex llm price
|
|
28172
27949
|
genex llm status
|
|
28173
27950
|
genex llm cancel <id>
|
|
@@ -28404,24 +28181,22 @@ ${c.bold("How the allowance works")}
|
|
|
28404
28181
|
|
|
28405
28182
|
Every number printed is live from your account; prices can change without a deploy.
|
|
28406
28183
|
`,
|
|
28407
|
-
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what
|
|
28184
|
+
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what should the game declare?
|
|
28408
28185
|
|
|
28409
28186
|
${c.bold("Usage")}
|
|
28410
28187
|
genex llm models [--all] [--json] Which models this stand serves, their provider rates per
|
|
28411
|
-
million tokens, the
|
|
28412
|
-
|
|
28413
|
-
|
|
28414
|
-
|
|
28415
|
-
|
|
28416
|
-
|
|
28417
|
-
|
|
28418
|
-
|
|
28419
|
-
|
|
28420
|
-
|
|
28421
|
-
|
|
28422
|
-
|
|
28423
|
-
allowance does not count. Needs a folder linked to a
|
|
28424
|
-
game you own, and a balance that covers one attempt.
|
|
28188
|
+
million tokens, and the headroom it recommends over a
|
|
28189
|
+
benchmark. The catalog is synced from OpenRouter, so the
|
|
28190
|
+
FEATURED short list is printed by default and --all
|
|
28191
|
+
prints every row (--json always carries every row). Off
|
|
28192
|
+
on this stand: it says so and exits 0 \u2014 build the feature
|
|
28193
|
+
without a model rather than promising a 404.
|
|
28194
|
+
genex llm bench "<prompt>" --max-coins <n> --user-approved [options]
|
|
28195
|
+
Run the real model <n> times through the development
|
|
28196
|
+
lane and report what each attempt was CHARGED. Refused
|
|
28197
|
+
without both flags, before any network call: this spends
|
|
28198
|
+
YOUR OWN COIN, a wallet the build's asset allowance
|
|
28199
|
+
cannot see. Needs a folder linked to a game you own.
|
|
28425
28200
|
genex llm price [--json] The recommendation from the last bench in this folder.
|
|
28426
28201
|
Reads a file; spends nothing.
|
|
28427
28202
|
genex llm status [--json] This project's open development calls: active, or
|
|
@@ -28438,32 +28213,29 @@ ${c.bold("Bench options")}
|
|
|
28438
28213
|
--samples <n> How many real attempts (default ${DEFAULT_BENCH_SAMPLES}). More samples, better p95.
|
|
28439
28214
|
--schema <file> A JSON file holding the output schema; implies JSON output.
|
|
28440
28215
|
--json-output Ask for JSON without a schema. --text Ask for text (the default).
|
|
28441
|
-
--max-
|
|
28216
|
+
--max-coins <n> The per-attempt ceiling YOU approved \u2014 a bench runs on your own coin,
|
|
28442
28217
|
never a player's. The run's worst case is that number times --samples,
|
|
28443
|
-
and it is printed before anything starts.
|
|
28444
|
-
name, read as the same number.)
|
|
28218
|
+
and it is printed before anything starts.
|
|
28445
28219
|
--timeout <seconds> Bounds the wait for each attempt.
|
|
28446
28220
|
--json One machine-readable object; every key is always present, null when
|
|
28447
28221
|
the CLI could not learn it.
|
|
28448
28222
|
|
|
28449
28223
|
${c.bold("Why a benchmark and not an estimate")}
|
|
28450
|
-
|
|
28451
|
-
|
|
28452
|
-
|
|
28453
|
-
|
|
28454
|
-
|
|
28455
|
-
|
|
28456
|
-
|
|
28457
|
-
|
|
28458
|
-
|
|
28459
|
-
recommendation, because those multipliers are not the CLI's to invent.
|
|
28224
|
+
\`estimateCoins\` is a PRICE, not a guess: declaring it charges it, in full, even when the
|
|
28225
|
+
attempt fails, is cancelled, or stops at its budget. And three layers sit between the
|
|
28226
|
+
provider's rate card and that number \u2014 the provider's cost, the platform's tariff (already
|
|
28227
|
+
inside every charged coin), and your headroom. The bench itself is billed to YOU, on the
|
|
28228
|
+
development lane; the player-funded lane is what the published game uses. Adding a margin
|
|
28229
|
+
to the provider's rate silently drops the tariff and under-prices every call, so the
|
|
28230
|
+
recommendation is built from CHARGED COINS and the headroom comes from the server. A stand
|
|
28231
|
+
that serves no headroom gets no recommendation, because the multiplier is not the CLI's to
|
|
28232
|
+
invent.
|
|
28460
28233
|
|
|
28461
|
-
|
|
28462
|
-
|
|
28463
|
-
A sample cut off at your --max-
|
|
28464
|
-
|
|
28465
|
-
|
|
28466
|
-
fee included, which is the honest estimate a budget's disclosure is built on.
|
|
28234
|
+
The declared price also decides how LONG the answer may be: it sizes each call's output
|
|
28235
|
+
allowance. So the recommendation is never below the smallest price that leaves room for
|
|
28236
|
+
the benchmarked answer, as the server computes it. A sample cut off at your --max-coins
|
|
28237
|
+
(provider_token_limit) is not a sample \u2014 re-run with a higher --max-coins; an answer longer
|
|
28238
|
+
than one call on this stand may produce gets no price at all \u2014 ask for a shorter one.
|
|
28467
28239
|
|
|
28468
28240
|
Results land in ${LLM_BENCH_FILE} \u2014 gitignored, machine-local, re-read by \`genex llm price\`.
|
|
28469
28241
|
`,
|
|
@@ -28678,13 +28450,11 @@ function parseArgs(argv) {
|
|
|
28678
28450
|
"--limit",
|
|
28679
28451
|
// `genex budget --assets <credits>` — the allowance the player approved.
|
|
28680
28452
|
"--assets",
|
|
28681
|
-
// `genex llm bench` value flags. `--max-
|
|
28682
|
-
// shape as `--assets
|
|
28683
|
-
// count a bench. `--max-coins` is its deprecated name (same number).
|
|
28453
|
+
// `genex llm bench` value flags. `--max-coins` is the COIN half of the
|
|
28454
|
+
// same approval shape as `--assets` (a different economy, its own gate).
|
|
28684
28455
|
"--model",
|
|
28685
28456
|
"--samples",
|
|
28686
28457
|
"--schema",
|
|
28687
|
-
"--max-credits",
|
|
28688
28458
|
"--max-coins",
|
|
28689
28459
|
// `genex player install --label <name>` — what this machine is called in
|
|
28690
28460
|
// the dashboard. Its own key rather than the generic 2nd positional,
|
|
@@ -29202,14 +28972,12 @@ function applyValueFlag(options, flag, value) {
|
|
|
29202
28972
|
options.samples = n;
|
|
29203
28973
|
break;
|
|
29204
28974
|
}
|
|
29205
|
-
case "--max-credits":
|
|
29206
28975
|
case "--max-coins": {
|
|
29207
28976
|
const n = Number(value);
|
|
29208
28977
|
if (!Number.isInteger(n) || n < 1) {
|
|
29209
|
-
throw new Error(`Invalid
|
|
28978
|
+
throw new Error(`Invalid --max-coins value: ${value} (whole coin, 1 or more)`);
|
|
29210
28979
|
}
|
|
29211
|
-
|
|
29212
|
-
else options.maxCoins = n;
|
|
28980
|
+
options.maxCoins = n;
|
|
29213
28981
|
break;
|
|
29214
28982
|
}
|
|
29215
28983
|
case "--timeout": {
|
|
@@ -29507,9 +29275,9 @@ async function main() {
|
|
|
29507
29275
|
await runBudget(parsed.options);
|
|
29508
29276
|
break;
|
|
29509
29277
|
// The in-game model lane: is it live here, what does one call really
|
|
29510
|
-
// cost, what
|
|
29511
|
-
//
|
|
29512
|
-
//
|
|
29278
|
+
// cost, what should the game declare. `bench` is the only CLI command
|
|
29279
|
+
// that spends COIN, so it carries its own --max-coins/--user-approved
|
|
29280
|
+
// gate — the credits allowance cannot see that wallet.
|
|
29513
29281
|
case "llm":
|
|
29514
29282
|
await runLlm(parsed.options);
|
|
29515
29283
|
break;
|