@genex-ai/cli-demo 1.36.5-dev.857 → 1.36.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{blender-mcp-WSURPZET.js → blender-mcp-QVEGHE5L.js} +2 -2
- package/dist/{blender-serve-EH2MOW3U.js → blender-serve-66OHC4ML.js} +1 -1
- package/dist/{chunk-EABIVAGT.js → chunk-4BBTPRJW.js} +2 -2
- package/dist/{chunk-K4EHXOZK.js → chunk-QI7FIYBY.js} +1 -1
- package/dist/index.js +216 -448
- package/package.json +1 -1
- package/templates/skills/genex-game-director/SKILL.md +1 -1
- package/templates/skills/genex-getting-started/SKILL.md +2 -2
- package/templates/skills/genex-llm-in-games/SKILL.md +72 -209
- package/templates/skills/genex-llm-in-games/references/pricing.md +85 -126
- package/templates/skills/genex-tool-llm/SKILL.md +9 -11
- package/templates/skills/genex-updates/SKILL.md +1 -1
package/dist/index.js
CHANGED
|
@@ -57,7 +57,7 @@ import {
|
|
|
57
57
|
writeSecretFile,
|
|
58
58
|
writeUserToken,
|
|
59
59
|
writeWorkspace
|
|
60
|
-
} from "./chunk-
|
|
60
|
+
} from "./chunk-4BBTPRJW.js";
|
|
61
61
|
import {
|
|
62
62
|
CLI_CHANNEL,
|
|
63
63
|
DEFAULT_API_URL,
|
|
@@ -78,7 +78,7 @@ import {
|
|
|
78
78
|
normalizeApiOrigin,
|
|
79
79
|
originKey,
|
|
80
80
|
resolveAgentTargets
|
|
81
|
-
} from "./chunk-
|
|
81
|
+
} from "./chunk-QI7FIYBY.js";
|
|
82
82
|
|
|
83
83
|
// src/instrument.ts
|
|
84
84
|
import * as Sentry from "@sentry/node";
|
|
@@ -981,7 +981,7 @@ Important note: put soul into your creations, with many details and love. Aim to
|
|
|
981
981
|
21. A turn ends in exactly one of two ways: on a question the player must answer before the next step can be chosen, or on a handoff \u2014 the draft link, what changed, what to try, and what is still open. Never close a turn on a promise. "I'll keep building", "while I keep working", "adding that now" are things you say and then DO before you stop; if you are stopping, say that you have stopped and what remains. While the Build plan's \`Now:\` line has open work and no answer is needed from the player, the turn is not over: preview, hand off, and start the next milestone in the same turn, until the requested outcome is reached or the platform's own budget ends the session. If a stop was forced on you anyway, the first line of your next turn is the promise you left and where it stands.
|
|
982
982
|
22. **What the player sees must read as the thing it is**, in this game's own style: a building as that building, a person as a person, a prop as that prop, a surface as its material \u2014 and a coloured box, a capsule, or a flat grey block standing in for one of them is never its finished version. Generation is the default route to that bar wherever code will not honestly reach it: the player's character, whatever the request names, whatever the player walks up to, enters, or interacts with, and the music and sound underneath \u2014 reach for the belt on your own judgment, without stopping to ask; a status line saying what you queued is enough, and \`--no-wait\` keeps you building while it lands. Procedural code is a first-class engine for what is structural, repeated, distant, or parametric \u2014 terrain, sky, fences, paving, modular kits, filler \u2014 and for anything else only when the result meets the same bar and you have looked at it in a capture before its row says \`landed\`. Start with one reusable implementation per distinct required gameplay or visual role. Background populations default to reusable procedural bodies and motion or compatible existing rigged assets. Catalog clips need a compatible skeleton and may be paid. Reserve paid custom bodies for the player and characters examined or interacted with closely, while honoring explicit user requests. Add variants only for an unmet requirement; unspent allowance is not unfinished work. Allowance questions use timeoutPolicy no-consent: silence does not raise the allowance. Record role, reuse and completion criteria in the existing Assets table. Decide per object, not per category, and write the route on each Assets row in \`DESIGN.md\` \u2014 the paid flow for generated pieces, the procedural flow with its local path for code-built ones. An Assets table with no generation in it is a decision, not a default: record it as \`Generation: none \u2014 <why code alone reaches the bar here>\` or generate.
|
|
983
983
|
23. **The screenshots you take to look at your own work go in \`.genex/scratch/\`.** Captures, render comparisons, before/after strips, traces and metrics dumps are how you SEE what you built \u2014 they are not part of what you built, and \`genex preview\` pushes the whole folder to the game's source, every time, forever. \`.genex/scratch/\` already exists for this and is already ignored, so there is nothing to set up: write the capture as \`.genex/scratch/arena-before.png\` and read it back from there. Never put them in a folder of your own at the top level: \`progress/\`, \`reports/\`, \`shots/\` and their kind are pushed like source and become permanent weight in the game and in every remix of it \u2014 one real game reached 11 GB and 10,332 committed screenshots exactly this way. If you find such a folder already there, add it to \`.gitignore\`; never delete the player's files (law 18). One file in \`.genex/scratch/\` IS read, by the platform: \`.genex/scratch/cover.png\` is the game's cover \u2014 the frame you would put on its poster: real gameplay at its signature moment, well lit, no menus, popups or busy HUD, landscape 16:9, and never any text (the page sets the title beside it). The next \`genex preview\` makes it the cover; save a better moment over it whenever one comes along, and if your browser tool can set the cover itself, use that instead. Before the first publish, and whenever the game's look changes, load \`$genex-cover\`: it chooses, stages, lights and judges that frame.
|
|
984
|
-
24. **Benchmark before declaring an in-game LLM
|
|
984
|
+
24. **Benchmark before declaring an in-game LLM price.** If the game calls a language model while the player plays \u2014 an NPC answering in its own words, a quest written for this save, a judge reading what the player typed \u2014 the PLAYER pays for it, and \`estimateCoins\` is the FIXED price of a started attempt: charged whether it succeeds, fails, is canceled, or stops at its own budget. Never pick that number by judgement or from a vendor's rate card. Run the real prompt on your own coins \u2014 \`npx genex llm bench "<the prompt>" --schema <file> --samples 3 --max-coins <n> --user-approved\` \u2014 and declare the figure it recommends, recorded in \`DESIGN.md\` with its model and date; re-benchmark whenever the prompt, the schema or the model changes, because a prompt edit is a price change. Check the lane first with \`npx genex llm models\`: where it is off the routes 404, so build the feature behind a graceful unavailable state and say so in the handoff. A repeated call is a standing budget the player approves once (\`requestSpendGrant()\`, then \`generate({ grantId })\`), never a popup per turn, and its disclosed call rate is computed from the game's own loop. Model output is data the game validates against its own expectation \u2014 never executed, and never authority over coin, items, entitlements or rewards. Load \`$genex-llm-in-games\` before writing any of it.
|
|
985
985
|
${CONTRACT_END}
|
|
986
986
|
`;
|
|
987
987
|
var TOOLS_CONTRACT_HEAD = `# Genex Tools (always in effect in this folder)
|
|
@@ -995,7 +995,7 @@ var TOOLS_CONTRACT_OFFER = `5. **When the game is built and playable \u2014 neve
|
|
|
995
995
|
var TOOLS_CONTRACT_HOSTED_LAWS = `5. **This game is hosted on Genex now, and publishing is glue \u2014 never a rebuild.** The game ships exactly as the user built it: \`initEmbed()\` first in the boot code (the \`genex-threejs-embed-auth\` card has the call; the renderer draws before any \`await waitForPlayer()\`), a static build into \`dist/\` with relative asset paths, then \`npx genex preview\`. Never restructure, reformat or "improve" the game to ship it, never scaffold a new app around it, and never start a design document or a build plan for it \u2014 the user's own process is theirs. Load the \`genex-tool-publish\` card before the first preview; it owns the vocabulary, the links and the limits.
|
|
996
996
|
6. **Ship first, then report.** \`preview\` prints a preflight \u2014 phone memory, a missing volume slider, an asset nobody wired, a viewport line. It never blocks a deploy, so push the build as it is, hand over the link, THEN relay each preflight line to the user in one plain sentence with an offer to fix it, and fix one only on their yes. The preflight is a report for the user, not a to-do list for you.
|
|
997
997
|
7. **The link is the game's page, and there are two versions.** After every preview give the user \`<dashboard>/draft/<slug>\` \u2014 \`dashboardOrigins[0]\` and \`slug\` from \`.genex/project.json\` \u2014 never localhost, a file path or the bare play origin. \`preview\` updates the draft and never touches what players are on; the first release is \`npx genex publish\` (it lists the game); after that "publish it", "update it" and "yes" all mean \`npx genex promote\` \u2014 the exact draft build, no rebuild. Ask once per round of work, in one line, and keep working while you wait.`;
|
|
998
|
-
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with
|
|
998
|
+
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with Genex coins or their own Claude/ChatGPT subscription, and approves it on a Genex sheet; your game just calls \`generate()\`. Want it that way?`;
|
|
999
999
|
function toolsContractLlmLaw(n, hosted) {
|
|
1000
1000
|
const onYes = hosted ? "On a yes, load `$genex-tool-llm` and then `$genex-llm-in-games`, which owns the build." : "On a yes, load `$genex-tool-llm`, which owns the rest: it checks the lane with `npx genex llm models` first, then runs `npx genex init --convert` (a hosted game is a static browser build \u2014 a local server holding a key can never ship).";
|
|
1001
1001
|
return `${n}. **A model running while people PLAY is a platform feature \u2014 never something you wire onto the user's own meter.** When a request implies one (NPCs that talk or decide in their own words, content written from what the player types, a prompt box in the game, "let the player pick a model"), recognise it, say in ONE line: "${TOOLS_CONTRACT_LLM_OFFER}" \u2014 then ASK and wait. ${onYes} On a no, build the authored version instead. Either way, never ship a game that calls a model on a key or an account of the user's own: every visitor would spend their money with nobody approving it, and the credential is readable in the bundle. Player-funded generation is text and JSON only \u2014 3D, images, video and audio stay the asset lanes above, on the user's meter.`;
|
|
@@ -22948,7 +22948,7 @@ async function runBlender(opts) {
|
|
|
22948
22948
|
return 1;
|
|
22949
22949
|
}
|
|
22950
22950
|
if (sub === "serve") {
|
|
22951
|
-
const { serveLocalBlender } = await import("./blender-serve-
|
|
22951
|
+
const { serveLocalBlender } = await import("./blender-serve-66OHC4ML.js");
|
|
22952
22952
|
const port = Number(process.env.GENEX_BLENDER_PORT ?? 8088);
|
|
22953
22953
|
log.step(`Starting a local Blender service on port ${port}`);
|
|
22954
22954
|
log.plain(
|
|
@@ -22957,7 +22957,7 @@ async function runBlender(opts) {
|
|
|
22957
22957
|
return serveLocalBlender({ port, log });
|
|
22958
22958
|
}
|
|
22959
22959
|
if (sub === "mcp") {
|
|
22960
|
-
const { runBlenderMcp } = await import("./blender-mcp-
|
|
22960
|
+
const { runBlenderMcp } = await import("./blender-mcp-QVEGHE5L.js");
|
|
22961
22961
|
return runBlenderMcp();
|
|
22962
22962
|
}
|
|
22963
22963
|
if (sub === "seat") {
|
|
@@ -23465,12 +23465,7 @@ import fs35 from "fs/promises";
|
|
|
23465
23465
|
import path36 from "path";
|
|
23466
23466
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
23467
23467
|
var SUBS3 = ["models", "bench", "price", "status", "cancel"];
|
|
23468
|
-
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN
|
|
23469
|
-
var LLM_MAX_COINS_DEPRECATED = "--max-coins is deprecated: it is read as --max-credits, the same number \u2014 a bench spends credits now.";
|
|
23470
|
-
function benchBalanceShortLine(spendable, maxCredits) {
|
|
23471
|
-
return `Your spendable credits (${spendable}) cannot cover one attempt at --max-credits ${maxCredits} \u2014 nothing was sent, nothing was spent. Add credits, or re-run with a lower --max-credits once the person at the keyboard agrees to it.`;
|
|
23472
|
-
}
|
|
23473
|
-
var LLM_BENCH_UNVERIFIED_LINE = "Credits pay only for a verified email, and this account's is not verified yet (credits_unverified) \u2014 nothing was sent, nothing was spent. Verify it, then re-run.";
|
|
23468
|
+
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN coin \u2014 your wallet, not a player's, and one this build's asset allowance cannot see. Re-run with --max-coins <n> --user-approved only after the person at the keyboard agreed to the number.";
|
|
23474
23469
|
var LLM_LANE_OFF_LINE = "The runtime LLM lane is off on this stand \u2014 in-game generate() answers 404 here, and nothing can be benchmarked.";
|
|
23475
23470
|
var LLM_TOOLS_CONVERT_HINT = [
|
|
23476
23471
|
"In-game model calls need a hosted game. This folder is a Genex Tools workspace, so nothing here",
|
|
@@ -23483,7 +23478,7 @@ function serverSlotCounts(body) {
|
|
|
23483
23478
|
return { held: body.slotsHeld, limit: body.slotLimit };
|
|
23484
23479
|
}
|
|
23485
23480
|
var PENDING_SLOT_RELEASE_MINUTES = 10;
|
|
23486
|
-
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so
|
|
23481
|
+
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so its coin stays held until the bill resolves; such a row stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`;
|
|
23487
23482
|
var LLM_STATUS_NO_COUNT_LINE = "This stand does not report how many slots are held across your projects, so no count is shown.";
|
|
23488
23483
|
var LLM_CANCEL_AWAITING_BILL = "this call already stopped; its bill is awaiting the provider";
|
|
23489
23484
|
var LLM_CANCEL_ALREADY_FINAL = "this call is already final";
|
|
@@ -23497,33 +23492,11 @@ var DEFAULT_SAMPLE_TIMEOUT_SEC = 180;
|
|
|
23497
23492
|
function parseProviderKey(value) {
|
|
23498
23493
|
return value === "ok" || value === "rejected" || value === "unknown" ? value : null;
|
|
23499
23494
|
}
|
|
23500
|
-
function bpsOrNull(value) {
|
|
23501
|
-
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
23502
|
-
}
|
|
23503
|
-
function standingCreditGrantOrNull(value) {
|
|
23504
|
-
if (!value || typeof value !== "object") return null;
|
|
23505
|
-
const v = value;
|
|
23506
|
-
const whole = (x) => typeof x === "number" && Number.isSafeInteger(x) && x >= 0;
|
|
23507
|
-
if (!whole(v.minCredits) || !whole(v.maxCredits) || !whole(v.stepCredits) || !whole(v.defaultCredits)) return null;
|
|
23508
|
-
return {
|
|
23509
|
-
minCredits: v.minCredits,
|
|
23510
|
-
maxCredits: v.maxCredits,
|
|
23511
|
-
stepCredits: v.stepCredits,
|
|
23512
|
-
defaultCredits: v.defaultCredits,
|
|
23513
|
-
...whole(v.minUsdCents) ? { minUsdCents: v.minUsdCents } : {},
|
|
23514
|
-
...whole(v.maxUsdCents) ? { maxUsdCents: v.maxUsdCents } : {},
|
|
23515
|
-
...whole(v.stepUsdCents) ? { stepUsdCents: v.stepUsdCents } : {}
|
|
23516
|
-
};
|
|
23517
|
-
}
|
|
23518
23495
|
async function readRuntimeLane(apiUrl, token) {
|
|
23519
23496
|
const empty = (state) => ({
|
|
23520
23497
|
state,
|
|
23521
23498
|
models: null,
|
|
23522
23499
|
recommendedDeclaredHeadroomBps: null,
|
|
23523
|
-
currency: null,
|
|
23524
|
-
feeBps: null,
|
|
23525
|
-
usdCentsPerCredit: null,
|
|
23526
|
-
standingCreditGrant: null,
|
|
23527
23500
|
externalProviders: null,
|
|
23528
23501
|
total: null,
|
|
23529
23502
|
featuredCount: null,
|
|
@@ -23542,11 +23515,7 @@ async function readRuntimeLane(apiUrl, token) {
|
|
|
23542
23515
|
return {
|
|
23543
23516
|
state: "live",
|
|
23544
23517
|
models: body.models,
|
|
23545
|
-
recommendedDeclaredHeadroomBps:
|
|
23546
|
-
currency: body.currency === "credits" || body.currency === "coins" ? body.currency : null,
|
|
23547
|
-
feeBps: bpsOrNull(body.feeBps),
|
|
23548
|
-
usdCentsPerCredit: typeof body.usdCentsPerCredit === "number" && Number.isSafeInteger(body.usdCentsPerCredit) && body.usdCentsPerCredit >= 1 ? body.usdCentsPerCredit : null,
|
|
23549
|
-
standingCreditGrant: standingCreditGrantOrNull(body.standingCreditGrant),
|
|
23518
|
+
recommendedDeclaredHeadroomBps: typeof body.recommendedDeclaredHeadroomBps === "number" && Number.isFinite(body.recommendedDeclaredHeadroomBps) ? body.recommendedDeclaredHeadroomBps : null,
|
|
23550
23519
|
externalProviders: Array.isArray(body.externalProviders) ? body.externalProviders : null,
|
|
23551
23520
|
total: typeof body.total === "number" && Number.isFinite(body.total) ? body.total : null,
|
|
23552
23521
|
featuredCount: typeof body.featuredCount === "number" && Number.isFinite(body.featuredCount) ? body.featuredCount : null,
|
|
@@ -23562,45 +23531,18 @@ function percentile2(values, p) {
|
|
|
23562
23531
|
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
23563
23532
|
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
23564
23533
|
}
|
|
23565
|
-
function
|
|
23566
|
-
|
|
23567
|
-
const sorted = [...values].sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
23568
|
-
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
23569
|
-
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
23534
|
+
function recommendEstimateCoins(p95, headroomBps) {
|
|
23535
|
+
return Math.max(1, Math.ceil(p95 * (1e4 + headroomBps) / 1e4));
|
|
23570
23536
|
}
|
|
23571
|
-
|
|
23572
|
-
|
|
23573
|
-
if (costPicos <= 0n) return 1;
|
|
23574
|
-
const num = costPicos * BigInt(1e4 + terms.feeBps) * BigInt(1e4 + terms.headroomBps);
|
|
23575
|
-
const den = 100000000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
23576
|
-
return Math.max(1, Number((num + den - 1n) / den));
|
|
23577
|
-
}
|
|
23578
|
-
function averageCreditsPerCall(costsPicos, terms) {
|
|
23579
|
-
if (costsPicos.length === 0) return null;
|
|
23580
|
-
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
23581
|
-
return Number(sum * BigInt(1e4 + terms.feeBps)) / Number(BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT);
|
|
23582
|
-
}
|
|
23583
|
-
function perCallEstimateCreditsFor(costsPicos, terms) {
|
|
23584
|
-
if (costsPicos.length === 0) return null;
|
|
23585
|
-
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
23586
|
-
const num = sum * BigInt(1e4 + terms.feeBps);
|
|
23587
|
-
const den = BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
23588
|
-
return Math.max(1, Number((num + den - 1n) / den));
|
|
23589
|
-
}
|
|
23590
|
-
function formatCredits(value) {
|
|
23591
|
-
if (!Number.isFinite(value)) return "\u2014";
|
|
23592
|
-
if (value >= 100) return String(Math.round(value));
|
|
23593
|
-
return String(Number(value.toPrecision(value < 1 ? 2 : 3)));
|
|
23594
|
-
}
|
|
23595
|
-
function usdFromPicos(picos) {
|
|
23596
|
-
if (picos === null || picos === void 0) return "unknown";
|
|
23597
|
-
const value = typeof picos === "bigint" ? picos : /^\d+$/.test(picos) ? BigInt(picos) : null;
|
|
23598
|
-
if (value === null) return "unknown";
|
|
23599
|
-
return `$${(Number(value) / 1e12).toFixed(6)}`;
|
|
23537
|
+
function recommendPerCallMaxCoins(p95, max, headroomBps) {
|
|
23538
|
+
return Math.max(recommendEstimateCoins(p95, headroomBps), recommendEstimateCoins(max, headroomBps));
|
|
23600
23539
|
}
|
|
23601
23540
|
function usdPerMillion(value) {
|
|
23602
23541
|
return `$${value.toFixed(value < 1 ? 4 : 2)}`;
|
|
23603
23542
|
}
|
|
23543
|
+
function usd(value) {
|
|
23544
|
+
return value === null ? "unknown" : `$${value.toFixed(6)}`;
|
|
23545
|
+
}
|
|
23604
23546
|
async function runLlm(opts = {}) {
|
|
23605
23547
|
const log = createLogger({ quiet: opts.quiet || opts.json });
|
|
23606
23548
|
const cwd = opts.cwd ?? process.cwd();
|
|
@@ -23626,23 +23568,16 @@ async function runLlm(opts = {}) {
|
|
|
23626
23568
|
return;
|
|
23627
23569
|
}
|
|
23628
23570
|
if (sub === "bench") {
|
|
23629
|
-
if (opts.maxCoins
|
|
23630
|
-
`);
|
|
23631
|
-
if (opts.maxCoins !== void 0 && opts.maxCredits !== void 0 && opts.maxCoins !== opts.maxCredits) {
|
|
23632
|
-
fail5("llm bench", `Pass --max-credits alone \u2014 --max-coins is its deprecated name, and the two disagree (${opts.maxCredits} vs ${opts.maxCoins}).`);
|
|
23633
|
-
return;
|
|
23634
|
-
}
|
|
23635
|
-
const approvedCredits = opts.maxCredits ?? opts.maxCoins;
|
|
23636
|
-
if (approvedCredits === void 0 || opts.userApproved !== true) {
|
|
23571
|
+
if (opts.maxCoins === void 0 || opts.userApproved !== true) {
|
|
23637
23572
|
fail5("llm bench", LLM_BENCH_APPROVAL_REQUIRED);
|
|
23638
23573
|
return;
|
|
23639
23574
|
}
|
|
23640
|
-
if (!Number.isInteger(
|
|
23641
|
-
fail5("llm bench", `--max-
|
|
23575
|
+
if (!Number.isInteger(opts.maxCoins) || opts.maxCoins < 1) {
|
|
23576
|
+
fail5("llm bench", `--max-coins takes a whole number of coin, 1 or more (got ${String(opts.maxCoins)}).`);
|
|
23642
23577
|
return;
|
|
23643
23578
|
}
|
|
23644
23579
|
if (!opts.benchPrompt?.trim()) {
|
|
23645
|
-
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-
|
|
23580
|
+
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-coins <n> --user-approved.');
|
|
23646
23581
|
return;
|
|
23647
23582
|
}
|
|
23648
23583
|
if (opts.jsonOutput && opts.textOutput) {
|
|
@@ -23734,10 +23669,6 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
23734
23669
|
models: null,
|
|
23735
23670
|
externalProviders: null,
|
|
23736
23671
|
recommendedDeclaredHeadroomBps: null,
|
|
23737
|
-
currency: null,
|
|
23738
|
-
feeBps: null,
|
|
23739
|
-
usdCentsPerCredit: null,
|
|
23740
|
-
standingCreditGrant: null,
|
|
23741
23672
|
total: null,
|
|
23742
23673
|
featuredCount: null,
|
|
23743
23674
|
// The stand was never asked, so the verdict is unknown — but the key is
|
|
@@ -23763,12 +23694,6 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
23763
23694
|
models: lane.models,
|
|
23764
23695
|
externalProviders: lane.externalProviders,
|
|
23765
23696
|
recommendedDeclaredHeadroomBps: lane.recommendedDeclaredHeadroomBps,
|
|
23766
|
-
// The credit terms a call is billed on, the server's — null on a stand
|
|
23767
|
-
// that predates credit billing, never a number the CLI filled in.
|
|
23768
|
-
currency: lane.currency,
|
|
23769
|
-
feeBps: lane.feeBps,
|
|
23770
|
-
usdCentsPerCredit: lane.usdCentsPerCredit,
|
|
23771
|
-
standingCreditGrant: lane.standingCreditGrant,
|
|
23772
23697
|
total: lane.total ?? lane.models?.length ?? null,
|
|
23773
23698
|
featuredCount: lane.featuredCount ?? (lane.models ? lane.models.filter((m) => m.featured).length : null),
|
|
23774
23699
|
// `null` when the lane is not live or the server predates the probe.
|
|
@@ -23815,9 +23740,8 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
23815
23740
|
);
|
|
23816
23741
|
for (const m of shown) {
|
|
23817
23742
|
const plan = m.personalPlan ? ` \xB7 personal plan: ${m.personalPlan}` : "";
|
|
23818
|
-
const kind = m.kind === "typesafe" ? " \xB7 judge (classifier): judge() under a budget, not generate() or bench" : "";
|
|
23819
23743
|
log.plain(
|
|
23820
|
-
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}
|
|
23744
|
+
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}`
|
|
23821
23745
|
);
|
|
23822
23746
|
}
|
|
23823
23747
|
if (hiddenCount > 0) {
|
|
@@ -23833,104 +23757,74 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
23833
23757
|
}
|
|
23834
23758
|
log.plain("");
|
|
23835
23759
|
if (lane.recommendedDeclaredHeadroomBps === null) {
|
|
23836
|
-
log.dim(" This stand serves no recommended headroom, so no
|
|
23760
|
+
log.dim(" This stand serves no recommended headroom, so no price can be recommended from a bench.");
|
|
23837
23761
|
} else {
|
|
23838
|
-
log.dim(` Recommended headroom over
|
|
23839
|
-
}
|
|
23840
|
-
if (lane.feeBps === null) {
|
|
23841
|
-
log.dim(" This stand serves no platform fee \u2014 it predates credit billing, so no ceiling can be recommended from a bench.");
|
|
23842
|
-
} else {
|
|
23843
|
-
log.dim(` In-game calls are billed in the player's credits, as used: the provider's cost plus this stand's platform fee (${lane.feeBps} bps).`);
|
|
23762
|
+
log.dim(` Recommended headroom over benchmarked coins on this stand: ${lane.recommendedDeclaredHeadroomBps} bps.`);
|
|
23844
23763
|
}
|
|
23845
23764
|
if (toolsOnly) {
|
|
23846
23765
|
printConvertHint();
|
|
23847
23766
|
return;
|
|
23848
23767
|
}
|
|
23849
|
-
log.dim(" Those per-million rates are the PROVIDER's
|
|
23850
|
-
log.dim(`
|
|
23768
|
+
log.dim(" Those per-million rates are the PROVIDER's; what your game is charged is the platform");
|
|
23769
|
+
log.dim(` tariff on top, which only a real call reveals: ${c.cyan('genex llm bench "<prompt>" --max-coins <n> --user-approved')}`);
|
|
23851
23770
|
}
|
|
23852
23771
|
function wholeOrNull(value) {
|
|
23853
23772
|
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
23854
23773
|
}
|
|
23855
|
-
function costPicosOf(usage) {
|
|
23856
|
-
if (typeof usage?.costUsdPicos === "string" && /^\d+$/.test(usage.costUsdPicos)) return usage.costUsdPicos;
|
|
23857
|
-
if (typeof usage?.costUsd === "number" && Number.isFinite(usage.costUsd) && usage.costUsd >= 0) {
|
|
23858
|
-
return String(BigInt(Math.round(usage.costUsd * 1e12)));
|
|
23859
|
-
}
|
|
23860
|
-
return null;
|
|
23861
|
-
}
|
|
23862
23774
|
function sampleFrom(id, settled) {
|
|
23863
23775
|
const usage = settled?.usage;
|
|
23864
|
-
const picos = costPicosOf(usage);
|
|
23865
23776
|
return {
|
|
23866
23777
|
id,
|
|
23867
23778
|
status: settled?.status ?? null,
|
|
23868
23779
|
billingStatus: settled?.billingStatus ?? null,
|
|
23869
|
-
|
|
23870
|
-
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd :
|
|
23871
|
-
costUsdPicos: picos,
|
|
23780
|
+
chargedCoins: typeof settled?.chargedCoins === "number" ? settled.chargedCoins : null,
|
|
23781
|
+
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd : null,
|
|
23872
23782
|
error: settled?.error ?? (settled ? null : "timed_out"),
|
|
23873
23783
|
providerMessage: typeof settled?.providerMessage === "string" && settled.providerMessage.trim() ? settled.providerMessage.trim() : null,
|
|
23874
23784
|
inputTokens: wholeOrNull(usage?.inputTokens),
|
|
23875
23785
|
outputTokens: wholeOrNull(usage?.outputTokens),
|
|
23876
23786
|
maxOutputTokens: wholeOrNull(usage?.maxOutputTokens),
|
|
23877
|
-
|
|
23787
|
+
fitCoins: typeof usage?.fitCoins === "number" && Number.isSafeInteger(usage.fitCoins) && usage.fitCoins >= 1 ? usage.fitCoins : null,
|
|
23878
23788
|
fitExceedsCap: typeof usage?.fitExceedsCap === "boolean" ? usage.fitExceedsCap : null,
|
|
23879
23789
|
truncated: typeof usage?.truncated === "boolean" ? usage.truncated : settled?.error === "provider_token_limit" ? true : null
|
|
23880
23790
|
};
|
|
23881
23791
|
}
|
|
23882
23792
|
var PROVIDER_TOKEN_LIMIT = "provider_token_limit";
|
|
23883
|
-
function benchRecommendation(results,
|
|
23884
|
-
const settled = results.filter(
|
|
23885
|
-
|
|
23886
|
-
);
|
|
23887
|
-
const
|
|
23888
|
-
const
|
|
23889
|
-
const fits = settled.map((r) => r.fitCredits).filter((v) => typeof v === "number");
|
|
23890
|
-
const costP95 = percentilePicos(costs, 95);
|
|
23891
|
-
const costMax = costs.length ? costs.reduce((a, b) => b > a ? b : a) : null;
|
|
23793
|
+
function benchRecommendation(results, headroomBps) {
|
|
23794
|
+
const settled = results.filter((r) => r.status === "succeeded" && r.billingStatus === "final" && typeof r.chargedCoins === "number");
|
|
23795
|
+
const charged = settled.map((r) => r.chargedCoins);
|
|
23796
|
+
const fits = settled.map((r) => r.fitCoins).filter((v) => typeof v === "number");
|
|
23797
|
+
const p95 = percentile2(charged, 95);
|
|
23798
|
+
const max = charged.length ? Math.max(...charged) : null;
|
|
23892
23799
|
const fitP95 = percentile2(fits, 95);
|
|
23893
23800
|
const fitMax = fits.length ? Math.max(...fits) : null;
|
|
23894
23801
|
const exceedsStandCap = results.some((r) => r.fitExceedsCap === true);
|
|
23895
23802
|
const cutOffSamples = results.filter((r) => r.error === PROVIDER_TOKEN_LIMIT && r.fitExceedsCap !== true).length;
|
|
23896
23803
|
const noRecommendationReason = exceedsStandCap ? "answer_exceeds_stand_cap" : cutOffSamples > 0 ? "answer_cut_off" : null;
|
|
23897
|
-
const
|
|
23898
|
-
const
|
|
23899
|
-
const
|
|
23900
|
-
const billing = feeBps !== null && usdCentsPerCredit !== null ? { feeBps, usdCentsPerCredit } : null;
|
|
23901
|
-
const costBased = full !== null && costP95 !== null ? ceilingCreditsFor(costP95, full) : null;
|
|
23902
|
-
const recommended = costBased === null || noRecommendationReason !== null ? null : Math.max(costBased, fitP95 ?? 0);
|
|
23903
|
-
const ceiling = recommended === null || costMax === null || full === null ? null : Math.max(ceilingCreditsFor(costMax, full), recommended, fitMax ?? 0);
|
|
23904
|
-
const average = billing !== null ? averageCreditsPerCall(costs, billing) : null;
|
|
23905
|
-
const estimate = ceiling === null || billing === null ? null : Math.min(ceiling, perCallEstimateCreditsFor(costs, billing) ?? 1);
|
|
23804
|
+
const chargedBased = p95 !== null && headroomBps !== null ? recommendEstimateCoins(p95, headroomBps) : null;
|
|
23805
|
+
const recommended = chargedBased === null || noRecommendationReason !== null ? null : Math.max(chargedBased, fitP95 ?? 0);
|
|
23806
|
+
const ceiling = recommended === null || p95 === null || max === null || headroomBps === null ? null : Math.max(recommendPerCallMaxCoins(p95, max, headroomBps), recommended, fitMax ?? 0);
|
|
23906
23807
|
return {
|
|
23907
|
-
costs,
|
|
23908
23808
|
charged,
|
|
23909
|
-
|
|
23910
|
-
|
|
23911
|
-
|
|
23912
|
-
chargedP50: percentile2(charged, 50),
|
|
23913
|
-
chargedP95: percentile2(charged, 95),
|
|
23914
|
-
chargedMax: charged.length ? Math.max(...charged) : null,
|
|
23809
|
+
p50: percentile2(charged, 50),
|
|
23810
|
+
p95,
|
|
23811
|
+
max,
|
|
23915
23812
|
fits,
|
|
23916
23813
|
fitP95,
|
|
23917
23814
|
fitMax,
|
|
23918
|
-
|
|
23919
|
-
|
|
23920
|
-
|
|
23921
|
-
|
|
23922
|
-
recommendedPerCallEstimateCredits: estimate,
|
|
23923
|
-
lengthRaisedCeiling: recommended !== null && costBased !== null && recommended > costBased,
|
|
23815
|
+
chargedBasedEstimateCoins: chargedBased,
|
|
23816
|
+
recommendedEstimateCoins: recommended,
|
|
23817
|
+
recommendedPerCallMaxCoins: ceiling,
|
|
23818
|
+
lengthRaisedPrice: recommended !== null && chargedBased !== null && recommended > chargedBased,
|
|
23924
23819
|
cutOffSamples,
|
|
23925
23820
|
exceedsStandCap,
|
|
23926
|
-
noRecommendationReason
|
|
23927
|
-
missingTerms
|
|
23821
|
+
noRecommendationReason
|
|
23928
23822
|
};
|
|
23929
23823
|
}
|
|
23930
|
-
var LLM_LENGTH_RAISED_LINE = "The
|
|
23931
|
-
var LLM_EXCEEDS_STAND_CAP_LINE = "No
|
|
23932
|
-
function benchCutOffLine(cutOff,
|
|
23933
|
-
return `No
|
|
23824
|
+
var LLM_LENGTH_RAISED_LINE = "The declared price also pays for how long the answer may be: at the charged-based price the answer would be cut off, so declare this one.";
|
|
23825
|
+
var LLM_EXCEEDS_STAND_CAP_LINE = "No price recommended: this answer is longer than one call on this stand may produce. Ask for a shorter answer \u2014 fewer fields, shorter strings, a length the prompt states \u2014 and benchmark again.";
|
|
23826
|
+
function benchCutOffLine(cutOff, maxCoins) {
|
|
23827
|
+
return `No price recommended: ${cutOff} sample${cutOff === 1 ? " was" : "s were"} cut off at this ceiling (--max-coins ${maxCoins}) before the answer was finished, so its real length is unknown. Re-run with a higher --max-coins.`;
|
|
23934
23828
|
}
|
|
23935
23829
|
var ACTIVE_STATUSES = /* @__PURE__ */ new Set(["requires_confirmation", "queued", "dispatching", "awaiting_external"]);
|
|
23936
23830
|
function rowState(row) {
|
|
@@ -23941,22 +23835,7 @@ function rowState(row) {
|
|
|
23941
23835
|
function rowHoldsSlot(row) {
|
|
23942
23836
|
if (typeof row.slotHeld === "boolean") return row.slotHeld;
|
|
23943
23837
|
const state = rowState(row);
|
|
23944
|
-
return state === "active" || state === "awaiting bill" && (
|
|
23945
|
-
}
|
|
23946
|
-
function rowMoney(row) {
|
|
23947
|
-
const credits = row.currency === "credits" || row.currency === void 0 && typeof row.reservedCredits === "number" && typeof row.reservedCoins !== "number";
|
|
23948
|
-
if (credits) {
|
|
23949
|
-
return {
|
|
23950
|
-
unit: "credits",
|
|
23951
|
-
reserved: typeof row.reservedCredits === "number" ? row.reservedCredits : null,
|
|
23952
|
-
charged: typeof row.chargedCredits === "number" ? row.chargedCredits : null
|
|
23953
|
-
};
|
|
23954
|
-
}
|
|
23955
|
-
return {
|
|
23956
|
-
unit: "coin",
|
|
23957
|
-
reserved: typeof row.reservedCoins === "number" ? row.reservedCoins : null,
|
|
23958
|
-
charged: typeof row.chargedCoins === "number" ? row.chargedCoins : null
|
|
23959
|
-
};
|
|
23838
|
+
return state === "active" || state === "awaiting bill" && (row.reservedCoins ?? 0) > 0;
|
|
23960
23839
|
}
|
|
23961
23840
|
function providerRefusalStatus(error) {
|
|
23962
23841
|
const m = /^provider_http_(\d{3})$/.exec(error ?? "");
|
|
@@ -23975,7 +23854,7 @@ function formatAge(iso, now = Date.now()) {
|
|
|
23975
23854
|
}
|
|
23976
23855
|
async function runBench(args) {
|
|
23977
23856
|
const { apiUrl, token, projectId, cwd, opts, log } = args;
|
|
23978
|
-
const
|
|
23857
|
+
const maxCoins = opts.maxCoins;
|
|
23979
23858
|
const samples = opts.samples ?? DEFAULT_BENCH_SAMPLES;
|
|
23980
23859
|
const prompt = opts.benchPrompt.trim();
|
|
23981
23860
|
const auth = { Authorization: `Bearer ${token}`, "Content-Type": "application/json" };
|
|
@@ -23997,7 +23876,7 @@ async function runBench(args) {
|
|
|
23997
23876
|
const lane = await readRuntimeLane(apiUrl, token);
|
|
23998
23877
|
if (lane.state !== "live" || !lane.models || lane.models.length === 0) {
|
|
23999
23878
|
if (lane.state === "off") {
|
|
24000
|
-
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
23879
|
+
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCoins, opts), status: "off", error: null });
|
|
24001
23880
|
else {
|
|
24002
23881
|
log.plain(LLM_LANE_OFF_LINE);
|
|
24003
23882
|
log.dim(" Nothing was spent.");
|
|
@@ -24024,27 +23903,17 @@ async function runBench(args) {
|
|
|
24024
23903
|
);
|
|
24025
23904
|
return;
|
|
24026
23905
|
}
|
|
24027
|
-
const
|
|
24028
|
-
if (walletBefore?.emailVerified === false) {
|
|
24029
|
-
benchFailed(opts, log, LLM_BENCH_UNVERIFIED_LINE);
|
|
24030
|
-
return;
|
|
24031
|
-
}
|
|
24032
|
-
if (walletBefore !== null && walletBefore.spendable < maxCredits) {
|
|
24033
|
-
benchFailed(opts, log, benchBalanceShortLine(walletBefore.spendable, maxCredits));
|
|
24034
|
-
return;
|
|
24035
|
-
}
|
|
24036
|
-
const balanceBefore = walletBefore?.spendable ?? null;
|
|
23906
|
+
const balanceBefore = await readCoinBalance(apiUrl, token);
|
|
24037
23907
|
if (!opts.json) {
|
|
24038
23908
|
log.plain(c.bold("genex llm bench"));
|
|
24039
23909
|
log.dim(` ${apiUrl}`);
|
|
24040
23910
|
log.plain("");
|
|
24041
23911
|
log.plain(` Model ${c.cyan(modelId)}`);
|
|
24042
23912
|
log.plain(` Samples ${samples} real attempt${samples === 1 ? "" : "s"}, ${outputFormat} output`);
|
|
24043
|
-
log.plain(` Approved ${
|
|
23913
|
+
log.plain(` Approved ${maxCoins} coin per attempt \u2014 at worst ${maxCoins * samples} coin for this run`);
|
|
24044
23914
|
log.plain(
|
|
24045
|
-
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore}
|
|
23915
|
+
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore} coin spendable`}`
|
|
24046
23916
|
);
|
|
24047
|
-
log.dim(" Billed as used: each attempt is charged its real cost plus the platform fee, rounded up to a whole credit.");
|
|
24048
23917
|
log.plain("");
|
|
24049
23918
|
}
|
|
24050
23919
|
const base = `${apiUrl}/api/runtime/development/projects/${encodeURIComponent(projectId)}/generations`;
|
|
@@ -24070,7 +23939,7 @@ async function runBench(args) {
|
|
|
24070
23939
|
const res = await call(base, {
|
|
24071
23940
|
method: "POST",
|
|
24072
23941
|
body: JSON.stringify({
|
|
24073
|
-
|
|
23942
|
+
maxCoins,
|
|
24074
23943
|
request: {
|
|
24075
23944
|
idempotencyKey: `bench-${randomUUID2()}`,
|
|
24076
23945
|
modelId,
|
|
@@ -24100,14 +23969,13 @@ async function runBench(args) {
|
|
|
24100
23969
|
if (refusedAt !== null) providerRefused++;
|
|
24101
23970
|
if (!opts.json) {
|
|
24102
23971
|
const length = row.outputTokens !== null ? ` \xB7 ${row.outputTokens}${row.maxOutputTokens !== null ? ` of ${row.maxOutputTokens}` : ""} tokens out` : "";
|
|
24103
|
-
const charged = row.chargedCredits === null ? "\u2014" : String(row.chargedCredits);
|
|
24104
23972
|
log.plain(
|
|
24105
|
-
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${
|
|
23973
|
+
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${String(row.chargedCoins ?? "\u2014").padStart(4)} coin charged \xB7 provider ${usd(row.costUsd)}${length}${row.error ? ` \xB7 ${row.error}` : ""}`
|
|
24106
23974
|
);
|
|
24107
23975
|
if (refusedAt !== null) printProviderRefusal(log, refusedAt, row.providerMessage, row.status === "unknown");
|
|
24108
23976
|
if (row.error === PROVIDER_TOKEN_LIMIT) {
|
|
24109
23977
|
log.dim(
|
|
24110
|
-
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no
|
|
23978
|
+
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no price makes room for it; it is not a sample." : ` The answer was cut off at this ceiling (--max-coins ${maxCoins}) before it was finished \u2014 it was charged, and it is not a sample.`
|
|
24111
23979
|
);
|
|
24112
23980
|
}
|
|
24113
23981
|
}
|
|
@@ -24115,19 +23983,16 @@ async function runBench(args) {
|
|
|
24115
23983
|
} finally {
|
|
24116
23984
|
process.removeListener("SIGINT", onSigint);
|
|
24117
23985
|
}
|
|
24118
|
-
const
|
|
24119
|
-
|
|
24120
|
-
|
|
24121
|
-
|
|
24122
|
-
|
|
24123
|
-
const
|
|
24124
|
-
const settledCount = rec.charged.length;
|
|
23986
|
+
const headroomBps = lane.recommendedDeclaredHeadroomBps;
|
|
23987
|
+
const rec = benchRecommendation(results, headroomBps);
|
|
23988
|
+
const { charged, p50, p95, max } = rec;
|
|
23989
|
+
const settledCount = charged.length;
|
|
23990
|
+
const recommended = rec.recommendedEstimateCoins;
|
|
23991
|
+
const ceiling = rec.recommendedPerCallMaxCoins;
|
|
24125
23992
|
const lengthBlocked = rec.noRecommendationReason !== null;
|
|
24126
|
-
const balanceAfter =
|
|
24127
|
-
const picosOrNull = (v) => v === null ? null : String(v);
|
|
23993
|
+
const balanceAfter = await readCoinBalance(apiUrl, token);
|
|
24128
23994
|
const record = {
|
|
24129
|
-
v:
|
|
24130
|
-
currency: "credits",
|
|
23995
|
+
v: 1,
|
|
24131
23996
|
ranAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
24132
23997
|
apiUrl,
|
|
24133
23998
|
projectId,
|
|
@@ -24136,34 +24001,28 @@ async function runBench(args) {
|
|
|
24136
24001
|
outputFormat,
|
|
24137
24002
|
samples,
|
|
24138
24003
|
completed: settledCount,
|
|
24139
|
-
|
|
24140
|
-
|
|
24141
|
-
|
|
24142
|
-
|
|
24143
|
-
|
|
24144
|
-
|
|
24004
|
+
chargedCoins: charged,
|
|
24005
|
+
p50,
|
|
24006
|
+
p95,
|
|
24007
|
+
max,
|
|
24008
|
+
recommendedDeclaredHeadroomBps: headroomBps,
|
|
24009
|
+
recommendedEstimateCoins: recommended,
|
|
24010
|
+
recommendedPerCallMaxCoins: ceiling,
|
|
24011
|
+
maxCoinsPerSample: maxCoins,
|
|
24012
|
+
fitCoins: rec.fits,
|
|
24145
24013
|
fitP95: rec.fitP95,
|
|
24146
|
-
|
|
24147
|
-
|
|
24148
|
-
feeBps: terms.feeBps,
|
|
24149
|
-
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
24150
|
-
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
24151
|
-
recommendedMaxCredits: rec.recommendedMaxCredits,
|
|
24152
|
-
recommendedPerCallMaxCredits: rec.recommendedPerCallMaxCredits,
|
|
24153
|
-
averageCreditsPerCall: rec.averageCreditsPerCall,
|
|
24154
|
-
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
24155
|
-
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
24014
|
+
chargedBasedEstimateCoins: rec.chargedBasedEstimateCoins,
|
|
24015
|
+
lengthRaisedPrice: rec.lengthRaisedPrice,
|
|
24156
24016
|
cutOffSamples: rec.cutOffSamples,
|
|
24157
24017
|
noRecommendationReason: rec.noRecommendationReason
|
|
24158
24018
|
};
|
|
24159
24019
|
const savedTo = await saveBench(cwd, record);
|
|
24160
24020
|
if (opts.json) {
|
|
24161
|
-
const ok =
|
|
24021
|
+
const ok = charged.length > 0 && !lengthBlocked;
|
|
24162
24022
|
writeJsonLine({
|
|
24163
24023
|
command: "llm bench",
|
|
24164
24024
|
status: ok ? "ok" : "failed",
|
|
24165
24025
|
error: ok ? null : rec.noRecommendationReason ?? "no_sample_settled",
|
|
24166
|
-
currency: "credits",
|
|
24167
24026
|
apiUrl,
|
|
24168
24027
|
projectId,
|
|
24169
24028
|
modelId,
|
|
@@ -24172,54 +24031,35 @@ async function runBench(args) {
|
|
|
24172
24031
|
schemaPath: opts.schemaPath ?? null,
|
|
24173
24032
|
samples,
|
|
24174
24033
|
completed: settledCount,
|
|
24175
|
-
|
|
24176
|
-
// Spendable credits before and after the run; null when unreadable.
|
|
24034
|
+
maxCoinsPerSample: maxCoins,
|
|
24177
24035
|
balanceBefore,
|
|
24178
24036
|
balanceAfter,
|
|
24179
24037
|
results,
|
|
24180
24038
|
// The create-time refusal that ended the run, or null. Beside the
|
|
24181
24039
|
// samples, never among them: no attempt existed and nothing was spent.
|
|
24182
24040
|
refusal: refusal2,
|
|
24183
|
-
|
|
24184
|
-
//
|
|
24185
|
-
costUsdPicos: record.cost,
|
|
24186
|
-
costUsd: {
|
|
24187
|
-
p50: rec.costP50 === null ? null : Number(rec.costP50) / 1e12,
|
|
24188
|
-
p95: rec.costP95 === null ? null : Number(rec.costP95) / 1e12,
|
|
24189
|
-
max: rec.costMax === null ? null : Number(rec.costMax) / 1e12
|
|
24190
|
-
},
|
|
24191
|
-
chargedCredits: record.charged,
|
|
24192
|
-
// The answer's LENGTH, priced by the server per sample (`fitCredits`,
|
|
24041
|
+
chargedCoins: { p50, p95, max },
|
|
24042
|
+
// The answer's LENGTH, priced by the server per sample (`fitCoins`,
|
|
24193
24043
|
// headroom already on the tokens), over the settled samples.
|
|
24194
|
-
|
|
24195
|
-
|
|
24196
|
-
|
|
24197
|
-
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
24198
|
-
missingTerms: rec.missingTerms,
|
|
24199
|
-
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
24200
|
-
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
24044
|
+
fitCoins: { p95: rec.fitP95, max: rec.fitMax },
|
|
24045
|
+
chargedBasedEstimateCoins: rec.chargedBasedEstimateCoins,
|
|
24046
|
+
lengthRaisedPrice: rec.lengthRaisedPrice,
|
|
24201
24047
|
cutOffSamples: rec.cutOffSamples,
|
|
24202
24048
|
exceedsStandCap: rec.exceedsStandCap,
|
|
24203
24049
|
noRecommendationReason: rec.noRecommendationReason,
|
|
24204
|
-
|
|
24205
|
-
|
|
24206
|
-
|
|
24207
|
-
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
24050
|
+
recommendedDeclaredHeadroomBps: headroomBps,
|
|
24051
|
+
recommendedEstimateCoins: recommended,
|
|
24052
|
+
recommendedPerCallMaxCoins: ceiling,
|
|
24208
24053
|
savedTo
|
|
24209
24054
|
});
|
|
24210
24055
|
if (!ok) process.exitCode = 1;
|
|
24211
24056
|
return;
|
|
24212
24057
|
}
|
|
24213
24058
|
log.plain("");
|
|
24214
|
-
const printMeasured = () => {
|
|
24215
|
-
log.plain(c.bold(" Real cost per call"));
|
|
24216
|
-
log.plain(` p50 ${usdFromPicos(rec.costP50)} p95 ${usdFromPicos(rec.costP95)} max ${usdFromPicos(rec.costMax)} (${settledCount} of ${samples} settled)`);
|
|
24217
|
-
log.plain(c.bold(" Charged credits"));
|
|
24218
|
-
log.plain(` p50 ${rec.chargedP50} p95 ${rec.chargedP95} max ${rec.chargedMax} (whole credits, rounded up per call)`);
|
|
24219
|
-
};
|
|
24220
24059
|
if (lengthBlocked) {
|
|
24221
|
-
if (
|
|
24222
|
-
|
|
24060
|
+
if (charged.length > 0) {
|
|
24061
|
+
log.plain(c.bold(" Charged coins"));
|
|
24062
|
+
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
24223
24063
|
log.plain("");
|
|
24224
24064
|
}
|
|
24225
24065
|
printRecommendation(log, record);
|
|
@@ -24227,16 +24067,17 @@ async function runBench(args) {
|
|
|
24227
24067
|
process.exitCode = 1;
|
|
24228
24068
|
return;
|
|
24229
24069
|
}
|
|
24230
|
-
if (
|
|
24070
|
+
if (charged.length === 0) {
|
|
24231
24071
|
log.error(
|
|
24232
|
-
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to
|
|
24072
|
+
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to price from." : " No sample settled, so there is nothing to price from."
|
|
24233
24073
|
);
|
|
24234
24074
|
process.exitCode = 1;
|
|
24235
24075
|
return;
|
|
24236
24076
|
}
|
|
24237
|
-
|
|
24077
|
+
log.plain(c.bold(" Charged coins"));
|
|
24078
|
+
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
24238
24079
|
if (balanceBefore !== null && balanceAfter !== null) {
|
|
24239
|
-
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter}
|
|
24080
|
+
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter} coin spendable`);
|
|
24240
24081
|
}
|
|
24241
24082
|
log.plain("");
|
|
24242
24083
|
printRecommendation(log, record);
|
|
@@ -24248,58 +24089,43 @@ function printRecommendation(log, record) {
|
|
|
24248
24089
|
return;
|
|
24249
24090
|
}
|
|
24250
24091
|
if (record.noRecommendationReason === "answer_cut_off") {
|
|
24251
|
-
log.warn(` ${benchCutOffLine(record.cutOffSamples
|
|
24092
|
+
log.warn(` ${benchCutOffLine(record.cutOffSamples ?? 1, record.maxCoinsPerSample ?? 0)}`);
|
|
24252
24093
|
return;
|
|
24253
24094
|
}
|
|
24254
|
-
|
|
24255
|
-
|
|
24256
|
-
log.warn(" No ceiling recommended.");
|
|
24095
|
+
if (record.recommendedEstimateCoins === null || record.p95 === null) {
|
|
24096
|
+
log.warn(" No price recommended.");
|
|
24257
24097
|
log.dim(
|
|
24258
|
-
record.
|
|
24098
|
+
record.recommendedDeclaredHeadroomBps === null ? " This stand served no recommended headroom, and the multiplier is the server's to set \u2014" : " No sample settled, so there is no p95 to build a price on \u2014"
|
|
24259
24099
|
);
|
|
24260
24100
|
log.dim(" declaring a number from this run would be a guess dressed as a measurement.");
|
|
24261
|
-
printAverage(log, record, attempts);
|
|
24262
24101
|
return;
|
|
24263
24102
|
}
|
|
24264
|
-
log.plain(c.bold(` Declare
|
|
24265
|
-
if (record.
|
|
24103
|
+
log.plain(c.bold(` Declare estimateCoins: ${record.recommendedEstimateCoins}`));
|
|
24104
|
+
if (record.lengthRaisedPrice) {
|
|
24266
24105
|
log.plain(` ${LLM_LENGTH_RAISED_LINE}`);
|
|
24267
24106
|
log.dim(
|
|
24268
|
-
` = the smallest
|
|
24107
|
+
` = the smallest price whose answer allowance holds the answer plus this stand's headroom (p95 ${record.fitP95 ?? "\u2014"}), above the`
|
|
24269
24108
|
);
|
|
24270
24109
|
log.dim(
|
|
24271
|
-
`
|
|
24110
|
+
` charged-based ${record.chargedBasedEstimateCoins ?? "\u2014"} (p95 of charged coins ${record.p95} plus ${record.recommendedDeclaredHeadroomBps} bps), over ${record.completed} settled attempt${record.completed === 1 ? "" : "s"} on ${record.modelId}.`
|
|
24272
24111
|
);
|
|
24273
24112
|
} else {
|
|
24274
24113
|
log.dim(
|
|
24275
|
-
` = p95
|
|
24114
|
+
` = p95 of charged coins (${record.p95}) plus this stand's recommended headroom (${record.recommendedDeclaredHeadroomBps} bps),`
|
|
24276
24115
|
);
|
|
24277
|
-
log.dim(` measured over ${
|
|
24116
|
+
log.dim(` measured over ${record.completed} settled attempt${record.completed === 1 ? "" : "s"} on ${record.modelId}.`);
|
|
24278
24117
|
}
|
|
24279
|
-
log.dim(" That number is a
|
|
24280
|
-
|
|
24281
|
-
|
|
24282
|
-
log.plain(c.bold(` Grant perCallMaxCredits: ${record.recommendedPerCallMaxCredits}`));
|
|
24118
|
+
log.dim(" That number is a PRICE: a started attempt is charged in full, including one that fails.");
|
|
24119
|
+
if (record.recommendedPerCallMaxCoins != null) {
|
|
24120
|
+
log.plain(c.bold(` Grant perCallMaxCoins: ${record.recommendedPerCallMaxCoins}`));
|
|
24283
24121
|
log.dim(
|
|
24284
|
-
record.
|
|
24122
|
+
record.lengthRaisedPrice ? ` = the same headroom over the worst sample (${record.max}) or the longest answer's price, whichever is larger, and never below the price above it.` : ` = the same headroom over the worst sample (${record.max}), and never below the price above it.`
|
|
24285
24123
|
);
|
|
24286
|
-
|
|
24287
|
-
printAverage(log, record, attempts);
|
|
24288
|
-
}
|
|
24289
|
-
function printAverage(log, record, attempts) {
|
|
24290
|
-
if (record.averageCreditsPerCall === null) return;
|
|
24291
|
-
log.plain(` Per-call estimate: about ${formatCredits(record.averageCreditsPerCall)} credits per call`);
|
|
24292
|
-
log.dim(` = the average real cost with the platform fee, no headroom, over ${attempts}.`);
|
|
24293
|
-
if (record.recommendedPerCallEstimateCredits !== null) {
|
|
24294
|
-
log.plain(c.bold(` Grant perCallEstimateCredits: ${record.recommendedPerCallEstimateCredits}`));
|
|
24295
|
-
log.dim(` = that average rounded up to a whole credit. disclosure.estimatedCreditsPerPeriod = your calls per period \xD7 ${formatCredits(record.averageCreditsPerCall)}, rounded up.`);
|
|
24124
|
+
log.dim(" The ceiling is the price's room to be wrong; declaring one under the price is refused.");
|
|
24296
24125
|
}
|
|
24297
24126
|
}
|
|
24298
24127
|
async function explainBenchRefusal(res, call, base, log, opts, index, settledSoFar) {
|
|
24299
|
-
if (printedStructuredError(res)) {
|
|
24300
|
-
const printed = await res.json().catch(() => ({}));
|
|
24301
|
-
return { stop: true, refusal: { status: res.status, error: typeof printed.error === "string" ? printed.error : null, slotsHeld: null, slotLimit: null } };
|
|
24302
|
-
}
|
|
24128
|
+
if (printedStructuredError(res)) return { stop: true, refusal: { status: res.status, error: null, slotsHeld: null, slotLimit: null } };
|
|
24303
24129
|
const body = await res.json().catch(() => ({}));
|
|
24304
24130
|
const refusal2 = { status: res.status, error: body.error ?? null, slotsHeld: null, slotLimit: null };
|
|
24305
24131
|
if (res.status === 429 && body.error === "generation_limit") {
|
|
@@ -24309,10 +24135,6 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24309
24135
|
if (!opts.json) log.error(` Sample ${index + 1} ${benchSlotsHeldSentence(counts)}`);
|
|
24310
24136
|
return { stop: true, refusal: refusal2 };
|
|
24311
24137
|
}
|
|
24312
|
-
if (res.status === 403 && body.error === "credits_unverified") {
|
|
24313
|
-
if (!opts.json) log.error(` ${LLM_BENCH_UNVERIFIED_LINE}`);
|
|
24314
|
-
return { stop: true, refusal: refusal2 };
|
|
24315
|
-
}
|
|
24316
24138
|
if (opts.json) return { stop: res.status !== 429, refusal: refusal2 };
|
|
24317
24139
|
if (res.status === 404 && body.error === "not_found") {
|
|
24318
24140
|
log.error(` ${LLM_LANE_OFF_LINE}`);
|
|
@@ -24328,7 +24150,7 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24328
24150
|
return { stop: true, refusal: refusal2 };
|
|
24329
24151
|
}
|
|
24330
24152
|
if (res.status === 402) {
|
|
24331
|
-
log.error(" Not enough
|
|
24153
|
+
log.error(" Not enough coin to start the attempt.");
|
|
24332
24154
|
return { stop: true, refusal: refusal2 };
|
|
24333
24155
|
}
|
|
24334
24156
|
log.error(
|
|
@@ -24337,7 +24159,7 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24337
24159
|
return { stop: res.status >= 500 || res.status === 401 || res.status === 403, refusal: refusal2 };
|
|
24338
24160
|
}
|
|
24339
24161
|
function printProviderRefusal(log, status2, message, awaitingBill = false) {
|
|
24340
|
-
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so
|
|
24162
|
+
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so this call's coin stays held until it resolves (see \`genex llm status\`); it is not a sample.`);
|
|
24341
24163
|
else log.dim(" The provider refused this call at its door \u2014 no inference ran, it cost nothing, and it is not a sample.");
|
|
24342
24164
|
if (message) log.dim(` Provider said: ${message}`);
|
|
24343
24165
|
if (status2 === 401 || status2 === 403) {
|
|
@@ -24363,6 +24185,19 @@ async function pollSettled(call, url, timeoutSec) {
|
|
|
24363
24185
|
}
|
|
24364
24186
|
return last;
|
|
24365
24187
|
}
|
|
24188
|
+
async function readCoinBalance(apiUrl, token) {
|
|
24189
|
+
try {
|
|
24190
|
+
const res = await apiFetch(`${apiUrl}/api/coin/balance`, {
|
|
24191
|
+
headers: { Authorization: `Bearer ${token}` },
|
|
24192
|
+
signal: AbortSignal.timeout(6e3)
|
|
24193
|
+
});
|
|
24194
|
+
if (!res.ok) return null;
|
|
24195
|
+
const body = await res.json().catch(() => null);
|
|
24196
|
+
return typeof body?.spendable === "number" ? body.spendable : null;
|
|
24197
|
+
} catch {
|
|
24198
|
+
return null;
|
|
24199
|
+
}
|
|
24200
|
+
}
|
|
24366
24201
|
async function saveBench(cwd, record) {
|
|
24367
24202
|
try {
|
|
24368
24203
|
const file = path36.join(cwd, LLM_BENCH_FILE);
|
|
@@ -24378,10 +24213,9 @@ function benchFailed(opts, log, message) {
|
|
|
24378
24213
|
else log.error(message);
|
|
24379
24214
|
process.exitCode = 1;
|
|
24380
24215
|
}
|
|
24381
|
-
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
24216
|
+
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCoins, opts) {
|
|
24382
24217
|
return {
|
|
24383
24218
|
command: "llm bench",
|
|
24384
|
-
currency: "credits",
|
|
24385
24219
|
apiUrl,
|
|
24386
24220
|
projectId,
|
|
24387
24221
|
modelId: opts.modelId ?? null,
|
|
@@ -24390,28 +24224,21 @@ function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCre
|
|
|
24390
24224
|
schemaPath: opts.schemaPath ?? null,
|
|
24391
24225
|
samples,
|
|
24392
24226
|
completed: 0,
|
|
24393
|
-
|
|
24227
|
+
maxCoinsPerSample: maxCoins,
|
|
24394
24228
|
balanceBefore: null,
|
|
24395
24229
|
balanceAfter: null,
|
|
24396
24230
|
results: [],
|
|
24397
24231
|
refusal: null,
|
|
24398
|
-
|
|
24399
|
-
|
|
24400
|
-
|
|
24401
|
-
|
|
24402
|
-
recommendedDeclaredHeadroomBps: null,
|
|
24403
|
-
feeBps: null,
|
|
24404
|
-
usdCentsPerCredit: null,
|
|
24405
|
-
missingTerms: null,
|
|
24406
|
-
costBasedMaxCredits: null,
|
|
24407
|
-
lengthRaisedCeiling: false,
|
|
24232
|
+
chargedCoins: { p50: null, p95: null, max: null },
|
|
24233
|
+
fitCoins: { p95: null, max: null },
|
|
24234
|
+
chargedBasedEstimateCoins: null,
|
|
24235
|
+
lengthRaisedPrice: false,
|
|
24408
24236
|
cutOffSamples: 0,
|
|
24409
24237
|
exceedsStandCap: false,
|
|
24410
24238
|
noRecommendationReason: null,
|
|
24411
|
-
|
|
24412
|
-
|
|
24413
|
-
|
|
24414
|
-
recommendedPerCallEstimateCredits: null,
|
|
24239
|
+
recommendedDeclaredHeadroomBps: null,
|
|
24240
|
+
recommendedEstimateCoins: null,
|
|
24241
|
+
recommendedPerCallMaxCoins: null,
|
|
24415
24242
|
savedTo: null
|
|
24416
24243
|
};
|
|
24417
24244
|
}
|
|
@@ -24474,8 +24301,7 @@ async function reportStatus(args) {
|
|
|
24474
24301
|
for (const row of rows) {
|
|
24475
24302
|
const state = rowState(row);
|
|
24476
24303
|
const mark = state === "active" ? c.cyan("\u25CF") : state === "awaiting bill" ? c.yellow("\u25D0") : c.green("\u25CB");
|
|
24477
|
-
const
|
|
24478
|
-
const reserved = money2.reserved === null ? "reserved \u2014" : `${money2.reserved} ${money2.unit} reserved`;
|
|
24304
|
+
const reserved = typeof row.reservedCoins === "number" ? `${row.reservedCoins} coin reserved` : "reserved \u2014";
|
|
24479
24305
|
log.plain(
|
|
24480
24306
|
` ${mark} ${row.id ?? "?"} ${row.modelId ?? "?"} ${state.padEnd(13)} ${formatAge(row.createdAt, now).padStart(4)} old ${reserved}${row.error ? ` ${row.error}` : ""}${rowHoldsSlot(row) ? "" : c.dim(" (no slot)")}`
|
|
24481
24307
|
);
|
|
@@ -24547,11 +24373,10 @@ async function cancelCall(args) {
|
|
|
24547
24373
|
if (!opts.json) {
|
|
24548
24374
|
log.warn(` ${row.id} (${row.status ?? "?"}) \u2014 ${sentence}.`);
|
|
24549
24375
|
if (state === "awaiting bill") {
|
|
24550
|
-
log.dim(` Nothing to cancel:
|
|
24376
|
+
log.dim(` Nothing to cancel: its coin releases when the bill resolves, and it stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`);
|
|
24551
24377
|
if (row.providerMessage?.trim()) log.dim(` Provider said: ${row.providerMessage.trim()}`);
|
|
24552
24378
|
} else {
|
|
24553
|
-
|
|
24554
|
-
log.dim(` ${money2.charged === null ? "charge unknown" : `${money2.charged} ${money2.unit} charged`}${row.error ? ` \xB7 ${row.error}` : ""}.`);
|
|
24379
|
+
log.dim(` ${typeof row.chargedCoins === "number" ? `${row.chargedCoins} coin charged` : "charge unknown"}${row.error ? ` \xB7 ${row.error}` : ""}.`);
|
|
24555
24380
|
}
|
|
24556
24381
|
}
|
|
24557
24382
|
return done("ok", null, state === "awaiting bill" ? "awaiting_bill" : "already_final", row);
|
|
@@ -24573,126 +24398,78 @@ async function cancelCall(args) {
|
|
|
24573
24398
|
const after = await res.json().catch(() => null) ?? row;
|
|
24574
24399
|
if (!opts.json) {
|
|
24575
24400
|
log.success(` Cancelled ${row.id} (was ${row.status ?? "active"}).`);
|
|
24576
|
-
log.dim(`
|
|
24401
|
+
log.dim(` Its coin releases once the bill settles; ${c.cyan("genex llm status")} shows it until then.`);
|
|
24577
24402
|
}
|
|
24578
24403
|
return done("ok", null, "cancelled", after);
|
|
24579
24404
|
}
|
|
24580
|
-
async function
|
|
24581
|
-
let
|
|
24405
|
+
async function reportSavedPrice(cwd, opts, log) {
|
|
24406
|
+
let record = null;
|
|
24582
24407
|
try {
|
|
24583
|
-
|
|
24408
|
+
record = JSON.parse(await fs35.readFile(path36.join(cwd, LLM_BENCH_FILE), "utf8"));
|
|
24584
24409
|
} catch {
|
|
24585
|
-
|
|
24410
|
+
record = null;
|
|
24586
24411
|
}
|
|
24587
|
-
if (!
|
|
24588
|
-
const v = parsed;
|
|
24589
|
-
if (v.v === 2 && v.currency === "credits" && v.cost && v.charged) return { kind: "credits", record: parsed };
|
|
24590
|
-
return { kind: "coins", record: parsed };
|
|
24591
|
-
}
|
|
24592
|
-
var LLM_PRICE_COIN_TERMS_LINE = "This bench was measured under the old COIN terms, when a game declared a fixed price per call. In-game calls are now billed in the player's credits, as used, and a game declares a per-call CEILING (maxCredits) instead \u2014 a coin price is not that number. Re-run the bench for the figures to declare.";
|
|
24593
|
-
async function reportSavedPrice(cwd, opts, log) {
|
|
24594
|
-
const saved = await readSavedBench(cwd);
|
|
24595
|
-
const rerun = 'genex llm bench "<prompt>" --max-credits <n> --user-approved';
|
|
24596
|
-
const creditKeys = (record2) => ({
|
|
24597
|
-
costUsdPicos: record2?.cost ?? { p50: null, p95: null, max: null },
|
|
24598
|
-
chargedCredits: record2?.charged ?? { p50: null, p95: null, max: null },
|
|
24599
|
-
fitP95: record2?.fitP95 ?? null,
|
|
24600
|
-
fitMax: record2?.fitMax ?? null,
|
|
24601
|
-
recommendedDeclaredHeadroomBps: record2?.recommendedDeclaredHeadroomBps ?? null,
|
|
24602
|
-
feeBps: record2?.feeBps ?? null,
|
|
24603
|
-
usdCentsPerCredit: record2?.usdCentsPerCredit ?? null,
|
|
24604
|
-
costBasedMaxCredits: record2?.costBasedMaxCredits ?? null,
|
|
24605
|
-
lengthRaisedCeiling: record2?.lengthRaisedCeiling ?? false,
|
|
24606
|
-
cutOffSamples: record2?.cutOffSamples ?? 0,
|
|
24607
|
-
noRecommendationReason: record2?.noRecommendationReason ?? null,
|
|
24608
|
-
recommendedMaxCredits: record2?.recommendedMaxCredits ?? null,
|
|
24609
|
-
recommendedPerCallMaxCredits: record2?.recommendedPerCallMaxCredits ?? null,
|
|
24610
|
-
averageCreditsPerCall: record2?.averageCreditsPerCall ?? null,
|
|
24611
|
-
recommendedPerCallEstimateCredits: record2?.recommendedPerCallEstimateCredits ?? null
|
|
24612
|
-
});
|
|
24613
|
-
if (!saved) {
|
|
24412
|
+
if (!record) {
|
|
24614
24413
|
if (opts.json) {
|
|
24615
24414
|
writeJsonLine({
|
|
24616
24415
|
command: "llm price",
|
|
24617
24416
|
status: "failed",
|
|
24618
24417
|
error: "no_bench",
|
|
24619
|
-
currency: null,
|
|
24620
24418
|
ranAt: null,
|
|
24621
24419
|
modelId: null,
|
|
24622
24420
|
samples: null,
|
|
24623
24421
|
completed: null,
|
|
24624
|
-
|
|
24625
|
-
|
|
24422
|
+
p50: null,
|
|
24423
|
+
p95: null,
|
|
24424
|
+
max: null,
|
|
24425
|
+
fitP95: null,
|
|
24426
|
+
chargedBasedEstimateCoins: null,
|
|
24427
|
+
lengthRaisedPrice: false,
|
|
24428
|
+
cutOffSamples: 0,
|
|
24429
|
+
noRecommendationReason: null,
|
|
24430
|
+
recommendedDeclaredHeadroomBps: null,
|
|
24431
|
+
recommendedEstimateCoins: null,
|
|
24432
|
+
recommendedPerCallMaxCoins: null,
|
|
24626
24433
|
savedTo: null
|
|
24627
24434
|
});
|
|
24628
24435
|
} else {
|
|
24629
24436
|
log.error("Nothing has been benchmarked in this folder yet.");
|
|
24630
|
-
log.dim(` Run ${c.cyan(
|
|
24631
|
-
}
|
|
24632
|
-
process.exitCode = 1;
|
|
24633
|
-
return;
|
|
24634
|
-
}
|
|
24635
|
-
if (saved.kind === "coins") {
|
|
24636
|
-
const old = saved.record;
|
|
24637
|
-
const legacy = {
|
|
24638
|
-
p50: old.p50 ?? null,
|
|
24639
|
-
p95: old.p95 ?? null,
|
|
24640
|
-
max: old.max ?? null,
|
|
24641
|
-
recommendedEstimateCoins: old.recommendedEstimateCoins ?? null,
|
|
24642
|
-
recommendedPerCallMaxCoins: old.recommendedPerCallMaxCoins ?? null,
|
|
24643
|
-
maxCoinsPerSample: old.maxCoinsPerSample ?? null,
|
|
24644
|
-
noRecommendationReason: old.noRecommendationReason ?? null
|
|
24645
|
-
};
|
|
24646
|
-
if (opts.json) {
|
|
24647
|
-
writeJsonLine({
|
|
24648
|
-
command: "llm price",
|
|
24649
|
-
status: "failed",
|
|
24650
|
-
error: "coin_terms",
|
|
24651
|
-
currency: "coins",
|
|
24652
|
-
ranAt: old.ranAt ?? null,
|
|
24653
|
-
modelId: old.modelId ?? null,
|
|
24654
|
-
samples: old.samples ?? null,
|
|
24655
|
-
completed: old.completed ?? null,
|
|
24656
|
-
...creditKeys(null),
|
|
24657
|
-
legacyCoinTerms: legacy,
|
|
24658
|
-
savedTo: LLM_BENCH_FILE
|
|
24659
|
-
});
|
|
24660
|
-
} else {
|
|
24661
|
-
log.plain(c.bold("genex llm price"));
|
|
24662
|
-
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${old.ranAt ?? "at an unknown time"}${old.modelId ? ` on ${old.modelId}` : ""}`);
|
|
24663
|
-
log.plain("");
|
|
24664
|
-
log.warn(` ${LLM_PRICE_COIN_TERMS_LINE}`);
|
|
24665
|
-
const measured = legacy.p95 === null ? "no settled sample" : `charged coins p50 ${legacy.p50} \xB7 p95 ${legacy.p95} \xB7 max ${legacy.max}`;
|
|
24666
|
-
const recommended = legacy.recommendedEstimateCoins === null ? "no price recommended" : `it recommended estimateCoins ${legacy.recommendedEstimateCoins}${legacy.recommendedPerCallMaxCoins != null ? ` and perCallMaxCoins ${legacy.recommendedPerCallMaxCoins}` : ""}`;
|
|
24667
|
-
log.dim(` What it measured then: ${measured}; ${recommended}.`);
|
|
24668
|
-
log.dim(` Re-run: ${c.cyan(rerun)}`);
|
|
24437
|
+
log.dim(` Run ${c.cyan('genex llm bench "<prompt>" --max-coins <n> --user-approved')} first.`);
|
|
24669
24438
|
}
|
|
24670
24439
|
process.exitCode = 1;
|
|
24671
24440
|
return;
|
|
24672
24441
|
}
|
|
24673
|
-
const record = saved.record;
|
|
24674
24442
|
if (opts.json) {
|
|
24675
24443
|
writeJsonLine({
|
|
24676
24444
|
command: "llm price",
|
|
24677
|
-
status: record.
|
|
24678
|
-
error: record.
|
|
24679
|
-
currency: "credits",
|
|
24445
|
+
status: record.recommendedEstimateCoins === null ? "failed" : "ok",
|
|
24446
|
+
error: record.recommendedEstimateCoins === null ? record.noRecommendationReason ?? "no_recommendation" : null,
|
|
24680
24447
|
ranAt: record.ranAt ?? null,
|
|
24681
24448
|
modelId: record.modelId ?? null,
|
|
24682
24449
|
samples: record.samples ?? null,
|
|
24683
24450
|
completed: record.completed ?? null,
|
|
24684
|
-
|
|
24685
|
-
|
|
24451
|
+
p50: record.p50 ?? null,
|
|
24452
|
+
p95: record.p95 ?? null,
|
|
24453
|
+
max: record.max ?? null,
|
|
24454
|
+
// Absent from a file an older CLI wrote; null / false / 0 then, never missing.
|
|
24455
|
+
fitP95: record.fitP95 ?? null,
|
|
24456
|
+
chargedBasedEstimateCoins: record.chargedBasedEstimateCoins ?? null,
|
|
24457
|
+
lengthRaisedPrice: record.lengthRaisedPrice ?? false,
|
|
24458
|
+
cutOffSamples: record.cutOffSamples ?? 0,
|
|
24459
|
+
noRecommendationReason: record.noRecommendationReason ?? null,
|
|
24460
|
+
recommendedDeclaredHeadroomBps: record.recommendedDeclaredHeadroomBps ?? null,
|
|
24461
|
+
recommendedEstimateCoins: record.recommendedEstimateCoins ?? null,
|
|
24462
|
+
recommendedPerCallMaxCoins: record.recommendedPerCallMaxCoins ?? null,
|
|
24686
24463
|
savedTo: LLM_BENCH_FILE
|
|
24687
24464
|
});
|
|
24688
|
-
if (record.
|
|
24465
|
+
if (record.recommendedEstimateCoins === null) process.exitCode = 1;
|
|
24689
24466
|
return;
|
|
24690
24467
|
}
|
|
24691
24468
|
log.plain(c.bold("genex llm price"));
|
|
24692
24469
|
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${record.ranAt}`);
|
|
24693
24470
|
log.plain("");
|
|
24694
24471
|
printRecommendation(log, record);
|
|
24695
|
-
if (record.
|
|
24472
|
+
if (record.recommendedEstimateCoins === null) process.exitCode = 1;
|
|
24696
24473
|
}
|
|
24697
24474
|
function writeJsonLine(value) {
|
|
24698
24475
|
process.stdout.write(`${JSON.stringify(value)}
|
|
@@ -24952,7 +24729,7 @@ function runtimeRow(state, lane, toolsOnly = false) {
|
|
|
24952
24729
|
return {
|
|
24953
24730
|
label: "Runtime",
|
|
24954
24731
|
value: `in-game LLM lane live \xB7 ${n} model${n === 1 ? "" : "s"}${keyRefused ? ` \xB7 ${c.yellow("provider key refused")}` : ""}`,
|
|
24955
|
-
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : '
|
|
24732
|
+
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : 'Price a call before a game declares one: `npx genex llm bench "<prompt>" --max-coins <n> --user-approved`.'
|
|
24956
24733
|
};
|
|
24957
24734
|
}
|
|
24958
24735
|
case "off":
|
|
@@ -26908,16 +26685,16 @@ ${c.bold("Usage")}
|
|
|
26908
26685
|
status | cancel. "models" says whether this
|
|
26909
26686
|
stand serves the runtime LLM lane at all, and
|
|
26910
26687
|
at what provider rates. "bench" runs the real
|
|
26911
|
-
model on YOUR OWN
|
|
26912
|
-
attempt
|
|
26688
|
+
model on YOUR OWN coin and reports what each
|
|
26689
|
+
attempt was charged; it needs --max-coins <n>
|
|
26913
26690
|
--user-approved, like any spend the player has
|
|
26914
26691
|
to agree to. "price" reprints the last run's
|
|
26915
|
-
recommended
|
|
26692
|
+
recommended estimateCoins. "status" lists this
|
|
26916
26693
|
project's open calls and the slots they hold;
|
|
26917
26694
|
"cancel <id>" stops an active one. Declare a
|
|
26918
|
-
game's
|
|
26919
|
-
|
|
26920
|
-
|
|
26695
|
+
game's price from a bench, never from a guess:
|
|
26696
|
+
a started attempt is charged in full, including
|
|
26697
|
+
one that fails.
|
|
26921
26698
|
genex player <sub> [options] Answer the in-game model requests you approve, on your
|
|
26922
26699
|
OWN Claude or ChatGPT subscription: install | run |
|
|
26923
26700
|
status | stop | uninstall. "install" signs this machine
|
|
@@ -27372,7 +27149,7 @@ ${c.bold("Examples")}
|
|
|
27372
27149
|
genex shop remove sku_123
|
|
27373
27150
|
genex llm models
|
|
27374
27151
|
genex llm models --all
|
|
27375
|
-
genex llm bench "Reply with one short taunt." --samples 5 --max-
|
|
27152
|
+
genex llm bench "Reply with one short taunt." --samples 5 --max-coins 20 --user-approved
|
|
27376
27153
|
genex llm price
|
|
27377
27154
|
genex llm status
|
|
27378
27155
|
genex llm cancel <id>
|
|
@@ -27609,24 +27386,22 @@ ${c.bold("How the allowance works")}
|
|
|
27609
27386
|
|
|
27610
27387
|
Every number printed is live from your account; prices can change without a deploy.
|
|
27611
27388
|
`,
|
|
27612
|
-
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what
|
|
27389
|
+
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what should the game declare?
|
|
27613
27390
|
|
|
27614
27391
|
${c.bold("Usage")}
|
|
27615
27392
|
genex llm models [--all] [--json] Which models this stand serves, their provider rates per
|
|
27616
|
-
million tokens, the
|
|
27617
|
-
|
|
27618
|
-
|
|
27619
|
-
|
|
27620
|
-
|
|
27621
|
-
|
|
27622
|
-
|
|
27623
|
-
|
|
27624
|
-
|
|
27625
|
-
|
|
27626
|
-
|
|
27627
|
-
|
|
27628
|
-
allowance does not count. Needs a folder linked to a
|
|
27629
|
-
game you own, and a balance that covers one attempt.
|
|
27393
|
+
million tokens, and the headroom it recommends over a
|
|
27394
|
+
benchmark. The catalog is synced from OpenRouter, so the
|
|
27395
|
+
FEATURED short list is printed by default and --all
|
|
27396
|
+
prints every row (--json always carries every row). Off
|
|
27397
|
+
on this stand: it says so and exits 0 \u2014 build the feature
|
|
27398
|
+
without a model rather than promising a 404.
|
|
27399
|
+
genex llm bench "<prompt>" --max-coins <n> --user-approved [options]
|
|
27400
|
+
Run the real model <n> times through the development
|
|
27401
|
+
lane and report what each attempt was CHARGED. Refused
|
|
27402
|
+
without both flags, before any network call: this spends
|
|
27403
|
+
YOUR OWN COIN, a wallet the build's asset allowance
|
|
27404
|
+
cannot see. Needs a folder linked to a game you own.
|
|
27630
27405
|
genex llm price [--json] The recommendation from the last bench in this folder.
|
|
27631
27406
|
Reads a file; spends nothing.
|
|
27632
27407
|
genex llm status [--json] This project's open development calls: active, or
|
|
@@ -27643,32 +27418,29 @@ ${c.bold("Bench options")}
|
|
|
27643
27418
|
--samples <n> How many real attempts (default ${DEFAULT_BENCH_SAMPLES}). More samples, better p95.
|
|
27644
27419
|
--schema <file> A JSON file holding the output schema; implies JSON output.
|
|
27645
27420
|
--json-output Ask for JSON without a schema. --text Ask for text (the default).
|
|
27646
|
-
--max-
|
|
27421
|
+
--max-coins <n> The per-attempt ceiling YOU approved \u2014 a bench runs on your own coin,
|
|
27647
27422
|
never a player's. The run's worst case is that number times --samples,
|
|
27648
|
-
and it is printed before anything starts.
|
|
27649
|
-
name, read as the same number.)
|
|
27423
|
+
and it is printed before anything starts.
|
|
27650
27424
|
--timeout <seconds> Bounds the wait for each attempt.
|
|
27651
27425
|
--json One machine-readable object; every key is always present, null when
|
|
27652
27426
|
the CLI could not learn it.
|
|
27653
27427
|
|
|
27654
27428
|
${c.bold("Why a benchmark and not an estimate")}
|
|
27655
|
-
|
|
27656
|
-
|
|
27657
|
-
|
|
27658
|
-
|
|
27659
|
-
|
|
27660
|
-
|
|
27661
|
-
|
|
27662
|
-
|
|
27663
|
-
|
|
27664
|
-
recommendation, because those multipliers are not the CLI's to invent.
|
|
27429
|
+
\`estimateCoins\` is a PRICE, not a guess: declaring it charges it, in full, even when the
|
|
27430
|
+
attempt fails, is cancelled, or stops at its budget. And three layers sit between the
|
|
27431
|
+
provider's rate card and that number \u2014 the provider's cost, the platform's tariff (already
|
|
27432
|
+
inside every charged coin), and your headroom. The bench itself is billed to YOU, on the
|
|
27433
|
+
development lane; the player-funded lane is what the published game uses. Adding a margin
|
|
27434
|
+
to the provider's rate silently drops the tariff and under-prices every call, so the
|
|
27435
|
+
recommendation is built from CHARGED COINS and the headroom comes from the server. A stand
|
|
27436
|
+
that serves no headroom gets no recommendation, because the multiplier is not the CLI's to
|
|
27437
|
+
invent.
|
|
27665
27438
|
|
|
27666
|
-
|
|
27667
|
-
|
|
27668
|
-
A sample cut off at your --max-
|
|
27669
|
-
|
|
27670
|
-
|
|
27671
|
-
fee included, which is the honest estimate a budget's disclosure is built on.
|
|
27439
|
+
The declared price also decides how LONG the answer may be: it sizes each call's output
|
|
27440
|
+
allowance. So the recommendation is never below the smallest price that leaves room for
|
|
27441
|
+
the benchmarked answer, as the server computes it. A sample cut off at your --max-coins
|
|
27442
|
+
(provider_token_limit) is not a sample \u2014 re-run with a higher --max-coins; an answer longer
|
|
27443
|
+
than one call on this stand may produce gets no price at all \u2014 ask for a shorter one.
|
|
27672
27444
|
|
|
27673
27445
|
Results land in ${LLM_BENCH_FILE} \u2014 gitignored, machine-local, re-read by \`genex llm price\`.
|
|
27674
27446
|
`,
|
|
@@ -27883,13 +27655,11 @@ function parseArgs(argv) {
|
|
|
27883
27655
|
"--limit",
|
|
27884
27656
|
// `genex budget --assets <credits>` — the allowance the player approved.
|
|
27885
27657
|
"--assets",
|
|
27886
|
-
// `genex llm bench` value flags. `--max-
|
|
27887
|
-
// shape as `--assets
|
|
27888
|
-
// count a bench. `--max-coins` is its deprecated name (same number).
|
|
27658
|
+
// `genex llm bench` value flags. `--max-coins` is the COIN half of the
|
|
27659
|
+
// same approval shape as `--assets` (a different economy, its own gate).
|
|
27889
27660
|
"--model",
|
|
27890
27661
|
"--samples",
|
|
27891
27662
|
"--schema",
|
|
27892
|
-
"--max-credits",
|
|
27893
27663
|
"--max-coins",
|
|
27894
27664
|
// `genex player install --label <name>` — what this machine is called in
|
|
27895
27665
|
// the dashboard. Its own key rather than the generic 2nd positional,
|
|
@@ -28404,14 +28174,12 @@ function applyValueFlag(options, flag, value) {
|
|
|
28404
28174
|
options.samples = n;
|
|
28405
28175
|
break;
|
|
28406
28176
|
}
|
|
28407
|
-
case "--max-credits":
|
|
28408
28177
|
case "--max-coins": {
|
|
28409
28178
|
const n = Number(value);
|
|
28410
28179
|
if (!Number.isInteger(n) || n < 1) {
|
|
28411
|
-
throw new Error(`Invalid
|
|
28180
|
+
throw new Error(`Invalid --max-coins value: ${value} (whole coin, 1 or more)`);
|
|
28412
28181
|
}
|
|
28413
|
-
|
|
28414
|
-
else options.maxCoins = n;
|
|
28182
|
+
options.maxCoins = n;
|
|
28415
28183
|
break;
|
|
28416
28184
|
}
|
|
28417
28185
|
case "--timeout": {
|
|
@@ -28703,9 +28471,9 @@ async function main() {
|
|
|
28703
28471
|
await runBudget(parsed.options);
|
|
28704
28472
|
break;
|
|
28705
28473
|
// The in-game model lane: is it live here, what does one call really
|
|
28706
|
-
// cost, what
|
|
28707
|
-
//
|
|
28708
|
-
//
|
|
28474
|
+
// cost, what should the game declare. `bench` is the only CLI command
|
|
28475
|
+
// that spends COIN, so it carries its own --max-coins/--user-approved
|
|
28476
|
+
// gate — the credits allowance cannot see that wallet.
|
|
28709
28477
|
case "llm":
|
|
28710
28478
|
await runLlm(parsed.options);
|
|
28711
28479
|
break;
|