@genex-ai/cli-demo 1.36.7-dev.895 → 1.36.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{blender-mcp-WSURPZET.js → blender-mcp-QVEGHE5L.js} +2 -2
- package/dist/{blender-serve-EH2MOW3U.js → blender-serve-66OHC4ML.js} +1 -1
- package/dist/{chunk-EABIVAGT.js → chunk-4BBTPRJW.js} +2 -2
- package/dist/{chunk-K4EHXOZK.js → chunk-QI7FIYBY.js} +1 -1
- package/dist/index.js +216 -448
- package/package.json +1 -1
- package/templates/skills/genex-game-director/SKILL.md +1 -1
- package/templates/skills/genex-getting-started/SKILL.md +2 -2
- package/templates/skills/genex-llm-in-games/SKILL.md +72 -209
- package/templates/skills/genex-llm-in-games/references/pricing.md +85 -126
- package/templates/skills/genex-monetization/SKILL.md +9 -1
- package/templates/skills/genex-tool-llm/SKILL.md +9 -11
- package/templates/skills/genex-updates/SKILL.md +1 -1
package/dist/index.js
CHANGED
|
@@ -57,7 +57,7 @@ import {
|
|
|
57
57
|
writeSecretFile,
|
|
58
58
|
writeUserToken,
|
|
59
59
|
writeWorkspace
|
|
60
|
-
} from "./chunk-
|
|
60
|
+
} from "./chunk-4BBTPRJW.js";
|
|
61
61
|
import {
|
|
62
62
|
CLI_CHANNEL,
|
|
63
63
|
DEFAULT_API_URL,
|
|
@@ -78,7 +78,7 @@ import {
|
|
|
78
78
|
normalizeApiOrigin,
|
|
79
79
|
originKey,
|
|
80
80
|
resolveAgentTargets
|
|
81
|
-
} from "./chunk-
|
|
81
|
+
} from "./chunk-QI7FIYBY.js";
|
|
82
82
|
|
|
83
83
|
// src/instrument.ts
|
|
84
84
|
import * as Sentry from "@sentry/node";
|
|
@@ -981,7 +981,7 @@ Important note: put soul into your creations, with many details and love. Aim to
|
|
|
981
981
|
21. A turn ends in exactly one of two ways: on a question the player must answer before the next step can be chosen, or on a handoff \u2014 the draft link, what changed, what to try, and what is still open. Never close a turn on a promise. "I'll keep building", "while I keep working", "adding that now" are things you say and then DO before you stop; if you are stopping, say that you have stopped and what remains. While the Build plan's \`Now:\` line has open work and no answer is needed from the player, the turn is not over: preview, hand off, and start the next milestone in the same turn, until the requested outcome is reached or the platform's own budget ends the session. If a stop was forced on you anyway, the first line of your next turn is the promise you left and where it stands.
|
|
982
982
|
22. **What the player sees must read as the thing it is**, in this game's own style: a building as that building, a person as a person, a prop as that prop, a surface as its material \u2014 and a coloured box, a capsule, or a flat grey block standing in for one of them is never its finished version. Generation is the default route to that bar wherever code will not honestly reach it: the player's character, whatever the request names, whatever the player walks up to, enters, or interacts with, and the music and sound underneath \u2014 reach for the belt on your own judgment, without stopping to ask; a status line saying what you queued is enough, and \`--no-wait\` keeps you building while it lands. Procedural code is a first-class engine for what is structural, repeated, distant, or parametric \u2014 terrain, sky, fences, paving, modular kits, filler \u2014 and for anything else only when the result meets the same bar and you have looked at it in a capture before its row says \`landed\`. Start with one reusable implementation per distinct required gameplay or visual role. Background populations default to reusable procedural bodies and motion or compatible existing rigged assets. Catalog clips need a compatible skeleton and may be paid. Reserve paid custom bodies for the player and characters examined or interacted with closely, while honoring explicit user requests. Add variants only for an unmet requirement; unspent allowance is not unfinished work. Allowance questions use timeoutPolicy no-consent: silence does not raise the allowance. Record role, reuse and completion criteria in the existing Assets table. Decide per object, not per category, and write the route on each Assets row in \`DESIGN.md\` \u2014 the paid flow for generated pieces, the procedural flow with its local path for code-built ones. An Assets table with no generation in it is a decision, not a default: record it as \`Generation: none \u2014 <why code alone reaches the bar here>\` or generate.
|
|
983
983
|
23. **The screenshots you take to look at your own work go in \`.genex/scratch/\`.** Captures, render comparisons, before/after strips, traces and metrics dumps are how you SEE what you built \u2014 they are not part of what you built, and \`genex preview\` pushes the whole folder to the game's source, every time, forever. \`.genex/scratch/\` already exists for this and is already ignored, so there is nothing to set up: write the capture as \`.genex/scratch/arena-before.png\` and read it back from there. Never put them in a folder of your own at the top level: \`progress/\`, \`reports/\`, \`shots/\` and their kind are pushed like source and become permanent weight in the game and in every remix of it \u2014 one real game reached 11 GB and 10,332 committed screenshots exactly this way. If you find such a folder already there, add it to \`.gitignore\`; never delete the player's files (law 18). One file in \`.genex/scratch/\` IS read, by the platform: \`.genex/scratch/cover.png\` is the game's cover \u2014 the frame you would put on its poster: real gameplay at its signature moment, well lit, no menus, popups or busy HUD, landscape 16:9, and never any text (the page sets the title beside it). The next \`genex preview\` makes it the cover; save a better moment over it whenever one comes along, and if your browser tool can set the cover itself, use that instead. Before the first publish, and whenever the game's look changes, load \`$genex-cover\`: it chooses, stages, lights and judges that frame.
|
|
984
|
-
24. **Benchmark before declaring an in-game LLM
|
|
984
|
+
24. **Benchmark before declaring an in-game LLM price.** If the game calls a language model while the player plays \u2014 an NPC answering in its own words, a quest written for this save, a judge reading what the player typed \u2014 the PLAYER pays for it, and \`estimateCoins\` is the FIXED price of a started attempt: charged whether it succeeds, fails, is canceled, or stops at its own budget. Never pick that number by judgement or from a vendor's rate card. Run the real prompt on your own coins \u2014 \`npx genex llm bench "<the prompt>" --schema <file> --samples 3 --max-coins <n> --user-approved\` \u2014 and declare the figure it recommends, recorded in \`DESIGN.md\` with its model and date; re-benchmark whenever the prompt, the schema or the model changes, because a prompt edit is a price change. Check the lane first with \`npx genex llm models\`: where it is off the routes 404, so build the feature behind a graceful unavailable state and say so in the handoff. A repeated call is a standing budget the player approves once (\`requestSpendGrant()\`, then \`generate({ grantId })\`), never a popup per turn, and its disclosed call rate is computed from the game's own loop. Model output is data the game validates against its own expectation \u2014 never executed, and never authority over coin, items, entitlements or rewards. Load \`$genex-llm-in-games\` before writing any of it.
|
|
985
985
|
${CONTRACT_END}
|
|
986
986
|
`;
|
|
987
987
|
var TOOLS_CONTRACT_HEAD = `# Genex Tools (always in effect in this folder)
|
|
@@ -995,7 +995,7 @@ var TOOLS_CONTRACT_OFFER = `5. **When the game is built and playable \u2014 neve
|
|
|
995
995
|
var TOOLS_CONTRACT_HOSTED_LAWS = `5. **This game is hosted on Genex now, and publishing is glue \u2014 never a rebuild.** The game ships exactly as the user built it: \`initEmbed()\` first in the boot code (the \`genex-threejs-embed-auth\` card has the call; the renderer draws before any \`await waitForPlayer()\`), a static build into \`dist/\` with relative asset paths, then \`npx genex preview\`. Never restructure, reformat or "improve" the game to ship it, never scaffold a new app around it, and never start a design document or a build plan for it \u2014 the user's own process is theirs. Load the \`genex-tool-publish\` card before the first preview; it owns the vocabulary, the links and the limits.
|
|
996
996
|
6. **Ship first, then report.** \`preview\` prints a preflight \u2014 phone memory, a missing volume slider, an asset nobody wired, a viewport line. It never blocks a deploy, so push the build as it is, hand over the link, THEN relay each preflight line to the user in one plain sentence with an offer to fix it, and fix one only on their yes. The preflight is a report for the user, not a to-do list for you.
|
|
997
997
|
7. **The link is the game's page, and there are two versions.** After every preview give the user \`<dashboard>/draft/<slug>\` \u2014 \`dashboardOrigins[0]\` and \`slug\` from \`.genex/project.json\` \u2014 never localhost, a file path or the bare play origin. \`preview\` updates the draft and never touches what players are on; the first release is \`npx genex publish\` (it lists the game); after that "publish it", "update it" and "yes" all mean \`npx genex promote\` \u2014 the exact draft build, no rebuild. Ask once per round of work, in one line, and keep working while you wait.`;
|
|
998
|
-
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with
|
|
998
|
+
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with Genex coins or their own Claude/ChatGPT subscription, and approves it on a Genex sheet; your game just calls \`generate()\`. Want it that way?`;
|
|
999
999
|
function toolsContractLlmLaw(n, hosted) {
|
|
1000
1000
|
const onYes = hosted ? "On a yes, load `$genex-tool-llm` and then `$genex-llm-in-games`, which owns the build." : "On a yes, load `$genex-tool-llm`, which owns the rest: it checks the lane with `npx genex llm models` first, then runs `npx genex init --convert` (a hosted game is a static browser build \u2014 a local server holding a key can never ship).";
|
|
1001
1001
|
return `${n}. **A model running while people PLAY is a platform feature \u2014 never something you wire onto the user's own meter.** When a request implies one (NPCs that talk or decide in their own words, content written from what the player types, a prompt box in the game, "let the player pick a model"), recognise it, say in ONE line: "${TOOLS_CONTRACT_LLM_OFFER}" \u2014 then ASK and wait. ${onYes} On a no, build the authored version instead. Either way, never ship a game that calls a model on a key or an account of the user's own: every visitor would spend their money with nobody approving it, and the credential is readable in the bundle. Player-funded generation is text and JSON only \u2014 3D, images, video and audio stay the asset lanes above, on the user's meter.`;
|
|
@@ -23441,7 +23441,7 @@ async function runBlender(opts) {
|
|
|
23441
23441
|
return 1;
|
|
23442
23442
|
}
|
|
23443
23443
|
if (sub === "serve") {
|
|
23444
|
-
const { serveLocalBlender } = await import("./blender-serve-
|
|
23444
|
+
const { serveLocalBlender } = await import("./blender-serve-66OHC4ML.js");
|
|
23445
23445
|
const port = Number(process.env.GENEX_BLENDER_PORT ?? 8088);
|
|
23446
23446
|
log.step(`Starting a local Blender service on port ${port}`);
|
|
23447
23447
|
log.plain(
|
|
@@ -23450,7 +23450,7 @@ async function runBlender(opts) {
|
|
|
23450
23450
|
return serveLocalBlender({ port, log });
|
|
23451
23451
|
}
|
|
23452
23452
|
if (sub === "mcp") {
|
|
23453
|
-
const { runBlenderMcp } = await import("./blender-mcp-
|
|
23453
|
+
const { runBlenderMcp } = await import("./blender-mcp-QVEGHE5L.js");
|
|
23454
23454
|
return runBlenderMcp();
|
|
23455
23455
|
}
|
|
23456
23456
|
if (sub === "seat") {
|
|
@@ -23958,12 +23958,7 @@ import fs35 from "fs/promises";
|
|
|
23958
23958
|
import path36 from "path";
|
|
23959
23959
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
23960
23960
|
var SUBS3 = ["models", "bench", "price", "status", "cancel"];
|
|
23961
|
-
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN
|
|
23962
|
-
var LLM_MAX_COINS_DEPRECATED = "--max-coins is deprecated: it is read as --max-credits, the same number \u2014 a bench spends credits now.";
|
|
23963
|
-
function benchBalanceShortLine(spendable, maxCredits) {
|
|
23964
|
-
return `Your spendable credits (${spendable}) cannot cover one attempt at --max-credits ${maxCredits} \u2014 nothing was sent, nothing was spent. Add credits, or re-run with a lower --max-credits once the person at the keyboard agrees to it.`;
|
|
23965
|
-
}
|
|
23966
|
-
var LLM_BENCH_UNVERIFIED_LINE = "Credits pay only for a verified email, and this account's is not verified yet (credits_unverified) \u2014 nothing was sent, nothing was spent. Verify it, then re-run.";
|
|
23961
|
+
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN coin \u2014 your wallet, not a player's, and one this build's asset allowance cannot see. Re-run with --max-coins <n> --user-approved only after the person at the keyboard agreed to the number.";
|
|
23967
23962
|
var LLM_LANE_OFF_LINE = "The runtime LLM lane is off on this stand \u2014 in-game generate() answers 404 here, and nothing can be benchmarked.";
|
|
23968
23963
|
var LLM_TOOLS_CONVERT_HINT = [
|
|
23969
23964
|
"In-game model calls need a hosted game. This folder is a Genex Tools workspace, so nothing here",
|
|
@@ -23976,7 +23971,7 @@ function serverSlotCounts(body) {
|
|
|
23976
23971
|
return { held: body.slotsHeld, limit: body.slotLimit };
|
|
23977
23972
|
}
|
|
23978
23973
|
var PENDING_SLOT_RELEASE_MINUTES = 10;
|
|
23979
|
-
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so
|
|
23974
|
+
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so its coin stays held until the bill resolves; such a row stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`;
|
|
23980
23975
|
var LLM_STATUS_NO_COUNT_LINE = "This stand does not report how many slots are held across your projects, so no count is shown.";
|
|
23981
23976
|
var LLM_CANCEL_AWAITING_BILL = "this call already stopped; its bill is awaiting the provider";
|
|
23982
23977
|
var LLM_CANCEL_ALREADY_FINAL = "this call is already final";
|
|
@@ -23990,33 +23985,11 @@ var DEFAULT_SAMPLE_TIMEOUT_SEC = 180;
|
|
|
23990
23985
|
function parseProviderKey(value) {
|
|
23991
23986
|
return value === "ok" || value === "rejected" || value === "unknown" ? value : null;
|
|
23992
23987
|
}
|
|
23993
|
-
function bpsOrNull(value) {
|
|
23994
|
-
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
23995
|
-
}
|
|
23996
|
-
function standingCreditGrantOrNull(value) {
|
|
23997
|
-
if (!value || typeof value !== "object") return null;
|
|
23998
|
-
const v = value;
|
|
23999
|
-
const whole = (x) => typeof x === "number" && Number.isSafeInteger(x) && x >= 0;
|
|
24000
|
-
if (!whole(v.minCredits) || !whole(v.maxCredits) || !whole(v.stepCredits) || !whole(v.defaultCredits)) return null;
|
|
24001
|
-
return {
|
|
24002
|
-
minCredits: v.minCredits,
|
|
24003
|
-
maxCredits: v.maxCredits,
|
|
24004
|
-
stepCredits: v.stepCredits,
|
|
24005
|
-
defaultCredits: v.defaultCredits,
|
|
24006
|
-
...whole(v.minUsdCents) ? { minUsdCents: v.minUsdCents } : {},
|
|
24007
|
-
...whole(v.maxUsdCents) ? { maxUsdCents: v.maxUsdCents } : {},
|
|
24008
|
-
...whole(v.stepUsdCents) ? { stepUsdCents: v.stepUsdCents } : {}
|
|
24009
|
-
};
|
|
24010
|
-
}
|
|
24011
23988
|
async function readRuntimeLane(apiUrl, token) {
|
|
24012
23989
|
const empty = (state) => ({
|
|
24013
23990
|
state,
|
|
24014
23991
|
models: null,
|
|
24015
23992
|
recommendedDeclaredHeadroomBps: null,
|
|
24016
|
-
currency: null,
|
|
24017
|
-
feeBps: null,
|
|
24018
|
-
usdCentsPerCredit: null,
|
|
24019
|
-
standingCreditGrant: null,
|
|
24020
23993
|
externalProviders: null,
|
|
24021
23994
|
total: null,
|
|
24022
23995
|
featuredCount: null,
|
|
@@ -24035,11 +24008,7 @@ async function readRuntimeLane(apiUrl, token) {
|
|
|
24035
24008
|
return {
|
|
24036
24009
|
state: "live",
|
|
24037
24010
|
models: body.models,
|
|
24038
|
-
recommendedDeclaredHeadroomBps:
|
|
24039
|
-
currency: body.currency === "credits" || body.currency === "coins" ? body.currency : null,
|
|
24040
|
-
feeBps: bpsOrNull(body.feeBps),
|
|
24041
|
-
usdCentsPerCredit: typeof body.usdCentsPerCredit === "number" && Number.isSafeInteger(body.usdCentsPerCredit) && body.usdCentsPerCredit >= 1 ? body.usdCentsPerCredit : null,
|
|
24042
|
-
standingCreditGrant: standingCreditGrantOrNull(body.standingCreditGrant),
|
|
24011
|
+
recommendedDeclaredHeadroomBps: typeof body.recommendedDeclaredHeadroomBps === "number" && Number.isFinite(body.recommendedDeclaredHeadroomBps) ? body.recommendedDeclaredHeadroomBps : null,
|
|
24043
24012
|
externalProviders: Array.isArray(body.externalProviders) ? body.externalProviders : null,
|
|
24044
24013
|
total: typeof body.total === "number" && Number.isFinite(body.total) ? body.total : null,
|
|
24045
24014
|
featuredCount: typeof body.featuredCount === "number" && Number.isFinite(body.featuredCount) ? body.featuredCount : null,
|
|
@@ -24055,45 +24024,18 @@ function percentile2(values, p) {
|
|
|
24055
24024
|
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
24056
24025
|
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
24057
24026
|
}
|
|
24058
|
-
function
|
|
24059
|
-
|
|
24060
|
-
const sorted = [...values].sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
24061
|
-
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
24062
|
-
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
24027
|
+
function recommendEstimateCoins(p95, headroomBps) {
|
|
24028
|
+
return Math.max(1, Math.ceil(p95 * (1e4 + headroomBps) / 1e4));
|
|
24063
24029
|
}
|
|
24064
|
-
|
|
24065
|
-
|
|
24066
|
-
if (costPicos <= 0n) return 1;
|
|
24067
|
-
const num = costPicos * BigInt(1e4 + terms.feeBps) * BigInt(1e4 + terms.headroomBps);
|
|
24068
|
-
const den = 100000000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
24069
|
-
return Math.max(1, Number((num + den - 1n) / den));
|
|
24070
|
-
}
|
|
24071
|
-
function averageCreditsPerCall(costsPicos, terms) {
|
|
24072
|
-
if (costsPicos.length === 0) return null;
|
|
24073
|
-
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
24074
|
-
return Number(sum * BigInt(1e4 + terms.feeBps)) / Number(BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT);
|
|
24075
|
-
}
|
|
24076
|
-
function perCallEstimateCreditsFor(costsPicos, terms) {
|
|
24077
|
-
if (costsPicos.length === 0) return null;
|
|
24078
|
-
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
24079
|
-
const num = sum * BigInt(1e4 + terms.feeBps);
|
|
24080
|
-
const den = BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
24081
|
-
return Math.max(1, Number((num + den - 1n) / den));
|
|
24082
|
-
}
|
|
24083
|
-
function formatCredits(value) {
|
|
24084
|
-
if (!Number.isFinite(value)) return "\u2014";
|
|
24085
|
-
if (value >= 100) return String(Math.round(value));
|
|
24086
|
-
return String(Number(value.toPrecision(value < 1 ? 2 : 3)));
|
|
24087
|
-
}
|
|
24088
|
-
function usdFromPicos(picos) {
|
|
24089
|
-
if (picos === null || picos === void 0) return "unknown";
|
|
24090
|
-
const value = typeof picos === "bigint" ? picos : /^\d+$/.test(picos) ? BigInt(picos) : null;
|
|
24091
|
-
if (value === null) return "unknown";
|
|
24092
|
-
return `$${(Number(value) / 1e12).toFixed(6)}`;
|
|
24030
|
+
function recommendPerCallMaxCoins(p95, max, headroomBps) {
|
|
24031
|
+
return Math.max(recommendEstimateCoins(p95, headroomBps), recommendEstimateCoins(max, headroomBps));
|
|
24093
24032
|
}
|
|
24094
24033
|
function usdPerMillion(value) {
|
|
24095
24034
|
return `$${value.toFixed(value < 1 ? 4 : 2)}`;
|
|
24096
24035
|
}
|
|
24036
|
+
function usd(value) {
|
|
24037
|
+
return value === null ? "unknown" : `$${value.toFixed(6)}`;
|
|
24038
|
+
}
|
|
24097
24039
|
async function runLlm(opts = {}) {
|
|
24098
24040
|
const log = createLogger({ quiet: opts.quiet || opts.json });
|
|
24099
24041
|
const cwd = opts.cwd ?? process.cwd();
|
|
@@ -24119,23 +24061,16 @@ async function runLlm(opts = {}) {
|
|
|
24119
24061
|
return;
|
|
24120
24062
|
}
|
|
24121
24063
|
if (sub === "bench") {
|
|
24122
|
-
if (opts.maxCoins
|
|
24123
|
-
`);
|
|
24124
|
-
if (opts.maxCoins !== void 0 && opts.maxCredits !== void 0 && opts.maxCoins !== opts.maxCredits) {
|
|
24125
|
-
fail5("llm bench", `Pass --max-credits alone \u2014 --max-coins is its deprecated name, and the two disagree (${opts.maxCredits} vs ${opts.maxCoins}).`);
|
|
24126
|
-
return;
|
|
24127
|
-
}
|
|
24128
|
-
const approvedCredits = opts.maxCredits ?? opts.maxCoins;
|
|
24129
|
-
if (approvedCredits === void 0 || opts.userApproved !== true) {
|
|
24064
|
+
if (opts.maxCoins === void 0 || opts.userApproved !== true) {
|
|
24130
24065
|
fail5("llm bench", LLM_BENCH_APPROVAL_REQUIRED);
|
|
24131
24066
|
return;
|
|
24132
24067
|
}
|
|
24133
|
-
if (!Number.isInteger(
|
|
24134
|
-
fail5("llm bench", `--max-
|
|
24068
|
+
if (!Number.isInteger(opts.maxCoins) || opts.maxCoins < 1) {
|
|
24069
|
+
fail5("llm bench", `--max-coins takes a whole number of coin, 1 or more (got ${String(opts.maxCoins)}).`);
|
|
24135
24070
|
return;
|
|
24136
24071
|
}
|
|
24137
24072
|
if (!opts.benchPrompt?.trim()) {
|
|
24138
|
-
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-
|
|
24073
|
+
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-coins <n> --user-approved.');
|
|
24139
24074
|
return;
|
|
24140
24075
|
}
|
|
24141
24076
|
if (opts.jsonOutput && opts.textOutput) {
|
|
@@ -24227,10 +24162,6 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24227
24162
|
models: null,
|
|
24228
24163
|
externalProviders: null,
|
|
24229
24164
|
recommendedDeclaredHeadroomBps: null,
|
|
24230
|
-
currency: null,
|
|
24231
|
-
feeBps: null,
|
|
24232
|
-
usdCentsPerCredit: null,
|
|
24233
|
-
standingCreditGrant: null,
|
|
24234
24165
|
total: null,
|
|
24235
24166
|
featuredCount: null,
|
|
24236
24167
|
// The stand was never asked, so the verdict is unknown — but the key is
|
|
@@ -24256,12 +24187,6 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24256
24187
|
models: lane.models,
|
|
24257
24188
|
externalProviders: lane.externalProviders,
|
|
24258
24189
|
recommendedDeclaredHeadroomBps: lane.recommendedDeclaredHeadroomBps,
|
|
24259
|
-
// The credit terms a call is billed on, the server's — null on a stand
|
|
24260
|
-
// that predates credit billing, never a number the CLI filled in.
|
|
24261
|
-
currency: lane.currency,
|
|
24262
|
-
feeBps: lane.feeBps,
|
|
24263
|
-
usdCentsPerCredit: lane.usdCentsPerCredit,
|
|
24264
|
-
standingCreditGrant: lane.standingCreditGrant,
|
|
24265
24190
|
total: lane.total ?? lane.models?.length ?? null,
|
|
24266
24191
|
featuredCount: lane.featuredCount ?? (lane.models ? lane.models.filter((m) => m.featured).length : null),
|
|
24267
24192
|
// `null` when the lane is not live or the server predates the probe.
|
|
@@ -24308,9 +24233,8 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24308
24233
|
);
|
|
24309
24234
|
for (const m of shown) {
|
|
24310
24235
|
const plan = m.personalPlan ? ` \xB7 personal plan: ${m.personalPlan}` : "";
|
|
24311
|
-
const kind = m.kind === "typesafe" ? " \xB7 judge (classifier): judge() under a budget, not generate() or bench" : "";
|
|
24312
24236
|
log.plain(
|
|
24313
|
-
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}
|
|
24237
|
+
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}`
|
|
24314
24238
|
);
|
|
24315
24239
|
}
|
|
24316
24240
|
if (hiddenCount > 0) {
|
|
@@ -24326,104 +24250,74 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24326
24250
|
}
|
|
24327
24251
|
log.plain("");
|
|
24328
24252
|
if (lane.recommendedDeclaredHeadroomBps === null) {
|
|
24329
|
-
log.dim(" This stand serves no recommended headroom, so no
|
|
24253
|
+
log.dim(" This stand serves no recommended headroom, so no price can be recommended from a bench.");
|
|
24330
24254
|
} else {
|
|
24331
|
-
log.dim(` Recommended headroom over
|
|
24332
|
-
}
|
|
24333
|
-
if (lane.feeBps === null) {
|
|
24334
|
-
log.dim(" This stand serves no platform fee \u2014 it predates credit billing, so no ceiling can be recommended from a bench.");
|
|
24335
|
-
} else {
|
|
24336
|
-
log.dim(` In-game calls are billed in the player's credits, as used: the provider's cost plus this stand's platform fee (${lane.feeBps} bps).`);
|
|
24255
|
+
log.dim(` Recommended headroom over benchmarked coins on this stand: ${lane.recommendedDeclaredHeadroomBps} bps.`);
|
|
24337
24256
|
}
|
|
24338
24257
|
if (toolsOnly) {
|
|
24339
24258
|
printConvertHint();
|
|
24340
24259
|
return;
|
|
24341
24260
|
}
|
|
24342
|
-
log.dim(" Those per-million rates are the PROVIDER's
|
|
24343
|
-
log.dim(`
|
|
24261
|
+
log.dim(" Those per-million rates are the PROVIDER's; what your game is charged is the platform");
|
|
24262
|
+
log.dim(` tariff on top, which only a real call reveals: ${c.cyan('genex llm bench "<prompt>" --max-coins <n> --user-approved')}`);
|
|
24344
24263
|
}
|
|
24345
24264
|
function wholeOrNull(value) {
|
|
24346
24265
|
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
24347
24266
|
}
|
|
24348
|
-
function costPicosOf(usage) {
|
|
24349
|
-
if (typeof usage?.costUsdPicos === "string" && /^\d+$/.test(usage.costUsdPicos)) return usage.costUsdPicos;
|
|
24350
|
-
if (typeof usage?.costUsd === "number" && Number.isFinite(usage.costUsd) && usage.costUsd >= 0) {
|
|
24351
|
-
return String(BigInt(Math.round(usage.costUsd * 1e12)));
|
|
24352
|
-
}
|
|
24353
|
-
return null;
|
|
24354
|
-
}
|
|
24355
24267
|
function sampleFrom(id, settled) {
|
|
24356
24268
|
const usage = settled?.usage;
|
|
24357
|
-
const picos = costPicosOf(usage);
|
|
24358
24269
|
return {
|
|
24359
24270
|
id,
|
|
24360
24271
|
status: settled?.status ?? null,
|
|
24361
24272
|
billingStatus: settled?.billingStatus ?? null,
|
|
24362
|
-
|
|
24363
|
-
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd :
|
|
24364
|
-
costUsdPicos: picos,
|
|
24273
|
+
chargedCoins: typeof settled?.chargedCoins === "number" ? settled.chargedCoins : null,
|
|
24274
|
+
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd : null,
|
|
24365
24275
|
error: settled?.error ?? (settled ? null : "timed_out"),
|
|
24366
24276
|
providerMessage: typeof settled?.providerMessage === "string" && settled.providerMessage.trim() ? settled.providerMessage.trim() : null,
|
|
24367
24277
|
inputTokens: wholeOrNull(usage?.inputTokens),
|
|
24368
24278
|
outputTokens: wholeOrNull(usage?.outputTokens),
|
|
24369
24279
|
maxOutputTokens: wholeOrNull(usage?.maxOutputTokens),
|
|
24370
|
-
|
|
24280
|
+
fitCoins: typeof usage?.fitCoins === "number" && Number.isSafeInteger(usage.fitCoins) && usage.fitCoins >= 1 ? usage.fitCoins : null,
|
|
24371
24281
|
fitExceedsCap: typeof usage?.fitExceedsCap === "boolean" ? usage.fitExceedsCap : null,
|
|
24372
24282
|
truncated: typeof usage?.truncated === "boolean" ? usage.truncated : settled?.error === "provider_token_limit" ? true : null
|
|
24373
24283
|
};
|
|
24374
24284
|
}
|
|
24375
24285
|
var PROVIDER_TOKEN_LIMIT = "provider_token_limit";
|
|
24376
|
-
function benchRecommendation(results,
|
|
24377
|
-
const settled = results.filter(
|
|
24378
|
-
|
|
24379
|
-
);
|
|
24380
|
-
const
|
|
24381
|
-
const
|
|
24382
|
-
const fits = settled.map((r) => r.fitCredits).filter((v) => typeof v === "number");
|
|
24383
|
-
const costP95 = percentilePicos(costs, 95);
|
|
24384
|
-
const costMax = costs.length ? costs.reduce((a, b) => b > a ? b : a) : null;
|
|
24286
|
+
function benchRecommendation(results, headroomBps) {
|
|
24287
|
+
const settled = results.filter((r) => r.status === "succeeded" && r.billingStatus === "final" && typeof r.chargedCoins === "number");
|
|
24288
|
+
const charged = settled.map((r) => r.chargedCoins);
|
|
24289
|
+
const fits = settled.map((r) => r.fitCoins).filter((v) => typeof v === "number");
|
|
24290
|
+
const p95 = percentile2(charged, 95);
|
|
24291
|
+
const max = charged.length ? Math.max(...charged) : null;
|
|
24385
24292
|
const fitP95 = percentile2(fits, 95);
|
|
24386
24293
|
const fitMax = fits.length ? Math.max(...fits) : null;
|
|
24387
24294
|
const exceedsStandCap = results.some((r) => r.fitExceedsCap === true);
|
|
24388
24295
|
const cutOffSamples = results.filter((r) => r.error === PROVIDER_TOKEN_LIMIT && r.fitExceedsCap !== true).length;
|
|
24389
24296
|
const noRecommendationReason = exceedsStandCap ? "answer_exceeds_stand_cap" : cutOffSamples > 0 ? "answer_cut_off" : null;
|
|
24390
|
-
const
|
|
24391
|
-
const
|
|
24392
|
-
const
|
|
24393
|
-
const billing = feeBps !== null && usdCentsPerCredit !== null ? { feeBps, usdCentsPerCredit } : null;
|
|
24394
|
-
const costBased = full !== null && costP95 !== null ? ceilingCreditsFor(costP95, full) : null;
|
|
24395
|
-
const recommended = costBased === null || noRecommendationReason !== null ? null : Math.max(costBased, fitP95 ?? 0);
|
|
24396
|
-
const ceiling = recommended === null || costMax === null || full === null ? null : Math.max(ceilingCreditsFor(costMax, full), recommended, fitMax ?? 0);
|
|
24397
|
-
const average = billing !== null ? averageCreditsPerCall(costs, billing) : null;
|
|
24398
|
-
const estimate = ceiling === null || billing === null ? null : Math.min(ceiling, perCallEstimateCreditsFor(costs, billing) ?? 1);
|
|
24297
|
+
const chargedBased = p95 !== null && headroomBps !== null ? recommendEstimateCoins(p95, headroomBps) : null;
|
|
24298
|
+
const recommended = chargedBased === null || noRecommendationReason !== null ? null : Math.max(chargedBased, fitP95 ?? 0);
|
|
24299
|
+
const ceiling = recommended === null || p95 === null || max === null || headroomBps === null ? null : Math.max(recommendPerCallMaxCoins(p95, max, headroomBps), recommended, fitMax ?? 0);
|
|
24399
24300
|
return {
|
|
24400
|
-
costs,
|
|
24401
24301
|
charged,
|
|
24402
|
-
|
|
24403
|
-
|
|
24404
|
-
|
|
24405
|
-
chargedP50: percentile2(charged, 50),
|
|
24406
|
-
chargedP95: percentile2(charged, 95),
|
|
24407
|
-
chargedMax: charged.length ? Math.max(...charged) : null,
|
|
24302
|
+
p50: percentile2(charged, 50),
|
|
24303
|
+
p95,
|
|
24304
|
+
max,
|
|
24408
24305
|
fits,
|
|
24409
24306
|
fitP95,
|
|
24410
24307
|
fitMax,
|
|
24411
|
-
|
|
24412
|
-
|
|
24413
|
-
|
|
24414
|
-
|
|
24415
|
-
recommendedPerCallEstimateCredits: estimate,
|
|
24416
|
-
lengthRaisedCeiling: recommended !== null && costBased !== null && recommended > costBased,
|
|
24308
|
+
chargedBasedEstimateCoins: chargedBased,
|
|
24309
|
+
recommendedEstimateCoins: recommended,
|
|
24310
|
+
recommendedPerCallMaxCoins: ceiling,
|
|
24311
|
+
lengthRaisedPrice: recommended !== null && chargedBased !== null && recommended > chargedBased,
|
|
24417
24312
|
cutOffSamples,
|
|
24418
24313
|
exceedsStandCap,
|
|
24419
|
-
noRecommendationReason
|
|
24420
|
-
missingTerms
|
|
24314
|
+
noRecommendationReason
|
|
24421
24315
|
};
|
|
24422
24316
|
}
|
|
24423
|
-
var LLM_LENGTH_RAISED_LINE = "The
|
|
24424
|
-
var LLM_EXCEEDS_STAND_CAP_LINE = "No
|
|
24425
|
-
function benchCutOffLine(cutOff,
|
|
24426
|
-
return `No
|
|
24317
|
+
var LLM_LENGTH_RAISED_LINE = "The declared price also pays for how long the answer may be: at the charged-based price the answer would be cut off, so declare this one.";
|
|
24318
|
+
var LLM_EXCEEDS_STAND_CAP_LINE = "No price recommended: this answer is longer than one call on this stand may produce. Ask for a shorter answer \u2014 fewer fields, shorter strings, a length the prompt states \u2014 and benchmark again.";
|
|
24319
|
+
function benchCutOffLine(cutOff, maxCoins) {
|
|
24320
|
+
return `No price recommended: ${cutOff} sample${cutOff === 1 ? " was" : "s were"} cut off at this ceiling (--max-coins ${maxCoins}) before the answer was finished, so its real length is unknown. Re-run with a higher --max-coins.`;
|
|
24427
24321
|
}
|
|
24428
24322
|
var ACTIVE_STATUSES = /* @__PURE__ */ new Set(["requires_confirmation", "queued", "dispatching", "awaiting_external"]);
|
|
24429
24323
|
function rowState(row) {
|
|
@@ -24434,22 +24328,7 @@ function rowState(row) {
|
|
|
24434
24328
|
function rowHoldsSlot(row) {
|
|
24435
24329
|
if (typeof row.slotHeld === "boolean") return row.slotHeld;
|
|
24436
24330
|
const state = rowState(row);
|
|
24437
|
-
return state === "active" || state === "awaiting bill" && (
|
|
24438
|
-
}
|
|
24439
|
-
function rowMoney(row) {
|
|
24440
|
-
const credits = row.currency === "credits" || row.currency === void 0 && typeof row.reservedCredits === "number" && typeof row.reservedCoins !== "number";
|
|
24441
|
-
if (credits) {
|
|
24442
|
-
return {
|
|
24443
|
-
unit: "credits",
|
|
24444
|
-
reserved: typeof row.reservedCredits === "number" ? row.reservedCredits : null,
|
|
24445
|
-
charged: typeof row.chargedCredits === "number" ? row.chargedCredits : null
|
|
24446
|
-
};
|
|
24447
|
-
}
|
|
24448
|
-
return {
|
|
24449
|
-
unit: "coin",
|
|
24450
|
-
reserved: typeof row.reservedCoins === "number" ? row.reservedCoins : null,
|
|
24451
|
-
charged: typeof row.chargedCoins === "number" ? row.chargedCoins : null
|
|
24452
|
-
};
|
|
24331
|
+
return state === "active" || state === "awaiting bill" && (row.reservedCoins ?? 0) > 0;
|
|
24453
24332
|
}
|
|
24454
24333
|
function providerRefusalStatus(error) {
|
|
24455
24334
|
const m = /^provider_http_(\d{3})$/.exec(error ?? "");
|
|
@@ -24468,7 +24347,7 @@ function formatAge(iso, now = Date.now()) {
|
|
|
24468
24347
|
}
|
|
24469
24348
|
async function runBench(args) {
|
|
24470
24349
|
const { apiUrl, token, projectId, cwd, opts, log } = args;
|
|
24471
|
-
const
|
|
24350
|
+
const maxCoins = opts.maxCoins;
|
|
24472
24351
|
const samples = opts.samples ?? DEFAULT_BENCH_SAMPLES;
|
|
24473
24352
|
const prompt = opts.benchPrompt.trim();
|
|
24474
24353
|
const auth = { Authorization: `Bearer ${token}`, "Content-Type": "application/json" };
|
|
@@ -24490,7 +24369,7 @@ async function runBench(args) {
|
|
|
24490
24369
|
const lane = await readRuntimeLane(apiUrl, token);
|
|
24491
24370
|
if (lane.state !== "live" || !lane.models || lane.models.length === 0) {
|
|
24492
24371
|
if (lane.state === "off") {
|
|
24493
|
-
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
24372
|
+
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCoins, opts), status: "off", error: null });
|
|
24494
24373
|
else {
|
|
24495
24374
|
log.plain(LLM_LANE_OFF_LINE);
|
|
24496
24375
|
log.dim(" Nothing was spent.");
|
|
@@ -24517,27 +24396,17 @@ async function runBench(args) {
|
|
|
24517
24396
|
);
|
|
24518
24397
|
return;
|
|
24519
24398
|
}
|
|
24520
|
-
const
|
|
24521
|
-
if (walletBefore?.emailVerified === false) {
|
|
24522
|
-
benchFailed(opts, log, LLM_BENCH_UNVERIFIED_LINE);
|
|
24523
|
-
return;
|
|
24524
|
-
}
|
|
24525
|
-
if (walletBefore !== null && walletBefore.spendable < maxCredits) {
|
|
24526
|
-
benchFailed(opts, log, benchBalanceShortLine(walletBefore.spendable, maxCredits));
|
|
24527
|
-
return;
|
|
24528
|
-
}
|
|
24529
|
-
const balanceBefore = walletBefore?.spendable ?? null;
|
|
24399
|
+
const balanceBefore = await readCoinBalance(apiUrl, token);
|
|
24530
24400
|
if (!opts.json) {
|
|
24531
24401
|
log.plain(c.bold("genex llm bench"));
|
|
24532
24402
|
log.dim(` ${apiUrl}`);
|
|
24533
24403
|
log.plain("");
|
|
24534
24404
|
log.plain(` Model ${c.cyan(modelId)}`);
|
|
24535
24405
|
log.plain(` Samples ${samples} real attempt${samples === 1 ? "" : "s"}, ${outputFormat} output`);
|
|
24536
|
-
log.plain(` Approved ${
|
|
24406
|
+
log.plain(` Approved ${maxCoins} coin per attempt \u2014 at worst ${maxCoins * samples} coin for this run`);
|
|
24537
24407
|
log.plain(
|
|
24538
|
-
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore}
|
|
24408
|
+
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore} coin spendable`}`
|
|
24539
24409
|
);
|
|
24540
|
-
log.dim(" Billed as used: each attempt is charged its real cost plus the platform fee, rounded up to a whole credit.");
|
|
24541
24410
|
log.plain("");
|
|
24542
24411
|
}
|
|
24543
24412
|
const base = `${apiUrl}/api/runtime/development/projects/${encodeURIComponent(projectId)}/generations`;
|
|
@@ -24563,7 +24432,7 @@ async function runBench(args) {
|
|
|
24563
24432
|
const res = await call(base, {
|
|
24564
24433
|
method: "POST",
|
|
24565
24434
|
body: JSON.stringify({
|
|
24566
|
-
|
|
24435
|
+
maxCoins,
|
|
24567
24436
|
request: {
|
|
24568
24437
|
idempotencyKey: `bench-${randomUUID2()}`,
|
|
24569
24438
|
modelId,
|
|
@@ -24593,14 +24462,13 @@ async function runBench(args) {
|
|
|
24593
24462
|
if (refusedAt !== null) providerRefused++;
|
|
24594
24463
|
if (!opts.json) {
|
|
24595
24464
|
const length = row.outputTokens !== null ? ` \xB7 ${row.outputTokens}${row.maxOutputTokens !== null ? ` of ${row.maxOutputTokens}` : ""} tokens out` : "";
|
|
24596
|
-
const charged = row.chargedCredits === null ? "\u2014" : String(row.chargedCredits);
|
|
24597
24465
|
log.plain(
|
|
24598
|
-
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${
|
|
24466
|
+
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${String(row.chargedCoins ?? "\u2014").padStart(4)} coin charged \xB7 provider ${usd(row.costUsd)}${length}${row.error ? ` \xB7 ${row.error}` : ""}`
|
|
24599
24467
|
);
|
|
24600
24468
|
if (refusedAt !== null) printProviderRefusal(log, refusedAt, row.providerMessage, row.status === "unknown");
|
|
24601
24469
|
if (row.error === PROVIDER_TOKEN_LIMIT) {
|
|
24602
24470
|
log.dim(
|
|
24603
|
-
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no
|
|
24471
|
+
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no price makes room for it; it is not a sample." : ` The answer was cut off at this ceiling (--max-coins ${maxCoins}) before it was finished \u2014 it was charged, and it is not a sample.`
|
|
24604
24472
|
);
|
|
24605
24473
|
}
|
|
24606
24474
|
}
|
|
@@ -24608,19 +24476,16 @@ async function runBench(args) {
|
|
|
24608
24476
|
} finally {
|
|
24609
24477
|
process.removeListener("SIGINT", onSigint);
|
|
24610
24478
|
}
|
|
24611
|
-
const
|
|
24612
|
-
|
|
24613
|
-
|
|
24614
|
-
|
|
24615
|
-
|
|
24616
|
-
const
|
|
24617
|
-
const settledCount = rec.charged.length;
|
|
24479
|
+
const headroomBps = lane.recommendedDeclaredHeadroomBps;
|
|
24480
|
+
const rec = benchRecommendation(results, headroomBps);
|
|
24481
|
+
const { charged, p50, p95, max } = rec;
|
|
24482
|
+
const settledCount = charged.length;
|
|
24483
|
+
const recommended = rec.recommendedEstimateCoins;
|
|
24484
|
+
const ceiling = rec.recommendedPerCallMaxCoins;
|
|
24618
24485
|
const lengthBlocked = rec.noRecommendationReason !== null;
|
|
24619
|
-
const balanceAfter =
|
|
24620
|
-
const picosOrNull = (v) => v === null ? null : String(v);
|
|
24486
|
+
const balanceAfter = await readCoinBalance(apiUrl, token);
|
|
24621
24487
|
const record = {
|
|
24622
|
-
v:
|
|
24623
|
-
currency: "credits",
|
|
24488
|
+
v: 1,
|
|
24624
24489
|
ranAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
24625
24490
|
apiUrl,
|
|
24626
24491
|
projectId,
|
|
@@ -24629,34 +24494,28 @@ async function runBench(args) {
|
|
|
24629
24494
|
outputFormat,
|
|
24630
24495
|
samples,
|
|
24631
24496
|
completed: settledCount,
|
|
24632
|
-
|
|
24633
|
-
|
|
24634
|
-
|
|
24635
|
-
|
|
24636
|
-
|
|
24637
|
-
|
|
24497
|
+
chargedCoins: charged,
|
|
24498
|
+
p50,
|
|
24499
|
+
p95,
|
|
24500
|
+
max,
|
|
24501
|
+
recommendedDeclaredHeadroomBps: headroomBps,
|
|
24502
|
+
recommendedEstimateCoins: recommended,
|
|
24503
|
+
recommendedPerCallMaxCoins: ceiling,
|
|
24504
|
+
maxCoinsPerSample: maxCoins,
|
|
24505
|
+
fitCoins: rec.fits,
|
|
24638
24506
|
fitP95: rec.fitP95,
|
|
24639
|
-
|
|
24640
|
-
|
|
24641
|
-
feeBps: terms.feeBps,
|
|
24642
|
-
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
24643
|
-
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
24644
|
-
recommendedMaxCredits: rec.recommendedMaxCredits,
|
|
24645
|
-
recommendedPerCallMaxCredits: rec.recommendedPerCallMaxCredits,
|
|
24646
|
-
averageCreditsPerCall: rec.averageCreditsPerCall,
|
|
24647
|
-
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
24648
|
-
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
24507
|
+
chargedBasedEstimateCoins: rec.chargedBasedEstimateCoins,
|
|
24508
|
+
lengthRaisedPrice: rec.lengthRaisedPrice,
|
|
24649
24509
|
cutOffSamples: rec.cutOffSamples,
|
|
24650
24510
|
noRecommendationReason: rec.noRecommendationReason
|
|
24651
24511
|
};
|
|
24652
24512
|
const savedTo = await saveBench(cwd, record);
|
|
24653
24513
|
if (opts.json) {
|
|
24654
|
-
const ok =
|
|
24514
|
+
const ok = charged.length > 0 && !lengthBlocked;
|
|
24655
24515
|
writeJsonLine({
|
|
24656
24516
|
command: "llm bench",
|
|
24657
24517
|
status: ok ? "ok" : "failed",
|
|
24658
24518
|
error: ok ? null : rec.noRecommendationReason ?? "no_sample_settled",
|
|
24659
|
-
currency: "credits",
|
|
24660
24519
|
apiUrl,
|
|
24661
24520
|
projectId,
|
|
24662
24521
|
modelId,
|
|
@@ -24665,54 +24524,35 @@ async function runBench(args) {
|
|
|
24665
24524
|
schemaPath: opts.schemaPath ?? null,
|
|
24666
24525
|
samples,
|
|
24667
24526
|
completed: settledCount,
|
|
24668
|
-
|
|
24669
|
-
// Spendable credits before and after the run; null when unreadable.
|
|
24527
|
+
maxCoinsPerSample: maxCoins,
|
|
24670
24528
|
balanceBefore,
|
|
24671
24529
|
balanceAfter,
|
|
24672
24530
|
results,
|
|
24673
24531
|
// The create-time refusal that ended the run, or null. Beside the
|
|
24674
24532
|
// samples, never among them: no attempt existed and nothing was spent.
|
|
24675
24533
|
refusal: refusal2,
|
|
24676
|
-
|
|
24677
|
-
//
|
|
24678
|
-
costUsdPicos: record.cost,
|
|
24679
|
-
costUsd: {
|
|
24680
|
-
p50: rec.costP50 === null ? null : Number(rec.costP50) / 1e12,
|
|
24681
|
-
p95: rec.costP95 === null ? null : Number(rec.costP95) / 1e12,
|
|
24682
|
-
max: rec.costMax === null ? null : Number(rec.costMax) / 1e12
|
|
24683
|
-
},
|
|
24684
|
-
chargedCredits: record.charged,
|
|
24685
|
-
// The answer's LENGTH, priced by the server per sample (`fitCredits`,
|
|
24534
|
+
chargedCoins: { p50, p95, max },
|
|
24535
|
+
// The answer's LENGTH, priced by the server per sample (`fitCoins`,
|
|
24686
24536
|
// headroom already on the tokens), over the settled samples.
|
|
24687
|
-
|
|
24688
|
-
|
|
24689
|
-
|
|
24690
|
-
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
24691
|
-
missingTerms: rec.missingTerms,
|
|
24692
|
-
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
24693
|
-
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
24537
|
+
fitCoins: { p95: rec.fitP95, max: rec.fitMax },
|
|
24538
|
+
chargedBasedEstimateCoins: rec.chargedBasedEstimateCoins,
|
|
24539
|
+
lengthRaisedPrice: rec.lengthRaisedPrice,
|
|
24694
24540
|
cutOffSamples: rec.cutOffSamples,
|
|
24695
24541
|
exceedsStandCap: rec.exceedsStandCap,
|
|
24696
24542
|
noRecommendationReason: rec.noRecommendationReason,
|
|
24697
|
-
|
|
24698
|
-
|
|
24699
|
-
|
|
24700
|
-
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
24543
|
+
recommendedDeclaredHeadroomBps: headroomBps,
|
|
24544
|
+
recommendedEstimateCoins: recommended,
|
|
24545
|
+
recommendedPerCallMaxCoins: ceiling,
|
|
24701
24546
|
savedTo
|
|
24702
24547
|
});
|
|
24703
24548
|
if (!ok) process.exitCode = 1;
|
|
24704
24549
|
return;
|
|
24705
24550
|
}
|
|
24706
24551
|
log.plain("");
|
|
24707
|
-
const printMeasured = () => {
|
|
24708
|
-
log.plain(c.bold(" Real cost per call"));
|
|
24709
|
-
log.plain(` p50 ${usdFromPicos(rec.costP50)} p95 ${usdFromPicos(rec.costP95)} max ${usdFromPicos(rec.costMax)} (${settledCount} of ${samples} settled)`);
|
|
24710
|
-
log.plain(c.bold(" Charged credits"));
|
|
24711
|
-
log.plain(` p50 ${rec.chargedP50} p95 ${rec.chargedP95} max ${rec.chargedMax} (whole credits, rounded up per call)`);
|
|
24712
|
-
};
|
|
24713
24552
|
if (lengthBlocked) {
|
|
24714
|
-
if (
|
|
24715
|
-
|
|
24553
|
+
if (charged.length > 0) {
|
|
24554
|
+
log.plain(c.bold(" Charged coins"));
|
|
24555
|
+
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
24716
24556
|
log.plain("");
|
|
24717
24557
|
}
|
|
24718
24558
|
printRecommendation(log, record);
|
|
@@ -24720,16 +24560,17 @@ async function runBench(args) {
|
|
|
24720
24560
|
process.exitCode = 1;
|
|
24721
24561
|
return;
|
|
24722
24562
|
}
|
|
24723
|
-
if (
|
|
24563
|
+
if (charged.length === 0) {
|
|
24724
24564
|
log.error(
|
|
24725
|
-
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to
|
|
24565
|
+
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to price from." : " No sample settled, so there is nothing to price from."
|
|
24726
24566
|
);
|
|
24727
24567
|
process.exitCode = 1;
|
|
24728
24568
|
return;
|
|
24729
24569
|
}
|
|
24730
|
-
|
|
24570
|
+
log.plain(c.bold(" Charged coins"));
|
|
24571
|
+
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
24731
24572
|
if (balanceBefore !== null && balanceAfter !== null) {
|
|
24732
|
-
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter}
|
|
24573
|
+
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter} coin spendable`);
|
|
24733
24574
|
}
|
|
24734
24575
|
log.plain("");
|
|
24735
24576
|
printRecommendation(log, record);
|
|
@@ -24741,58 +24582,43 @@ function printRecommendation(log, record) {
|
|
|
24741
24582
|
return;
|
|
24742
24583
|
}
|
|
24743
24584
|
if (record.noRecommendationReason === "answer_cut_off") {
|
|
24744
|
-
log.warn(` ${benchCutOffLine(record.cutOffSamples
|
|
24585
|
+
log.warn(` ${benchCutOffLine(record.cutOffSamples ?? 1, record.maxCoinsPerSample ?? 0)}`);
|
|
24745
24586
|
return;
|
|
24746
24587
|
}
|
|
24747
|
-
|
|
24748
|
-
|
|
24749
|
-
log.warn(" No ceiling recommended.");
|
|
24588
|
+
if (record.recommendedEstimateCoins === null || record.p95 === null) {
|
|
24589
|
+
log.warn(" No price recommended.");
|
|
24750
24590
|
log.dim(
|
|
24751
|
-
record.
|
|
24591
|
+
record.recommendedDeclaredHeadroomBps === null ? " This stand served no recommended headroom, and the multiplier is the server's to set \u2014" : " No sample settled, so there is no p95 to build a price on \u2014"
|
|
24752
24592
|
);
|
|
24753
24593
|
log.dim(" declaring a number from this run would be a guess dressed as a measurement.");
|
|
24754
|
-
printAverage(log, record, attempts);
|
|
24755
24594
|
return;
|
|
24756
24595
|
}
|
|
24757
|
-
log.plain(c.bold(` Declare
|
|
24758
|
-
if (record.
|
|
24596
|
+
log.plain(c.bold(` Declare estimateCoins: ${record.recommendedEstimateCoins}`));
|
|
24597
|
+
if (record.lengthRaisedPrice) {
|
|
24759
24598
|
log.plain(` ${LLM_LENGTH_RAISED_LINE}`);
|
|
24760
24599
|
log.dim(
|
|
24761
|
-
` = the smallest
|
|
24600
|
+
` = the smallest price whose answer allowance holds the answer plus this stand's headroom (p95 ${record.fitP95 ?? "\u2014"}), above the`
|
|
24762
24601
|
);
|
|
24763
24602
|
log.dim(
|
|
24764
|
-
`
|
|
24603
|
+
` charged-based ${record.chargedBasedEstimateCoins ?? "\u2014"} (p95 of charged coins ${record.p95} plus ${record.recommendedDeclaredHeadroomBps} bps), over ${record.completed} settled attempt${record.completed === 1 ? "" : "s"} on ${record.modelId}.`
|
|
24765
24604
|
);
|
|
24766
24605
|
} else {
|
|
24767
24606
|
log.dim(
|
|
24768
|
-
` = p95
|
|
24607
|
+
` = p95 of charged coins (${record.p95}) plus this stand's recommended headroom (${record.recommendedDeclaredHeadroomBps} bps),`
|
|
24769
24608
|
);
|
|
24770
|
-
log.dim(` measured over ${
|
|
24609
|
+
log.dim(` measured over ${record.completed} settled attempt${record.completed === 1 ? "" : "s"} on ${record.modelId}.`);
|
|
24771
24610
|
}
|
|
24772
|
-
log.dim(" That number is a
|
|
24773
|
-
|
|
24774
|
-
|
|
24775
|
-
log.plain(c.bold(` Grant perCallMaxCredits: ${record.recommendedPerCallMaxCredits}`));
|
|
24611
|
+
log.dim(" That number is a PRICE: a started attempt is charged in full, including one that fails.");
|
|
24612
|
+
if (record.recommendedPerCallMaxCoins != null) {
|
|
24613
|
+
log.plain(c.bold(` Grant perCallMaxCoins: ${record.recommendedPerCallMaxCoins}`));
|
|
24776
24614
|
log.dim(
|
|
24777
|
-
record.
|
|
24615
|
+
record.lengthRaisedPrice ? ` = the same headroom over the worst sample (${record.max}) or the longest answer's price, whichever is larger, and never below the price above it.` : ` = the same headroom over the worst sample (${record.max}), and never below the price above it.`
|
|
24778
24616
|
);
|
|
24779
|
-
|
|
24780
|
-
printAverage(log, record, attempts);
|
|
24781
|
-
}
|
|
24782
|
-
function printAverage(log, record, attempts) {
|
|
24783
|
-
if (record.averageCreditsPerCall === null) return;
|
|
24784
|
-
log.plain(` Per-call estimate: about ${formatCredits(record.averageCreditsPerCall)} credits per call`);
|
|
24785
|
-
log.dim(` = the average real cost with the platform fee, no headroom, over ${attempts}.`);
|
|
24786
|
-
if (record.recommendedPerCallEstimateCredits !== null) {
|
|
24787
|
-
log.plain(c.bold(` Grant perCallEstimateCredits: ${record.recommendedPerCallEstimateCredits}`));
|
|
24788
|
-
log.dim(` = that average rounded up to a whole credit. disclosure.estimatedCreditsPerPeriod = your calls per period \xD7 ${formatCredits(record.averageCreditsPerCall)}, rounded up.`);
|
|
24617
|
+
log.dim(" The ceiling is the price's room to be wrong; declaring one under the price is refused.");
|
|
24789
24618
|
}
|
|
24790
24619
|
}
|
|
24791
24620
|
async function explainBenchRefusal(res, call, base, log, opts, index, settledSoFar) {
|
|
24792
|
-
if (printedStructuredError(res)) {
|
|
24793
|
-
const printed = await res.json().catch(() => ({}));
|
|
24794
|
-
return { stop: true, refusal: { status: res.status, error: typeof printed.error === "string" ? printed.error : null, slotsHeld: null, slotLimit: null } };
|
|
24795
|
-
}
|
|
24621
|
+
if (printedStructuredError(res)) return { stop: true, refusal: { status: res.status, error: null, slotsHeld: null, slotLimit: null } };
|
|
24796
24622
|
const body = await res.json().catch(() => ({}));
|
|
24797
24623
|
const refusal2 = { status: res.status, error: body.error ?? null, slotsHeld: null, slotLimit: null };
|
|
24798
24624
|
if (res.status === 429 && body.error === "generation_limit") {
|
|
@@ -24802,10 +24628,6 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24802
24628
|
if (!opts.json) log.error(` Sample ${index + 1} ${benchSlotsHeldSentence(counts)}`);
|
|
24803
24629
|
return { stop: true, refusal: refusal2 };
|
|
24804
24630
|
}
|
|
24805
|
-
if (res.status === 403 && body.error === "credits_unverified") {
|
|
24806
|
-
if (!opts.json) log.error(` ${LLM_BENCH_UNVERIFIED_LINE}`);
|
|
24807
|
-
return { stop: true, refusal: refusal2 };
|
|
24808
|
-
}
|
|
24809
24631
|
if (opts.json) return { stop: res.status !== 429, refusal: refusal2 };
|
|
24810
24632
|
if (res.status === 404 && body.error === "not_found") {
|
|
24811
24633
|
log.error(` ${LLM_LANE_OFF_LINE}`);
|
|
@@ -24821,7 +24643,7 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24821
24643
|
return { stop: true, refusal: refusal2 };
|
|
24822
24644
|
}
|
|
24823
24645
|
if (res.status === 402) {
|
|
24824
|
-
log.error(" Not enough
|
|
24646
|
+
log.error(" Not enough coin to start the attempt.");
|
|
24825
24647
|
return { stop: true, refusal: refusal2 };
|
|
24826
24648
|
}
|
|
24827
24649
|
log.error(
|
|
@@ -24830,7 +24652,7 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24830
24652
|
return { stop: res.status >= 500 || res.status === 401 || res.status === 403, refusal: refusal2 };
|
|
24831
24653
|
}
|
|
24832
24654
|
function printProviderRefusal(log, status2, message, awaitingBill = false) {
|
|
24833
|
-
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so
|
|
24655
|
+
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so this call's coin stays held until it resolves (see \`genex llm status\`); it is not a sample.`);
|
|
24834
24656
|
else log.dim(" The provider refused this call at its door \u2014 no inference ran, it cost nothing, and it is not a sample.");
|
|
24835
24657
|
if (message) log.dim(` Provider said: ${message}`);
|
|
24836
24658
|
if (status2 === 401 || status2 === 403) {
|
|
@@ -24856,6 +24678,19 @@ async function pollSettled(call, url, timeoutSec) {
|
|
|
24856
24678
|
}
|
|
24857
24679
|
return last;
|
|
24858
24680
|
}
|
|
24681
|
+
async function readCoinBalance(apiUrl, token) {
|
|
24682
|
+
try {
|
|
24683
|
+
const res = await apiFetch(`${apiUrl}/api/coin/balance`, {
|
|
24684
|
+
headers: { Authorization: `Bearer ${token}` },
|
|
24685
|
+
signal: AbortSignal.timeout(6e3)
|
|
24686
|
+
});
|
|
24687
|
+
if (!res.ok) return null;
|
|
24688
|
+
const body = await res.json().catch(() => null);
|
|
24689
|
+
return typeof body?.spendable === "number" ? body.spendable : null;
|
|
24690
|
+
} catch {
|
|
24691
|
+
return null;
|
|
24692
|
+
}
|
|
24693
|
+
}
|
|
24859
24694
|
async function saveBench(cwd, record) {
|
|
24860
24695
|
try {
|
|
24861
24696
|
const file = path36.join(cwd, LLM_BENCH_FILE);
|
|
@@ -24871,10 +24706,9 @@ function benchFailed(opts, log, message) {
|
|
|
24871
24706
|
else log.error(message);
|
|
24872
24707
|
process.exitCode = 1;
|
|
24873
24708
|
}
|
|
24874
|
-
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
24709
|
+
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCoins, opts) {
|
|
24875
24710
|
return {
|
|
24876
24711
|
command: "llm bench",
|
|
24877
|
-
currency: "credits",
|
|
24878
24712
|
apiUrl,
|
|
24879
24713
|
projectId,
|
|
24880
24714
|
modelId: opts.modelId ?? null,
|
|
@@ -24883,28 +24717,21 @@ function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCre
|
|
|
24883
24717
|
schemaPath: opts.schemaPath ?? null,
|
|
24884
24718
|
samples,
|
|
24885
24719
|
completed: 0,
|
|
24886
|
-
|
|
24720
|
+
maxCoinsPerSample: maxCoins,
|
|
24887
24721
|
balanceBefore: null,
|
|
24888
24722
|
balanceAfter: null,
|
|
24889
24723
|
results: [],
|
|
24890
24724
|
refusal: null,
|
|
24891
|
-
|
|
24892
|
-
|
|
24893
|
-
|
|
24894
|
-
|
|
24895
|
-
recommendedDeclaredHeadroomBps: null,
|
|
24896
|
-
feeBps: null,
|
|
24897
|
-
usdCentsPerCredit: null,
|
|
24898
|
-
missingTerms: null,
|
|
24899
|
-
costBasedMaxCredits: null,
|
|
24900
|
-
lengthRaisedCeiling: false,
|
|
24725
|
+
chargedCoins: { p50: null, p95: null, max: null },
|
|
24726
|
+
fitCoins: { p95: null, max: null },
|
|
24727
|
+
chargedBasedEstimateCoins: null,
|
|
24728
|
+
lengthRaisedPrice: false,
|
|
24901
24729
|
cutOffSamples: 0,
|
|
24902
24730
|
exceedsStandCap: false,
|
|
24903
24731
|
noRecommendationReason: null,
|
|
24904
|
-
|
|
24905
|
-
|
|
24906
|
-
|
|
24907
|
-
recommendedPerCallEstimateCredits: null,
|
|
24732
|
+
recommendedDeclaredHeadroomBps: null,
|
|
24733
|
+
recommendedEstimateCoins: null,
|
|
24734
|
+
recommendedPerCallMaxCoins: null,
|
|
24908
24735
|
savedTo: null
|
|
24909
24736
|
};
|
|
24910
24737
|
}
|
|
@@ -24967,8 +24794,7 @@ async function reportStatus(args) {
|
|
|
24967
24794
|
for (const row of rows) {
|
|
24968
24795
|
const state = rowState(row);
|
|
24969
24796
|
const mark = state === "active" ? c.cyan("\u25CF") : state === "awaiting bill" ? c.yellow("\u25D0") : c.green("\u25CB");
|
|
24970
|
-
const
|
|
24971
|
-
const reserved = money2.reserved === null ? "reserved \u2014" : `${money2.reserved} ${money2.unit} reserved`;
|
|
24797
|
+
const reserved = typeof row.reservedCoins === "number" ? `${row.reservedCoins} coin reserved` : "reserved \u2014";
|
|
24972
24798
|
log.plain(
|
|
24973
24799
|
` ${mark} ${row.id ?? "?"} ${row.modelId ?? "?"} ${state.padEnd(13)} ${formatAge(row.createdAt, now).padStart(4)} old ${reserved}${row.error ? ` ${row.error}` : ""}${rowHoldsSlot(row) ? "" : c.dim(" (no slot)")}`
|
|
24974
24800
|
);
|
|
@@ -25040,11 +24866,10 @@ async function cancelCall(args) {
|
|
|
25040
24866
|
if (!opts.json) {
|
|
25041
24867
|
log.warn(` ${row.id} (${row.status ?? "?"}) \u2014 ${sentence}.`);
|
|
25042
24868
|
if (state === "awaiting bill") {
|
|
25043
|
-
log.dim(` Nothing to cancel:
|
|
24869
|
+
log.dim(` Nothing to cancel: its coin releases when the bill resolves, and it stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`);
|
|
25044
24870
|
if (row.providerMessage?.trim()) log.dim(` Provider said: ${row.providerMessage.trim()}`);
|
|
25045
24871
|
} else {
|
|
25046
|
-
|
|
25047
|
-
log.dim(` ${money2.charged === null ? "charge unknown" : `${money2.charged} ${money2.unit} charged`}${row.error ? ` \xB7 ${row.error}` : ""}.`);
|
|
24872
|
+
log.dim(` ${typeof row.chargedCoins === "number" ? `${row.chargedCoins} coin charged` : "charge unknown"}${row.error ? ` \xB7 ${row.error}` : ""}.`);
|
|
25048
24873
|
}
|
|
25049
24874
|
}
|
|
25050
24875
|
return done("ok", null, state === "awaiting bill" ? "awaiting_bill" : "already_final", row);
|
|
@@ -25066,126 +24891,78 @@ async function cancelCall(args) {
|
|
|
25066
24891
|
const after = await res.json().catch(() => null) ?? row;
|
|
25067
24892
|
if (!opts.json) {
|
|
25068
24893
|
log.success(` Cancelled ${row.id} (was ${row.status ?? "active"}).`);
|
|
25069
|
-
log.dim(`
|
|
24894
|
+
log.dim(` Its coin releases once the bill settles; ${c.cyan("genex llm status")} shows it until then.`);
|
|
25070
24895
|
}
|
|
25071
24896
|
return done("ok", null, "cancelled", after);
|
|
25072
24897
|
}
|
|
25073
|
-
async function
|
|
25074
|
-
let
|
|
24898
|
+
async function reportSavedPrice(cwd, opts, log) {
|
|
24899
|
+
let record = null;
|
|
25075
24900
|
try {
|
|
25076
|
-
|
|
24901
|
+
record = JSON.parse(await fs35.readFile(path36.join(cwd, LLM_BENCH_FILE), "utf8"));
|
|
25077
24902
|
} catch {
|
|
25078
|
-
|
|
24903
|
+
record = null;
|
|
25079
24904
|
}
|
|
25080
|
-
if (!
|
|
25081
|
-
const v = parsed;
|
|
25082
|
-
if (v.v === 2 && v.currency === "credits" && v.cost && v.charged) return { kind: "credits", record: parsed };
|
|
25083
|
-
return { kind: "coins", record: parsed };
|
|
25084
|
-
}
|
|
25085
|
-
var LLM_PRICE_COIN_TERMS_LINE = "This bench was measured under the old COIN terms, when a game declared a fixed price per call. In-game calls are now billed in the player's credits, as used, and a game declares a per-call CEILING (maxCredits) instead \u2014 a coin price is not that number. Re-run the bench for the figures to declare.";
|
|
25086
|
-
async function reportSavedPrice(cwd, opts, log) {
|
|
25087
|
-
const saved = await readSavedBench(cwd);
|
|
25088
|
-
const rerun = 'genex llm bench "<prompt>" --max-credits <n> --user-approved';
|
|
25089
|
-
const creditKeys = (record2) => ({
|
|
25090
|
-
costUsdPicos: record2?.cost ?? { p50: null, p95: null, max: null },
|
|
25091
|
-
chargedCredits: record2?.charged ?? { p50: null, p95: null, max: null },
|
|
25092
|
-
fitP95: record2?.fitP95 ?? null,
|
|
25093
|
-
fitMax: record2?.fitMax ?? null,
|
|
25094
|
-
recommendedDeclaredHeadroomBps: record2?.recommendedDeclaredHeadroomBps ?? null,
|
|
25095
|
-
feeBps: record2?.feeBps ?? null,
|
|
25096
|
-
usdCentsPerCredit: record2?.usdCentsPerCredit ?? null,
|
|
25097
|
-
costBasedMaxCredits: record2?.costBasedMaxCredits ?? null,
|
|
25098
|
-
lengthRaisedCeiling: record2?.lengthRaisedCeiling ?? false,
|
|
25099
|
-
cutOffSamples: record2?.cutOffSamples ?? 0,
|
|
25100
|
-
noRecommendationReason: record2?.noRecommendationReason ?? null,
|
|
25101
|
-
recommendedMaxCredits: record2?.recommendedMaxCredits ?? null,
|
|
25102
|
-
recommendedPerCallMaxCredits: record2?.recommendedPerCallMaxCredits ?? null,
|
|
25103
|
-
averageCreditsPerCall: record2?.averageCreditsPerCall ?? null,
|
|
25104
|
-
recommendedPerCallEstimateCredits: record2?.recommendedPerCallEstimateCredits ?? null
|
|
25105
|
-
});
|
|
25106
|
-
if (!saved) {
|
|
24905
|
+
if (!record) {
|
|
25107
24906
|
if (opts.json) {
|
|
25108
24907
|
writeJsonLine({
|
|
25109
24908
|
command: "llm price",
|
|
25110
24909
|
status: "failed",
|
|
25111
24910
|
error: "no_bench",
|
|
25112
|
-
currency: null,
|
|
25113
24911
|
ranAt: null,
|
|
25114
24912
|
modelId: null,
|
|
25115
24913
|
samples: null,
|
|
25116
24914
|
completed: null,
|
|
25117
|
-
|
|
25118
|
-
|
|
24915
|
+
p50: null,
|
|
24916
|
+
p95: null,
|
|
24917
|
+
max: null,
|
|
24918
|
+
fitP95: null,
|
|
24919
|
+
chargedBasedEstimateCoins: null,
|
|
24920
|
+
lengthRaisedPrice: false,
|
|
24921
|
+
cutOffSamples: 0,
|
|
24922
|
+
noRecommendationReason: null,
|
|
24923
|
+
recommendedDeclaredHeadroomBps: null,
|
|
24924
|
+
recommendedEstimateCoins: null,
|
|
24925
|
+
recommendedPerCallMaxCoins: null,
|
|
25119
24926
|
savedTo: null
|
|
25120
24927
|
});
|
|
25121
24928
|
} else {
|
|
25122
24929
|
log.error("Nothing has been benchmarked in this folder yet.");
|
|
25123
|
-
log.dim(` Run ${c.cyan(
|
|
25124
|
-
}
|
|
25125
|
-
process.exitCode = 1;
|
|
25126
|
-
return;
|
|
25127
|
-
}
|
|
25128
|
-
if (saved.kind === "coins") {
|
|
25129
|
-
const old = saved.record;
|
|
25130
|
-
const legacy = {
|
|
25131
|
-
p50: old.p50 ?? null,
|
|
25132
|
-
p95: old.p95 ?? null,
|
|
25133
|
-
max: old.max ?? null,
|
|
25134
|
-
recommendedEstimateCoins: old.recommendedEstimateCoins ?? null,
|
|
25135
|
-
recommendedPerCallMaxCoins: old.recommendedPerCallMaxCoins ?? null,
|
|
25136
|
-
maxCoinsPerSample: old.maxCoinsPerSample ?? null,
|
|
25137
|
-
noRecommendationReason: old.noRecommendationReason ?? null
|
|
25138
|
-
};
|
|
25139
|
-
if (opts.json) {
|
|
25140
|
-
writeJsonLine({
|
|
25141
|
-
command: "llm price",
|
|
25142
|
-
status: "failed",
|
|
25143
|
-
error: "coin_terms",
|
|
25144
|
-
currency: "coins",
|
|
25145
|
-
ranAt: old.ranAt ?? null,
|
|
25146
|
-
modelId: old.modelId ?? null,
|
|
25147
|
-
samples: old.samples ?? null,
|
|
25148
|
-
completed: old.completed ?? null,
|
|
25149
|
-
...creditKeys(null),
|
|
25150
|
-
legacyCoinTerms: legacy,
|
|
25151
|
-
savedTo: LLM_BENCH_FILE
|
|
25152
|
-
});
|
|
25153
|
-
} else {
|
|
25154
|
-
log.plain(c.bold("genex llm price"));
|
|
25155
|
-
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${old.ranAt ?? "at an unknown time"}${old.modelId ? ` on ${old.modelId}` : ""}`);
|
|
25156
|
-
log.plain("");
|
|
25157
|
-
log.warn(` ${LLM_PRICE_COIN_TERMS_LINE}`);
|
|
25158
|
-
const measured = legacy.p95 === null ? "no settled sample" : `charged coins p50 ${legacy.p50} \xB7 p95 ${legacy.p95} \xB7 max ${legacy.max}`;
|
|
25159
|
-
const recommended = legacy.recommendedEstimateCoins === null ? "no price recommended" : `it recommended estimateCoins ${legacy.recommendedEstimateCoins}${legacy.recommendedPerCallMaxCoins != null ? ` and perCallMaxCoins ${legacy.recommendedPerCallMaxCoins}` : ""}`;
|
|
25160
|
-
log.dim(` What it measured then: ${measured}; ${recommended}.`);
|
|
25161
|
-
log.dim(` Re-run: ${c.cyan(rerun)}`);
|
|
24930
|
+
log.dim(` Run ${c.cyan('genex llm bench "<prompt>" --max-coins <n> --user-approved')} first.`);
|
|
25162
24931
|
}
|
|
25163
24932
|
process.exitCode = 1;
|
|
25164
24933
|
return;
|
|
25165
24934
|
}
|
|
25166
|
-
const record = saved.record;
|
|
25167
24935
|
if (opts.json) {
|
|
25168
24936
|
writeJsonLine({
|
|
25169
24937
|
command: "llm price",
|
|
25170
|
-
status: record.
|
|
25171
|
-
error: record.
|
|
25172
|
-
currency: "credits",
|
|
24938
|
+
status: record.recommendedEstimateCoins === null ? "failed" : "ok",
|
|
24939
|
+
error: record.recommendedEstimateCoins === null ? record.noRecommendationReason ?? "no_recommendation" : null,
|
|
25173
24940
|
ranAt: record.ranAt ?? null,
|
|
25174
24941
|
modelId: record.modelId ?? null,
|
|
25175
24942
|
samples: record.samples ?? null,
|
|
25176
24943
|
completed: record.completed ?? null,
|
|
25177
|
-
|
|
25178
|
-
|
|
24944
|
+
p50: record.p50 ?? null,
|
|
24945
|
+
p95: record.p95 ?? null,
|
|
24946
|
+
max: record.max ?? null,
|
|
24947
|
+
// Absent from a file an older CLI wrote; null / false / 0 then, never missing.
|
|
24948
|
+
fitP95: record.fitP95 ?? null,
|
|
24949
|
+
chargedBasedEstimateCoins: record.chargedBasedEstimateCoins ?? null,
|
|
24950
|
+
lengthRaisedPrice: record.lengthRaisedPrice ?? false,
|
|
24951
|
+
cutOffSamples: record.cutOffSamples ?? 0,
|
|
24952
|
+
noRecommendationReason: record.noRecommendationReason ?? null,
|
|
24953
|
+
recommendedDeclaredHeadroomBps: record.recommendedDeclaredHeadroomBps ?? null,
|
|
24954
|
+
recommendedEstimateCoins: record.recommendedEstimateCoins ?? null,
|
|
24955
|
+
recommendedPerCallMaxCoins: record.recommendedPerCallMaxCoins ?? null,
|
|
25179
24956
|
savedTo: LLM_BENCH_FILE
|
|
25180
24957
|
});
|
|
25181
|
-
if (record.
|
|
24958
|
+
if (record.recommendedEstimateCoins === null) process.exitCode = 1;
|
|
25182
24959
|
return;
|
|
25183
24960
|
}
|
|
25184
24961
|
log.plain(c.bold("genex llm price"));
|
|
25185
24962
|
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${record.ranAt}`);
|
|
25186
24963
|
log.plain("");
|
|
25187
24964
|
printRecommendation(log, record);
|
|
25188
|
-
if (record.
|
|
24965
|
+
if (record.recommendedEstimateCoins === null) process.exitCode = 1;
|
|
25189
24966
|
}
|
|
25190
24967
|
function writeJsonLine(value) {
|
|
25191
24968
|
process.stdout.write(`${JSON.stringify(value)}
|
|
@@ -25445,7 +25222,7 @@ function runtimeRow(state, lane, toolsOnly = false) {
|
|
|
25445
25222
|
return {
|
|
25446
25223
|
label: "Runtime",
|
|
25447
25224
|
value: `in-game LLM lane live \xB7 ${n} model${n === 1 ? "" : "s"}${keyRefused ? ` \xB7 ${c.yellow("provider key refused")}` : ""}`,
|
|
25448
|
-
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : '
|
|
25225
|
+
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : 'Price a call before a game declares one: `npx genex llm bench "<prompt>" --max-coins <n> --user-approved`.'
|
|
25449
25226
|
};
|
|
25450
25227
|
}
|
|
25451
25228
|
case "off":
|
|
@@ -27401,16 +27178,16 @@ ${c.bold("Usage")}
|
|
|
27401
27178
|
status | cancel. "models" says whether this
|
|
27402
27179
|
stand serves the runtime LLM lane at all, and
|
|
27403
27180
|
at what provider rates. "bench" runs the real
|
|
27404
|
-
model on YOUR OWN
|
|
27405
|
-
attempt
|
|
27181
|
+
model on YOUR OWN coin and reports what each
|
|
27182
|
+
attempt was charged; it needs --max-coins <n>
|
|
27406
27183
|
--user-approved, like any spend the player has
|
|
27407
27184
|
to agree to. "price" reprints the last run's
|
|
27408
|
-
recommended
|
|
27185
|
+
recommended estimateCoins. "status" lists this
|
|
27409
27186
|
project's open calls and the slots they hold;
|
|
27410
27187
|
"cancel <id>" stops an active one. Declare a
|
|
27411
|
-
game's
|
|
27412
|
-
|
|
27413
|
-
|
|
27188
|
+
game's price from a bench, never from a guess:
|
|
27189
|
+
a started attempt is charged in full, including
|
|
27190
|
+
one that fails.
|
|
27414
27191
|
genex player <sub> [options] Answer the in-game model requests you approve, on your
|
|
27415
27192
|
OWN Claude or ChatGPT subscription: install | run |
|
|
27416
27193
|
status | stop | uninstall. "install" signs this machine
|
|
@@ -27865,7 +27642,7 @@ ${c.bold("Examples")}
|
|
|
27865
27642
|
genex shop remove sku_123
|
|
27866
27643
|
genex llm models
|
|
27867
27644
|
genex llm models --all
|
|
27868
|
-
genex llm bench "Reply with one short taunt." --samples 5 --max-
|
|
27645
|
+
genex llm bench "Reply with one short taunt." --samples 5 --max-coins 20 --user-approved
|
|
27869
27646
|
genex llm price
|
|
27870
27647
|
genex llm status
|
|
27871
27648
|
genex llm cancel <id>
|
|
@@ -28102,24 +27879,22 @@ ${c.bold("How the allowance works")}
|
|
|
28102
27879
|
|
|
28103
27880
|
Every number printed is live from your account; prices can change without a deploy.
|
|
28104
27881
|
`,
|
|
28105
|
-
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what
|
|
27882
|
+
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what should the game declare?
|
|
28106
27883
|
|
|
28107
27884
|
${c.bold("Usage")}
|
|
28108
27885
|
genex llm models [--all] [--json] Which models this stand serves, their provider rates per
|
|
28109
|
-
million tokens, the
|
|
28110
|
-
|
|
28111
|
-
|
|
28112
|
-
|
|
28113
|
-
|
|
28114
|
-
|
|
28115
|
-
|
|
28116
|
-
|
|
28117
|
-
|
|
28118
|
-
|
|
28119
|
-
|
|
28120
|
-
|
|
28121
|
-
allowance does not count. Needs a folder linked to a
|
|
28122
|
-
game you own, and a balance that covers one attempt.
|
|
27886
|
+
million tokens, and the headroom it recommends over a
|
|
27887
|
+
benchmark. The catalog is synced from OpenRouter, so the
|
|
27888
|
+
FEATURED short list is printed by default and --all
|
|
27889
|
+
prints every row (--json always carries every row). Off
|
|
27890
|
+
on this stand: it says so and exits 0 \u2014 build the feature
|
|
27891
|
+
without a model rather than promising a 404.
|
|
27892
|
+
genex llm bench "<prompt>" --max-coins <n> --user-approved [options]
|
|
27893
|
+
Run the real model <n> times through the development
|
|
27894
|
+
lane and report what each attempt was CHARGED. Refused
|
|
27895
|
+
without both flags, before any network call: this spends
|
|
27896
|
+
YOUR OWN COIN, a wallet the build's asset allowance
|
|
27897
|
+
cannot see. Needs a folder linked to a game you own.
|
|
28123
27898
|
genex llm price [--json] The recommendation from the last bench in this folder.
|
|
28124
27899
|
Reads a file; spends nothing.
|
|
28125
27900
|
genex llm status [--json] This project's open development calls: active, or
|
|
@@ -28136,32 +27911,29 @@ ${c.bold("Bench options")}
|
|
|
28136
27911
|
--samples <n> How many real attempts (default ${DEFAULT_BENCH_SAMPLES}). More samples, better p95.
|
|
28137
27912
|
--schema <file> A JSON file holding the output schema; implies JSON output.
|
|
28138
27913
|
--json-output Ask for JSON without a schema. --text Ask for text (the default).
|
|
28139
|
-
--max-
|
|
27914
|
+
--max-coins <n> The per-attempt ceiling YOU approved \u2014 a bench runs on your own coin,
|
|
28140
27915
|
never a player's. The run's worst case is that number times --samples,
|
|
28141
|
-
and it is printed before anything starts.
|
|
28142
|
-
name, read as the same number.)
|
|
27916
|
+
and it is printed before anything starts.
|
|
28143
27917
|
--timeout <seconds> Bounds the wait for each attempt.
|
|
28144
27918
|
--json One machine-readable object; every key is always present, null when
|
|
28145
27919
|
the CLI could not learn it.
|
|
28146
27920
|
|
|
28147
27921
|
${c.bold("Why a benchmark and not an estimate")}
|
|
28148
|
-
|
|
28149
|
-
|
|
28150
|
-
|
|
28151
|
-
|
|
28152
|
-
|
|
28153
|
-
|
|
28154
|
-
|
|
28155
|
-
|
|
28156
|
-
|
|
28157
|
-
recommendation, because those multipliers are not the CLI's to invent.
|
|
27922
|
+
\`estimateCoins\` is a PRICE, not a guess: declaring it charges it, in full, even when the
|
|
27923
|
+
attempt fails, is cancelled, or stops at its budget. And three layers sit between the
|
|
27924
|
+
provider's rate card and that number \u2014 the provider's cost, the platform's tariff (already
|
|
27925
|
+
inside every charged coin), and your headroom. The bench itself is billed to YOU, on the
|
|
27926
|
+
development lane; the player-funded lane is what the published game uses. Adding a margin
|
|
27927
|
+
to the provider's rate silently drops the tariff and under-prices every call, so the
|
|
27928
|
+
recommendation is built from CHARGED COINS and the headroom comes from the server. A stand
|
|
27929
|
+
that serves no headroom gets no recommendation, because the multiplier is not the CLI's to
|
|
27930
|
+
invent.
|
|
28158
27931
|
|
|
28159
|
-
|
|
28160
|
-
|
|
28161
|
-
A sample cut off at your --max-
|
|
28162
|
-
|
|
28163
|
-
|
|
28164
|
-
fee included, which is the honest estimate a budget's disclosure is built on.
|
|
27932
|
+
The declared price also decides how LONG the answer may be: it sizes each call's output
|
|
27933
|
+
allowance. So the recommendation is never below the smallest price that leaves room for
|
|
27934
|
+
the benchmarked answer, as the server computes it. A sample cut off at your --max-coins
|
|
27935
|
+
(provider_token_limit) is not a sample \u2014 re-run with a higher --max-coins; an answer longer
|
|
27936
|
+
than one call on this stand may produce gets no price at all \u2014 ask for a shorter one.
|
|
28165
27937
|
|
|
28166
27938
|
Results land in ${LLM_BENCH_FILE} \u2014 gitignored, machine-local, re-read by \`genex llm price\`.
|
|
28167
27939
|
`,
|
|
@@ -28376,13 +28148,11 @@ function parseArgs(argv) {
|
|
|
28376
28148
|
"--limit",
|
|
28377
28149
|
// `genex budget --assets <credits>` — the allowance the player approved.
|
|
28378
28150
|
"--assets",
|
|
28379
|
-
// `genex llm bench` value flags. `--max-
|
|
28380
|
-
// shape as `--assets
|
|
28381
|
-
// count a bench. `--max-coins` is its deprecated name (same number).
|
|
28151
|
+
// `genex llm bench` value flags. `--max-coins` is the COIN half of the
|
|
28152
|
+
// same approval shape as `--assets` (a different economy, its own gate).
|
|
28382
28153
|
"--model",
|
|
28383
28154
|
"--samples",
|
|
28384
28155
|
"--schema",
|
|
28385
|
-
"--max-credits",
|
|
28386
28156
|
"--max-coins",
|
|
28387
28157
|
// `genex player install --label <name>` — what this machine is called in
|
|
28388
28158
|
// the dashboard. Its own key rather than the generic 2nd positional,
|
|
@@ -28897,14 +28667,12 @@ function applyValueFlag(options, flag, value) {
|
|
|
28897
28667
|
options.samples = n;
|
|
28898
28668
|
break;
|
|
28899
28669
|
}
|
|
28900
|
-
case "--max-credits":
|
|
28901
28670
|
case "--max-coins": {
|
|
28902
28671
|
const n = Number(value);
|
|
28903
28672
|
if (!Number.isInteger(n) || n < 1) {
|
|
28904
|
-
throw new Error(`Invalid
|
|
28673
|
+
throw new Error(`Invalid --max-coins value: ${value} (whole coin, 1 or more)`);
|
|
28905
28674
|
}
|
|
28906
|
-
|
|
28907
|
-
else options.maxCoins = n;
|
|
28675
|
+
options.maxCoins = n;
|
|
28908
28676
|
break;
|
|
28909
28677
|
}
|
|
28910
28678
|
case "--timeout": {
|
|
@@ -29196,9 +28964,9 @@ async function main() {
|
|
|
29196
28964
|
await runBudget(parsed.options);
|
|
29197
28965
|
break;
|
|
29198
28966
|
// The in-game model lane: is it live here, what does one call really
|
|
29199
|
-
// cost, what
|
|
29200
|
-
//
|
|
29201
|
-
//
|
|
28967
|
+
// cost, what should the game declare. `bench` is the only CLI command
|
|
28968
|
+
// that spends COIN, so it carries its own --max-coins/--user-approved
|
|
28969
|
+
// gate — the credits allowance cannot see that wallet.
|
|
29202
28970
|
case "llm":
|
|
29203
28971
|
await runLlm(parsed.options);
|
|
29204
28972
|
break;
|