@genex-ai/cli-demo 1.36.8-dev.910 → 1.36.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{blender-mcp-WSURPZET.js → blender-mcp-QVEGHE5L.js} +2 -2
- package/dist/{blender-serve-EH2MOW3U.js → blender-serve-66OHC4ML.js} +1 -1
- package/dist/{chunk-EABIVAGT.js → chunk-4BBTPRJW.js} +2 -2
- package/dist/{chunk-K4EHXOZK.js → chunk-QI7FIYBY.js} +1 -1
- package/dist/index.js +216 -448
- package/package.json +1 -1
- package/templates/skills/genex-game-director/SKILL.md +1 -1
- package/templates/skills/genex-getting-started/SKILL.md +2 -2
- package/templates/skills/genex-llm-in-games/SKILL.md +72 -209
- package/templates/skills/genex-llm-in-games/references/pricing.md +85 -126
- package/templates/skills/genex-monetization/SKILL.md +9 -1
- package/templates/skills/genex-tool-llm/SKILL.md +9 -11
- package/templates/skills/genex-updates/SKILL.md +1 -1
package/dist/index.js
CHANGED
|
@@ -57,7 +57,7 @@ import {
|
|
|
57
57
|
writeSecretFile,
|
|
58
58
|
writeUserToken,
|
|
59
59
|
writeWorkspace
|
|
60
|
-
} from "./chunk-
|
|
60
|
+
} from "./chunk-4BBTPRJW.js";
|
|
61
61
|
import {
|
|
62
62
|
CLI_CHANNEL,
|
|
63
63
|
DEFAULT_API_URL,
|
|
@@ -78,7 +78,7 @@ import {
|
|
|
78
78
|
normalizeApiOrigin,
|
|
79
79
|
originKey,
|
|
80
80
|
resolveAgentTargets
|
|
81
|
-
} from "./chunk-
|
|
81
|
+
} from "./chunk-QI7FIYBY.js";
|
|
82
82
|
|
|
83
83
|
// src/instrument.ts
|
|
84
84
|
import * as Sentry from "@sentry/node";
|
|
@@ -981,7 +981,7 @@ Important note: put soul into your creations, with many details and love. Aim to
|
|
|
981
981
|
21. A turn ends in exactly one of two ways: on a question the player must answer before the next step can be chosen, or on a handoff \u2014 the draft link, what changed, what to try, and what is still open. Never close a turn on a promise. "I'll keep building", "while I keep working", "adding that now" are things you say and then DO before you stop; if you are stopping, say that you have stopped and what remains. While the Build plan's \`Now:\` line has open work and no answer is needed from the player, the turn is not over: preview, hand off, and start the next milestone in the same turn, until the requested outcome is reached or the platform's own budget ends the session. If a stop was forced on you anyway, the first line of your next turn is the promise you left and where it stands.
|
|
982
982
|
22. **What the player sees must read as the thing it is**, in this game's own style: a building as that building, a person as a person, a prop as that prop, a surface as its material \u2014 and a coloured box, a capsule, or a flat grey block standing in for one of them is never its finished version. Generation is the default route to that bar wherever code will not honestly reach it: the player's character, whatever the request names, whatever the player walks up to, enters, or interacts with, and the music and sound underneath \u2014 reach for the belt on your own judgment, without stopping to ask; a status line saying what you queued is enough, and \`--no-wait\` keeps you building while it lands. Procedural code is a first-class engine for what is structural, repeated, distant, or parametric \u2014 terrain, sky, fences, paving, modular kits, filler \u2014 and for anything else only when the result meets the same bar and you have looked at it in a capture before its row says \`landed\`. Start with one reusable implementation per distinct required gameplay or visual role. Background populations default to reusable procedural bodies and motion or compatible existing rigged assets. Catalog clips need a compatible skeleton and may be paid. Reserve paid custom bodies for the player and characters examined or interacted with closely, while honoring explicit user requests. Add variants only for an unmet requirement; unspent allowance is not unfinished work. Allowance questions use timeoutPolicy no-consent: silence does not raise the allowance. Record role, reuse and completion criteria in the existing Assets table. Decide per object, not per category, and write the route on each Assets row in \`DESIGN.md\` \u2014 the paid flow for generated pieces, the procedural flow with its local path for code-built ones. An Assets table with no generation in it is a decision, not a default: record it as \`Generation: none \u2014 <why code alone reaches the bar here>\` or generate.
|
|
983
983
|
23. **The screenshots you take to look at your own work go in \`.genex/scratch/\`.** Captures, render comparisons, before/after strips, traces and metrics dumps are how you SEE what you built \u2014 they are not part of what you built, and \`genex preview\` pushes the whole folder to the game's source, every time, forever. \`.genex/scratch/\` already exists for this and is already ignored, so there is nothing to set up: write the capture as \`.genex/scratch/arena-before.png\` and read it back from there. Never put them in a folder of your own at the top level: \`progress/\`, \`reports/\`, \`shots/\` and their kind are pushed like source and become permanent weight in the game and in every remix of it \u2014 one real game reached 11 GB and 10,332 committed screenshots exactly this way. If you find such a folder already there, add it to \`.gitignore\`; never delete the player's files (law 18). One file in \`.genex/scratch/\` IS read, by the platform: \`.genex/scratch/cover.png\` is the game's cover \u2014 the frame you would put on its poster: real gameplay at its signature moment, well lit, no menus, popups or busy HUD, landscape 16:9, and never any text (the page sets the title beside it). The next \`genex preview\` makes it the cover; save a better moment over it whenever one comes along, and if your browser tool can set the cover itself, use that instead. Before the first publish, and whenever the game's look changes, load \`$genex-cover\`: it chooses, stages, lights and judges that frame.
|
|
984
|
-
24. **Benchmark before declaring an in-game LLM
|
|
984
|
+
24. **Benchmark before declaring an in-game LLM price.** If the game calls a language model while the player plays \u2014 an NPC answering in its own words, a quest written for this save, a judge reading what the player typed \u2014 the PLAYER pays for it, and \`estimateCoins\` is the FIXED price of a started attempt: charged whether it succeeds, fails, is canceled, or stops at its own budget. Never pick that number by judgement or from a vendor's rate card. Run the real prompt on your own coins \u2014 \`npx genex llm bench "<the prompt>" --schema <file> --samples 3 --max-coins <n> --user-approved\` \u2014 and declare the figure it recommends, recorded in \`DESIGN.md\` with its model and date; re-benchmark whenever the prompt, the schema or the model changes, because a prompt edit is a price change. Check the lane first with \`npx genex llm models\`: where it is off the routes 404, so build the feature behind a graceful unavailable state and say so in the handoff. A repeated call is a standing budget the player approves once (\`requestSpendGrant()\`, then \`generate({ grantId })\`), never a popup per turn, and its disclosed call rate is computed from the game's own loop. Model output is data the game validates against its own expectation \u2014 never executed, and never authority over coin, items, entitlements or rewards. Load \`$genex-llm-in-games\` before writing any of it.
|
|
985
985
|
${CONTRACT_END}
|
|
986
986
|
`;
|
|
987
987
|
var TOOLS_CONTRACT_HEAD = `# Genex Tools (always in effect in this folder)
|
|
@@ -995,7 +995,7 @@ var TOOLS_CONTRACT_OFFER = `5. **When the game is built and playable \u2014 neve
|
|
|
995
995
|
var TOOLS_CONTRACT_HOSTED_LAWS = `5. **This game is hosted on Genex now, and publishing is glue \u2014 never a rebuild.** The game ships exactly as the user built it: \`initEmbed()\` first in the boot code (the \`genex-threejs-embed-auth\` card has the call; the renderer draws before any \`await waitForPlayer()\`), a static build into \`dist/\` with relative asset paths, then \`npx genex preview\`. Never restructure, reformat or "improve" the game to ship it, never scaffold a new app around it, and never start a design document or a build plan for it \u2014 the user's own process is theirs. Load the \`genex-tool-publish\` card before the first preview; it owns the vocabulary, the links and the limits.
|
|
996
996
|
6. **Ship first, then report.** \`preview\` prints a preflight \u2014 phone memory, a missing volume slider, an asset nobody wired, a viewport line. It never blocks a deploy, so push the build as it is, hand over the link, THEN relay each preflight line to the user in one plain sentence with an offer to fix it, and fix one only on their yes. The preflight is a report for the user, not a to-do list for you.
|
|
997
997
|
7. **The link is the game's page, and there are two versions.** After every preview give the user \`<dashboard>/draft/<slug>\` \u2014 \`dashboardOrigins[0]\` and \`slug\` from \`.genex/project.json\` \u2014 never localhost, a file path or the bare play origin. \`preview\` updates the draft and never touches what players are on; the first release is \`npx genex publish\` (it lists the game); after that "publish it", "update it" and "yes" all mean \`npx genex promote\` \u2014 the exact draft build, no rebuild. Ask once per round of work, in one line, and keep working while you wait.`;
|
|
998
|
-
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with
|
|
998
|
+
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with Genex coins or their own Claude/ChatGPT subscription, and approves it on a Genex sheet; your game just calls \`generate()\`. Want it that way?`;
|
|
999
999
|
function toolsContractLlmLaw(n, hosted) {
|
|
1000
1000
|
const onYes = hosted ? "On a yes, load `$genex-tool-llm` and then `$genex-llm-in-games`, which owns the build." : "On a yes, load `$genex-tool-llm`, which owns the rest: it checks the lane with `npx genex llm models` first, then runs `npx genex init --convert` (a hosted game is a static browser build \u2014 a local server holding a key can never ship).";
|
|
1001
1001
|
return `${n}. **A model running while people PLAY is a platform feature \u2014 never something you wire onto the user's own meter.** When a request implies one (NPCs that talk or decide in their own words, content written from what the player types, a prompt box in the game, "let the player pick a model"), recognise it, say in ONE line: "${TOOLS_CONTRACT_LLM_OFFER}" \u2014 then ASK and wait. ${onYes} On a no, build the authored version instead. Either way, never ship a game that calls a model on a key or an account of the user's own: every visitor would spend their money with nobody approving it, and the credential is readable in the bundle. Player-funded generation is text and JSON only \u2014 3D, images, video and audio stay the asset lanes above, on the user's meter.`;
|
|
@@ -23609,7 +23609,7 @@ async function runBlender(opts) {
|
|
|
23609
23609
|
return 1;
|
|
23610
23610
|
}
|
|
23611
23611
|
if (sub === "serve") {
|
|
23612
|
-
const { serveLocalBlender } = await import("./blender-serve-
|
|
23612
|
+
const { serveLocalBlender } = await import("./blender-serve-66OHC4ML.js");
|
|
23613
23613
|
const port = Number(process.env.GENEX_BLENDER_PORT ?? 8088);
|
|
23614
23614
|
log.step(`Starting a local Blender service on port ${port}`);
|
|
23615
23615
|
log.plain(
|
|
@@ -23618,7 +23618,7 @@ async function runBlender(opts) {
|
|
|
23618
23618
|
return serveLocalBlender({ port, log });
|
|
23619
23619
|
}
|
|
23620
23620
|
if (sub === "mcp") {
|
|
23621
|
-
const { runBlenderMcp } = await import("./blender-mcp-
|
|
23621
|
+
const { runBlenderMcp } = await import("./blender-mcp-QVEGHE5L.js");
|
|
23622
23622
|
return runBlenderMcp();
|
|
23623
23623
|
}
|
|
23624
23624
|
if (sub === "seat") {
|
|
@@ -24126,12 +24126,7 @@ import fs35 from "fs/promises";
|
|
|
24126
24126
|
import path36 from "path";
|
|
24127
24127
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
24128
24128
|
var SUBS3 = ["models", "bench", "price", "status", "cancel"];
|
|
24129
|
-
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN
|
|
24130
|
-
var LLM_MAX_COINS_DEPRECATED = "--max-coins is deprecated: it is read as --max-credits, the same number \u2014 a bench spends credits now.";
|
|
24131
|
-
function benchBalanceShortLine(spendable, maxCredits) {
|
|
24132
|
-
return `Your spendable credits (${spendable}) cannot cover one attempt at --max-credits ${maxCredits} \u2014 nothing was sent, nothing was spent. Add credits, or re-run with a lower --max-credits once the person at the keyboard agrees to it.`;
|
|
24133
|
-
}
|
|
24134
|
-
var LLM_BENCH_UNVERIFIED_LINE = "Credits pay only for a verified email, and this account's is not verified yet (credits_unverified) \u2014 nothing was sent, nothing was spent. Verify it, then re-run.";
|
|
24129
|
+
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN coin \u2014 your wallet, not a player's, and one this build's asset allowance cannot see. Re-run with --max-coins <n> --user-approved only after the person at the keyboard agreed to the number.";
|
|
24135
24130
|
var LLM_LANE_OFF_LINE = "The runtime LLM lane is off on this stand \u2014 in-game generate() answers 404 here, and nothing can be benchmarked.";
|
|
24136
24131
|
var LLM_TOOLS_CONVERT_HINT = [
|
|
24137
24132
|
"In-game model calls need a hosted game. This folder is a Genex Tools workspace, so nothing here",
|
|
@@ -24144,7 +24139,7 @@ function serverSlotCounts(body) {
|
|
|
24144
24139
|
return { held: body.slotsHeld, limit: body.slotLimit };
|
|
24145
24140
|
}
|
|
24146
24141
|
var PENDING_SLOT_RELEASE_MINUTES = 10;
|
|
24147
|
-
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so
|
|
24142
|
+
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so its coin stays held until the bill resolves; such a row stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`;
|
|
24148
24143
|
var LLM_STATUS_NO_COUNT_LINE = "This stand does not report how many slots are held across your projects, so no count is shown.";
|
|
24149
24144
|
var LLM_CANCEL_AWAITING_BILL = "this call already stopped; its bill is awaiting the provider";
|
|
24150
24145
|
var LLM_CANCEL_ALREADY_FINAL = "this call is already final";
|
|
@@ -24158,33 +24153,11 @@ var DEFAULT_SAMPLE_TIMEOUT_SEC = 180;
|
|
|
24158
24153
|
function parseProviderKey(value) {
|
|
24159
24154
|
return value === "ok" || value === "rejected" || value === "unknown" ? value : null;
|
|
24160
24155
|
}
|
|
24161
|
-
function bpsOrNull(value) {
|
|
24162
|
-
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
24163
|
-
}
|
|
24164
|
-
function standingCreditGrantOrNull(value) {
|
|
24165
|
-
if (!value || typeof value !== "object") return null;
|
|
24166
|
-
const v = value;
|
|
24167
|
-
const whole = (x) => typeof x === "number" && Number.isSafeInteger(x) && x >= 0;
|
|
24168
|
-
if (!whole(v.minCredits) || !whole(v.maxCredits) || !whole(v.stepCredits) || !whole(v.defaultCredits)) return null;
|
|
24169
|
-
return {
|
|
24170
|
-
minCredits: v.minCredits,
|
|
24171
|
-
maxCredits: v.maxCredits,
|
|
24172
|
-
stepCredits: v.stepCredits,
|
|
24173
|
-
defaultCredits: v.defaultCredits,
|
|
24174
|
-
...whole(v.minUsdCents) ? { minUsdCents: v.minUsdCents } : {},
|
|
24175
|
-
...whole(v.maxUsdCents) ? { maxUsdCents: v.maxUsdCents } : {},
|
|
24176
|
-
...whole(v.stepUsdCents) ? { stepUsdCents: v.stepUsdCents } : {}
|
|
24177
|
-
};
|
|
24178
|
-
}
|
|
24179
24156
|
async function readRuntimeLane(apiUrl, token) {
|
|
24180
24157
|
const empty = (state) => ({
|
|
24181
24158
|
state,
|
|
24182
24159
|
models: null,
|
|
24183
24160
|
recommendedDeclaredHeadroomBps: null,
|
|
24184
|
-
currency: null,
|
|
24185
|
-
feeBps: null,
|
|
24186
|
-
usdCentsPerCredit: null,
|
|
24187
|
-
standingCreditGrant: null,
|
|
24188
24161
|
externalProviders: null,
|
|
24189
24162
|
total: null,
|
|
24190
24163
|
featuredCount: null,
|
|
@@ -24203,11 +24176,7 @@ async function readRuntimeLane(apiUrl, token) {
|
|
|
24203
24176
|
return {
|
|
24204
24177
|
state: "live",
|
|
24205
24178
|
models: body.models,
|
|
24206
|
-
recommendedDeclaredHeadroomBps:
|
|
24207
|
-
currency: body.currency === "credits" || body.currency === "coins" ? body.currency : null,
|
|
24208
|
-
feeBps: bpsOrNull(body.feeBps),
|
|
24209
|
-
usdCentsPerCredit: typeof body.usdCentsPerCredit === "number" && Number.isSafeInteger(body.usdCentsPerCredit) && body.usdCentsPerCredit >= 1 ? body.usdCentsPerCredit : null,
|
|
24210
|
-
standingCreditGrant: standingCreditGrantOrNull(body.standingCreditGrant),
|
|
24179
|
+
recommendedDeclaredHeadroomBps: typeof body.recommendedDeclaredHeadroomBps === "number" && Number.isFinite(body.recommendedDeclaredHeadroomBps) ? body.recommendedDeclaredHeadroomBps : null,
|
|
24211
24180
|
externalProviders: Array.isArray(body.externalProviders) ? body.externalProviders : null,
|
|
24212
24181
|
total: typeof body.total === "number" && Number.isFinite(body.total) ? body.total : null,
|
|
24213
24182
|
featuredCount: typeof body.featuredCount === "number" && Number.isFinite(body.featuredCount) ? body.featuredCount : null,
|
|
@@ -24223,45 +24192,18 @@ function percentile2(values, p) {
|
|
|
24223
24192
|
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
24224
24193
|
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
24225
24194
|
}
|
|
24226
|
-
function
|
|
24227
|
-
|
|
24228
|
-
const sorted = [...values].sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
24229
|
-
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
24230
|
-
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
24195
|
+
function recommendEstimateCoins(p95, headroomBps) {
|
|
24196
|
+
return Math.max(1, Math.ceil(p95 * (1e4 + headroomBps) / 1e4));
|
|
24231
24197
|
}
|
|
24232
|
-
|
|
24233
|
-
|
|
24234
|
-
if (costPicos <= 0n) return 1;
|
|
24235
|
-
const num = costPicos * BigInt(1e4 + terms.feeBps) * BigInt(1e4 + terms.headroomBps);
|
|
24236
|
-
const den = 100000000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
24237
|
-
return Math.max(1, Number((num + den - 1n) / den));
|
|
24238
|
-
}
|
|
24239
|
-
function averageCreditsPerCall(costsPicos, terms) {
|
|
24240
|
-
if (costsPicos.length === 0) return null;
|
|
24241
|
-
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
24242
|
-
return Number(sum * BigInt(1e4 + terms.feeBps)) / Number(BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT);
|
|
24243
|
-
}
|
|
24244
|
-
function perCallEstimateCreditsFor(costsPicos, terms) {
|
|
24245
|
-
if (costsPicos.length === 0) return null;
|
|
24246
|
-
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
24247
|
-
const num = sum * BigInt(1e4 + terms.feeBps);
|
|
24248
|
-
const den = BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
24249
|
-
return Math.max(1, Number((num + den - 1n) / den));
|
|
24250
|
-
}
|
|
24251
|
-
function formatCredits(value) {
|
|
24252
|
-
if (!Number.isFinite(value)) return "\u2014";
|
|
24253
|
-
if (value >= 100) return String(Math.round(value));
|
|
24254
|
-
return String(Number(value.toPrecision(value < 1 ? 2 : 3)));
|
|
24255
|
-
}
|
|
24256
|
-
function usdFromPicos(picos) {
|
|
24257
|
-
if (picos === null || picos === void 0) return "unknown";
|
|
24258
|
-
const value = typeof picos === "bigint" ? picos : /^\d+$/.test(picos) ? BigInt(picos) : null;
|
|
24259
|
-
if (value === null) return "unknown";
|
|
24260
|
-
return `$${(Number(value) / 1e12).toFixed(6)}`;
|
|
24198
|
+
function recommendPerCallMaxCoins(p95, max, headroomBps) {
|
|
24199
|
+
return Math.max(recommendEstimateCoins(p95, headroomBps), recommendEstimateCoins(max, headroomBps));
|
|
24261
24200
|
}
|
|
24262
24201
|
function usdPerMillion(value) {
|
|
24263
24202
|
return `$${value.toFixed(value < 1 ? 4 : 2)}`;
|
|
24264
24203
|
}
|
|
24204
|
+
function usd(value) {
|
|
24205
|
+
return value === null ? "unknown" : `$${value.toFixed(6)}`;
|
|
24206
|
+
}
|
|
24265
24207
|
async function runLlm(opts = {}) {
|
|
24266
24208
|
const log = createLogger({ quiet: opts.quiet || opts.json });
|
|
24267
24209
|
const cwd = opts.cwd ?? process.cwd();
|
|
@@ -24287,23 +24229,16 @@ async function runLlm(opts = {}) {
|
|
|
24287
24229
|
return;
|
|
24288
24230
|
}
|
|
24289
24231
|
if (sub === "bench") {
|
|
24290
|
-
if (opts.maxCoins
|
|
24291
|
-
`);
|
|
24292
|
-
if (opts.maxCoins !== void 0 && opts.maxCredits !== void 0 && opts.maxCoins !== opts.maxCredits) {
|
|
24293
|
-
fail5("llm bench", `Pass --max-credits alone \u2014 --max-coins is its deprecated name, and the two disagree (${opts.maxCredits} vs ${opts.maxCoins}).`);
|
|
24294
|
-
return;
|
|
24295
|
-
}
|
|
24296
|
-
const approvedCredits = opts.maxCredits ?? opts.maxCoins;
|
|
24297
|
-
if (approvedCredits === void 0 || opts.userApproved !== true) {
|
|
24232
|
+
if (opts.maxCoins === void 0 || opts.userApproved !== true) {
|
|
24298
24233
|
fail5("llm bench", LLM_BENCH_APPROVAL_REQUIRED);
|
|
24299
24234
|
return;
|
|
24300
24235
|
}
|
|
24301
|
-
if (!Number.isInteger(
|
|
24302
|
-
fail5("llm bench", `--max-
|
|
24236
|
+
if (!Number.isInteger(opts.maxCoins) || opts.maxCoins < 1) {
|
|
24237
|
+
fail5("llm bench", `--max-coins takes a whole number of coin, 1 or more (got ${String(opts.maxCoins)}).`);
|
|
24303
24238
|
return;
|
|
24304
24239
|
}
|
|
24305
24240
|
if (!opts.benchPrompt?.trim()) {
|
|
24306
|
-
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-
|
|
24241
|
+
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-coins <n> --user-approved.');
|
|
24307
24242
|
return;
|
|
24308
24243
|
}
|
|
24309
24244
|
if (opts.jsonOutput && opts.textOutput) {
|
|
@@ -24395,10 +24330,6 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24395
24330
|
models: null,
|
|
24396
24331
|
externalProviders: null,
|
|
24397
24332
|
recommendedDeclaredHeadroomBps: null,
|
|
24398
|
-
currency: null,
|
|
24399
|
-
feeBps: null,
|
|
24400
|
-
usdCentsPerCredit: null,
|
|
24401
|
-
standingCreditGrant: null,
|
|
24402
24333
|
total: null,
|
|
24403
24334
|
featuredCount: null,
|
|
24404
24335
|
// The stand was never asked, so the verdict is unknown — but the key is
|
|
@@ -24424,12 +24355,6 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24424
24355
|
models: lane.models,
|
|
24425
24356
|
externalProviders: lane.externalProviders,
|
|
24426
24357
|
recommendedDeclaredHeadroomBps: lane.recommendedDeclaredHeadroomBps,
|
|
24427
|
-
// The credit terms a call is billed on, the server's — null on a stand
|
|
24428
|
-
// that predates credit billing, never a number the CLI filled in.
|
|
24429
|
-
currency: lane.currency,
|
|
24430
|
-
feeBps: lane.feeBps,
|
|
24431
|
-
usdCentsPerCredit: lane.usdCentsPerCredit,
|
|
24432
|
-
standingCreditGrant: lane.standingCreditGrant,
|
|
24433
24358
|
total: lane.total ?? lane.models?.length ?? null,
|
|
24434
24359
|
featuredCount: lane.featuredCount ?? (lane.models ? lane.models.filter((m) => m.featured).length : null),
|
|
24435
24360
|
// `null` when the lane is not live or the server predates the probe.
|
|
@@ -24476,9 +24401,8 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24476
24401
|
);
|
|
24477
24402
|
for (const m of shown) {
|
|
24478
24403
|
const plan = m.personalPlan ? ` \xB7 personal plan: ${m.personalPlan}` : "";
|
|
24479
|
-
const kind = m.kind === "typesafe" ? " \xB7 judge (classifier): judge() under a budget, not generate() or bench" : "";
|
|
24480
24404
|
log.plain(
|
|
24481
|
-
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}
|
|
24405
|
+
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}`
|
|
24482
24406
|
);
|
|
24483
24407
|
}
|
|
24484
24408
|
if (hiddenCount > 0) {
|
|
@@ -24494,104 +24418,74 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
24494
24418
|
}
|
|
24495
24419
|
log.plain("");
|
|
24496
24420
|
if (lane.recommendedDeclaredHeadroomBps === null) {
|
|
24497
|
-
log.dim(" This stand serves no recommended headroom, so no
|
|
24421
|
+
log.dim(" This stand serves no recommended headroom, so no price can be recommended from a bench.");
|
|
24498
24422
|
} else {
|
|
24499
|
-
log.dim(` Recommended headroom over
|
|
24500
|
-
}
|
|
24501
|
-
if (lane.feeBps === null) {
|
|
24502
|
-
log.dim(" This stand serves no platform fee \u2014 it predates credit billing, so no ceiling can be recommended from a bench.");
|
|
24503
|
-
} else {
|
|
24504
|
-
log.dim(` In-game calls are billed in the player's credits, as used: the provider's cost plus this stand's platform fee (${lane.feeBps} bps).`);
|
|
24423
|
+
log.dim(` Recommended headroom over benchmarked coins on this stand: ${lane.recommendedDeclaredHeadroomBps} bps.`);
|
|
24505
24424
|
}
|
|
24506
24425
|
if (toolsOnly) {
|
|
24507
24426
|
printConvertHint();
|
|
24508
24427
|
return;
|
|
24509
24428
|
}
|
|
24510
|
-
log.dim(" Those per-million rates are the PROVIDER's
|
|
24511
|
-
log.dim(`
|
|
24429
|
+
log.dim(" Those per-million rates are the PROVIDER's; what your game is charged is the platform");
|
|
24430
|
+
log.dim(` tariff on top, which only a real call reveals: ${c.cyan('genex llm bench "<prompt>" --max-coins <n> --user-approved')}`);
|
|
24512
24431
|
}
|
|
24513
24432
|
function wholeOrNull(value) {
|
|
24514
24433
|
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
24515
24434
|
}
|
|
24516
|
-
function costPicosOf(usage) {
|
|
24517
|
-
if (typeof usage?.costUsdPicos === "string" && /^\d+$/.test(usage.costUsdPicos)) return usage.costUsdPicos;
|
|
24518
|
-
if (typeof usage?.costUsd === "number" && Number.isFinite(usage.costUsd) && usage.costUsd >= 0) {
|
|
24519
|
-
return String(BigInt(Math.round(usage.costUsd * 1e12)));
|
|
24520
|
-
}
|
|
24521
|
-
return null;
|
|
24522
|
-
}
|
|
24523
24435
|
function sampleFrom(id, settled) {
|
|
24524
24436
|
const usage = settled?.usage;
|
|
24525
|
-
const picos = costPicosOf(usage);
|
|
24526
24437
|
return {
|
|
24527
24438
|
id,
|
|
24528
24439
|
status: settled?.status ?? null,
|
|
24529
24440
|
billingStatus: settled?.billingStatus ?? null,
|
|
24530
|
-
|
|
24531
|
-
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd :
|
|
24532
|
-
costUsdPicos: picos,
|
|
24441
|
+
chargedCoins: typeof settled?.chargedCoins === "number" ? settled.chargedCoins : null,
|
|
24442
|
+
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd : null,
|
|
24533
24443
|
error: settled?.error ?? (settled ? null : "timed_out"),
|
|
24534
24444
|
providerMessage: typeof settled?.providerMessage === "string" && settled.providerMessage.trim() ? settled.providerMessage.trim() : null,
|
|
24535
24445
|
inputTokens: wholeOrNull(usage?.inputTokens),
|
|
24536
24446
|
outputTokens: wholeOrNull(usage?.outputTokens),
|
|
24537
24447
|
maxOutputTokens: wholeOrNull(usage?.maxOutputTokens),
|
|
24538
|
-
|
|
24448
|
+
fitCoins: typeof usage?.fitCoins === "number" && Number.isSafeInteger(usage.fitCoins) && usage.fitCoins >= 1 ? usage.fitCoins : null,
|
|
24539
24449
|
fitExceedsCap: typeof usage?.fitExceedsCap === "boolean" ? usage.fitExceedsCap : null,
|
|
24540
24450
|
truncated: typeof usage?.truncated === "boolean" ? usage.truncated : settled?.error === "provider_token_limit" ? true : null
|
|
24541
24451
|
};
|
|
24542
24452
|
}
|
|
24543
24453
|
var PROVIDER_TOKEN_LIMIT = "provider_token_limit";
|
|
24544
|
-
function benchRecommendation(results,
|
|
24545
|
-
const settled = results.filter(
|
|
24546
|
-
|
|
24547
|
-
);
|
|
24548
|
-
const
|
|
24549
|
-
const
|
|
24550
|
-
const fits = settled.map((r) => r.fitCredits).filter((v) => typeof v === "number");
|
|
24551
|
-
const costP95 = percentilePicos(costs, 95);
|
|
24552
|
-
const costMax = costs.length ? costs.reduce((a, b) => b > a ? b : a) : null;
|
|
24454
|
+
function benchRecommendation(results, headroomBps) {
|
|
24455
|
+
const settled = results.filter((r) => r.status === "succeeded" && r.billingStatus === "final" && typeof r.chargedCoins === "number");
|
|
24456
|
+
const charged = settled.map((r) => r.chargedCoins);
|
|
24457
|
+
const fits = settled.map((r) => r.fitCoins).filter((v) => typeof v === "number");
|
|
24458
|
+
const p95 = percentile2(charged, 95);
|
|
24459
|
+
const max = charged.length ? Math.max(...charged) : null;
|
|
24553
24460
|
const fitP95 = percentile2(fits, 95);
|
|
24554
24461
|
const fitMax = fits.length ? Math.max(...fits) : null;
|
|
24555
24462
|
const exceedsStandCap = results.some((r) => r.fitExceedsCap === true);
|
|
24556
24463
|
const cutOffSamples = results.filter((r) => r.error === PROVIDER_TOKEN_LIMIT && r.fitExceedsCap !== true).length;
|
|
24557
24464
|
const noRecommendationReason = exceedsStandCap ? "answer_exceeds_stand_cap" : cutOffSamples > 0 ? "answer_cut_off" : null;
|
|
24558
|
-
const
|
|
24559
|
-
const
|
|
24560
|
-
const
|
|
24561
|
-
const billing = feeBps !== null && usdCentsPerCredit !== null ? { feeBps, usdCentsPerCredit } : null;
|
|
24562
|
-
const costBased = full !== null && costP95 !== null ? ceilingCreditsFor(costP95, full) : null;
|
|
24563
|
-
const recommended = costBased === null || noRecommendationReason !== null ? null : Math.max(costBased, fitP95 ?? 0);
|
|
24564
|
-
const ceiling = recommended === null || costMax === null || full === null ? null : Math.max(ceilingCreditsFor(costMax, full), recommended, fitMax ?? 0);
|
|
24565
|
-
const average = billing !== null ? averageCreditsPerCall(costs, billing) : null;
|
|
24566
|
-
const estimate = ceiling === null || billing === null ? null : Math.min(ceiling, perCallEstimateCreditsFor(costs, billing) ?? 1);
|
|
24465
|
+
const chargedBased = p95 !== null && headroomBps !== null ? recommendEstimateCoins(p95, headroomBps) : null;
|
|
24466
|
+
const recommended = chargedBased === null || noRecommendationReason !== null ? null : Math.max(chargedBased, fitP95 ?? 0);
|
|
24467
|
+
const ceiling = recommended === null || p95 === null || max === null || headroomBps === null ? null : Math.max(recommendPerCallMaxCoins(p95, max, headroomBps), recommended, fitMax ?? 0);
|
|
24567
24468
|
return {
|
|
24568
|
-
costs,
|
|
24569
24469
|
charged,
|
|
24570
|
-
|
|
24571
|
-
|
|
24572
|
-
|
|
24573
|
-
chargedP50: percentile2(charged, 50),
|
|
24574
|
-
chargedP95: percentile2(charged, 95),
|
|
24575
|
-
chargedMax: charged.length ? Math.max(...charged) : null,
|
|
24470
|
+
p50: percentile2(charged, 50),
|
|
24471
|
+
p95,
|
|
24472
|
+
max,
|
|
24576
24473
|
fits,
|
|
24577
24474
|
fitP95,
|
|
24578
24475
|
fitMax,
|
|
24579
|
-
|
|
24580
|
-
|
|
24581
|
-
|
|
24582
|
-
|
|
24583
|
-
recommendedPerCallEstimateCredits: estimate,
|
|
24584
|
-
lengthRaisedCeiling: recommended !== null && costBased !== null && recommended > costBased,
|
|
24476
|
+
chargedBasedEstimateCoins: chargedBased,
|
|
24477
|
+
recommendedEstimateCoins: recommended,
|
|
24478
|
+
recommendedPerCallMaxCoins: ceiling,
|
|
24479
|
+
lengthRaisedPrice: recommended !== null && chargedBased !== null && recommended > chargedBased,
|
|
24585
24480
|
cutOffSamples,
|
|
24586
24481
|
exceedsStandCap,
|
|
24587
|
-
noRecommendationReason
|
|
24588
|
-
missingTerms
|
|
24482
|
+
noRecommendationReason
|
|
24589
24483
|
};
|
|
24590
24484
|
}
|
|
24591
|
-
var LLM_LENGTH_RAISED_LINE = "The
|
|
24592
|
-
var LLM_EXCEEDS_STAND_CAP_LINE = "No
|
|
24593
|
-
function benchCutOffLine(cutOff,
|
|
24594
|
-
return `No
|
|
24485
|
+
var LLM_LENGTH_RAISED_LINE = "The declared price also pays for how long the answer may be: at the charged-based price the answer would be cut off, so declare this one.";
|
|
24486
|
+
var LLM_EXCEEDS_STAND_CAP_LINE = "No price recommended: this answer is longer than one call on this stand may produce. Ask for a shorter answer \u2014 fewer fields, shorter strings, a length the prompt states \u2014 and benchmark again.";
|
|
24487
|
+
function benchCutOffLine(cutOff, maxCoins) {
|
|
24488
|
+
return `No price recommended: ${cutOff} sample${cutOff === 1 ? " was" : "s were"} cut off at this ceiling (--max-coins ${maxCoins}) before the answer was finished, so its real length is unknown. Re-run with a higher --max-coins.`;
|
|
24595
24489
|
}
|
|
24596
24490
|
var ACTIVE_STATUSES = /* @__PURE__ */ new Set(["requires_confirmation", "queued", "dispatching", "awaiting_external"]);
|
|
24597
24491
|
function rowState(row) {
|
|
@@ -24602,22 +24496,7 @@ function rowState(row) {
|
|
|
24602
24496
|
function rowHoldsSlot(row) {
|
|
24603
24497
|
if (typeof row.slotHeld === "boolean") return row.slotHeld;
|
|
24604
24498
|
const state = rowState(row);
|
|
24605
|
-
return state === "active" || state === "awaiting bill" && (
|
|
24606
|
-
}
|
|
24607
|
-
function rowMoney(row) {
|
|
24608
|
-
const credits = row.currency === "credits" || row.currency === void 0 && typeof row.reservedCredits === "number" && typeof row.reservedCoins !== "number";
|
|
24609
|
-
if (credits) {
|
|
24610
|
-
return {
|
|
24611
|
-
unit: "credits",
|
|
24612
|
-
reserved: typeof row.reservedCredits === "number" ? row.reservedCredits : null,
|
|
24613
|
-
charged: typeof row.chargedCredits === "number" ? row.chargedCredits : null
|
|
24614
|
-
};
|
|
24615
|
-
}
|
|
24616
|
-
return {
|
|
24617
|
-
unit: "coin",
|
|
24618
|
-
reserved: typeof row.reservedCoins === "number" ? row.reservedCoins : null,
|
|
24619
|
-
charged: typeof row.chargedCoins === "number" ? row.chargedCoins : null
|
|
24620
|
-
};
|
|
24499
|
+
return state === "active" || state === "awaiting bill" && (row.reservedCoins ?? 0) > 0;
|
|
24621
24500
|
}
|
|
24622
24501
|
function providerRefusalStatus(error) {
|
|
24623
24502
|
const m = /^provider_http_(\d{3})$/.exec(error ?? "");
|
|
@@ -24636,7 +24515,7 @@ function formatAge(iso, now = Date.now()) {
|
|
|
24636
24515
|
}
|
|
24637
24516
|
async function runBench(args) {
|
|
24638
24517
|
const { apiUrl, token, projectId, cwd, opts, log } = args;
|
|
24639
|
-
const
|
|
24518
|
+
const maxCoins = opts.maxCoins;
|
|
24640
24519
|
const samples = opts.samples ?? DEFAULT_BENCH_SAMPLES;
|
|
24641
24520
|
const prompt = opts.benchPrompt.trim();
|
|
24642
24521
|
const auth = { Authorization: `Bearer ${token}`, "Content-Type": "application/json" };
|
|
@@ -24658,7 +24537,7 @@ async function runBench(args) {
|
|
|
24658
24537
|
const lane = await readRuntimeLane(apiUrl, token);
|
|
24659
24538
|
if (lane.state !== "live" || !lane.models || lane.models.length === 0) {
|
|
24660
24539
|
if (lane.state === "off") {
|
|
24661
|
-
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
24540
|
+
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCoins, opts), status: "off", error: null });
|
|
24662
24541
|
else {
|
|
24663
24542
|
log.plain(LLM_LANE_OFF_LINE);
|
|
24664
24543
|
log.dim(" Nothing was spent.");
|
|
@@ -24685,27 +24564,17 @@ async function runBench(args) {
|
|
|
24685
24564
|
);
|
|
24686
24565
|
return;
|
|
24687
24566
|
}
|
|
24688
|
-
const
|
|
24689
|
-
if (walletBefore?.emailVerified === false) {
|
|
24690
|
-
benchFailed(opts, log, LLM_BENCH_UNVERIFIED_LINE);
|
|
24691
|
-
return;
|
|
24692
|
-
}
|
|
24693
|
-
if (walletBefore !== null && walletBefore.spendable < maxCredits) {
|
|
24694
|
-
benchFailed(opts, log, benchBalanceShortLine(walletBefore.spendable, maxCredits));
|
|
24695
|
-
return;
|
|
24696
|
-
}
|
|
24697
|
-
const balanceBefore = walletBefore?.spendable ?? null;
|
|
24567
|
+
const balanceBefore = await readCoinBalance(apiUrl, token);
|
|
24698
24568
|
if (!opts.json) {
|
|
24699
24569
|
log.plain(c.bold("genex llm bench"));
|
|
24700
24570
|
log.dim(` ${apiUrl}`);
|
|
24701
24571
|
log.plain("");
|
|
24702
24572
|
log.plain(` Model ${c.cyan(modelId)}`);
|
|
24703
24573
|
log.plain(` Samples ${samples} real attempt${samples === 1 ? "" : "s"}, ${outputFormat} output`);
|
|
24704
|
-
log.plain(` Approved ${
|
|
24574
|
+
log.plain(` Approved ${maxCoins} coin per attempt \u2014 at worst ${maxCoins * samples} coin for this run`);
|
|
24705
24575
|
log.plain(
|
|
24706
|
-
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore}
|
|
24576
|
+
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore} coin spendable`}`
|
|
24707
24577
|
);
|
|
24708
|
-
log.dim(" Billed as used: each attempt is charged its real cost plus the platform fee, rounded up to a whole credit.");
|
|
24709
24578
|
log.plain("");
|
|
24710
24579
|
}
|
|
24711
24580
|
const base = `${apiUrl}/api/runtime/development/projects/${encodeURIComponent(projectId)}/generations`;
|
|
@@ -24731,7 +24600,7 @@ async function runBench(args) {
|
|
|
24731
24600
|
const res = await call(base, {
|
|
24732
24601
|
method: "POST",
|
|
24733
24602
|
body: JSON.stringify({
|
|
24734
|
-
|
|
24603
|
+
maxCoins,
|
|
24735
24604
|
request: {
|
|
24736
24605
|
idempotencyKey: `bench-${randomUUID2()}`,
|
|
24737
24606
|
modelId,
|
|
@@ -24761,14 +24630,13 @@ async function runBench(args) {
|
|
|
24761
24630
|
if (refusedAt !== null) providerRefused++;
|
|
24762
24631
|
if (!opts.json) {
|
|
24763
24632
|
const length = row.outputTokens !== null ? ` \xB7 ${row.outputTokens}${row.maxOutputTokens !== null ? ` of ${row.maxOutputTokens}` : ""} tokens out` : "";
|
|
24764
|
-
const charged = row.chargedCredits === null ? "\u2014" : String(row.chargedCredits);
|
|
24765
24633
|
log.plain(
|
|
24766
|
-
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${
|
|
24634
|
+
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${String(row.chargedCoins ?? "\u2014").padStart(4)} coin charged \xB7 provider ${usd(row.costUsd)}${length}${row.error ? ` \xB7 ${row.error}` : ""}`
|
|
24767
24635
|
);
|
|
24768
24636
|
if (refusedAt !== null) printProviderRefusal(log, refusedAt, row.providerMessage, row.status === "unknown");
|
|
24769
24637
|
if (row.error === PROVIDER_TOKEN_LIMIT) {
|
|
24770
24638
|
log.dim(
|
|
24771
|
-
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no
|
|
24639
|
+
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no price makes room for it; it is not a sample." : ` The answer was cut off at this ceiling (--max-coins ${maxCoins}) before it was finished \u2014 it was charged, and it is not a sample.`
|
|
24772
24640
|
);
|
|
24773
24641
|
}
|
|
24774
24642
|
}
|
|
@@ -24776,19 +24644,16 @@ async function runBench(args) {
|
|
|
24776
24644
|
} finally {
|
|
24777
24645
|
process.removeListener("SIGINT", onSigint);
|
|
24778
24646
|
}
|
|
24779
|
-
const
|
|
24780
|
-
|
|
24781
|
-
|
|
24782
|
-
|
|
24783
|
-
|
|
24784
|
-
const
|
|
24785
|
-
const settledCount = rec.charged.length;
|
|
24647
|
+
const headroomBps = lane.recommendedDeclaredHeadroomBps;
|
|
24648
|
+
const rec = benchRecommendation(results, headroomBps);
|
|
24649
|
+
const { charged, p50, p95, max } = rec;
|
|
24650
|
+
const settledCount = charged.length;
|
|
24651
|
+
const recommended = rec.recommendedEstimateCoins;
|
|
24652
|
+
const ceiling = rec.recommendedPerCallMaxCoins;
|
|
24786
24653
|
const lengthBlocked = rec.noRecommendationReason !== null;
|
|
24787
|
-
const balanceAfter =
|
|
24788
|
-
const picosOrNull = (v) => v === null ? null : String(v);
|
|
24654
|
+
const balanceAfter = await readCoinBalance(apiUrl, token);
|
|
24789
24655
|
const record = {
|
|
24790
|
-
v:
|
|
24791
|
-
currency: "credits",
|
|
24656
|
+
v: 1,
|
|
24792
24657
|
ranAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
24793
24658
|
apiUrl,
|
|
24794
24659
|
projectId,
|
|
@@ -24797,34 +24662,28 @@ async function runBench(args) {
|
|
|
24797
24662
|
outputFormat,
|
|
24798
24663
|
samples,
|
|
24799
24664
|
completed: settledCount,
|
|
24800
|
-
|
|
24801
|
-
|
|
24802
|
-
|
|
24803
|
-
|
|
24804
|
-
|
|
24805
|
-
|
|
24665
|
+
chargedCoins: charged,
|
|
24666
|
+
p50,
|
|
24667
|
+
p95,
|
|
24668
|
+
max,
|
|
24669
|
+
recommendedDeclaredHeadroomBps: headroomBps,
|
|
24670
|
+
recommendedEstimateCoins: recommended,
|
|
24671
|
+
recommendedPerCallMaxCoins: ceiling,
|
|
24672
|
+
maxCoinsPerSample: maxCoins,
|
|
24673
|
+
fitCoins: rec.fits,
|
|
24806
24674
|
fitP95: rec.fitP95,
|
|
24807
|
-
|
|
24808
|
-
|
|
24809
|
-
feeBps: terms.feeBps,
|
|
24810
|
-
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
24811
|
-
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
24812
|
-
recommendedMaxCredits: rec.recommendedMaxCredits,
|
|
24813
|
-
recommendedPerCallMaxCredits: rec.recommendedPerCallMaxCredits,
|
|
24814
|
-
averageCreditsPerCall: rec.averageCreditsPerCall,
|
|
24815
|
-
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
24816
|
-
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
24675
|
+
chargedBasedEstimateCoins: rec.chargedBasedEstimateCoins,
|
|
24676
|
+
lengthRaisedPrice: rec.lengthRaisedPrice,
|
|
24817
24677
|
cutOffSamples: rec.cutOffSamples,
|
|
24818
24678
|
noRecommendationReason: rec.noRecommendationReason
|
|
24819
24679
|
};
|
|
24820
24680
|
const savedTo = await saveBench(cwd, record);
|
|
24821
24681
|
if (opts.json) {
|
|
24822
|
-
const ok =
|
|
24682
|
+
const ok = charged.length > 0 && !lengthBlocked;
|
|
24823
24683
|
writeJsonLine({
|
|
24824
24684
|
command: "llm bench",
|
|
24825
24685
|
status: ok ? "ok" : "failed",
|
|
24826
24686
|
error: ok ? null : rec.noRecommendationReason ?? "no_sample_settled",
|
|
24827
|
-
currency: "credits",
|
|
24828
24687
|
apiUrl,
|
|
24829
24688
|
projectId,
|
|
24830
24689
|
modelId,
|
|
@@ -24833,54 +24692,35 @@ async function runBench(args) {
|
|
|
24833
24692
|
schemaPath: opts.schemaPath ?? null,
|
|
24834
24693
|
samples,
|
|
24835
24694
|
completed: settledCount,
|
|
24836
|
-
|
|
24837
|
-
// Spendable credits before and after the run; null when unreadable.
|
|
24695
|
+
maxCoinsPerSample: maxCoins,
|
|
24838
24696
|
balanceBefore,
|
|
24839
24697
|
balanceAfter,
|
|
24840
24698
|
results,
|
|
24841
24699
|
// The create-time refusal that ended the run, or null. Beside the
|
|
24842
24700
|
// samples, never among them: no attempt existed and nothing was spent.
|
|
24843
24701
|
refusal: refusal2,
|
|
24844
|
-
|
|
24845
|
-
//
|
|
24846
|
-
costUsdPicos: record.cost,
|
|
24847
|
-
costUsd: {
|
|
24848
|
-
p50: rec.costP50 === null ? null : Number(rec.costP50) / 1e12,
|
|
24849
|
-
p95: rec.costP95 === null ? null : Number(rec.costP95) / 1e12,
|
|
24850
|
-
max: rec.costMax === null ? null : Number(rec.costMax) / 1e12
|
|
24851
|
-
},
|
|
24852
|
-
chargedCredits: record.charged,
|
|
24853
|
-
// The answer's LENGTH, priced by the server per sample (`fitCredits`,
|
|
24702
|
+
chargedCoins: { p50, p95, max },
|
|
24703
|
+
// The answer's LENGTH, priced by the server per sample (`fitCoins`,
|
|
24854
24704
|
// headroom already on the tokens), over the settled samples.
|
|
24855
|
-
|
|
24856
|
-
|
|
24857
|
-
|
|
24858
|
-
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
24859
|
-
missingTerms: rec.missingTerms,
|
|
24860
|
-
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
24861
|
-
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
24705
|
+
fitCoins: { p95: rec.fitP95, max: rec.fitMax },
|
|
24706
|
+
chargedBasedEstimateCoins: rec.chargedBasedEstimateCoins,
|
|
24707
|
+
lengthRaisedPrice: rec.lengthRaisedPrice,
|
|
24862
24708
|
cutOffSamples: rec.cutOffSamples,
|
|
24863
24709
|
exceedsStandCap: rec.exceedsStandCap,
|
|
24864
24710
|
noRecommendationReason: rec.noRecommendationReason,
|
|
24865
|
-
|
|
24866
|
-
|
|
24867
|
-
|
|
24868
|
-
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
24711
|
+
recommendedDeclaredHeadroomBps: headroomBps,
|
|
24712
|
+
recommendedEstimateCoins: recommended,
|
|
24713
|
+
recommendedPerCallMaxCoins: ceiling,
|
|
24869
24714
|
savedTo
|
|
24870
24715
|
});
|
|
24871
24716
|
if (!ok) process.exitCode = 1;
|
|
24872
24717
|
return;
|
|
24873
24718
|
}
|
|
24874
24719
|
log.plain("");
|
|
24875
|
-
const printMeasured = () => {
|
|
24876
|
-
log.plain(c.bold(" Real cost per call"));
|
|
24877
|
-
log.plain(` p50 ${usdFromPicos(rec.costP50)} p95 ${usdFromPicos(rec.costP95)} max ${usdFromPicos(rec.costMax)} (${settledCount} of ${samples} settled)`);
|
|
24878
|
-
log.plain(c.bold(" Charged credits"));
|
|
24879
|
-
log.plain(` p50 ${rec.chargedP50} p95 ${rec.chargedP95} max ${rec.chargedMax} (whole credits, rounded up per call)`);
|
|
24880
|
-
};
|
|
24881
24720
|
if (lengthBlocked) {
|
|
24882
|
-
if (
|
|
24883
|
-
|
|
24721
|
+
if (charged.length > 0) {
|
|
24722
|
+
log.plain(c.bold(" Charged coins"));
|
|
24723
|
+
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
24884
24724
|
log.plain("");
|
|
24885
24725
|
}
|
|
24886
24726
|
printRecommendation(log, record);
|
|
@@ -24888,16 +24728,17 @@ async function runBench(args) {
|
|
|
24888
24728
|
process.exitCode = 1;
|
|
24889
24729
|
return;
|
|
24890
24730
|
}
|
|
24891
|
-
if (
|
|
24731
|
+
if (charged.length === 0) {
|
|
24892
24732
|
log.error(
|
|
24893
|
-
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to
|
|
24733
|
+
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to price from." : " No sample settled, so there is nothing to price from."
|
|
24894
24734
|
);
|
|
24895
24735
|
process.exitCode = 1;
|
|
24896
24736
|
return;
|
|
24897
24737
|
}
|
|
24898
|
-
|
|
24738
|
+
log.plain(c.bold(" Charged coins"));
|
|
24739
|
+
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
24899
24740
|
if (balanceBefore !== null && balanceAfter !== null) {
|
|
24900
|
-
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter}
|
|
24741
|
+
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter} coin spendable`);
|
|
24901
24742
|
}
|
|
24902
24743
|
log.plain("");
|
|
24903
24744
|
printRecommendation(log, record);
|
|
@@ -24909,58 +24750,43 @@ function printRecommendation(log, record) {
|
|
|
24909
24750
|
return;
|
|
24910
24751
|
}
|
|
24911
24752
|
if (record.noRecommendationReason === "answer_cut_off") {
|
|
24912
|
-
log.warn(` ${benchCutOffLine(record.cutOffSamples
|
|
24753
|
+
log.warn(` ${benchCutOffLine(record.cutOffSamples ?? 1, record.maxCoinsPerSample ?? 0)}`);
|
|
24913
24754
|
return;
|
|
24914
24755
|
}
|
|
24915
|
-
|
|
24916
|
-
|
|
24917
|
-
log.warn(" No ceiling recommended.");
|
|
24756
|
+
if (record.recommendedEstimateCoins === null || record.p95 === null) {
|
|
24757
|
+
log.warn(" No price recommended.");
|
|
24918
24758
|
log.dim(
|
|
24919
|
-
record.
|
|
24759
|
+
record.recommendedDeclaredHeadroomBps === null ? " This stand served no recommended headroom, and the multiplier is the server's to set \u2014" : " No sample settled, so there is no p95 to build a price on \u2014"
|
|
24920
24760
|
);
|
|
24921
24761
|
log.dim(" declaring a number from this run would be a guess dressed as a measurement.");
|
|
24922
|
-
printAverage(log, record, attempts);
|
|
24923
24762
|
return;
|
|
24924
24763
|
}
|
|
24925
|
-
log.plain(c.bold(` Declare
|
|
24926
|
-
if (record.
|
|
24764
|
+
log.plain(c.bold(` Declare estimateCoins: ${record.recommendedEstimateCoins}`));
|
|
24765
|
+
if (record.lengthRaisedPrice) {
|
|
24927
24766
|
log.plain(` ${LLM_LENGTH_RAISED_LINE}`);
|
|
24928
24767
|
log.dim(
|
|
24929
|
-
` = the smallest
|
|
24768
|
+
` = the smallest price whose answer allowance holds the answer plus this stand's headroom (p95 ${record.fitP95 ?? "\u2014"}), above the`
|
|
24930
24769
|
);
|
|
24931
24770
|
log.dim(
|
|
24932
|
-
`
|
|
24771
|
+
` charged-based ${record.chargedBasedEstimateCoins ?? "\u2014"} (p95 of charged coins ${record.p95} plus ${record.recommendedDeclaredHeadroomBps} bps), over ${record.completed} settled attempt${record.completed === 1 ? "" : "s"} on ${record.modelId}.`
|
|
24933
24772
|
);
|
|
24934
24773
|
} else {
|
|
24935
24774
|
log.dim(
|
|
24936
|
-
` = p95
|
|
24775
|
+
` = p95 of charged coins (${record.p95}) plus this stand's recommended headroom (${record.recommendedDeclaredHeadroomBps} bps),`
|
|
24937
24776
|
);
|
|
24938
|
-
log.dim(` measured over ${
|
|
24777
|
+
log.dim(` measured over ${record.completed} settled attempt${record.completed === 1 ? "" : "s"} on ${record.modelId}.`);
|
|
24939
24778
|
}
|
|
24940
|
-
log.dim(" That number is a
|
|
24941
|
-
|
|
24942
|
-
|
|
24943
|
-
log.plain(c.bold(` Grant perCallMaxCredits: ${record.recommendedPerCallMaxCredits}`));
|
|
24779
|
+
log.dim(" That number is a PRICE: a started attempt is charged in full, including one that fails.");
|
|
24780
|
+
if (record.recommendedPerCallMaxCoins != null) {
|
|
24781
|
+
log.plain(c.bold(` Grant perCallMaxCoins: ${record.recommendedPerCallMaxCoins}`));
|
|
24944
24782
|
log.dim(
|
|
24945
|
-
record.
|
|
24783
|
+
record.lengthRaisedPrice ? ` = the same headroom over the worst sample (${record.max}) or the longest answer's price, whichever is larger, and never below the price above it.` : ` = the same headroom over the worst sample (${record.max}), and never below the price above it.`
|
|
24946
24784
|
);
|
|
24947
|
-
|
|
24948
|
-
printAverage(log, record, attempts);
|
|
24949
|
-
}
|
|
24950
|
-
function printAverage(log, record, attempts) {
|
|
24951
|
-
if (record.averageCreditsPerCall === null) return;
|
|
24952
|
-
log.plain(` Per-call estimate: about ${formatCredits(record.averageCreditsPerCall)} credits per call`);
|
|
24953
|
-
log.dim(` = the average real cost with the platform fee, no headroom, over ${attempts}.`);
|
|
24954
|
-
if (record.recommendedPerCallEstimateCredits !== null) {
|
|
24955
|
-
log.plain(c.bold(` Grant perCallEstimateCredits: ${record.recommendedPerCallEstimateCredits}`));
|
|
24956
|
-
log.dim(` = that average rounded up to a whole credit. disclosure.estimatedCreditsPerPeriod = your calls per period \xD7 ${formatCredits(record.averageCreditsPerCall)}, rounded up.`);
|
|
24785
|
+
log.dim(" The ceiling is the price's room to be wrong; declaring one under the price is refused.");
|
|
24957
24786
|
}
|
|
24958
24787
|
}
|
|
24959
24788
|
async function explainBenchRefusal(res, call, base, log, opts, index, settledSoFar) {
|
|
24960
|
-
if (printedStructuredError(res)) {
|
|
24961
|
-
const printed = await res.json().catch(() => ({}));
|
|
24962
|
-
return { stop: true, refusal: { status: res.status, error: typeof printed.error === "string" ? printed.error : null, slotsHeld: null, slotLimit: null } };
|
|
24963
|
-
}
|
|
24789
|
+
if (printedStructuredError(res)) return { stop: true, refusal: { status: res.status, error: null, slotsHeld: null, slotLimit: null } };
|
|
24964
24790
|
const body = await res.json().catch(() => ({}));
|
|
24965
24791
|
const refusal2 = { status: res.status, error: body.error ?? null, slotsHeld: null, slotLimit: null };
|
|
24966
24792
|
if (res.status === 429 && body.error === "generation_limit") {
|
|
@@ -24970,10 +24796,6 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24970
24796
|
if (!opts.json) log.error(` Sample ${index + 1} ${benchSlotsHeldSentence(counts)}`);
|
|
24971
24797
|
return { stop: true, refusal: refusal2 };
|
|
24972
24798
|
}
|
|
24973
|
-
if (res.status === 403 && body.error === "credits_unverified") {
|
|
24974
|
-
if (!opts.json) log.error(` ${LLM_BENCH_UNVERIFIED_LINE}`);
|
|
24975
|
-
return { stop: true, refusal: refusal2 };
|
|
24976
|
-
}
|
|
24977
24799
|
if (opts.json) return { stop: res.status !== 429, refusal: refusal2 };
|
|
24978
24800
|
if (res.status === 404 && body.error === "not_found") {
|
|
24979
24801
|
log.error(` ${LLM_LANE_OFF_LINE}`);
|
|
@@ -24989,7 +24811,7 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24989
24811
|
return { stop: true, refusal: refusal2 };
|
|
24990
24812
|
}
|
|
24991
24813
|
if (res.status === 402) {
|
|
24992
|
-
log.error(" Not enough
|
|
24814
|
+
log.error(" Not enough coin to start the attempt.");
|
|
24993
24815
|
return { stop: true, refusal: refusal2 };
|
|
24994
24816
|
}
|
|
24995
24817
|
log.error(
|
|
@@ -24998,7 +24820,7 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24998
24820
|
return { stop: res.status >= 500 || res.status === 401 || res.status === 403, refusal: refusal2 };
|
|
24999
24821
|
}
|
|
25000
24822
|
function printProviderRefusal(log, status2, message, awaitingBill = false) {
|
|
25001
|
-
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so
|
|
24823
|
+
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so this call's coin stays held until it resolves (see \`genex llm status\`); it is not a sample.`);
|
|
25002
24824
|
else log.dim(" The provider refused this call at its door \u2014 no inference ran, it cost nothing, and it is not a sample.");
|
|
25003
24825
|
if (message) log.dim(` Provider said: ${message}`);
|
|
25004
24826
|
if (status2 === 401 || status2 === 403) {
|
|
@@ -25024,6 +24846,19 @@ async function pollSettled(call, url, timeoutSec) {
|
|
|
25024
24846
|
}
|
|
25025
24847
|
return last;
|
|
25026
24848
|
}
|
|
24849
|
+
async function readCoinBalance(apiUrl, token) {
|
|
24850
|
+
try {
|
|
24851
|
+
const res = await apiFetch(`${apiUrl}/api/coin/balance`, {
|
|
24852
|
+
headers: { Authorization: `Bearer ${token}` },
|
|
24853
|
+
signal: AbortSignal.timeout(6e3)
|
|
24854
|
+
});
|
|
24855
|
+
if (!res.ok) return null;
|
|
24856
|
+
const body = await res.json().catch(() => null);
|
|
24857
|
+
return typeof body?.spendable === "number" ? body.spendable : null;
|
|
24858
|
+
} catch {
|
|
24859
|
+
return null;
|
|
24860
|
+
}
|
|
24861
|
+
}
|
|
25027
24862
|
async function saveBench(cwd, record) {
|
|
25028
24863
|
try {
|
|
25029
24864
|
const file = path36.join(cwd, LLM_BENCH_FILE);
|
|
@@ -25039,10 +24874,9 @@ function benchFailed(opts, log, message) {
|
|
|
25039
24874
|
else log.error(message);
|
|
25040
24875
|
process.exitCode = 1;
|
|
25041
24876
|
}
|
|
25042
|
-
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
24877
|
+
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCoins, opts) {
|
|
25043
24878
|
return {
|
|
25044
24879
|
command: "llm bench",
|
|
25045
|
-
currency: "credits",
|
|
25046
24880
|
apiUrl,
|
|
25047
24881
|
projectId,
|
|
25048
24882
|
modelId: opts.modelId ?? null,
|
|
@@ -25051,28 +24885,21 @@ function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCre
|
|
|
25051
24885
|
schemaPath: opts.schemaPath ?? null,
|
|
25052
24886
|
samples,
|
|
25053
24887
|
completed: 0,
|
|
25054
|
-
|
|
24888
|
+
maxCoinsPerSample: maxCoins,
|
|
25055
24889
|
balanceBefore: null,
|
|
25056
24890
|
balanceAfter: null,
|
|
25057
24891
|
results: [],
|
|
25058
24892
|
refusal: null,
|
|
25059
|
-
|
|
25060
|
-
|
|
25061
|
-
|
|
25062
|
-
|
|
25063
|
-
recommendedDeclaredHeadroomBps: null,
|
|
25064
|
-
feeBps: null,
|
|
25065
|
-
usdCentsPerCredit: null,
|
|
25066
|
-
missingTerms: null,
|
|
25067
|
-
costBasedMaxCredits: null,
|
|
25068
|
-
lengthRaisedCeiling: false,
|
|
24893
|
+
chargedCoins: { p50: null, p95: null, max: null },
|
|
24894
|
+
fitCoins: { p95: null, max: null },
|
|
24895
|
+
chargedBasedEstimateCoins: null,
|
|
24896
|
+
lengthRaisedPrice: false,
|
|
25069
24897
|
cutOffSamples: 0,
|
|
25070
24898
|
exceedsStandCap: false,
|
|
25071
24899
|
noRecommendationReason: null,
|
|
25072
|
-
|
|
25073
|
-
|
|
25074
|
-
|
|
25075
|
-
recommendedPerCallEstimateCredits: null,
|
|
24900
|
+
recommendedDeclaredHeadroomBps: null,
|
|
24901
|
+
recommendedEstimateCoins: null,
|
|
24902
|
+
recommendedPerCallMaxCoins: null,
|
|
25076
24903
|
savedTo: null
|
|
25077
24904
|
};
|
|
25078
24905
|
}
|
|
@@ -25135,8 +24962,7 @@ async function reportStatus(args) {
|
|
|
25135
24962
|
for (const row of rows) {
|
|
25136
24963
|
const state = rowState(row);
|
|
25137
24964
|
const mark = state === "active" ? c.cyan("\u25CF") : state === "awaiting bill" ? c.yellow("\u25D0") : c.green("\u25CB");
|
|
25138
|
-
const
|
|
25139
|
-
const reserved = money2.reserved === null ? "reserved \u2014" : `${money2.reserved} ${money2.unit} reserved`;
|
|
24965
|
+
const reserved = typeof row.reservedCoins === "number" ? `${row.reservedCoins} coin reserved` : "reserved \u2014";
|
|
25140
24966
|
log.plain(
|
|
25141
24967
|
` ${mark} ${row.id ?? "?"} ${row.modelId ?? "?"} ${state.padEnd(13)} ${formatAge(row.createdAt, now).padStart(4)} old ${reserved}${row.error ? ` ${row.error}` : ""}${rowHoldsSlot(row) ? "" : c.dim(" (no slot)")}`
|
|
25142
24968
|
);
|
|
@@ -25208,11 +25034,10 @@ async function cancelCall(args) {
|
|
|
25208
25034
|
if (!opts.json) {
|
|
25209
25035
|
log.warn(` ${row.id} (${row.status ?? "?"}) \u2014 ${sentence}.`);
|
|
25210
25036
|
if (state === "awaiting bill") {
|
|
25211
|
-
log.dim(` Nothing to cancel:
|
|
25037
|
+
log.dim(` Nothing to cancel: its coin releases when the bill resolves, and it stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`);
|
|
25212
25038
|
if (row.providerMessage?.trim()) log.dim(` Provider said: ${row.providerMessage.trim()}`);
|
|
25213
25039
|
} else {
|
|
25214
|
-
|
|
25215
|
-
log.dim(` ${money2.charged === null ? "charge unknown" : `${money2.charged} ${money2.unit} charged`}${row.error ? ` \xB7 ${row.error}` : ""}.`);
|
|
25040
|
+
log.dim(` ${typeof row.chargedCoins === "number" ? `${row.chargedCoins} coin charged` : "charge unknown"}${row.error ? ` \xB7 ${row.error}` : ""}.`);
|
|
25216
25041
|
}
|
|
25217
25042
|
}
|
|
25218
25043
|
return done("ok", null, state === "awaiting bill" ? "awaiting_bill" : "already_final", row);
|
|
@@ -25234,126 +25059,78 @@ async function cancelCall(args) {
|
|
|
25234
25059
|
const after = await res.json().catch(() => null) ?? row;
|
|
25235
25060
|
if (!opts.json) {
|
|
25236
25061
|
log.success(` Cancelled ${row.id} (was ${row.status ?? "active"}).`);
|
|
25237
|
-
log.dim(`
|
|
25062
|
+
log.dim(` Its coin releases once the bill settles; ${c.cyan("genex llm status")} shows it until then.`);
|
|
25238
25063
|
}
|
|
25239
25064
|
return done("ok", null, "cancelled", after);
|
|
25240
25065
|
}
|
|
25241
|
-
async function
|
|
25242
|
-
let
|
|
25066
|
+
async function reportSavedPrice(cwd, opts, log) {
|
|
25067
|
+
let record = null;
|
|
25243
25068
|
try {
|
|
25244
|
-
|
|
25069
|
+
record = JSON.parse(await fs35.readFile(path36.join(cwd, LLM_BENCH_FILE), "utf8"));
|
|
25245
25070
|
} catch {
|
|
25246
|
-
|
|
25071
|
+
record = null;
|
|
25247
25072
|
}
|
|
25248
|
-
if (!
|
|
25249
|
-
const v = parsed;
|
|
25250
|
-
if (v.v === 2 && v.currency === "credits" && v.cost && v.charged) return { kind: "credits", record: parsed };
|
|
25251
|
-
return { kind: "coins", record: parsed };
|
|
25252
|
-
}
|
|
25253
|
-
var LLM_PRICE_COIN_TERMS_LINE = "This bench was measured under the old COIN terms, when a game declared a fixed price per call. In-game calls are now billed in the player's credits, as used, and a game declares a per-call CEILING (maxCredits) instead \u2014 a coin price is not that number. Re-run the bench for the figures to declare.";
|
|
25254
|
-
async function reportSavedPrice(cwd, opts, log) {
|
|
25255
|
-
const saved = await readSavedBench(cwd);
|
|
25256
|
-
const rerun = 'genex llm bench "<prompt>" --max-credits <n> --user-approved';
|
|
25257
|
-
const creditKeys = (record2) => ({
|
|
25258
|
-
costUsdPicos: record2?.cost ?? { p50: null, p95: null, max: null },
|
|
25259
|
-
chargedCredits: record2?.charged ?? { p50: null, p95: null, max: null },
|
|
25260
|
-
fitP95: record2?.fitP95 ?? null,
|
|
25261
|
-
fitMax: record2?.fitMax ?? null,
|
|
25262
|
-
recommendedDeclaredHeadroomBps: record2?.recommendedDeclaredHeadroomBps ?? null,
|
|
25263
|
-
feeBps: record2?.feeBps ?? null,
|
|
25264
|
-
usdCentsPerCredit: record2?.usdCentsPerCredit ?? null,
|
|
25265
|
-
costBasedMaxCredits: record2?.costBasedMaxCredits ?? null,
|
|
25266
|
-
lengthRaisedCeiling: record2?.lengthRaisedCeiling ?? false,
|
|
25267
|
-
cutOffSamples: record2?.cutOffSamples ?? 0,
|
|
25268
|
-
noRecommendationReason: record2?.noRecommendationReason ?? null,
|
|
25269
|
-
recommendedMaxCredits: record2?.recommendedMaxCredits ?? null,
|
|
25270
|
-
recommendedPerCallMaxCredits: record2?.recommendedPerCallMaxCredits ?? null,
|
|
25271
|
-
averageCreditsPerCall: record2?.averageCreditsPerCall ?? null,
|
|
25272
|
-
recommendedPerCallEstimateCredits: record2?.recommendedPerCallEstimateCredits ?? null
|
|
25273
|
-
});
|
|
25274
|
-
if (!saved) {
|
|
25073
|
+
if (!record) {
|
|
25275
25074
|
if (opts.json) {
|
|
25276
25075
|
writeJsonLine({
|
|
25277
25076
|
command: "llm price",
|
|
25278
25077
|
status: "failed",
|
|
25279
25078
|
error: "no_bench",
|
|
25280
|
-
currency: null,
|
|
25281
25079
|
ranAt: null,
|
|
25282
25080
|
modelId: null,
|
|
25283
25081
|
samples: null,
|
|
25284
25082
|
completed: null,
|
|
25285
|
-
|
|
25286
|
-
|
|
25083
|
+
p50: null,
|
|
25084
|
+
p95: null,
|
|
25085
|
+
max: null,
|
|
25086
|
+
fitP95: null,
|
|
25087
|
+
chargedBasedEstimateCoins: null,
|
|
25088
|
+
lengthRaisedPrice: false,
|
|
25089
|
+
cutOffSamples: 0,
|
|
25090
|
+
noRecommendationReason: null,
|
|
25091
|
+
recommendedDeclaredHeadroomBps: null,
|
|
25092
|
+
recommendedEstimateCoins: null,
|
|
25093
|
+
recommendedPerCallMaxCoins: null,
|
|
25287
25094
|
savedTo: null
|
|
25288
25095
|
});
|
|
25289
25096
|
} else {
|
|
25290
25097
|
log.error("Nothing has been benchmarked in this folder yet.");
|
|
25291
|
-
log.dim(` Run ${c.cyan(
|
|
25292
|
-
}
|
|
25293
|
-
process.exitCode = 1;
|
|
25294
|
-
return;
|
|
25295
|
-
}
|
|
25296
|
-
if (saved.kind === "coins") {
|
|
25297
|
-
const old = saved.record;
|
|
25298
|
-
const legacy = {
|
|
25299
|
-
p50: old.p50 ?? null,
|
|
25300
|
-
p95: old.p95 ?? null,
|
|
25301
|
-
max: old.max ?? null,
|
|
25302
|
-
recommendedEstimateCoins: old.recommendedEstimateCoins ?? null,
|
|
25303
|
-
recommendedPerCallMaxCoins: old.recommendedPerCallMaxCoins ?? null,
|
|
25304
|
-
maxCoinsPerSample: old.maxCoinsPerSample ?? null,
|
|
25305
|
-
noRecommendationReason: old.noRecommendationReason ?? null
|
|
25306
|
-
};
|
|
25307
|
-
if (opts.json) {
|
|
25308
|
-
writeJsonLine({
|
|
25309
|
-
command: "llm price",
|
|
25310
|
-
status: "failed",
|
|
25311
|
-
error: "coin_terms",
|
|
25312
|
-
currency: "coins",
|
|
25313
|
-
ranAt: old.ranAt ?? null,
|
|
25314
|
-
modelId: old.modelId ?? null,
|
|
25315
|
-
samples: old.samples ?? null,
|
|
25316
|
-
completed: old.completed ?? null,
|
|
25317
|
-
...creditKeys(null),
|
|
25318
|
-
legacyCoinTerms: legacy,
|
|
25319
|
-
savedTo: LLM_BENCH_FILE
|
|
25320
|
-
});
|
|
25321
|
-
} else {
|
|
25322
|
-
log.plain(c.bold("genex llm price"));
|
|
25323
|
-
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${old.ranAt ?? "at an unknown time"}${old.modelId ? ` on ${old.modelId}` : ""}`);
|
|
25324
|
-
log.plain("");
|
|
25325
|
-
log.warn(` ${LLM_PRICE_COIN_TERMS_LINE}`);
|
|
25326
|
-
const measured = legacy.p95 === null ? "no settled sample" : `charged coins p50 ${legacy.p50} \xB7 p95 ${legacy.p95} \xB7 max ${legacy.max}`;
|
|
25327
|
-
const recommended = legacy.recommendedEstimateCoins === null ? "no price recommended" : `it recommended estimateCoins ${legacy.recommendedEstimateCoins}${legacy.recommendedPerCallMaxCoins != null ? ` and perCallMaxCoins ${legacy.recommendedPerCallMaxCoins}` : ""}`;
|
|
25328
|
-
log.dim(` What it measured then: ${measured}; ${recommended}.`);
|
|
25329
|
-
log.dim(` Re-run: ${c.cyan(rerun)}`);
|
|
25098
|
+
log.dim(` Run ${c.cyan('genex llm bench "<prompt>" --max-coins <n> --user-approved')} first.`);
|
|
25330
25099
|
}
|
|
25331
25100
|
process.exitCode = 1;
|
|
25332
25101
|
return;
|
|
25333
25102
|
}
|
|
25334
|
-
const record = saved.record;
|
|
25335
25103
|
if (opts.json) {
|
|
25336
25104
|
writeJsonLine({
|
|
25337
25105
|
command: "llm price",
|
|
25338
|
-
status: record.
|
|
25339
|
-
error: record.
|
|
25340
|
-
currency: "credits",
|
|
25106
|
+
status: record.recommendedEstimateCoins === null ? "failed" : "ok",
|
|
25107
|
+
error: record.recommendedEstimateCoins === null ? record.noRecommendationReason ?? "no_recommendation" : null,
|
|
25341
25108
|
ranAt: record.ranAt ?? null,
|
|
25342
25109
|
modelId: record.modelId ?? null,
|
|
25343
25110
|
samples: record.samples ?? null,
|
|
25344
25111
|
completed: record.completed ?? null,
|
|
25345
|
-
|
|
25346
|
-
|
|
25112
|
+
p50: record.p50 ?? null,
|
|
25113
|
+
p95: record.p95 ?? null,
|
|
25114
|
+
max: record.max ?? null,
|
|
25115
|
+
// Absent from a file an older CLI wrote; null / false / 0 then, never missing.
|
|
25116
|
+
fitP95: record.fitP95 ?? null,
|
|
25117
|
+
chargedBasedEstimateCoins: record.chargedBasedEstimateCoins ?? null,
|
|
25118
|
+
lengthRaisedPrice: record.lengthRaisedPrice ?? false,
|
|
25119
|
+
cutOffSamples: record.cutOffSamples ?? 0,
|
|
25120
|
+
noRecommendationReason: record.noRecommendationReason ?? null,
|
|
25121
|
+
recommendedDeclaredHeadroomBps: record.recommendedDeclaredHeadroomBps ?? null,
|
|
25122
|
+
recommendedEstimateCoins: record.recommendedEstimateCoins ?? null,
|
|
25123
|
+
recommendedPerCallMaxCoins: record.recommendedPerCallMaxCoins ?? null,
|
|
25347
25124
|
savedTo: LLM_BENCH_FILE
|
|
25348
25125
|
});
|
|
25349
|
-
if (record.
|
|
25126
|
+
if (record.recommendedEstimateCoins === null) process.exitCode = 1;
|
|
25350
25127
|
return;
|
|
25351
25128
|
}
|
|
25352
25129
|
log.plain(c.bold("genex llm price"));
|
|
25353
25130
|
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${record.ranAt}`);
|
|
25354
25131
|
log.plain("");
|
|
25355
25132
|
printRecommendation(log, record);
|
|
25356
|
-
if (record.
|
|
25133
|
+
if (record.recommendedEstimateCoins === null) process.exitCode = 1;
|
|
25357
25134
|
}
|
|
25358
25135
|
function writeJsonLine(value) {
|
|
25359
25136
|
process.stdout.write(`${JSON.stringify(value)}
|
|
@@ -25613,7 +25390,7 @@ function runtimeRow(state, lane, toolsOnly = false) {
|
|
|
25613
25390
|
return {
|
|
25614
25391
|
label: "Runtime",
|
|
25615
25392
|
value: `in-game LLM lane live \xB7 ${n} model${n === 1 ? "" : "s"}${keyRefused ? ` \xB7 ${c.yellow("provider key refused")}` : ""}`,
|
|
25616
|
-
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : '
|
|
25393
|
+
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : 'Price a call before a game declares one: `npx genex llm bench "<prompt>" --max-coins <n> --user-approved`.'
|
|
25617
25394
|
};
|
|
25618
25395
|
}
|
|
25619
25396
|
case "off":
|
|
@@ -27569,16 +27346,16 @@ ${c.bold("Usage")}
|
|
|
27569
27346
|
status | cancel. "models" says whether this
|
|
27570
27347
|
stand serves the runtime LLM lane at all, and
|
|
27571
27348
|
at what provider rates. "bench" runs the real
|
|
27572
|
-
model on YOUR OWN
|
|
27573
|
-
attempt
|
|
27349
|
+
model on YOUR OWN coin and reports what each
|
|
27350
|
+
attempt was charged; it needs --max-coins <n>
|
|
27574
27351
|
--user-approved, like any spend the player has
|
|
27575
27352
|
to agree to. "price" reprints the last run's
|
|
27576
|
-
recommended
|
|
27353
|
+
recommended estimateCoins. "status" lists this
|
|
27577
27354
|
project's open calls and the slots they hold;
|
|
27578
27355
|
"cancel <id>" stops an active one. Declare a
|
|
27579
|
-
game's
|
|
27580
|
-
|
|
27581
|
-
|
|
27356
|
+
game's price from a bench, never from a guess:
|
|
27357
|
+
a started attempt is charged in full, including
|
|
27358
|
+
one that fails.
|
|
27582
27359
|
genex player <sub> [options] Answer the in-game model requests you approve, on your
|
|
27583
27360
|
OWN Claude or ChatGPT subscription: install | run |
|
|
27584
27361
|
status | stop | uninstall. "install" signs this machine
|
|
@@ -28040,7 +27817,7 @@ ${c.bold("Examples")}
|
|
|
28040
27817
|
genex shop remove sku_123
|
|
28041
27818
|
genex llm models
|
|
28042
27819
|
genex llm models --all
|
|
28043
|
-
genex llm bench "Reply with one short taunt." --samples 5 --max-
|
|
27820
|
+
genex llm bench "Reply with one short taunt." --samples 5 --max-coins 20 --user-approved
|
|
28044
27821
|
genex llm price
|
|
28045
27822
|
genex llm status
|
|
28046
27823
|
genex llm cancel <id>
|
|
@@ -28277,24 +28054,22 @@ ${c.bold("How the allowance works")}
|
|
|
28277
28054
|
|
|
28278
28055
|
Every number printed is live from your account; prices can change without a deploy.
|
|
28279
28056
|
`,
|
|
28280
|
-
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what
|
|
28057
|
+
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what should the game declare?
|
|
28281
28058
|
|
|
28282
28059
|
${c.bold("Usage")}
|
|
28283
28060
|
genex llm models [--all] [--json] Which models this stand serves, their provider rates per
|
|
28284
|
-
million tokens, the
|
|
28285
|
-
|
|
28286
|
-
|
|
28287
|
-
|
|
28288
|
-
|
|
28289
|
-
|
|
28290
|
-
|
|
28291
|
-
|
|
28292
|
-
|
|
28293
|
-
|
|
28294
|
-
|
|
28295
|
-
|
|
28296
|
-
allowance does not count. Needs a folder linked to a
|
|
28297
|
-
game you own, and a balance that covers one attempt.
|
|
28061
|
+
million tokens, and the headroom it recommends over a
|
|
28062
|
+
benchmark. The catalog is synced from OpenRouter, so the
|
|
28063
|
+
FEATURED short list is printed by default and --all
|
|
28064
|
+
prints every row (--json always carries every row). Off
|
|
28065
|
+
on this stand: it says so and exits 0 \u2014 build the feature
|
|
28066
|
+
without a model rather than promising a 404.
|
|
28067
|
+
genex llm bench "<prompt>" --max-coins <n> --user-approved [options]
|
|
28068
|
+
Run the real model <n> times through the development
|
|
28069
|
+
lane and report what each attempt was CHARGED. Refused
|
|
28070
|
+
without both flags, before any network call: this spends
|
|
28071
|
+
YOUR OWN COIN, a wallet the build's asset allowance
|
|
28072
|
+
cannot see. Needs a folder linked to a game you own.
|
|
28298
28073
|
genex llm price [--json] The recommendation from the last bench in this folder.
|
|
28299
28074
|
Reads a file; spends nothing.
|
|
28300
28075
|
genex llm status [--json] This project's open development calls: active, or
|
|
@@ -28311,32 +28086,29 @@ ${c.bold("Bench options")}
|
|
|
28311
28086
|
--samples <n> How many real attempts (default ${DEFAULT_BENCH_SAMPLES}). More samples, better p95.
|
|
28312
28087
|
--schema <file> A JSON file holding the output schema; implies JSON output.
|
|
28313
28088
|
--json-output Ask for JSON without a schema. --text Ask for text (the default).
|
|
28314
|
-
--max-
|
|
28089
|
+
--max-coins <n> The per-attempt ceiling YOU approved \u2014 a bench runs on your own coin,
|
|
28315
28090
|
never a player's. The run's worst case is that number times --samples,
|
|
28316
|
-
and it is printed before anything starts.
|
|
28317
|
-
name, read as the same number.)
|
|
28091
|
+
and it is printed before anything starts.
|
|
28318
28092
|
--timeout <seconds> Bounds the wait for each attempt.
|
|
28319
28093
|
--json One machine-readable object; every key is always present, null when
|
|
28320
28094
|
the CLI could not learn it.
|
|
28321
28095
|
|
|
28322
28096
|
${c.bold("Why a benchmark and not an estimate")}
|
|
28323
|
-
|
|
28324
|
-
|
|
28325
|
-
|
|
28326
|
-
|
|
28327
|
-
|
|
28328
|
-
|
|
28329
|
-
|
|
28330
|
-
|
|
28331
|
-
|
|
28332
|
-
recommendation, because those multipliers are not the CLI's to invent.
|
|
28097
|
+
\`estimateCoins\` is a PRICE, not a guess: declaring it charges it, in full, even when the
|
|
28098
|
+
attempt fails, is cancelled, or stops at its budget. And three layers sit between the
|
|
28099
|
+
provider's rate card and that number \u2014 the provider's cost, the platform's tariff (already
|
|
28100
|
+
inside every charged coin), and your headroom. The bench itself is billed to YOU, on the
|
|
28101
|
+
development lane; the player-funded lane is what the published game uses. Adding a margin
|
|
28102
|
+
to the provider's rate silently drops the tariff and under-prices every call, so the
|
|
28103
|
+
recommendation is built from CHARGED COINS and the headroom comes from the server. A stand
|
|
28104
|
+
that serves no headroom gets no recommendation, because the multiplier is not the CLI's to
|
|
28105
|
+
invent.
|
|
28333
28106
|
|
|
28334
|
-
|
|
28335
|
-
|
|
28336
|
-
A sample cut off at your --max-
|
|
28337
|
-
|
|
28338
|
-
|
|
28339
|
-
fee included, which is the honest estimate a budget's disclosure is built on.
|
|
28107
|
+
The declared price also decides how LONG the answer may be: it sizes each call's output
|
|
28108
|
+
allowance. So the recommendation is never below the smallest price that leaves room for
|
|
28109
|
+
the benchmarked answer, as the server computes it. A sample cut off at your --max-coins
|
|
28110
|
+
(provider_token_limit) is not a sample \u2014 re-run with a higher --max-coins; an answer longer
|
|
28111
|
+
than one call on this stand may produce gets no price at all \u2014 ask for a shorter one.
|
|
28340
28112
|
|
|
28341
28113
|
Results land in ${LLM_BENCH_FILE} \u2014 gitignored, machine-local, re-read by \`genex llm price\`.
|
|
28342
28114
|
`,
|
|
@@ -28551,13 +28323,11 @@ function parseArgs(argv) {
|
|
|
28551
28323
|
"--limit",
|
|
28552
28324
|
// `genex budget --assets <credits>` — the allowance the player approved.
|
|
28553
28325
|
"--assets",
|
|
28554
|
-
// `genex llm bench` value flags. `--max-
|
|
28555
|
-
// shape as `--assets
|
|
28556
|
-
// count a bench. `--max-coins` is its deprecated name (same number).
|
|
28326
|
+
// `genex llm bench` value flags. `--max-coins` is the COIN half of the
|
|
28327
|
+
// same approval shape as `--assets` (a different economy, its own gate).
|
|
28557
28328
|
"--model",
|
|
28558
28329
|
"--samples",
|
|
28559
28330
|
"--schema",
|
|
28560
|
-
"--max-credits",
|
|
28561
28331
|
"--max-coins",
|
|
28562
28332
|
// `genex player install --label <name>` — what this machine is called in
|
|
28563
28333
|
// the dashboard. Its own key rather than the generic 2nd positional,
|
|
@@ -29075,14 +28845,12 @@ function applyValueFlag(options, flag, value) {
|
|
|
29075
28845
|
options.samples = n;
|
|
29076
28846
|
break;
|
|
29077
28847
|
}
|
|
29078
|
-
case "--max-credits":
|
|
29079
28848
|
case "--max-coins": {
|
|
29080
28849
|
const n = Number(value);
|
|
29081
28850
|
if (!Number.isInteger(n) || n < 1) {
|
|
29082
|
-
throw new Error(`Invalid
|
|
28851
|
+
throw new Error(`Invalid --max-coins value: ${value} (whole coin, 1 or more)`);
|
|
29083
28852
|
}
|
|
29084
|
-
|
|
29085
|
-
else options.maxCoins = n;
|
|
28853
|
+
options.maxCoins = n;
|
|
29086
28854
|
break;
|
|
29087
28855
|
}
|
|
29088
28856
|
case "--timeout": {
|
|
@@ -29380,9 +29148,9 @@ async function main() {
|
|
|
29380
29148
|
await runBudget(parsed.options);
|
|
29381
29149
|
break;
|
|
29382
29150
|
// The in-game model lane: is it live here, what does one call really
|
|
29383
|
-
// cost, what
|
|
29384
|
-
//
|
|
29385
|
-
//
|
|
29151
|
+
// cost, what should the game declare. `bench` is the only CLI command
|
|
29152
|
+
// that spends COIN, so it carries its own --max-coins/--user-approved
|
|
29153
|
+
// gate — the credits allowance cannot see that wallet.
|
|
29386
29154
|
case "llm":
|
|
29387
29155
|
await runLlm(parsed.options);
|
|
29388
29156
|
break;
|