@genex-ai/cli-demo 1.36.15 → 1.36.16-dev.1039
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{blender-mcp-QVEGHE5L.js → blender-mcp-WSURPZET.js} +2 -2
- package/dist/{blender-serve-66OHC4ML.js → blender-serve-EH2MOW3U.js} +1 -1
- package/dist/{chunk-4BBTPRJW.js → chunk-EABIVAGT.js} +2 -2
- package/dist/{chunk-QI7FIYBY.js → chunk-K4EHXOZK.js} +1 -1
- package/dist/index.js +489 -239
- package/package.json +1 -1
- package/templates/skills/genex-game-director/SKILL.md +3 -3
- package/templates/skills/genex-getting-started/SKILL.md +2 -2
- package/templates/skills/genex-items/SKILL.md +27 -10
- package/templates/skills/genex-llm-in-games/SKILL.md +210 -73
- package/templates/skills/genex-llm-in-games/references/pricing.md +126 -85
- package/templates/skills/genex-monetization/SKILL.md +114 -86
- package/templates/skills/genex-tool-llm/SKILL.md +11 -9
- package/templates/skills/genex-updates/SKILL.md +1 -1
package/dist/index.js
CHANGED
|
@@ -57,7 +57,7 @@ import {
|
|
|
57
57
|
writeSecretFile,
|
|
58
58
|
writeUserToken,
|
|
59
59
|
writeWorkspace
|
|
60
|
-
} from "./chunk-
|
|
60
|
+
} from "./chunk-EABIVAGT.js";
|
|
61
61
|
import {
|
|
62
62
|
CLI_CHANNEL,
|
|
63
63
|
DEFAULT_API_URL,
|
|
@@ -78,7 +78,7 @@ import {
|
|
|
78
78
|
normalizeApiOrigin,
|
|
79
79
|
originKey,
|
|
80
80
|
resolveAgentTargets
|
|
81
|
-
} from "./chunk-
|
|
81
|
+
} from "./chunk-K4EHXOZK.js";
|
|
82
82
|
|
|
83
83
|
// src/instrument.ts
|
|
84
84
|
import { setTimeout as sleep } from "timers/promises";
|
|
@@ -1129,12 +1129,12 @@ Important note: put soul into your creations, with many details and love. Aim to
|
|
|
1129
1129
|
16. Never add debug-only code that fakes gameplay \u2014 no hidden test modes, special URL parameters, forced-visible flags or auth mocks. The visual-validation skill permits a development-only final-mix analyser for a deliberately triggered audible event; remove it from published builds. The \`genex-cover\` skill's development-only cover shot (\`?cover=<name>\` on the dev server) is a camera, not a bypass: it never reaches a published build, never touches sign-in, saves or progress, and is never evidence that the game works. \`?genex_local_test=1\` is the platform's own supported mode and is fine; your own bypass is not. (The multiplayer skill's small build identifier, token-free status line, and connected-quorum watchdog are production supportability, not a bypass \u2014 keep those.)
|
|
1130
1130
|
17. Input directions match their labels: A/\u2190 moves or turns the player screen-LEFT, D/\u2192 screen-RIGHT, mouse-up looks up, and drag-pan axes share ONE convention. The cursor is either the gameplay tool (RTS, card, builder) or locked away during play \u2014 keyboard-only games included. Check it in every milestone's smoke pass.
|
|
1131
1131
|
18. NEVER delete, empty, move, rename, or overwrite anything you did not create yourself. This folder may hold the player's own reference images, notes, sketches, or an earlier attempt \u2014 files that exist nowhere else and have no undo, no trash, no backup. A non-empty folder is normal and is NEVER something to clean up, and "start clean" is never a reason. That rules out \`rm\`/\`rm -rf\`, \`git clean\`, \`git checkout -- .\`, \`git reset --hard\` over their work, deleting to resolve a conflict or a stuck interactive prompt, and every setup tool's offer to empty a directory (\`--force\`, \`--overwrite\`, "Remove existing files") \u2014 scaffold into a fresh subfolder and copy in instead. You may add files and edit the ones you wrote. If a step genuinely cannot continue without removing something of theirs, STOP and ask, naming the exact files, and wait for a yes \u2014 "it looks like junk" is never that yes. This binds hardest during setup, where it runs fast and automatically before the player has asked for anything at all.
|
|
1132
|
-
19. If the game sells anything, it sells it for
|
|
1132
|
+
19. If the game sells anything for real money, it sells it for Nex \u2014 Genex's own currency, which players buy with real money and spend in the Genex store \u2014 at a fixed, visible price, and NEVER sells chance. A price in Nex, or a request to sell something for real money that names no currency ("paid characters", "sell skins"), means a Genex store item: build it into the game, then list it with \`npx genex shop add \u2026 --json\` without asking (\`$genex-items\`); a price in coins, gold, gems or any other name is a currency of the game's own, earned by playing. Run this test on any purchasable thing before building it: does the player pay (with Nex, or with anything Nex bought, directly or indirectly), is the outcome uncertain when they pay, and is there a prize they wanted \u2014 all three yes means it is paid randomness, and you build the deterministic version instead. That rules out loot boxes, gacha, mystery boxes, crates, card packs, prize wheels and raffles; wagering, staking, betting, coinflips and casino or slot mechanics denominated in Nex; and donation prompts, tip jars or any player-to-player Nex transfer, because Nex buys goods and never just moves. Randomness the player EARNS by playing is gameplay, not commerce \u2014 an enemy dropping a random item, a chest found in the level, a procedural layout, a crit roll \u2014 and is completely fine. Every Nex price renders its real-money equivalent beside it (the server sends one with every item), item prices sit on the platform's price grid so no player is left holding change they cannot spend, and nothing in a shop carries a countdown, "limited time", or a stock counter. When a request crosses one of these lines, name the mechanic, give the one-sentence reason, propose a specific compliant alternative, and build that \u2014 never the banned version "as an option", never a partial one, and never after asking the player to confirm they want it. Load \`$genex-monetization\` before building a shop.
|
|
1133
1133
|
20. When the player names a specific thing to integrate \u2014 a library, a controller, an asset, a part of an existing game, or a tagged reference the platform placed in \`.genex/refs/<slug>/\` \u2014 integrate the real thing: read its source first and build from it, never a lighter look-alike written from scratch because it was faster. A tagged reference owns the subsystem it was tagged for, outranking the default lane and the vendored controllers for that subsystem. If you genuinely can't use it, say so in one plain line and ask how to proceed.
|
|
1134
1134
|
21. A turn ends in exactly one of two ways: on a question the player must answer before the next step can be chosen, or on a handoff \u2014 the draft link, what changed, what to try, and what is still open. Never close a turn on a promise. "I'll keep building", "while I keep working", "adding that now" are things you say and then DO before you stop; if you are stopping, say that you have stopped and what remains. While the Build plan's \`Now:\` line has open work and no answer is needed from the player, the turn is not over: preview, hand off, and start the next milestone in the same turn, until the requested outcome is reached or the platform's own budget ends the session. If a stop was forced on you anyway, the first line of your next turn is the promise you left and where it stands.
|
|
1135
1135
|
22. **What the player sees must read as the thing it is**, in this game's own style: a building as that building, a person as a person, a prop as that prop, a surface as its material \u2014 and a coloured box, a capsule, or a flat grey block standing in for one of them is never its finished version. Generation is the default route to that bar wherever code will not honestly reach it: the player's character, whatever the request names, whatever the player walks up to, enters, or interacts with, and the music and sound underneath \u2014 reach for the belt on your own judgment, without stopping to ask; a status line saying what you queued is enough, and \`--no-wait\` keeps you building while it lands. Procedural code is a first-class engine for what is structural, repeated, distant, or parametric \u2014 terrain, sky, fences, paving, modular kits, filler \u2014 and for anything else only when the result meets the same bar and you have looked at it in a capture before its row says \`landed\`. Start with one reusable implementation per distinct required gameplay or visual role. Background populations default to reusable procedural bodies and motion or compatible existing rigged assets. Catalog clips need a compatible skeleton and may be paid. Reserve paid custom bodies for the player and characters examined or interacted with closely, while honoring explicit user requests. Add variants only for an unmet requirement; unspent allowance is not unfinished work. Allowance questions use timeoutPolicy no-consent: silence does not raise the allowance. Record role, reuse and completion criteria in the existing Assets table. Decide per object, not per category, and write the route on each Assets row in \`DESIGN.md\` \u2014 the paid flow for generated pieces, the procedural flow with its local path for code-built ones. An Assets table with no generation in it is a decision, not a default: record it as \`Generation: none \u2014 <why code alone reaches the bar here>\` or generate.
|
|
1136
1136
|
23. **The screenshots you take to look at your own work go in \`.genex/scratch/\`.** Captures, render comparisons, before/after strips, traces and metrics dumps are how you SEE what you built \u2014 they are not part of what you built, and \`genex preview\` pushes the whole folder to the game's source, every time, forever. \`.genex/scratch/\` already exists for this and is already ignored, so there is nothing to set up: write the capture as \`.genex/scratch/arena-before.png\` and read it back from there. Never put them in a folder of your own at the top level: \`progress/\`, \`reports/\`, \`shots/\` and their kind are pushed like source and become permanent weight in the game and in every remix of it \u2014 one real game reached 11 GB and 10,332 committed screenshots exactly this way. If you find such a folder already there, add it to \`.gitignore\`; never delete the player's files (law 18). One file in \`.genex/scratch/\` IS read, by the platform: \`.genex/scratch/cover.png\` is the game's cover \u2014 the frame you would put on its poster: real gameplay at its signature moment, well lit, no menus, popups or busy HUD, landscape 16:9, and never any text (the page sets the title beside it). The next \`genex preview\` makes it the cover; save a better moment over it whenever one comes along, and if your browser tool can set the cover itself, use that instead. Before the first publish, and whenever the game's look changes, load \`$genex-cover\`: it chooses, stages, lights and judges that frame.
|
|
1137
|
-
24. **Benchmark before declaring an in-game LLM
|
|
1137
|
+
24. **Benchmark before declaring an in-game LLM ceiling.** If the game calls a language model while the player plays \u2014 an NPC answering in its own words, a quest written for this save, a judge reading what the player typed \u2014 the PLAYER pays for it from their Genex credits, billed as used, and the game declares no price, only a per-call CEILING: \`maxCredits\` on \`generate()\`, \`perCallMaxCredits\` on a standing budget. That ceiling bounds what one call may cost AND funds how long its answer may be, so never pick it by judgement or from a vendor's rate card. Run the real prompt on your own credits \u2014 \`npx genex llm bench "<the prompt>" --schema <file> --samples 3 --max-credits <n> --user-approved\` \u2014 and declare the figures it prints, recorded in \`DESIGN.md\` with its model and date; re-benchmark whenever the prompt, the schema or the model changes, because a prompt edit changes the ceiling. Check the lane first with \`npx genex llm models\`: where it is off the routes 404, so build the feature behind a graceful unavailable state and say so in the handoff. A repeated call is a standing budget the player approves once (\`requestSpendGrant()\`, then \`generate({ grantId })\`), never a popup per turn, and its disclosed call rate and credits per period are computed from the game's own loop and the bench's measured average. The coin spellings (\`estimateCoins\`, \`perCallMaxCoins\`) are deprecated \u2014 write the credit names. Model output is data the game validates against its own expectation \u2014 never executed, and never authority over Nex, credits, items, entitlements or rewards. Load \`$genex-llm-in-games\` before writing any of it.
|
|
1138
1138
|
${CONTRACT_END}
|
|
1139
1139
|
`;
|
|
1140
1140
|
var TOOLS_CONTRACT_HEAD = `# Genex Tools (always in effect in this folder)
|
|
@@ -1148,7 +1148,7 @@ var TOOLS_CONTRACT_OFFER = `5. **When the game is built and playable \u2014 neve
|
|
|
1148
1148
|
var TOOLS_CONTRACT_HOSTED_LAWS = `5. **This game is hosted on Genex now, and publishing is glue \u2014 never a rebuild.** The game ships exactly as the user built it: \`initEmbed()\` first in the boot code (the \`genex-threejs-embed-auth\` card has the call; the renderer draws before any \`await waitForPlayer()\`), a static build into \`dist/\` with relative asset paths, then \`npx genex preview\`. Never restructure, reformat or "improve" the game to ship it, never scaffold a new app around it, and never start a design document or a build plan for it \u2014 the user's own process is theirs. Load the \`genex-tool-publish\` card before the first preview; it owns the vocabulary, the links and the limits.
|
|
1149
1149
|
6. **Ship first, then report.** \`preview\` prints a preflight \u2014 phone memory, a missing volume slider, an asset nobody wired, a viewport line. It never blocks a deploy, so push the build as it is, hand over the link, THEN relay each preflight line to the user in one plain sentence with an offer to fix it, and fix one only on their yes. The preflight is a report for the user, not a to-do list for you.
|
|
1150
1150
|
7. **The link is the game's page, and there are two versions.** After every preview give the user \`<dashboard>/draft/<slug>\` \u2014 \`dashboardOrigins[0]\` and \`slug\` from \`.genex/project.json\` \u2014 never localhost, a file path or the bare play origin. \`preview\` updates the draft and never touches what players are on; the first release is \`npx genex publish\` (it lists the game); after that "publish it", "update it" and "yes" all mean \`npx genex promote\` \u2014 the exact draft build, no rebuild. Ask once per round of work, in one line, and keep working while you wait.`;
|
|
1151
|
-
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with Genex
|
|
1151
|
+
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with their Genex credits or their own Claude/ChatGPT subscription, and approves it on a Genex sheet; your game just calls \`generate()\`. Want it that way?`;
|
|
1152
1152
|
function toolsContractLlmLaw(n, hosted) {
|
|
1153
1153
|
const onYes = hosted ? "On a yes, load `$genex-tool-llm` and then `$genex-llm-in-games`, which owns the build." : "On a yes, load `$genex-tool-llm`, which owns the rest: it checks the lane with `npx genex llm models` first, then runs `npx genex init --convert` (a hosted game is a static browser build \u2014 a local server holding a key can never ship).";
|
|
1154
1154
|
return `${n}. **A model running while people PLAY is a platform feature \u2014 never something you wire onto the user's own meter.** When a request implies one (NPCs that talk or decide in their own words, content written from what the player types, a prompt box in the game, "let the player pick a model"), recognise it, say in ONE line: "${TOOLS_CONTRACT_LLM_OFFER}" \u2014 then ASK and wait. ${onYes} On a no, build the authored version instead. Either way, never ship a game that calls a model on a key or an account of the user's own: every visitor would spend their money with nobody approving it, and the credential is readable in the bundle. Player-funded generation is text and JSON only \u2014 3D, images, video and audio stay the asset lanes above, on the user's meter.`;
|
|
@@ -8350,7 +8350,7 @@ function checkFields(raw, label) {
|
|
|
8350
8350
|
if (description !== void 0 && (typeof description !== "string" || description.trim() === "" || description !== description.trim())) {
|
|
8351
8351
|
return bad("its description is not a trimmed, non-empty text");
|
|
8352
8352
|
}
|
|
8353
|
-
if (typeof priceCoins !== "number" || !Number.isSafeInteger(priceCoins) || priceCoins < 0) return bad("its price is not a whole
|
|
8353
|
+
if (typeof priceCoins !== "number" || !Number.isSafeInteger(priceCoins) || priceCoins < 0) return bad("its price is not a whole number of Nex, 0 or more");
|
|
8354
8354
|
if (type !== ItemType.Consumable && type !== ItemType.Durable) return bad("its type is neither consumable nor durable");
|
|
8355
8355
|
if (!Array.isArray(tags2) || !tags2.every((tag) => typeof tag === "string")) return bad("its tags are not a list of words");
|
|
8356
8356
|
if (typeof shareable !== "boolean") return bad("its sharing is not true or false");
|
|
@@ -8738,12 +8738,12 @@ function formatCoins(coins) {
|
|
|
8738
8738
|
}
|
|
8739
8739
|
function coinPrice(coins, displayUsdCents) {
|
|
8740
8740
|
if (coins === 0) return "Free";
|
|
8741
|
-
if (displayUsdCents === void 0) return `${formatCoins(coins)}
|
|
8742
|
-
const
|
|
8743
|
-
return `${formatCoins(coins)}
|
|
8741
|
+
if (displayUsdCents === void 0) return `${formatCoins(coins)} Nex`;
|
|
8742
|
+
const usd = (displayUsdCents / 100).toLocaleString("en-US", { minimumFractionDigits: 2, maximumFractionDigits: 2 });
|
|
8743
|
+
return `${formatCoins(coins)} Nex ($${usd})`;
|
|
8744
8744
|
}
|
|
8745
8745
|
function priceGridLine(grid, freeAllowed) {
|
|
8746
|
-
return `${[...freeAllowed ? ["Free"] : [], ...grid.map(formatCoins)].join(" \xB7 ")}
|
|
8746
|
+
return `${[...freeAllowed ? ["Free"] : [], ...grid.map(formatCoins)].join(" \xB7 ")} Nex`;
|
|
8747
8747
|
}
|
|
8748
8748
|
|
|
8749
8749
|
// src/lib/item-sync.ts
|
|
@@ -8752,7 +8752,7 @@ var ITEM_SYNC_VERSION = 1;
|
|
|
8752
8752
|
var ItemState = { Draft: "draft", OnSale: "on_sale", OffSale: "off_sale" };
|
|
8753
8753
|
var PRICE_CODES = /* @__PURE__ */ new Set(["price_off_grid", "bad_price"]);
|
|
8754
8754
|
var FIX_FLAG = {
|
|
8755
|
-
price: "--price <
|
|
8755
|
+
price: "--price <Nex>",
|
|
8756
8756
|
picture: "--image <picture.png>",
|
|
8757
8757
|
model: "--model <file.glb>",
|
|
8758
8758
|
package: "--package <dir>",
|
|
@@ -10065,7 +10065,7 @@ function printLine(draft) {
|
|
|
10065
10065
|
async function addLocal(opts, attachments, log, refuse) {
|
|
10066
10066
|
const name = opts.hostname?.trim();
|
|
10067
10067
|
if (!name) return refuse(usageRefusal(`\`genex shop add\` needs a name, e.g. ${c.cyan('genex shop add "Iron Key" --price 100')}.`));
|
|
10068
|
-
if (opts.price === void 0) return refuse(usageRefusal(`\`genex shop add\` needs ${c.cyan("--price")}, in
|
|
10068
|
+
if (opts.price === void 0) return refuse(usageRefusal(`\`genex shop add\` needs ${c.cyan("--price")}, in Nex (0 is Free).`));
|
|
10069
10069
|
const unsupported = localOnlyRefusal(opts);
|
|
10070
10070
|
if (unsupported) return refuse(unsupported);
|
|
10071
10071
|
const description = opts.description?.trim();
|
|
@@ -10357,7 +10357,7 @@ var ShopSession = class {
|
|
|
10357
10357
|
const named = grid && refusal2.error === "price_off_grid" && price !== void 0;
|
|
10358
10358
|
return {
|
|
10359
10359
|
...refusal2,
|
|
10360
|
-
...named ? { message: `${formatCoins(price)}
|
|
10360
|
+
...named ? { message: `${formatCoins(price)} Nex isn't one of this game's prices.` } : {},
|
|
10361
10361
|
hints: [...grid ? [`Prices available: ${grid}`] : [], `See the full list any time: ${c.cyan("genex shop list")}`]
|
|
10362
10362
|
};
|
|
10363
10363
|
}
|
|
@@ -10424,7 +10424,7 @@ var ShopSession = class {
|
|
|
10424
10424
|
const { log, opts } = this;
|
|
10425
10425
|
const name = opts.hostname?.trim();
|
|
10426
10426
|
if (!name) return this.refuse(usageRefusal(`\`genex shop add\` needs a name, e.g. ${c.cyan('genex shop add "Iron Key" --price 100')}.`));
|
|
10427
|
-
if (opts.price === void 0) return this.refuse(usageRefusal(`\`genex shop add\` needs ${c.cyan("--price")}, in
|
|
10427
|
+
if (opts.price === void 0) return this.refuse(usageRefusal(`\`genex shop add\` needs ${c.cyan("--price")}, in Nex (0 is Free).`));
|
|
10428
10428
|
const type = opts.type ?? (opts.shared ? "durable" : "consumable");
|
|
10429
10429
|
const res = await this.post(this.projectSkus, {
|
|
10430
10430
|
name,
|
|
@@ -10647,7 +10647,7 @@ var ItemsSession = class _ItemsSession {
|
|
|
10647
10647
|
for (const item of body.items) {
|
|
10648
10648
|
const reach = item.gamesCount === 1 ? "in 1 game" : `in ${item.gamesCount} games`;
|
|
10649
10649
|
log.plain(` ${c.cyan(item.id)} ${item.name} \u2014 ${coinPrice(item.priceCoins, item.priceDisplayUsdCents)} \xB7 ${reach} \xB7 by ${item.author.name}`);
|
|
10650
|
-
log.dim(` you get ${formatCoins(item.youGetPerSale)}
|
|
10650
|
+
log.dim(` you get ${formatCoins(item.youGetPerSale)} Nex per sale${item.tags.length ? ` \xB7 ${item.tags.join(", ")}` : ""}`);
|
|
10651
10651
|
if (item.description) log.dim(` ${item.description.length > 120 ? `${item.description.slice(0, 117)}\u2026` : item.description}`);
|
|
10652
10652
|
}
|
|
10653
10653
|
if (body.nextCursor) log.dim(" More match: narrow the search with more words or a --tag.");
|
|
@@ -26011,7 +26011,7 @@ async function runBlender(opts) {
|
|
|
26011
26011
|
return 1;
|
|
26012
26012
|
}
|
|
26013
26013
|
if (sub === "serve") {
|
|
26014
|
-
const { serveLocalBlender } = await import("./blender-serve-
|
|
26014
|
+
const { serveLocalBlender } = await import("./blender-serve-EH2MOW3U.js");
|
|
26015
26015
|
const port = Number(process.env.GENEX_BLENDER_PORT ?? 8088);
|
|
26016
26016
|
log.step(`Starting a local Blender service on port ${port}`);
|
|
26017
26017
|
log.plain(
|
|
@@ -26020,7 +26020,7 @@ async function runBlender(opts) {
|
|
|
26020
26020
|
return serveLocalBlender({ port, log });
|
|
26021
26021
|
}
|
|
26022
26022
|
if (sub === "mcp") {
|
|
26023
|
-
const { runBlenderMcp } = await import("./blender-mcp-
|
|
26023
|
+
const { runBlenderMcp } = await import("./blender-mcp-WSURPZET.js");
|
|
26024
26024
|
return runBlenderMcp();
|
|
26025
26025
|
}
|
|
26026
26026
|
if (sub === "seat") {
|
|
@@ -26528,7 +26528,12 @@ import fs39 from "fs/promises";
|
|
|
26528
26528
|
import path39 from "path";
|
|
26529
26529
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
26530
26530
|
var SUBS4 = ["models", "bench", "price", "status", "cancel"];
|
|
26531
|
-
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN
|
|
26531
|
+
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN credits \u2014 your balance, not a player's, and a spend this build's asset allowance does not count. Re-run with --max-credits <n> --user-approved only after the person at the keyboard agreed to the number.";
|
|
26532
|
+
var LLM_MAX_COINS_DEPRECATED = "--max-coins is deprecated: it is read as --max-credits, the same number \u2014 a bench spends credits now.";
|
|
26533
|
+
function benchBalanceShortLine(spendable, maxCredits) {
|
|
26534
|
+
return `Your spendable credits (${spendable}) cannot cover one attempt at --max-credits ${maxCredits} \u2014 nothing was sent, nothing was spent. Add credits, or re-run with a lower --max-credits once the person at the keyboard agrees to it.`;
|
|
26535
|
+
}
|
|
26536
|
+
var LLM_BENCH_UNVERIFIED_LINE = "Credits pay only for a verified email, and this account's is not verified yet (credits_unverified) \u2014 nothing was sent, nothing was spent. Verify it, then re-run.";
|
|
26532
26537
|
var LLM_LANE_OFF_LINE = "The runtime LLM lane is off on this stand \u2014 in-game generate() answers 404 here, and nothing can be benchmarked.";
|
|
26533
26538
|
var LLM_TOOLS_CONVERT_HINT = [
|
|
26534
26539
|
"In-game model calls need a hosted game. This folder is a Genex Tools workspace, so nothing here",
|
|
@@ -26541,7 +26546,7 @@ function serverSlotCounts(body) {
|
|
|
26541
26546
|
return { held: body.slotsHeld, limit: body.slotLimit };
|
|
26542
26547
|
}
|
|
26543
26548
|
var PENDING_SLOT_RELEASE_MINUTES = 10;
|
|
26544
|
-
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so
|
|
26549
|
+
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so what it reserved stays held until the bill resolves; such a row stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`;
|
|
26545
26550
|
var LLM_STATUS_NO_COUNT_LINE = "This stand does not report how many slots are held across your projects, so no count is shown.";
|
|
26546
26551
|
var LLM_CANCEL_AWAITING_BILL = "this call already stopped; its bill is awaiting the provider";
|
|
26547
26552
|
var LLM_CANCEL_ALREADY_FINAL = "this call is already final";
|
|
@@ -26555,11 +26560,33 @@ var DEFAULT_SAMPLE_TIMEOUT_SEC = 180;
|
|
|
26555
26560
|
function parseProviderKey(value) {
|
|
26556
26561
|
return value === "ok" || value === "rejected" || value === "unknown" ? value : null;
|
|
26557
26562
|
}
|
|
26563
|
+
function bpsOrNull(value) {
|
|
26564
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
26565
|
+
}
|
|
26566
|
+
function standingCreditGrantOrNull(value) {
|
|
26567
|
+
if (!value || typeof value !== "object") return null;
|
|
26568
|
+
const v = value;
|
|
26569
|
+
const whole = (x) => typeof x === "number" && Number.isSafeInteger(x) && x >= 0;
|
|
26570
|
+
if (!whole(v.minCredits) || !whole(v.maxCredits) || !whole(v.stepCredits) || !whole(v.defaultCredits)) return null;
|
|
26571
|
+
return {
|
|
26572
|
+
minCredits: v.minCredits,
|
|
26573
|
+
maxCredits: v.maxCredits,
|
|
26574
|
+
stepCredits: v.stepCredits,
|
|
26575
|
+
defaultCredits: v.defaultCredits,
|
|
26576
|
+
...whole(v.minUsdCents) ? { minUsdCents: v.minUsdCents } : {},
|
|
26577
|
+
...whole(v.maxUsdCents) ? { maxUsdCents: v.maxUsdCents } : {},
|
|
26578
|
+
...whole(v.stepUsdCents) ? { stepUsdCents: v.stepUsdCents } : {}
|
|
26579
|
+
};
|
|
26580
|
+
}
|
|
26558
26581
|
async function readRuntimeLane(apiUrl, token) {
|
|
26559
26582
|
const empty = (state) => ({
|
|
26560
26583
|
state,
|
|
26561
26584
|
models: null,
|
|
26562
26585
|
recommendedDeclaredHeadroomBps: null,
|
|
26586
|
+
currency: null,
|
|
26587
|
+
feeBps: null,
|
|
26588
|
+
usdCentsPerCredit: null,
|
|
26589
|
+
standingCreditGrant: null,
|
|
26563
26590
|
externalProviders: null,
|
|
26564
26591
|
total: null,
|
|
26565
26592
|
featuredCount: null,
|
|
@@ -26578,7 +26605,11 @@ async function readRuntimeLane(apiUrl, token) {
|
|
|
26578
26605
|
return {
|
|
26579
26606
|
state: "live",
|
|
26580
26607
|
models: body.models,
|
|
26581
|
-
recommendedDeclaredHeadroomBps:
|
|
26608
|
+
recommendedDeclaredHeadroomBps: bpsOrNull(body.recommendedDeclaredHeadroomBps),
|
|
26609
|
+
currency: body.currency === "credits" || body.currency === "coins" ? body.currency : null,
|
|
26610
|
+
feeBps: bpsOrNull(body.feeBps),
|
|
26611
|
+
usdCentsPerCredit: typeof body.usdCentsPerCredit === "number" && Number.isSafeInteger(body.usdCentsPerCredit) && body.usdCentsPerCredit >= 1 ? body.usdCentsPerCredit : null,
|
|
26612
|
+
standingCreditGrant: standingCreditGrantOrNull(body.standingCreditGrant),
|
|
26582
26613
|
externalProviders: Array.isArray(body.externalProviders) ? body.externalProviders : null,
|
|
26583
26614
|
total: typeof body.total === "number" && Number.isFinite(body.total) ? body.total : null,
|
|
26584
26615
|
featuredCount: typeof body.featuredCount === "number" && Number.isFinite(body.featuredCount) ? body.featuredCount : null,
|
|
@@ -26594,18 +26625,45 @@ function percentile2(values, p) {
|
|
|
26594
26625
|
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
26595
26626
|
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
26596
26627
|
}
|
|
26597
|
-
function
|
|
26598
|
-
|
|
26628
|
+
function percentilePicos(values, p) {
|
|
26629
|
+
if (values.length === 0) return null;
|
|
26630
|
+
const sorted = [...values].sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
26631
|
+
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
26632
|
+
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
26599
26633
|
}
|
|
26600
|
-
|
|
26601
|
-
|
|
26634
|
+
var PICOS_PER_CENT = 10000000000n;
|
|
26635
|
+
function ceilingCreditsFor(costPicos, terms) {
|
|
26636
|
+
if (costPicos <= 0n) return 1;
|
|
26637
|
+
const num = costPicos * BigInt(1e4 + terms.feeBps) * BigInt(1e4 + terms.headroomBps);
|
|
26638
|
+
const den = 100000000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
26639
|
+
return Math.max(1, Number((num + den - 1n) / den));
|
|
26640
|
+
}
|
|
26641
|
+
function averageCreditsPerCall(costsPicos, terms) {
|
|
26642
|
+
if (costsPicos.length === 0) return null;
|
|
26643
|
+
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
26644
|
+
return Number(sum * BigInt(1e4 + terms.feeBps)) / Number(BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT);
|
|
26645
|
+
}
|
|
26646
|
+
function perCallEstimateCreditsFor(costsPicos, terms) {
|
|
26647
|
+
if (costsPicos.length === 0) return null;
|
|
26648
|
+
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
26649
|
+
const num = sum * BigInt(1e4 + terms.feeBps);
|
|
26650
|
+
const den = BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
26651
|
+
return Math.max(1, Number((num + den - 1n) / den));
|
|
26652
|
+
}
|
|
26653
|
+
function formatCredits(value) {
|
|
26654
|
+
if (!Number.isFinite(value)) return "\u2014";
|
|
26655
|
+
if (value >= 100) return String(Math.round(value));
|
|
26656
|
+
return String(Number(value.toPrecision(value < 1 ? 2 : 3)));
|
|
26657
|
+
}
|
|
26658
|
+
function usdFromPicos(picos) {
|
|
26659
|
+
if (picos === null || picos === void 0) return "unknown";
|
|
26660
|
+
const value = typeof picos === "bigint" ? picos : /^\d+$/.test(picos) ? BigInt(picos) : null;
|
|
26661
|
+
if (value === null) return "unknown";
|
|
26662
|
+
return `$${(Number(value) / 1e12).toFixed(6)}`;
|
|
26602
26663
|
}
|
|
26603
26664
|
function usdPerMillion(value) {
|
|
26604
26665
|
return `$${value.toFixed(value < 1 ? 4 : 2)}`;
|
|
26605
26666
|
}
|
|
26606
|
-
function usd(value) {
|
|
26607
|
-
return value === null ? "unknown" : `$${value.toFixed(6)}`;
|
|
26608
|
-
}
|
|
26609
26667
|
async function runLlm(opts = {}) {
|
|
26610
26668
|
const log = createLogger({ quiet: opts.quiet || opts.json });
|
|
26611
26669
|
const cwd = opts.cwd ?? process.cwd();
|
|
@@ -26631,16 +26689,23 @@ async function runLlm(opts = {}) {
|
|
|
26631
26689
|
return;
|
|
26632
26690
|
}
|
|
26633
26691
|
if (sub === "bench") {
|
|
26634
|
-
if (opts.maxCoins
|
|
26692
|
+
if (opts.maxCoins !== void 0) process.stderr.write(`${c.yellow("!")} ${LLM_MAX_COINS_DEPRECATED}
|
|
26693
|
+
`);
|
|
26694
|
+
if (opts.maxCoins !== void 0 && opts.maxCredits !== void 0 && opts.maxCoins !== opts.maxCredits) {
|
|
26695
|
+
fail5("llm bench", `Pass --max-credits alone \u2014 --max-coins is its deprecated name, and the two disagree (${opts.maxCredits} vs ${opts.maxCoins}).`);
|
|
26696
|
+
return;
|
|
26697
|
+
}
|
|
26698
|
+
const approvedCredits = opts.maxCredits ?? opts.maxCoins;
|
|
26699
|
+
if (approvedCredits === void 0 || opts.userApproved !== true) {
|
|
26635
26700
|
fail5("llm bench", LLM_BENCH_APPROVAL_REQUIRED);
|
|
26636
26701
|
return;
|
|
26637
26702
|
}
|
|
26638
|
-
if (!Number.isInteger(
|
|
26639
|
-
fail5("llm bench", `--max-
|
|
26703
|
+
if (!Number.isInteger(approvedCredits) || approvedCredits < 1) {
|
|
26704
|
+
fail5("llm bench", `--max-credits takes a whole number of credits, 1 or more (got ${String(approvedCredits)}).`);
|
|
26640
26705
|
return;
|
|
26641
26706
|
}
|
|
26642
26707
|
if (!opts.benchPrompt?.trim()) {
|
|
26643
|
-
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-
|
|
26708
|
+
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-credits <n> --user-approved.');
|
|
26644
26709
|
return;
|
|
26645
26710
|
}
|
|
26646
26711
|
if (opts.jsonOutput && opts.textOutput) {
|
|
@@ -26732,6 +26797,10 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
26732
26797
|
models: null,
|
|
26733
26798
|
externalProviders: null,
|
|
26734
26799
|
recommendedDeclaredHeadroomBps: null,
|
|
26800
|
+
currency: null,
|
|
26801
|
+
feeBps: null,
|
|
26802
|
+
usdCentsPerCredit: null,
|
|
26803
|
+
standingCreditGrant: null,
|
|
26735
26804
|
total: null,
|
|
26736
26805
|
featuredCount: null,
|
|
26737
26806
|
// The stand was never asked, so the verdict is unknown — but the key is
|
|
@@ -26757,6 +26826,12 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
26757
26826
|
models: lane.models,
|
|
26758
26827
|
externalProviders: lane.externalProviders,
|
|
26759
26828
|
recommendedDeclaredHeadroomBps: lane.recommendedDeclaredHeadroomBps,
|
|
26829
|
+
// The credit terms a call is billed on, the server's — null on a stand
|
|
26830
|
+
// that predates credit billing, never a number the CLI filled in.
|
|
26831
|
+
currency: lane.currency,
|
|
26832
|
+
feeBps: lane.feeBps,
|
|
26833
|
+
usdCentsPerCredit: lane.usdCentsPerCredit,
|
|
26834
|
+
standingCreditGrant: lane.standingCreditGrant,
|
|
26760
26835
|
total: lane.total ?? lane.models?.length ?? null,
|
|
26761
26836
|
featuredCount: lane.featuredCount ?? (lane.models ? lane.models.filter((m) => m.featured).length : null),
|
|
26762
26837
|
// `null` when the lane is not live or the server predates the probe.
|
|
@@ -26803,8 +26878,9 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
26803
26878
|
);
|
|
26804
26879
|
for (const m of shown) {
|
|
26805
26880
|
const plan = m.personalPlan ? ` \xB7 personal plan: ${m.personalPlan}` : "";
|
|
26881
|
+
const kind = m.kind === "typesafe" ? " \xB7 judge (classifier): judge() under a budget, not generate() or bench" : "";
|
|
26806
26882
|
log.plain(
|
|
26807
|
-
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}`
|
|
26883
|
+
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}${kind}`
|
|
26808
26884
|
);
|
|
26809
26885
|
}
|
|
26810
26886
|
if (hiddenCount > 0) {
|
|
@@ -26820,74 +26896,104 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
26820
26896
|
}
|
|
26821
26897
|
log.plain("");
|
|
26822
26898
|
if (lane.recommendedDeclaredHeadroomBps === null) {
|
|
26823
|
-
log.dim(" This stand serves no recommended headroom, so no
|
|
26899
|
+
log.dim(" This stand serves no recommended headroom, so no ceiling can be recommended from a bench.");
|
|
26900
|
+
} else {
|
|
26901
|
+
log.dim(` Recommended headroom over a benchmarked call's cost on this stand: ${lane.recommendedDeclaredHeadroomBps} bps.`);
|
|
26902
|
+
}
|
|
26903
|
+
if (lane.feeBps === null) {
|
|
26904
|
+
log.dim(" This stand serves no platform fee \u2014 it predates credit billing, so no ceiling can be recommended from a bench.");
|
|
26824
26905
|
} else {
|
|
26825
|
-
log.dim(`
|
|
26906
|
+
log.dim(` In-game calls are billed in the player's credits, as used: the provider's cost plus this stand's platform fee (${lane.feeBps} bps).`);
|
|
26826
26907
|
}
|
|
26827
26908
|
if (toolsOnly) {
|
|
26828
26909
|
printConvertHint();
|
|
26829
26910
|
return;
|
|
26830
26911
|
}
|
|
26831
|
-
log.dim(" Those per-million rates are the PROVIDER's; what
|
|
26832
|
-
log.dim(`
|
|
26912
|
+
log.dim(" Those per-million rates are the PROVIDER's list; what one call of YOUR prompt really costs,");
|
|
26913
|
+
log.dim(` and the per-call ceiling to declare, only a real call measures: ${c.cyan('genex llm bench "<prompt>" --max-credits <n> --user-approved')}`);
|
|
26833
26914
|
}
|
|
26834
26915
|
function wholeOrNull(value) {
|
|
26835
26916
|
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
26836
26917
|
}
|
|
26918
|
+
function costPicosOf(usage) {
|
|
26919
|
+
if (typeof usage?.costUsdPicos === "string" && /^\d+$/.test(usage.costUsdPicos)) return usage.costUsdPicos;
|
|
26920
|
+
if (typeof usage?.costUsd === "number" && Number.isFinite(usage.costUsd) && usage.costUsd >= 0) {
|
|
26921
|
+
return String(BigInt(Math.round(usage.costUsd * 1e12)));
|
|
26922
|
+
}
|
|
26923
|
+
return null;
|
|
26924
|
+
}
|
|
26837
26925
|
function sampleFrom(id, settled) {
|
|
26838
26926
|
const usage = settled?.usage;
|
|
26927
|
+
const picos = costPicosOf(usage);
|
|
26839
26928
|
return {
|
|
26840
26929
|
id,
|
|
26841
26930
|
status: settled?.status ?? null,
|
|
26842
26931
|
billingStatus: settled?.billingStatus ?? null,
|
|
26843
|
-
|
|
26844
|
-
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd : null,
|
|
26932
|
+
chargedCredits: wholeOrNull(settled?.chargedCredits),
|
|
26933
|
+
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd : picos !== null ? Number(picos) / 1e12 : null,
|
|
26934
|
+
costUsdPicos: picos,
|
|
26845
26935
|
error: settled?.error ?? (settled ? null : "timed_out"),
|
|
26846
26936
|
providerMessage: typeof settled?.providerMessage === "string" && settled.providerMessage.trim() ? settled.providerMessage.trim() : null,
|
|
26847
26937
|
inputTokens: wholeOrNull(usage?.inputTokens),
|
|
26848
26938
|
outputTokens: wholeOrNull(usage?.outputTokens),
|
|
26849
26939
|
maxOutputTokens: wholeOrNull(usage?.maxOutputTokens),
|
|
26850
|
-
|
|
26940
|
+
fitCredits: typeof usage?.fitCredits === "number" && Number.isSafeInteger(usage.fitCredits) && usage.fitCredits >= 1 ? usage.fitCredits : null,
|
|
26851
26941
|
fitExceedsCap: typeof usage?.fitExceedsCap === "boolean" ? usage.fitExceedsCap : null,
|
|
26852
26942
|
truncated: typeof usage?.truncated === "boolean" ? usage.truncated : settled?.error === "provider_token_limit" ? true : null
|
|
26853
26943
|
};
|
|
26854
26944
|
}
|
|
26855
26945
|
var PROVIDER_TOKEN_LIMIT = "provider_token_limit";
|
|
26856
|
-
function benchRecommendation(results,
|
|
26857
|
-
const settled = results.filter(
|
|
26858
|
-
|
|
26859
|
-
|
|
26860
|
-
const
|
|
26861
|
-
const
|
|
26946
|
+
function benchRecommendation(results, terms) {
|
|
26947
|
+
const settled = results.filter(
|
|
26948
|
+
(r) => r.status === "succeeded" && r.billingStatus === "final" && r.chargedCredits !== null && r.costUsdPicos !== null
|
|
26949
|
+
);
|
|
26950
|
+
const costs = settled.map((r) => BigInt(r.costUsdPicos));
|
|
26951
|
+
const charged = settled.map((r) => r.chargedCredits);
|
|
26952
|
+
const fits = settled.map((r) => r.fitCredits).filter((v) => typeof v === "number");
|
|
26953
|
+
const costP95 = percentilePicos(costs, 95);
|
|
26954
|
+
const costMax = costs.length ? costs.reduce((a, b) => b > a ? b : a) : null;
|
|
26862
26955
|
const fitP95 = percentile2(fits, 95);
|
|
26863
26956
|
const fitMax = fits.length ? Math.max(...fits) : null;
|
|
26864
26957
|
const exceedsStandCap = results.some((r) => r.fitExceedsCap === true);
|
|
26865
26958
|
const cutOffSamples = results.filter((r) => r.error === PROVIDER_TOKEN_LIMIT && r.fitExceedsCap !== true).length;
|
|
26866
26959
|
const noRecommendationReason = exceedsStandCap ? "answer_exceeds_stand_cap" : cutOffSamples > 0 ? "answer_cut_off" : null;
|
|
26867
|
-
const
|
|
26868
|
-
const
|
|
26869
|
-
const
|
|
26960
|
+
const { headroomBps, feeBps, usdCentsPerCredit } = terms;
|
|
26961
|
+
const missingTerms = headroomBps === null ? "headroom" : feeBps === null || usdCentsPerCredit === null ? "credit_terms" : null;
|
|
26962
|
+
const full = headroomBps !== null && feeBps !== null && usdCentsPerCredit !== null ? { headroomBps, feeBps, usdCentsPerCredit } : null;
|
|
26963
|
+
const billing = feeBps !== null && usdCentsPerCredit !== null ? { feeBps, usdCentsPerCredit } : null;
|
|
26964
|
+
const costBased = full !== null && costP95 !== null ? ceilingCreditsFor(costP95, full) : null;
|
|
26965
|
+
const recommended = costBased === null || noRecommendationReason !== null ? null : Math.max(costBased, fitP95 ?? 0);
|
|
26966
|
+
const ceiling = recommended === null || costMax === null || full === null ? null : Math.max(ceilingCreditsFor(costMax, full), recommended, fitMax ?? 0);
|
|
26967
|
+
const average = billing !== null ? averageCreditsPerCall(costs, billing) : null;
|
|
26968
|
+
const estimate = ceiling === null || billing === null ? null : Math.min(ceiling, perCallEstimateCreditsFor(costs, billing) ?? 1);
|
|
26870
26969
|
return {
|
|
26970
|
+
costs,
|
|
26871
26971
|
charged,
|
|
26872
|
-
|
|
26873
|
-
|
|
26874
|
-
|
|
26972
|
+
costP50: percentilePicos(costs, 50),
|
|
26973
|
+
costP95,
|
|
26974
|
+
costMax,
|
|
26975
|
+
chargedP50: percentile2(charged, 50),
|
|
26976
|
+
chargedP95: percentile2(charged, 95),
|
|
26977
|
+
chargedMax: charged.length ? Math.max(...charged) : null,
|
|
26875
26978
|
fits,
|
|
26876
26979
|
fitP95,
|
|
26877
26980
|
fitMax,
|
|
26878
|
-
|
|
26879
|
-
|
|
26880
|
-
|
|
26881
|
-
|
|
26981
|
+
costBasedMaxCredits: costBased,
|
|
26982
|
+
recommendedMaxCredits: recommended,
|
|
26983
|
+
recommendedPerCallMaxCredits: ceiling,
|
|
26984
|
+
averageCreditsPerCall: average,
|
|
26985
|
+
recommendedPerCallEstimateCredits: estimate,
|
|
26986
|
+
lengthRaisedCeiling: recommended !== null && costBased !== null && recommended > costBased,
|
|
26882
26987
|
cutOffSamples,
|
|
26883
26988
|
exceedsStandCap,
|
|
26884
|
-
noRecommendationReason
|
|
26989
|
+
noRecommendationReason,
|
|
26990
|
+
missingTerms
|
|
26885
26991
|
};
|
|
26886
26992
|
}
|
|
26887
|
-
var LLM_LENGTH_RAISED_LINE = "The
|
|
26888
|
-
var LLM_EXCEEDS_STAND_CAP_LINE = "No
|
|
26889
|
-
function benchCutOffLine(cutOff,
|
|
26890
|
-
return `No
|
|
26993
|
+
var LLM_LENGTH_RAISED_LINE = "The ceiling also decides how long the answer may be: at the cost-based ceiling the answer would be cut off, so declare this one.";
|
|
26994
|
+
var LLM_EXCEEDS_STAND_CAP_LINE = "No ceiling recommended: this answer is longer than one call on this stand may produce. Ask for a shorter answer \u2014 fewer fields, shorter strings, a length the prompt states \u2014 and benchmark again.";
|
|
26995
|
+
function benchCutOffLine(cutOff, maxCredits) {
|
|
26996
|
+
return `No ceiling recommended: ${cutOff} sample${cutOff === 1 ? " was" : "s were"} cut off at this ceiling (--max-credits ${maxCredits}) before the answer was finished, so its real length is unknown. Re-run with a higher --max-credits.`;
|
|
26891
26997
|
}
|
|
26892
26998
|
var ACTIVE_STATUSES = /* @__PURE__ */ new Set(["requires_confirmation", "queued", "dispatching", "awaiting_external"]);
|
|
26893
26999
|
function rowState(row) {
|
|
@@ -26898,7 +27004,23 @@ function rowState(row) {
|
|
|
26898
27004
|
function rowHoldsSlot(row) {
|
|
26899
27005
|
if (typeof row.slotHeld === "boolean") return row.slotHeld;
|
|
26900
27006
|
const state = rowState(row);
|
|
26901
|
-
return state === "active" || state === "awaiting bill" && (row.reservedCoins ?? 0) > 0;
|
|
27007
|
+
return state === "active" || state === "awaiting bill" && ((row.reservedCoins ?? 0) > 0 || (row.reservedCredits ?? 0) > 0);
|
|
27008
|
+
}
|
|
27009
|
+
var UNIT_WORD = { credits: "credits", coin: "Nex" };
|
|
27010
|
+
function rowMoney(row) {
|
|
27011
|
+
const credits = row.currency === "credits" || row.currency === void 0 && typeof row.reservedCredits === "number" && typeof row.reservedCoins !== "number";
|
|
27012
|
+
if (credits) {
|
|
27013
|
+
return {
|
|
27014
|
+
unit: "credits",
|
|
27015
|
+
reserved: typeof row.reservedCredits === "number" ? row.reservedCredits : null,
|
|
27016
|
+
charged: typeof row.chargedCredits === "number" ? row.chargedCredits : null
|
|
27017
|
+
};
|
|
27018
|
+
}
|
|
27019
|
+
return {
|
|
27020
|
+
unit: "coin",
|
|
27021
|
+
reserved: typeof row.reservedCoins === "number" ? row.reservedCoins : null,
|
|
27022
|
+
charged: typeof row.chargedCoins === "number" ? row.chargedCoins : null
|
|
27023
|
+
};
|
|
26902
27024
|
}
|
|
26903
27025
|
function providerRefusalStatus(error) {
|
|
26904
27026
|
const m = /^provider_http_(\d{3})$/.exec(error ?? "");
|
|
@@ -26917,7 +27039,7 @@ function formatAge(iso, now = Date.now()) {
|
|
|
26917
27039
|
}
|
|
26918
27040
|
async function runBench(args) {
|
|
26919
27041
|
const { apiUrl, token, projectId, cwd, opts, log } = args;
|
|
26920
|
-
const
|
|
27042
|
+
const maxCredits = opts.maxCredits ?? opts.maxCoins;
|
|
26921
27043
|
const samples = opts.samples ?? DEFAULT_BENCH_SAMPLES;
|
|
26922
27044
|
const prompt = opts.benchPrompt.trim();
|
|
26923
27045
|
const auth = { Authorization: `Bearer ${token}`, "Content-Type": "application/json" };
|
|
@@ -26939,7 +27061,7 @@ async function runBench(args) {
|
|
|
26939
27061
|
const lane = await readRuntimeLane(apiUrl, token);
|
|
26940
27062
|
if (lane.state !== "live" || !lane.models || lane.models.length === 0) {
|
|
26941
27063
|
if (lane.state === "off") {
|
|
26942
|
-
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
27064
|
+
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCredits, opts), status: "off", error: null });
|
|
26943
27065
|
else {
|
|
26944
27066
|
log.plain(LLM_LANE_OFF_LINE);
|
|
26945
27067
|
log.dim(" Nothing was spent.");
|
|
@@ -26966,17 +27088,27 @@ async function runBench(args) {
|
|
|
26966
27088
|
);
|
|
26967
27089
|
return;
|
|
26968
27090
|
}
|
|
26969
|
-
const
|
|
27091
|
+
const walletBefore = await fetchCreditsSnapshot(apiUrl, token);
|
|
27092
|
+
if (walletBefore?.emailVerified === false) {
|
|
27093
|
+
benchFailed(opts, log, LLM_BENCH_UNVERIFIED_LINE);
|
|
27094
|
+
return;
|
|
27095
|
+
}
|
|
27096
|
+
if (walletBefore !== null && walletBefore.spendable < maxCredits) {
|
|
27097
|
+
benchFailed(opts, log, benchBalanceShortLine(walletBefore.spendable, maxCredits));
|
|
27098
|
+
return;
|
|
27099
|
+
}
|
|
27100
|
+
const balanceBefore = walletBefore?.spendable ?? null;
|
|
26970
27101
|
if (!opts.json) {
|
|
26971
27102
|
log.plain(c.bold("genex llm bench"));
|
|
26972
27103
|
log.dim(` ${apiUrl}`);
|
|
26973
27104
|
log.plain("");
|
|
26974
27105
|
log.plain(` Model ${c.cyan(modelId)}`);
|
|
26975
27106
|
log.plain(` Samples ${samples} real attempt${samples === 1 ? "" : "s"}, ${outputFormat} output`);
|
|
26976
|
-
log.plain(` Approved ${
|
|
27107
|
+
log.plain(` Approved ${maxCredits} credit${maxCredits === 1 ? "" : "s"} per attempt \u2014 at worst ${maxCredits * samples} credits for this run, from your own balance`);
|
|
26977
27108
|
log.plain(
|
|
26978
|
-
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore}
|
|
27109
|
+
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore} credits spendable`}`
|
|
26979
27110
|
);
|
|
27111
|
+
log.dim(" Billed as used: each attempt is charged its real cost plus the platform fee, rounded up to a whole credit.");
|
|
26980
27112
|
log.plain("");
|
|
26981
27113
|
}
|
|
26982
27114
|
const base = `${apiUrl}/api/runtime/development/projects/${encodeURIComponent(projectId)}/generations`;
|
|
@@ -27002,7 +27134,7 @@ async function runBench(args) {
|
|
|
27002
27134
|
const res = await call2(base, {
|
|
27003
27135
|
method: "POST",
|
|
27004
27136
|
body: JSON.stringify({
|
|
27005
|
-
|
|
27137
|
+
maxCredits,
|
|
27006
27138
|
request: {
|
|
27007
27139
|
idempotencyKey: `bench-${randomUUID2()}`,
|
|
27008
27140
|
modelId,
|
|
@@ -27032,13 +27164,14 @@ async function runBench(args) {
|
|
|
27032
27164
|
if (refusedAt !== null) providerRefused++;
|
|
27033
27165
|
if (!opts.json) {
|
|
27034
27166
|
const length = row.outputTokens !== null ? ` \xB7 ${row.outputTokens}${row.maxOutputTokens !== null ? ` of ${row.maxOutputTokens}` : ""} tokens out` : "";
|
|
27167
|
+
const charged = row.chargedCredits === null ? "\u2014" : String(row.chargedCredits);
|
|
27035
27168
|
log.plain(
|
|
27036
|
-
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${
|
|
27169
|
+
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${charged.padStart(4)} credit${row.chargedCredits === 1 ? "" : "s"} charged \xB7 cost ${usdFromPicos(row.costUsdPicos)}${length}${row.error ? ` \xB7 ${row.error}` : ""}`
|
|
27037
27170
|
);
|
|
27038
27171
|
if (refusedAt !== null) printProviderRefusal(log, refusedAt, row.providerMessage, row.status === "unknown");
|
|
27039
27172
|
if (row.error === PROVIDER_TOKEN_LIMIT) {
|
|
27040
27173
|
log.dim(
|
|
27041
|
-
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no
|
|
27174
|
+
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no ceiling makes room for it; it is not a sample." : ` The answer was cut off at this ceiling (--max-credits ${maxCredits}) before it was finished \u2014 it was charged, and it is not a sample.`
|
|
27042
27175
|
);
|
|
27043
27176
|
}
|
|
27044
27177
|
}
|
|
@@ -27046,16 +27179,19 @@ async function runBench(args) {
|
|
|
27046
27179
|
} finally {
|
|
27047
27180
|
process.removeListener("SIGINT", onSigint);
|
|
27048
27181
|
}
|
|
27049
|
-
const
|
|
27050
|
-
|
|
27051
|
-
|
|
27052
|
-
|
|
27053
|
-
|
|
27054
|
-
const
|
|
27182
|
+
const terms = {
|
|
27183
|
+
headroomBps: lane.recommendedDeclaredHeadroomBps,
|
|
27184
|
+
feeBps: lane.feeBps,
|
|
27185
|
+
usdCentsPerCredit: lane.usdCentsPerCredit
|
|
27186
|
+
};
|
|
27187
|
+
const rec = benchRecommendation(results, terms);
|
|
27188
|
+
const settledCount = rec.charged.length;
|
|
27055
27189
|
const lengthBlocked = rec.noRecommendationReason !== null;
|
|
27056
|
-
const balanceAfter = await
|
|
27190
|
+
const balanceAfter = (await fetchCreditsSnapshot(apiUrl, token))?.spendable ?? null;
|
|
27191
|
+
const picosOrNull = (v) => v === null ? null : String(v);
|
|
27057
27192
|
const record = {
|
|
27058
|
-
v:
|
|
27193
|
+
v: 2,
|
|
27194
|
+
currency: "credits",
|
|
27059
27195
|
ranAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
27060
27196
|
apiUrl,
|
|
27061
27197
|
projectId,
|
|
@@ -27064,28 +27200,34 @@ async function runBench(args) {
|
|
|
27064
27200
|
outputFormat,
|
|
27065
27201
|
samples,
|
|
27066
27202
|
completed: settledCount,
|
|
27067
|
-
|
|
27068
|
-
|
|
27069
|
-
|
|
27070
|
-
max,
|
|
27071
|
-
|
|
27072
|
-
|
|
27073
|
-
recommendedPerCallMaxCoins: ceiling,
|
|
27074
|
-
maxCoinsPerSample: maxCoins,
|
|
27075
|
-
fitCoins: rec.fits,
|
|
27203
|
+
maxCreditsPerSample: maxCredits,
|
|
27204
|
+
costUsdPicos: rec.costs.map(String),
|
|
27205
|
+
chargedCredits: rec.charged,
|
|
27206
|
+
cost: { p50: picosOrNull(rec.costP50), p95: picosOrNull(rec.costP95), max: picosOrNull(rec.costMax) },
|
|
27207
|
+
charged: { p50: rec.chargedP50, p95: rec.chargedP95, max: rec.chargedMax },
|
|
27208
|
+
fitCredits: rec.fits,
|
|
27076
27209
|
fitP95: rec.fitP95,
|
|
27077
|
-
|
|
27078
|
-
|
|
27210
|
+
fitMax: rec.fitMax,
|
|
27211
|
+
recommendedDeclaredHeadroomBps: terms.headroomBps,
|
|
27212
|
+
feeBps: terms.feeBps,
|
|
27213
|
+
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
27214
|
+
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
27215
|
+
recommendedMaxCredits: rec.recommendedMaxCredits,
|
|
27216
|
+
recommendedPerCallMaxCredits: rec.recommendedPerCallMaxCredits,
|
|
27217
|
+
averageCreditsPerCall: rec.averageCreditsPerCall,
|
|
27218
|
+
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
27219
|
+
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
27079
27220
|
cutOffSamples: rec.cutOffSamples,
|
|
27080
27221
|
noRecommendationReason: rec.noRecommendationReason
|
|
27081
27222
|
};
|
|
27082
27223
|
const savedTo = await saveBench(cwd, record);
|
|
27083
27224
|
if (opts.json) {
|
|
27084
|
-
const ok =
|
|
27225
|
+
const ok = settledCount > 0 && !lengthBlocked;
|
|
27085
27226
|
writeJsonLine({
|
|
27086
27227
|
command: "llm bench",
|
|
27087
27228
|
status: ok ? "ok" : "failed",
|
|
27088
27229
|
error: ok ? null : rec.noRecommendationReason ?? "no_sample_settled",
|
|
27230
|
+
currency: "credits",
|
|
27089
27231
|
apiUrl,
|
|
27090
27232
|
projectId,
|
|
27091
27233
|
modelId,
|
|
@@ -27094,35 +27236,54 @@ async function runBench(args) {
|
|
|
27094
27236
|
schemaPath: opts.schemaPath ?? null,
|
|
27095
27237
|
samples,
|
|
27096
27238
|
completed: settledCount,
|
|
27097
|
-
|
|
27239
|
+
maxCreditsPerSample: maxCredits,
|
|
27240
|
+
// Spendable credits before and after the run; null when unreadable.
|
|
27098
27241
|
balanceBefore,
|
|
27099
27242
|
balanceAfter,
|
|
27100
27243
|
results,
|
|
27101
27244
|
// The create-time refusal that ended the run, or null. Beside the
|
|
27102
27245
|
// samples, never among them: no attempt existed and nothing was spent.
|
|
27103
27246
|
refusal: refusal2,
|
|
27104
|
-
|
|
27105
|
-
//
|
|
27247
|
+
// Over the settled samples: the real provider cost (pico-dollar strings
|
|
27248
|
+
// and USD) and the whole credits charged.
|
|
27249
|
+
costUsdPicos: record.cost,
|
|
27250
|
+
costUsd: {
|
|
27251
|
+
p50: rec.costP50 === null ? null : Number(rec.costP50) / 1e12,
|
|
27252
|
+
p95: rec.costP95 === null ? null : Number(rec.costP95) / 1e12,
|
|
27253
|
+
max: rec.costMax === null ? null : Number(rec.costMax) / 1e12
|
|
27254
|
+
},
|
|
27255
|
+
chargedCredits: record.charged,
|
|
27256
|
+
// The answer's LENGTH, priced by the server per sample (`fitCredits`,
|
|
27106
27257
|
// headroom already on the tokens), over the settled samples.
|
|
27107
|
-
|
|
27108
|
-
|
|
27109
|
-
|
|
27258
|
+
fitCredits: { p95: rec.fitP95, max: rec.fitMax },
|
|
27259
|
+
recommendedDeclaredHeadroomBps: terms.headroomBps,
|
|
27260
|
+
feeBps: terms.feeBps,
|
|
27261
|
+
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
27262
|
+
missingTerms: rec.missingTerms,
|
|
27263
|
+
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
27264
|
+
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
27110
27265
|
cutOffSamples: rec.cutOffSamples,
|
|
27111
27266
|
exceedsStandCap: rec.exceedsStandCap,
|
|
27112
27267
|
noRecommendationReason: rec.noRecommendationReason,
|
|
27113
|
-
|
|
27114
|
-
|
|
27115
|
-
|
|
27268
|
+
recommendedMaxCredits: rec.recommendedMaxCredits,
|
|
27269
|
+
recommendedPerCallMaxCredits: rec.recommendedPerCallMaxCredits,
|
|
27270
|
+
averageCreditsPerCall: rec.averageCreditsPerCall,
|
|
27271
|
+
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
27116
27272
|
savedTo
|
|
27117
27273
|
});
|
|
27118
27274
|
if (!ok) process.exitCode = 1;
|
|
27119
27275
|
return;
|
|
27120
27276
|
}
|
|
27121
27277
|
log.plain("");
|
|
27278
|
+
const printMeasured = () => {
|
|
27279
|
+
log.plain(c.bold(" Real cost per call"));
|
|
27280
|
+
log.plain(` p50 ${usdFromPicos(rec.costP50)} p95 ${usdFromPicos(rec.costP95)} max ${usdFromPicos(rec.costMax)} (${settledCount} of ${samples} settled)`);
|
|
27281
|
+
log.plain(c.bold(" Charged credits"));
|
|
27282
|
+
log.plain(` p50 ${rec.chargedP50} p95 ${rec.chargedP95} max ${rec.chargedMax} (whole credits, rounded up per call)`);
|
|
27283
|
+
};
|
|
27122
27284
|
if (lengthBlocked) {
|
|
27123
|
-
if (
|
|
27124
|
-
|
|
27125
|
-
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
27285
|
+
if (settledCount > 0) {
|
|
27286
|
+
printMeasured();
|
|
27126
27287
|
log.plain("");
|
|
27127
27288
|
}
|
|
27128
27289
|
printRecommendation(log, record);
|
|
@@ -27130,17 +27291,16 @@ async function runBench(args) {
|
|
|
27130
27291
|
process.exitCode = 1;
|
|
27131
27292
|
return;
|
|
27132
27293
|
}
|
|
27133
|
-
if (
|
|
27294
|
+
if (settledCount === 0) {
|
|
27134
27295
|
log.error(
|
|
27135
|
-
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to
|
|
27296
|
+
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to measure from." : " No sample settled, so there is nothing to measure from."
|
|
27136
27297
|
);
|
|
27137
27298
|
process.exitCode = 1;
|
|
27138
27299
|
return;
|
|
27139
27300
|
}
|
|
27140
|
-
|
|
27141
|
-
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
27301
|
+
printMeasured();
|
|
27142
27302
|
if (balanceBefore !== null && balanceAfter !== null) {
|
|
27143
|
-
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter}
|
|
27303
|
+
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter} credits spendable`);
|
|
27144
27304
|
}
|
|
27145
27305
|
log.plain("");
|
|
27146
27306
|
printRecommendation(log, record);
|
|
@@ -27152,43 +27312,58 @@ function printRecommendation(log, record) {
|
|
|
27152
27312
|
return;
|
|
27153
27313
|
}
|
|
27154
27314
|
if (record.noRecommendationReason === "answer_cut_off") {
|
|
27155
|
-
log.warn(` ${benchCutOffLine(record.cutOffSamples
|
|
27315
|
+
log.warn(` ${benchCutOffLine(record.cutOffSamples || 1, record.maxCreditsPerSample)}`);
|
|
27156
27316
|
return;
|
|
27157
27317
|
}
|
|
27158
|
-
|
|
27159
|
-
|
|
27318
|
+
const attempts = `${record.completed} settled attempt${record.completed === 1 ? "" : "s"} on ${record.modelId}`;
|
|
27319
|
+
if (record.recommendedMaxCredits === null || record.cost.p95 === null) {
|
|
27320
|
+
log.warn(" No ceiling recommended.");
|
|
27160
27321
|
log.dim(
|
|
27161
|
-
record.recommendedDeclaredHeadroomBps === null ? " This stand served no recommended headroom, and the multiplier is the server's to set \u2014" : "
|
|
27322
|
+
record.cost.p95 === null ? " No sample settled, so there is no p95 to build a ceiling on \u2014" : record.recommendedDeclaredHeadroomBps === null ? " This stand served no recommended headroom, and the multiplier is the server's to set \u2014" : " This stand served no platform fee or credit value, and those are the server's to set \u2014"
|
|
27162
27323
|
);
|
|
27163
27324
|
log.dim(" declaring a number from this run would be a guess dressed as a measurement.");
|
|
27325
|
+
printAverage(log, record, attempts);
|
|
27164
27326
|
return;
|
|
27165
27327
|
}
|
|
27166
|
-
log.plain(c.bold(` Declare
|
|
27167
|
-
if (record.
|
|
27328
|
+
log.plain(c.bold(` Declare maxCredits: ${record.recommendedMaxCredits}`));
|
|
27329
|
+
if (record.lengthRaisedCeiling) {
|
|
27168
27330
|
log.plain(` ${LLM_LENGTH_RAISED_LINE}`);
|
|
27169
27331
|
log.dim(
|
|
27170
|
-
` = the smallest
|
|
27332
|
+
` = the smallest ceiling whose answer allowance holds the answer plus this stand's headroom (p95 ${record.fitP95 ?? "\u2014"}), above the`
|
|
27171
27333
|
);
|
|
27172
27334
|
log.dim(
|
|
27173
|
-
`
|
|
27335
|
+
` cost-based ${record.costBasedMaxCredits ?? "\u2014"} (p95 real cost ${usdFromPicos(record.cost.p95)} with the platform fee, ${record.feeBps} bps, and ${record.recommendedDeclaredHeadroomBps} bps of headroom), over ${attempts}.`
|
|
27174
27336
|
);
|
|
27175
27337
|
} else {
|
|
27176
27338
|
log.dim(
|
|
27177
|
-
` = p95
|
|
27339
|
+
` = p95 real cost (${usdFromPicos(record.cost.p95)}) with this stand's platform fee (${record.feeBps} bps) and recommended headroom (${record.recommendedDeclaredHeadroomBps} bps), in whole credits,`
|
|
27178
27340
|
);
|
|
27179
|
-
log.dim(` measured over ${
|
|
27341
|
+
log.dim(` measured over ${attempts}.`);
|
|
27180
27342
|
}
|
|
27181
|
-
log.dim(" That number is a
|
|
27182
|
-
|
|
27183
|
-
|
|
27343
|
+
log.dim(" That number is a CEILING, not a price: a call is billed its real cost as used, never more than it \u2014");
|
|
27344
|
+
log.dim(" and it also sizes how long the answer may be.");
|
|
27345
|
+
if (record.recommendedPerCallMaxCredits !== null) {
|
|
27346
|
+
log.plain(c.bold(` Grant perCallMaxCredits: ${record.recommendedPerCallMaxCredits}`));
|
|
27184
27347
|
log.dim(
|
|
27185
|
-
record.
|
|
27348
|
+
record.lengthRaisedCeiling ? ` = the same over the worst sample (${usdFromPicos(record.cost.max)}) or the longest answer's ceiling, whichever is larger, and never below the ceiling above it.` : ` = the same over the worst sample (${usdFromPicos(record.cost.max)}), and never below the ceiling above it.`
|
|
27186
27349
|
);
|
|
27187
|
-
|
|
27350
|
+
}
|
|
27351
|
+
printAverage(log, record, attempts);
|
|
27352
|
+
}
|
|
27353
|
+
function printAverage(log, record, attempts) {
|
|
27354
|
+
if (record.averageCreditsPerCall === null) return;
|
|
27355
|
+
log.plain(` Per-call estimate: about ${formatCredits(record.averageCreditsPerCall)} credits per call`);
|
|
27356
|
+
log.dim(` = the average real cost with the platform fee, no headroom, over ${attempts}.`);
|
|
27357
|
+
if (record.recommendedPerCallEstimateCredits !== null) {
|
|
27358
|
+
log.plain(c.bold(` Grant perCallEstimateCredits: ${record.recommendedPerCallEstimateCredits}`));
|
|
27359
|
+
log.dim(` = that average rounded up to a whole credit. disclosure.estimatedCreditsPerPeriod = your calls per period \xD7 ${formatCredits(record.averageCreditsPerCall)}, rounded up.`);
|
|
27188
27360
|
}
|
|
27189
27361
|
}
|
|
27190
27362
|
async function explainBenchRefusal(res, call2, base, log, opts, index, settledSoFar) {
|
|
27191
|
-
if (printedStructuredError(res))
|
|
27363
|
+
if (printedStructuredError(res)) {
|
|
27364
|
+
const printed = await res.json().catch(() => ({}));
|
|
27365
|
+
return { stop: true, refusal: { status: res.status, error: typeof printed.error === "string" ? printed.error : null, slotsHeld: null, slotLimit: null } };
|
|
27366
|
+
}
|
|
27192
27367
|
const body = await res.json().catch(() => ({}));
|
|
27193
27368
|
const refusal2 = { status: res.status, error: body.error ?? null, slotsHeld: null, slotLimit: null };
|
|
27194
27369
|
if (res.status === 429 && body.error === "generation_limit") {
|
|
@@ -27198,6 +27373,10 @@ async function explainBenchRefusal(res, call2, base, log, opts, index, settledSo
|
|
|
27198
27373
|
if (!opts.json) log.error(` Sample ${index + 1} ${benchSlotsHeldSentence(counts)}`);
|
|
27199
27374
|
return { stop: true, refusal: refusal2 };
|
|
27200
27375
|
}
|
|
27376
|
+
if (res.status === 403 && body.error === "credits_unverified") {
|
|
27377
|
+
if (!opts.json) log.error(` ${LLM_BENCH_UNVERIFIED_LINE}`);
|
|
27378
|
+
return { stop: true, refusal: refusal2 };
|
|
27379
|
+
}
|
|
27201
27380
|
if (opts.json) return { stop: res.status !== 429, refusal: refusal2 };
|
|
27202
27381
|
if (res.status === 404 && body.error === "not_found") {
|
|
27203
27382
|
log.error(` ${LLM_LANE_OFF_LINE}`);
|
|
@@ -27213,7 +27392,7 @@ async function explainBenchRefusal(res, call2, base, log, opts, index, settledSo
|
|
|
27213
27392
|
return { stop: true, refusal: refusal2 };
|
|
27214
27393
|
}
|
|
27215
27394
|
if (res.status === 402) {
|
|
27216
|
-
log.error(" Not enough
|
|
27395
|
+
log.error(" Not enough credits to start the attempt \u2014 a bench is paid from your own balance.");
|
|
27217
27396
|
return { stop: true, refusal: refusal2 };
|
|
27218
27397
|
}
|
|
27219
27398
|
log.error(
|
|
@@ -27222,7 +27401,7 @@ async function explainBenchRefusal(res, call2, base, log, opts, index, settledSo
|
|
|
27222
27401
|
return { stop: res.status >= 500 || res.status === 401 || res.status === 403, refusal: refusal2 };
|
|
27223
27402
|
}
|
|
27224
27403
|
function printProviderRefusal(log, status2, message, awaitingBill = false) {
|
|
27225
|
-
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so this call
|
|
27404
|
+
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so what this call reserved stays held until it resolves (see \`genex llm status\`); it is not a sample.`);
|
|
27226
27405
|
else log.dim(" The provider refused this call at its door \u2014 no inference ran, it cost nothing, and it is not a sample.");
|
|
27227
27406
|
if (message) log.dim(` Provider said: ${message}`);
|
|
27228
27407
|
if (status2 === 401 || status2 === 403) {
|
|
@@ -27248,19 +27427,6 @@ async function pollSettled(call2, url, timeoutSec) {
|
|
|
27248
27427
|
}
|
|
27249
27428
|
return last;
|
|
27250
27429
|
}
|
|
27251
|
-
async function readCoinBalance(apiUrl, token) {
|
|
27252
|
-
try {
|
|
27253
|
-
const res = await apiFetch(`${apiUrl}/api/coin/balance`, {
|
|
27254
|
-
headers: { Authorization: `Bearer ${token}` },
|
|
27255
|
-
signal: AbortSignal.timeout(6e3)
|
|
27256
|
-
});
|
|
27257
|
-
if (!res.ok) return null;
|
|
27258
|
-
const body = await res.json().catch(() => null);
|
|
27259
|
-
return typeof body?.spendable === "number" ? body.spendable : null;
|
|
27260
|
-
} catch {
|
|
27261
|
-
return null;
|
|
27262
|
-
}
|
|
27263
|
-
}
|
|
27264
27430
|
async function saveBench(cwd, record) {
|
|
27265
27431
|
try {
|
|
27266
27432
|
const file = path39.join(cwd, LLM_BENCH_FILE);
|
|
@@ -27276,9 +27442,10 @@ function benchFailed(opts, log, message) {
|
|
|
27276
27442
|
else log.error(message);
|
|
27277
27443
|
process.exitCode = 1;
|
|
27278
27444
|
}
|
|
27279
|
-
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
27445
|
+
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCredits, opts) {
|
|
27280
27446
|
return {
|
|
27281
27447
|
command: "llm bench",
|
|
27448
|
+
currency: "credits",
|
|
27282
27449
|
apiUrl,
|
|
27283
27450
|
projectId,
|
|
27284
27451
|
modelId: opts.modelId ?? null,
|
|
@@ -27287,21 +27454,28 @@ function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCoi
|
|
|
27287
27454
|
schemaPath: opts.schemaPath ?? null,
|
|
27288
27455
|
samples,
|
|
27289
27456
|
completed: 0,
|
|
27290
|
-
|
|
27457
|
+
maxCreditsPerSample: maxCredits,
|
|
27291
27458
|
balanceBefore: null,
|
|
27292
27459
|
balanceAfter: null,
|
|
27293
27460
|
results: [],
|
|
27294
27461
|
refusal: null,
|
|
27295
|
-
|
|
27296
|
-
|
|
27297
|
-
|
|
27298
|
-
|
|
27462
|
+
costUsdPicos: { p50: null, p95: null, max: null },
|
|
27463
|
+
costUsd: { p50: null, p95: null, max: null },
|
|
27464
|
+
chargedCredits: { p50: null, p95: null, max: null },
|
|
27465
|
+
fitCredits: { p95: null, max: null },
|
|
27466
|
+
recommendedDeclaredHeadroomBps: null,
|
|
27467
|
+
feeBps: null,
|
|
27468
|
+
usdCentsPerCredit: null,
|
|
27469
|
+
missingTerms: null,
|
|
27470
|
+
costBasedMaxCredits: null,
|
|
27471
|
+
lengthRaisedCeiling: false,
|
|
27299
27472
|
cutOffSamples: 0,
|
|
27300
27473
|
exceedsStandCap: false,
|
|
27301
27474
|
noRecommendationReason: null,
|
|
27302
|
-
|
|
27303
|
-
|
|
27304
|
-
|
|
27475
|
+
recommendedMaxCredits: null,
|
|
27476
|
+
recommendedPerCallMaxCredits: null,
|
|
27477
|
+
averageCreditsPerCall: null,
|
|
27478
|
+
recommendedPerCallEstimateCredits: null,
|
|
27305
27479
|
savedTo: null
|
|
27306
27480
|
};
|
|
27307
27481
|
}
|
|
@@ -27364,7 +27538,8 @@ async function reportStatus(args) {
|
|
|
27364
27538
|
for (const row of rows) {
|
|
27365
27539
|
const state = rowState(row);
|
|
27366
27540
|
const mark = state === "active" ? c.cyan("\u25CF") : state === "awaiting bill" ? c.yellow("\u25D0") : c.green("\u25CB");
|
|
27367
|
-
const
|
|
27541
|
+
const money = rowMoney(row);
|
|
27542
|
+
const reserved = money.reserved === null ? "reserved \u2014" : `${money.reserved} ${UNIT_WORD[money.unit]} reserved`;
|
|
27368
27543
|
log.plain(
|
|
27369
27544
|
` ${mark} ${row.id ?? "?"} ${row.modelId ?? "?"} ${state.padEnd(13)} ${formatAge(row.createdAt, now).padStart(4)} old ${reserved}${row.error ? ` ${row.error}` : ""}${rowHoldsSlot(row) ? "" : c.dim(" (no slot)")}`
|
|
27370
27545
|
);
|
|
@@ -27436,10 +27611,11 @@ async function cancelCall(args) {
|
|
|
27436
27611
|
if (!opts.json) {
|
|
27437
27612
|
log.warn(` ${row.id} (${row.status ?? "?"}) \u2014 ${sentence}.`);
|
|
27438
27613
|
if (state === "awaiting bill") {
|
|
27439
|
-
log.dim(` Nothing to cancel:
|
|
27614
|
+
log.dim(` Nothing to cancel: what it reserved releases when the bill resolves, and it stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`);
|
|
27440
27615
|
if (row.providerMessage?.trim()) log.dim(` Provider said: ${row.providerMessage.trim()}`);
|
|
27441
27616
|
} else {
|
|
27442
|
-
|
|
27617
|
+
const money = rowMoney(row);
|
|
27618
|
+
log.dim(` ${money.charged === null ? "charge unknown" : `${money.charged} ${UNIT_WORD[money.unit]} charged`}${row.error ? ` \xB7 ${row.error}` : ""}.`);
|
|
27443
27619
|
}
|
|
27444
27620
|
}
|
|
27445
27621
|
return done("ok", null, state === "awaiting bill" ? "awaiting_bill" : "already_final", row);
|
|
@@ -27461,78 +27637,126 @@ async function cancelCall(args) {
|
|
|
27461
27637
|
const after = await res.json().catch(() => null) ?? row;
|
|
27462
27638
|
if (!opts.json) {
|
|
27463
27639
|
log.success(` Cancelled ${row.id} (was ${row.status ?? "active"}).`);
|
|
27464
|
-
log.dim(`
|
|
27640
|
+
log.dim(` What it reserved releases once the bill settles; ${c.cyan("genex llm status")} shows it until then.`);
|
|
27465
27641
|
}
|
|
27466
27642
|
return done("ok", null, "cancelled", after);
|
|
27467
27643
|
}
|
|
27468
|
-
async function
|
|
27469
|
-
let
|
|
27644
|
+
async function readSavedBench(cwd) {
|
|
27645
|
+
let parsed;
|
|
27470
27646
|
try {
|
|
27471
|
-
|
|
27647
|
+
parsed = JSON.parse(await fs39.readFile(path39.join(cwd, LLM_BENCH_FILE), "utf8"));
|
|
27472
27648
|
} catch {
|
|
27473
|
-
|
|
27649
|
+
return null;
|
|
27474
27650
|
}
|
|
27475
|
-
if (!
|
|
27651
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null;
|
|
27652
|
+
const v = parsed;
|
|
27653
|
+
if (v.v === 2 && v.currency === "credits" && v.cost && v.charged) return { kind: "credits", record: parsed };
|
|
27654
|
+
return { kind: "coins", record: parsed };
|
|
27655
|
+
}
|
|
27656
|
+
var LLM_PRICE_COIN_TERMS_LINE = "This bench was measured under the old Nex terms, when a game declared a fixed Nex price per call. In-game calls are now billed in the player's credits, as used, and a game declares a per-call CEILING (maxCredits) instead \u2014 a Nex price is not that number. Re-run the bench for the figures to declare.";
|
|
27657
|
+
async function reportSavedPrice(cwd, opts, log) {
|
|
27658
|
+
const saved = await readSavedBench(cwd);
|
|
27659
|
+
const rerun = 'genex llm bench "<prompt>" --max-credits <n> --user-approved';
|
|
27660
|
+
const creditKeys = (record2) => ({
|
|
27661
|
+
costUsdPicos: record2?.cost ?? { p50: null, p95: null, max: null },
|
|
27662
|
+
chargedCredits: record2?.charged ?? { p50: null, p95: null, max: null },
|
|
27663
|
+
fitP95: record2?.fitP95 ?? null,
|
|
27664
|
+
fitMax: record2?.fitMax ?? null,
|
|
27665
|
+
recommendedDeclaredHeadroomBps: record2?.recommendedDeclaredHeadroomBps ?? null,
|
|
27666
|
+
feeBps: record2?.feeBps ?? null,
|
|
27667
|
+
usdCentsPerCredit: record2?.usdCentsPerCredit ?? null,
|
|
27668
|
+
costBasedMaxCredits: record2?.costBasedMaxCredits ?? null,
|
|
27669
|
+
lengthRaisedCeiling: record2?.lengthRaisedCeiling ?? false,
|
|
27670
|
+
cutOffSamples: record2?.cutOffSamples ?? 0,
|
|
27671
|
+
noRecommendationReason: record2?.noRecommendationReason ?? null,
|
|
27672
|
+
recommendedMaxCredits: record2?.recommendedMaxCredits ?? null,
|
|
27673
|
+
recommendedPerCallMaxCredits: record2?.recommendedPerCallMaxCredits ?? null,
|
|
27674
|
+
averageCreditsPerCall: record2?.averageCreditsPerCall ?? null,
|
|
27675
|
+
recommendedPerCallEstimateCredits: record2?.recommendedPerCallEstimateCredits ?? null
|
|
27676
|
+
});
|
|
27677
|
+
if (!saved) {
|
|
27476
27678
|
if (opts.json) {
|
|
27477
27679
|
writeJsonLine({
|
|
27478
27680
|
command: "llm price",
|
|
27479
27681
|
status: "failed",
|
|
27480
27682
|
error: "no_bench",
|
|
27683
|
+
currency: null,
|
|
27481
27684
|
ranAt: null,
|
|
27482
27685
|
modelId: null,
|
|
27483
27686
|
samples: null,
|
|
27484
27687
|
completed: null,
|
|
27485
|
-
|
|
27486
|
-
|
|
27487
|
-
max: null,
|
|
27488
|
-
fitP95: null,
|
|
27489
|
-
chargedBasedEstimateCoins: null,
|
|
27490
|
-
lengthRaisedPrice: false,
|
|
27491
|
-
cutOffSamples: 0,
|
|
27492
|
-
noRecommendationReason: null,
|
|
27493
|
-
recommendedDeclaredHeadroomBps: null,
|
|
27494
|
-
recommendedEstimateCoins: null,
|
|
27495
|
-
recommendedPerCallMaxCoins: null,
|
|
27688
|
+
...creditKeys(null),
|
|
27689
|
+
legacyCoinTerms: null,
|
|
27496
27690
|
savedTo: null
|
|
27497
27691
|
});
|
|
27498
27692
|
} else {
|
|
27499
27693
|
log.error("Nothing has been benchmarked in this folder yet.");
|
|
27500
|
-
log.dim(` Run ${c.cyan(
|
|
27694
|
+
log.dim(` Run ${c.cyan(rerun)} first.`);
|
|
27695
|
+
}
|
|
27696
|
+
process.exitCode = 1;
|
|
27697
|
+
return;
|
|
27698
|
+
}
|
|
27699
|
+
if (saved.kind === "coins") {
|
|
27700
|
+
const old = saved.record;
|
|
27701
|
+
const legacy = {
|
|
27702
|
+
p50: old.p50 ?? null,
|
|
27703
|
+
p95: old.p95 ?? null,
|
|
27704
|
+
max: old.max ?? null,
|
|
27705
|
+
recommendedEstimateCoins: old.recommendedEstimateCoins ?? null,
|
|
27706
|
+
recommendedPerCallMaxCoins: old.recommendedPerCallMaxCoins ?? null,
|
|
27707
|
+
maxCoinsPerSample: old.maxCoinsPerSample ?? null,
|
|
27708
|
+
noRecommendationReason: old.noRecommendationReason ?? null
|
|
27709
|
+
};
|
|
27710
|
+
if (opts.json) {
|
|
27711
|
+
writeJsonLine({
|
|
27712
|
+
command: "llm price",
|
|
27713
|
+
status: "failed",
|
|
27714
|
+
error: "coin_terms",
|
|
27715
|
+
currency: "coins",
|
|
27716
|
+
ranAt: old.ranAt ?? null,
|
|
27717
|
+
modelId: old.modelId ?? null,
|
|
27718
|
+
samples: old.samples ?? null,
|
|
27719
|
+
completed: old.completed ?? null,
|
|
27720
|
+
...creditKeys(null),
|
|
27721
|
+
legacyCoinTerms: legacy,
|
|
27722
|
+
savedTo: LLM_BENCH_FILE
|
|
27723
|
+
});
|
|
27724
|
+
} else {
|
|
27725
|
+
log.plain(c.bold("genex llm price"));
|
|
27726
|
+
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${old.ranAt ?? "at an unknown time"}${old.modelId ? ` on ${old.modelId}` : ""}`);
|
|
27727
|
+
log.plain("");
|
|
27728
|
+
log.warn(` ${LLM_PRICE_COIN_TERMS_LINE}`);
|
|
27729
|
+
const measured = legacy.p95 === null ? "no settled sample" : `charged Nex p50 ${legacy.p50} \xB7 p95 ${legacy.p95} \xB7 max ${legacy.max}`;
|
|
27730
|
+
const recommended = legacy.recommendedEstimateCoins === null ? "no price recommended" : `it recommended estimateCoins ${legacy.recommendedEstimateCoins}${legacy.recommendedPerCallMaxCoins != null ? ` and perCallMaxCoins ${legacy.recommendedPerCallMaxCoins}` : ""}`;
|
|
27731
|
+
log.dim(` What it measured then: ${measured}; ${recommended}.`);
|
|
27732
|
+
log.dim(` Re-run: ${c.cyan(rerun)}`);
|
|
27501
27733
|
}
|
|
27502
27734
|
process.exitCode = 1;
|
|
27503
27735
|
return;
|
|
27504
27736
|
}
|
|
27737
|
+
const record = saved.record;
|
|
27505
27738
|
if (opts.json) {
|
|
27506
27739
|
writeJsonLine({
|
|
27507
27740
|
command: "llm price",
|
|
27508
|
-
status: record.
|
|
27509
|
-
error: record.
|
|
27741
|
+
status: record.recommendedMaxCredits === null ? "failed" : "ok",
|
|
27742
|
+
error: record.recommendedMaxCredits === null ? record.noRecommendationReason ?? "no_recommendation" : null,
|
|
27743
|
+
currency: "credits",
|
|
27510
27744
|
ranAt: record.ranAt ?? null,
|
|
27511
27745
|
modelId: record.modelId ?? null,
|
|
27512
27746
|
samples: record.samples ?? null,
|
|
27513
27747
|
completed: record.completed ?? null,
|
|
27514
|
-
|
|
27515
|
-
|
|
27516
|
-
max: record.max ?? null,
|
|
27517
|
-
// Absent from a file an older CLI wrote; null / false / 0 then, never missing.
|
|
27518
|
-
fitP95: record.fitP95 ?? null,
|
|
27519
|
-
chargedBasedEstimateCoins: record.chargedBasedEstimateCoins ?? null,
|
|
27520
|
-
lengthRaisedPrice: record.lengthRaisedPrice ?? false,
|
|
27521
|
-
cutOffSamples: record.cutOffSamples ?? 0,
|
|
27522
|
-
noRecommendationReason: record.noRecommendationReason ?? null,
|
|
27523
|
-
recommendedDeclaredHeadroomBps: record.recommendedDeclaredHeadroomBps ?? null,
|
|
27524
|
-
recommendedEstimateCoins: record.recommendedEstimateCoins ?? null,
|
|
27525
|
-
recommendedPerCallMaxCoins: record.recommendedPerCallMaxCoins ?? null,
|
|
27748
|
+
...creditKeys(record),
|
|
27749
|
+
legacyCoinTerms: null,
|
|
27526
27750
|
savedTo: LLM_BENCH_FILE
|
|
27527
27751
|
});
|
|
27528
|
-
if (record.
|
|
27752
|
+
if (record.recommendedMaxCredits === null) process.exitCode = 1;
|
|
27529
27753
|
return;
|
|
27530
27754
|
}
|
|
27531
27755
|
log.plain(c.bold("genex llm price"));
|
|
27532
27756
|
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${record.ranAt}`);
|
|
27533
27757
|
log.plain("");
|
|
27534
27758
|
printRecommendation(log, record);
|
|
27535
|
-
if (record.
|
|
27759
|
+
if (record.recommendedMaxCredits === null) process.exitCode = 1;
|
|
27536
27760
|
}
|
|
27537
27761
|
function writeJsonLine(value) {
|
|
27538
27762
|
process.stdout.write(`${JSON.stringify(value)}
|
|
@@ -27792,7 +28016,7 @@ function runtimeRow(state, lane, toolsOnly = false) {
|
|
|
27792
28016
|
return {
|
|
27793
28017
|
label: "Runtime",
|
|
27794
28018
|
value: `in-game LLM lane live \xB7 ${n} model${n === 1 ? "" : "s"}${keyRefused ? ` \xB7 ${c.yellow("provider key refused")}` : ""}`,
|
|
27795
|
-
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : '
|
|
28019
|
+
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : 'Measure a call before a game declares its per-call ceiling: `npx genex llm bench "<prompt>" --max-credits <n> --user-approved`.'
|
|
27796
28020
|
};
|
|
27797
28021
|
}
|
|
27798
28022
|
case "off":
|
|
@@ -29436,7 +29660,7 @@ function oneParagraph(input) {
|
|
|
29436
29660
|
const runnerText = input.runners.length === 0 ? "no runner yet" : input.runners.map((r) => r === "claude" ? "Claude Code" : "Codex").join(" and ");
|
|
29437
29661
|
return [
|
|
29438
29662
|
` Genex approval stored in ${input.credentialFile} (only this machine can read it). It can`,
|
|
29439
|
-
" answer approved requests and spend nothing \u2014 no
|
|
29663
|
+
" answer approved requests and spend nothing \u2014 no Nex, no credits, no",
|
|
29440
29664
|
" account changes. Your Claude or ChatGPT sign-in is never read, copied or",
|
|
29441
29665
|
" sent anywhere; the watcher only starts the official CLI you already use.",
|
|
29442
29666
|
` Runners ${runnerText}`,
|
|
@@ -30189,7 +30413,8 @@ async function publishNativeBuild(ctx, body, log) {
|
|
|
30189
30413
|
async function listNativeBuilds(ctx, log) {
|
|
30190
30414
|
const res = await call(ctx, "/native-builds", { method: "GET" }, "list the builds", log);
|
|
30191
30415
|
if (!res) return null;
|
|
30192
|
-
|
|
30416
|
+
const body = await res.json();
|
|
30417
|
+
return { builds: body.builds, submissions: body.submissions ?? [] };
|
|
30193
30418
|
}
|
|
30194
30419
|
async function removeNativeBuild(ctx, platform, arch, log) {
|
|
30195
30420
|
const route = `/native-builds/${encodeURIComponent(platform)}/${encodeURIComponent(arch)}`;
|
|
@@ -30277,6 +30502,17 @@ function describeLine(d) {
|
|
|
30277
30502
|
if (d.engine) parts.push(d.engine);
|
|
30278
30503
|
return parts.join(" \xB7 ");
|
|
30279
30504
|
}
|
|
30505
|
+
function reportPublished(build, log) {
|
|
30506
|
+
const pair = `${build.platform} ${build.arch}`;
|
|
30507
|
+
if (build.review === "pending") {
|
|
30508
|
+
log.success(`Sent for review \u2014 Genex desktop players on ${pair} get version ${build.version} once Genex approves it.`);
|
|
30509
|
+
} else if (build.review === "rejected") {
|
|
30510
|
+
const why = build.reviewReason ? `: ${build.reviewReason}` : ".";
|
|
30511
|
+
log.warn(`Version ${build.version} for ${pair} was turned down in review${why} Change the build and publish it again.`);
|
|
30512
|
+
} else {
|
|
30513
|
+
log.success(`Live \u2014 Genex desktop players on ${pair} now get version ${build.version}.`);
|
|
30514
|
+
}
|
|
30515
|
+
}
|
|
30280
30516
|
async function runPublish2(opts, log) {
|
|
30281
30517
|
if (refusedInHostedSession(log)) return false;
|
|
30282
30518
|
if (!opts.nativePath) {
|
|
@@ -30316,7 +30552,7 @@ async function runPublish2(opts, log) {
|
|
|
30316
30552
|
log
|
|
30317
30553
|
);
|
|
30318
30554
|
if (!build) return false;
|
|
30319
|
-
|
|
30555
|
+
reportPublished(build, log);
|
|
30320
30556
|
log.dim(` ${build.url}`);
|
|
30321
30557
|
const listing = opts.gallery ? await listGame(ctx, opts, log) : void 0;
|
|
30322
30558
|
if (opts.json) {
|
|
@@ -30355,20 +30591,25 @@ async function listGame(ctx, opts, log) {
|
|
|
30355
30591
|
function buildLine(b) {
|
|
30356
30592
|
return `${c.bold(`${b.platform} ${b.arch}`)} ${b.version} ${b.entry} ${sizeLine(b.sizeBytes)} ${b.publishedAt}`;
|
|
30357
30593
|
}
|
|
30594
|
+
function heldLine(b) {
|
|
30595
|
+
const state = b.review === "rejected" ? `turned down${b.reviewReason ? `: ${b.reviewReason}` : ""}` : "in review";
|
|
30596
|
+
return `${buildLine(b)} ${c.dim(state)}`;
|
|
30597
|
+
}
|
|
30358
30598
|
async function runList2(opts, log) {
|
|
30359
30599
|
const ctx = await context3(opts, log);
|
|
30360
30600
|
if (!ctx) return false;
|
|
30361
|
-
const
|
|
30362
|
-
if (!
|
|
30601
|
+
const listed = await listNativeBuilds(ctx, log);
|
|
30602
|
+
if (!listed) return false;
|
|
30363
30603
|
if (opts.json) {
|
|
30364
|
-
console.log(JSON.stringify(
|
|
30604
|
+
console.log(JSON.stringify(listed));
|
|
30365
30605
|
return true;
|
|
30366
30606
|
}
|
|
30367
|
-
if (builds.length === 0) {
|
|
30607
|
+
if (listed.builds.length === 0 && listed.submissions.length === 0) {
|
|
30368
30608
|
log.plain("No native builds. Publish one with `genex native publish <path>`.");
|
|
30369
30609
|
return true;
|
|
30370
30610
|
}
|
|
30371
|
-
for (const b of builds) log.plain(buildLine(b));
|
|
30611
|
+
for (const b of listed.builds) log.plain(buildLine(b));
|
|
30612
|
+
for (const b of listed.submissions) log.plain(heldLine(b));
|
|
30372
30613
|
return true;
|
|
30373
30614
|
}
|
|
30374
30615
|
async function runRemove(opts, log) {
|
|
@@ -30469,16 +30710,16 @@ ${c.bold("Usage")}
|
|
|
30469
30710
|
status | cancel. "models" says whether this
|
|
30470
30711
|
stand serves the runtime LLM lane at all, and
|
|
30471
30712
|
at what provider rates. "bench" runs the real
|
|
30472
|
-
model on YOUR OWN
|
|
30473
|
-
attempt
|
|
30713
|
+
model on YOUR OWN credits and reports what each
|
|
30714
|
+
attempt really cost; it needs --max-credits <n>
|
|
30474
30715
|
--user-approved, like any spend the player has
|
|
30475
30716
|
to agree to. "price" reprints the last run's
|
|
30476
|
-
recommended
|
|
30717
|
+
recommended maxCredits. "status" lists this
|
|
30477
30718
|
project's open calls and the slots they hold;
|
|
30478
30719
|
"cancel <id>" stops an active one. Declare a
|
|
30479
|
-
game's
|
|
30480
|
-
a
|
|
30481
|
-
|
|
30720
|
+
game's per-call ceiling from a bench, never
|
|
30721
|
+
from a guess: the player is billed as used, and
|
|
30722
|
+
the ceiling also sizes how long an answer may be.
|
|
30482
30723
|
genex player <sub> [options] Answer the in-game model requests you approve, on your
|
|
30483
30724
|
OWN Claude or ChatGPT subscription: install | run |
|
|
30484
30725
|
status | stop | uninstall. "install" signs this machine
|
|
@@ -30550,8 +30791,8 @@ ${c.bold("Usage")}
|
|
|
30550
30791
|
the item id your game passes
|
|
30551
30792
|
to buy({ skuId }) \u2014 the platform owns the
|
|
30552
30793
|
catalog, so this is the only way one exists.
|
|
30553
|
-
Prices come off a fixed grid, printed on
|
|
30554
|
-
refusal. "test <id>" gives you a free copy of
|
|
30794
|
+
Prices in Nex come off a fixed grid, printed on
|
|
30795
|
+
any refusal. "test <id>" gives you a free copy of
|
|
30555
30796
|
your own item: you can't buy from your own game,
|
|
30556
30797
|
so this is how you check the shop works.
|
|
30557
30798
|
--icon <url> is the item's square art, shown in
|
|
@@ -30981,7 +31222,7 @@ ${c.bold("Examples")}
|
|
|
30981
31222
|
genex items add sku_456
|
|
30982
31223
|
genex llm models
|
|
30983
31224
|
genex llm models --all
|
|
30984
|
-
genex llm bench "Reply with one short taunt." --samples 5 --max-
|
|
31225
|
+
genex llm bench "Reply with one short taunt." --samples 5 --max-credits 20 --user-approved
|
|
30985
31226
|
genex llm price
|
|
30986
31227
|
genex llm status
|
|
30987
31228
|
genex llm cancel <id>
|
|
@@ -31218,22 +31459,24 @@ ${c.bold("How the allowance works")}
|
|
|
31218
31459
|
|
|
31219
31460
|
Every number printed is live from your account; prices can change without a deploy.
|
|
31220
31461
|
`,
|
|
31221
|
-
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what should the game declare?
|
|
31462
|
+
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what ceiling should the game declare?
|
|
31222
31463
|
|
|
31223
31464
|
${c.bold("Usage")}
|
|
31224
31465
|
genex llm models [--all] [--json] Which models this stand serves, their provider rates per
|
|
31225
|
-
million tokens,
|
|
31226
|
-
|
|
31227
|
-
|
|
31228
|
-
|
|
31229
|
-
on this stand: it says so
|
|
31230
|
-
|
|
31231
|
-
|
|
31232
|
-
|
|
31233
|
-
|
|
31234
|
-
|
|
31235
|
-
|
|
31236
|
-
|
|
31466
|
+
million tokens, the platform fee a call is billed at, and
|
|
31467
|
+
the headroom it recommends over a benchmark. The catalog
|
|
31468
|
+
is synced from OpenRouter, so the FEATURED short list is
|
|
31469
|
+
printed by default and --all prints every row (--json
|
|
31470
|
+
always carries every row). Off on this stand: it says so
|
|
31471
|
+
and exits 0 \u2014 build the feature without a model rather
|
|
31472
|
+
than promising a 404.
|
|
31473
|
+
genex llm bench "<prompt>" --max-credits <n> --user-approved [options]
|
|
31474
|
+
Run the real model through the development lane and
|
|
31475
|
+
report what each attempt really COST and was charged.
|
|
31476
|
+
Refused without both flags, before any network call:
|
|
31477
|
+
this spends YOUR OWN CREDITS, a spend the build's asset
|
|
31478
|
+
allowance does not count. Needs a folder linked to a
|
|
31479
|
+
game you own, and a balance that covers one attempt.
|
|
31237
31480
|
genex llm price [--json] The recommendation from the last bench in this folder.
|
|
31238
31481
|
Reads a file; spends nothing.
|
|
31239
31482
|
genex llm status [--json] This project's open development calls: active, or
|
|
@@ -31250,29 +31493,32 @@ ${c.bold("Bench options")}
|
|
|
31250
31493
|
--samples <n> How many real attempts (default ${DEFAULT_BENCH_SAMPLES}). More samples, better p95.
|
|
31251
31494
|
--schema <file> A JSON file holding the output schema; implies JSON output.
|
|
31252
31495
|
--json-output Ask for JSON without a schema. --text Ask for text (the default).
|
|
31253
|
-
--max-
|
|
31496
|
+
--max-credits <n> The per-attempt ceiling YOU approved \u2014 a bench runs on your own credits,
|
|
31254
31497
|
never a player's. The run's worst case is that number times --samples,
|
|
31255
|
-
and it is printed before anything starts.
|
|
31498
|
+
and it is printed before anything starts. (--max-coins is its deprecated
|
|
31499
|
+
name, read as the same number.)
|
|
31256
31500
|
--timeout <seconds> Bounds the wait for each attempt.
|
|
31257
31501
|
--json One machine-readable object; every key is always present, null when
|
|
31258
31502
|
the CLI could not learn it.
|
|
31259
31503
|
|
|
31260
31504
|
${c.bold("Why a benchmark and not an estimate")}
|
|
31261
|
-
|
|
31262
|
-
|
|
31263
|
-
|
|
31264
|
-
|
|
31265
|
-
|
|
31266
|
-
|
|
31267
|
-
|
|
31268
|
-
|
|
31269
|
-
|
|
31505
|
+
An in-game call is paid from the PLAYER's credits and billed as used: its real provider cost
|
|
31506
|
+
plus the platform fee, never more than the per-call CEILING the game declares (\`maxCredits\`,
|
|
31507
|
+
or \`perCallMaxCredits\` on a standing budget). The ceiling is not a price, but it is still a
|
|
31508
|
+
number with two jobs: it bounds what one call may cost, and it funds how LONG the answer may
|
|
31509
|
+
be. Three layers sit between the provider's rate card and it \u2014 the provider's cost for YOUR
|
|
31510
|
+
prompt, the platform fee, and your headroom \u2014 and only a real call measures the first. The
|
|
31511
|
+
bench is billed to YOU, on the development lane; the player-funded lane is what the
|
|
31512
|
+
published game uses. The recommendation is the p95 real cost with the server's fee and
|
|
31513
|
+
headroom applied, in whole credits; a stand that serves no headroom or no fee gets no
|
|
31514
|
+
recommendation, because those multipliers are not the CLI's to invent.
|
|
31270
31515
|
|
|
31271
|
-
|
|
31272
|
-
|
|
31273
|
-
|
|
31274
|
-
|
|
31275
|
-
|
|
31516
|
+
Because the ceiling sizes each call's output allowance, the recommendation is never below
|
|
31517
|
+
the smallest ceiling that leaves room for the benchmarked answer, as the server computes it.
|
|
31518
|
+
A sample cut off at your --max-credits (provider_token_limit) is not a sample \u2014 re-run with a
|
|
31519
|
+
higher --max-credits; an answer longer than one call on this stand may produce gets no
|
|
31520
|
+
ceiling at all \u2014 ask for a shorter one. The run also prints the measured average per call,
|
|
31521
|
+
fee included, which is the honest estimate a budget's disclosure is built on.
|
|
31276
31522
|
|
|
31277
31523
|
Results land in ${LLM_BENCH_FILE} \u2014 gitignored, machine-local, re-read by \`genex llm price\`.
|
|
31278
31524
|
`,
|
|
@@ -31298,7 +31544,7 @@ ${c.bold("What it does and does not touch")}
|
|
|
31298
31544
|
prints. Genex never sees your vendor account.
|
|
31299
31545
|
|
|
31300
31546
|
Its GENEX approval is a player-scoped one: it can answer requests you already approved and can
|
|
31301
|
-
spend nothing \u2014 no
|
|
31547
|
+
spend nothing \u2014 no Nex, no credits, no account changes. It lives in ~/.genex/player.json,
|
|
31302
31548
|
separate from your creator sign-in, and the dashboard can revoke it at any time.
|
|
31303
31549
|
|
|
31304
31550
|
The log under ~/.genex/logs/ carries ids, timings and outcomes \u2014 never what a game asked for
|
|
@@ -31527,11 +31773,13 @@ function parseArgs(argv) {
|
|
|
31527
31773
|
"--limit",
|
|
31528
31774
|
// `genex budget --assets <credits>` — the allowance the player approved.
|
|
31529
31775
|
"--assets",
|
|
31530
|
-
// `genex llm bench` value flags. `--max-
|
|
31531
|
-
//
|
|
31776
|
+
// `genex llm bench` value flags. `--max-credits` is the same approval
|
|
31777
|
+
// shape as `--assets`, but its own gate: the asset allowance does not
|
|
31778
|
+
// count a bench. `--max-coins` is its deprecated name (same number).
|
|
31532
31779
|
"--model",
|
|
31533
31780
|
"--samples",
|
|
31534
31781
|
"--schema",
|
|
31782
|
+
"--max-credits",
|
|
31535
31783
|
"--max-coins",
|
|
31536
31784
|
// `genex player install --label <name>` — what this machine is called in
|
|
31537
31785
|
// the dashboard. Its own key rather than the generic 2nd positional,
|
|
@@ -32102,12 +32350,14 @@ function applyValueFlag(options, flag, value) {
|
|
|
32102
32350
|
options.samples = n;
|
|
32103
32351
|
break;
|
|
32104
32352
|
}
|
|
32353
|
+
case "--max-credits":
|
|
32105
32354
|
case "--max-coins": {
|
|
32106
32355
|
const n = Number(value);
|
|
32107
32356
|
if (!Number.isInteger(n) || n < 1) {
|
|
32108
|
-
throw new Error(`Invalid
|
|
32357
|
+
throw new Error(`Invalid ${flag} value: ${value} (whole credits, 1 or more)`);
|
|
32109
32358
|
}
|
|
32110
|
-
options.
|
|
32359
|
+
if (flag === "--max-credits") options.maxCredits = n;
|
|
32360
|
+
else options.maxCoins = n;
|
|
32111
32361
|
break;
|
|
32112
32362
|
}
|
|
32113
32363
|
case "--timeout": {
|
|
@@ -32129,7 +32379,7 @@ function applyValueFlag(options, flag, value) {
|
|
|
32129
32379
|
case "--price": {
|
|
32130
32380
|
const n = Number(value);
|
|
32131
32381
|
if (value.trim() === "" || !Number.isInteger(n) || n < 0) {
|
|
32132
|
-
throw new Error(`Invalid --price value: ${value} (whole
|
|
32382
|
+
throw new Error(`Invalid --price value: ${value} (whole Nex, e.g. 100; 0 is Free)`);
|
|
32133
32383
|
}
|
|
32134
32384
|
options.price = n;
|
|
32135
32385
|
break;
|
|
@@ -32418,9 +32668,9 @@ async function main() {
|
|
|
32418
32668
|
await runBudget(parsed.options);
|
|
32419
32669
|
break;
|
|
32420
32670
|
// The in-game model lane: is it live here, what does one call really
|
|
32421
|
-
// cost, what should the game declare. `bench`
|
|
32422
|
-
//
|
|
32423
|
-
//
|
|
32671
|
+
// cost, what ceiling should the game declare. `bench` spends the
|
|
32672
|
+
// builder's own credits OUTSIDE the asset allowance, so it carries its
|
|
32673
|
+
// own --max-credits/--user-approved gate.
|
|
32424
32674
|
case "llm":
|
|
32425
32675
|
await runLlm(parsed.options);
|
|
32426
32676
|
break;
|