@genex-ai/cli-demo 1.36.0 → 1.36.1-dev.776
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{blender-mcp-NRJ5FI22.js → blender-mcp-ZRCMYTXI.js} +2 -2
- package/dist/{blender-serve-66OHC4ML.js → blender-serve-EH2MOW3U.js} +1 -1
- package/dist/{chunk-QI7FIYBY.js → chunk-K4EHXOZK.js} +1 -1
- package/dist/{chunk-GOP5FRSJ.js → chunk-Y6WO5FJZ.js} +2 -2
- package/dist/index.js +454 -503
- package/package.json +1 -1
- package/templates/skills/genex-ai-model/SKILL.md +4 -1
- package/templates/skills/genex-game-director/SKILL.md +1 -1
- package/templates/skills/genex-getting-started/SKILL.md +2 -2
- package/templates/skills/genex-llm-in-games/SKILL.md +209 -72
- package/templates/skills/genex-llm-in-games/references/pricing.md +126 -85
- package/templates/skills/genex-tool-llm/SKILL.md +11 -9
- package/templates/skills/genex-updates/SKILL.md +1 -1
package/dist/index.js
CHANGED
|
@@ -44,7 +44,7 @@ import {
|
|
|
44
44
|
writeSecretFile,
|
|
45
45
|
writeUserToken,
|
|
46
46
|
writeWorkspace
|
|
47
|
-
} from "./chunk-
|
|
47
|
+
} from "./chunk-Y6WO5FJZ.js";
|
|
48
48
|
import {
|
|
49
49
|
CLI_CHANNEL,
|
|
50
50
|
DEFAULT_API_URL,
|
|
@@ -65,7 +65,7 @@ import {
|
|
|
65
65
|
normalizeApiOrigin,
|
|
66
66
|
originKey,
|
|
67
67
|
resolveAgentTargets
|
|
68
|
-
} from "./chunk-
|
|
68
|
+
} from "./chunk-K4EHXOZK.js";
|
|
69
69
|
|
|
70
70
|
// src/instrument.ts
|
|
71
71
|
import * as Sentry from "@sentry/node";
|
|
@@ -904,7 +904,7 @@ Important note: put soul into your creations, with many details and love. Aim to
|
|
|
904
904
|
21. A turn ends in exactly one of two ways: on a question the player must answer before the next step can be chosen, or on a handoff \u2014 the draft link, what changed, what to try, and what is still open. Never close a turn on a promise. "I'll keep building", "while I keep working", "adding that now" are things you say and then DO before you stop; if you are stopping, say that you have stopped and what remains. While the Build plan's \`Now:\` line has open work and no answer is needed from the player, the turn is not over: preview, hand off, and start the next milestone in the same turn, until the requested outcome is reached or the platform's own budget ends the session. If a stop was forced on you anyway, the first line of your next turn is the promise you left and where it stands.
|
|
905
905
|
22. **What the player sees must read as the thing it is**, in this game's own style: a building as that building, a person as a person, a prop as that prop, a surface as its material \u2014 and a coloured box, a capsule, or a flat grey block standing in for one of them is never its finished version. Generation is the default route to that bar wherever code will not honestly reach it: the player's character, whatever the request names, whatever the player walks up to, enters, or interacts with, and the music and sound underneath \u2014 reach for the belt on your own judgment, without stopping to ask; a status line saying what you queued is enough, and \`--no-wait\` keeps you building while it lands. Procedural code is a first-class engine for what is structural, repeated, distant, or parametric \u2014 terrain, sky, fences, paving, modular kits, filler \u2014 and for anything else only when the result meets the same bar and you have looked at it in a capture before its row says \`landed\`. Start with one reusable implementation per distinct required gameplay or visual role. Background populations default to reusable procedural bodies and motion or compatible existing rigged assets. Catalog clips need a compatible skeleton and may be paid. Reserve paid custom bodies for the player and characters examined or interacted with closely, while honoring explicit user requests. Add variants only for an unmet requirement; unspent allowance is not unfinished work. Allowance questions use timeoutPolicy no-consent: silence does not raise the allowance. Record role, reuse and completion criteria in the existing Assets table. Decide per object, not per category, and write the route on each Assets row in \`DESIGN.md\` \u2014 the paid flow for generated pieces, the procedural flow with its local path for code-built ones. An Assets table with no generation in it is a decision, not a default: record it as \`Generation: none \u2014 <why code alone reaches the bar here>\` or generate.
|
|
906
906
|
23. **The screenshots you take to look at your own work go in \`.genex/scratch/\`.** Captures, render comparisons, before/after strips, traces and metrics dumps are how you SEE what you built \u2014 they are not part of what you built, and \`genex preview\` pushes the whole folder to the game's source, every time, forever. \`.genex/scratch/\` already exists for this and is already ignored, so there is nothing to set up: write the capture as \`.genex/scratch/arena-before.png\` and read it back from there. Never put them in a folder of your own at the top level: \`progress/\`, \`reports/\`, \`shots/\` and their kind are pushed like source and become permanent weight in the game and in every remix of it \u2014 one real game reached 11 GB and 10,332 committed screenshots exactly this way. If you find such a folder already there, add it to \`.gitignore\`; never delete the player's files (law 18). One file in \`.genex/scratch/\` IS read, by the platform: \`.genex/scratch/cover.png\` is the game's cover \u2014 the frame you would put on its poster: real gameplay at its signature moment, well lit, no menus, popups or busy HUD, landscape 16:9, and never any text (the page sets the title beside it). The next \`genex preview\` makes it the cover; save a better moment over it whenever one comes along, and if your browser tool can set the cover itself, use that instead.
|
|
907
|
-
24. **Benchmark before declaring an in-game LLM
|
|
907
|
+
24. **Benchmark before declaring an in-game LLM ceiling.** If the game calls a language model while the player plays \u2014 an NPC answering in its own words, a quest written for this save, a judge reading what the player typed \u2014 the PLAYER pays for it from their Genex credits, billed as used, and the game declares no price, only a per-call CEILING: \`maxCredits\` on \`generate()\`, \`perCallMaxCredits\` on a standing budget. That ceiling bounds what one call may cost AND funds how long its answer may be, so never pick it by judgement or from a vendor's rate card. Run the real prompt on your own credits \u2014 \`npx genex llm bench "<the prompt>" --schema <file> --samples 3 --max-credits <n> --user-approved\` \u2014 and declare the figures it prints, recorded in \`DESIGN.md\` with its model and date; re-benchmark whenever the prompt, the schema or the model changes, because a prompt edit changes the ceiling. Check the lane first with \`npx genex llm models\`: where it is off the routes 404, so build the feature behind a graceful unavailable state and say so in the handoff. A repeated call is a standing budget the player approves once (\`requestSpendGrant()\`, then \`generate({ grantId })\`), never a popup per turn, and its disclosed call rate and credits per period are computed from the game's own loop and the bench's measured average. The coin spellings (\`estimateCoins\`, \`perCallMaxCoins\`) are deprecated \u2014 write the credit names. Model output is data the game validates against its own expectation \u2014 never executed, and never authority over coin, credits, items, entitlements or rewards. Load \`$genex-llm-in-games\` before writing any of it.
|
|
908
908
|
${CONTRACT_END}
|
|
909
909
|
`;
|
|
910
910
|
var TOOLS_CONTRACT_HEAD = `# Genex Tools (always in effect in this folder)
|
|
@@ -918,7 +918,7 @@ var TOOLS_CONTRACT_OFFER = `5. **When the game is built and playable \u2014 neve
|
|
|
918
918
|
var TOOLS_CONTRACT_HOSTED_LAWS = `5. **This game is hosted on Genex now, and publishing is glue \u2014 never a rebuild.** The game ships exactly as the user built it: \`initEmbed()\` first in the boot code (the \`genex-threejs-embed-auth\` card has the call; the renderer draws before any \`await waitForPlayer()\`), a static build into \`dist/\` with relative asset paths, then \`npx genex preview\`. Never restructure, reformat or "improve" the game to ship it, never scaffold a new app around it, and never start a design document or a build plan for it \u2014 the user's own process is theirs. Load the \`genex-tool-publish\` card before the first preview; it owns the vocabulary, the links and the limits.
|
|
919
919
|
6. **Ship first, then report.** \`preview\` prints a preflight \u2014 phone memory, a missing volume slider, an asset nobody wired, a viewport line. It never blocks a deploy, so push the build as it is, hand over the link, THEN relay each preflight line to the user in one plain sentence with an offer to fix it, and fix one only on their yes. The preflight is a report for the user, not a to-do list for you.
|
|
920
920
|
7. **The link is the game's page, and there are two versions.** After every preview give the user \`<dashboard>/draft/<slug>\` \u2014 \`dashboardOrigins[0]\` and \`slug\` from \`.genex/project.json\` \u2014 never localhost, a file path or the bare play origin. \`preview\` updates the draft and never touches what players are on; the first release is \`npx genex publish\` (it lists the game); after that "publish it", "update it" and "yes" all mean \`npx genex promote\` \u2014 the exact draft build, no rebuild. Ask once per round of work, in one line, and keep working while you wait.`;
|
|
921
|
-
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with Genex
|
|
921
|
+
var TOOLS_CONTRACT_LLM_OFFER = `A model running while people play is built into the Genex platform \u2014 the player pays, with their Genex credits or their own Claude/ChatGPT subscription, and approves it on a Genex sheet; your game just calls \`generate()\`. Want it that way?`;
|
|
922
922
|
function toolsContractLlmLaw(n, hosted) {
|
|
923
923
|
const onYes = hosted ? "On a yes, load `$genex-tool-llm` and then `$genex-llm-in-games`, which owns the build." : "On a yes, load `$genex-tool-llm`, which owns the rest: it checks the lane with `npx genex llm models` first, then runs `npx genex init --convert` (a hosted game is a static browser build \u2014 a local server holding a key can never ship).";
|
|
924
924
|
return `${n}. **A model running while people PLAY is a platform feature \u2014 never something you wire onto the user's own meter.** When a request implies one (NPCs that talk or decide in their own words, content written from what the player types, a prompt box in the game, "let the player pick a model"), recognise it, say in ONE line: "${TOOLS_CONTRACT_LLM_OFFER}" \u2014 then ASK and wait. ${onYes} On a no, build the authored version instead. Either way, never ship a game that calls a model on a key or an account of the user's own: every visitor would spend their money with nobody approving it, and the credential is readable in the bundle. Player-funded generation is text and JSON only \u2014 3D, images, video and audio stay the asset lanes above, on the user's meter.`;
|
|
@@ -8680,6 +8680,7 @@ async function assetBudgetGate(input) {
|
|
|
8680
8680
|
// src/lib/generation-admission.ts
|
|
8681
8681
|
async function withAdmissionLock(cwd, work, waitMs = 3e4) {
|
|
8682
8682
|
const lock = path26.join(cwd, ".genex", "generation-admission.lock");
|
|
8683
|
+
await fs26.mkdir(path26.dirname(lock), { recursive: true });
|
|
8683
8684
|
const deadline = Date.now() + waitMs;
|
|
8684
8685
|
let file;
|
|
8685
8686
|
while (!file) {
|
|
@@ -16468,19 +16469,6 @@ var MESHY_ANIMATION_CATALOG = [
|
|
|
16468
16469
|
"isFree": false,
|
|
16469
16470
|
"createdAt": 1750829513123
|
|
16470
16471
|
},
|
|
16471
|
-
{
|
|
16472
|
-
"actionId": 464,
|
|
16473
|
-
"key": "Leap_and_Punch",
|
|
16474
|
-
"name": "Leap and Punch",
|
|
16475
|
-
"category": "BodyMovements",
|
|
16476
|
-
"subCategory": "Jumping",
|
|
16477
|
-
"previewUrl": "https://cdn.meshy.ai/webapp-assets/feature-demo/animation/preview/biped/Leap_and_Punch.gif",
|
|
16478
|
-
"rigType": "style_02",
|
|
16479
|
-
"tag": null,
|
|
16480
|
-
"isDefault": false,
|
|
16481
|
-
"isFree": false,
|
|
16482
|
-
"createdAt": 1750829513146
|
|
16483
|
-
},
|
|
16484
16472
|
{
|
|
16485
16473
|
"actionId": 465,
|
|
16486
16474
|
"key": "Leap_Right_and_Catch",
|
|
@@ -18223,58 +18211,6 @@ var MESHY_ANIMATION_CATALOG = [
|
|
|
18223
18211
|
"isFree": false,
|
|
18224
18212
|
"createdAt": 1750829520714
|
|
18225
18213
|
},
|
|
18226
|
-
{
|
|
18227
|
-
"actionId": 601,
|
|
18228
|
-
"key": "Backflip_inplace",
|
|
18229
|
-
"name": "Backflip",
|
|
18230
|
-
"category": "BodyMovements",
|
|
18231
|
-
"subCategory": "PerformingStunt",
|
|
18232
|
-
"previewUrl": "https://cdn.meshy.ai/webapp-assets/feature-demo/animation/preview/biped/Backflip.gif",
|
|
18233
|
-
"rigType": "style_02",
|
|
18234
|
-
"tag": "InPlace",
|
|
18235
|
-
"isDefault": false,
|
|
18236
|
-
"isFree": false,
|
|
18237
|
-
"createdAt": 1750829520865
|
|
18238
|
-
},
|
|
18239
|
-
{
|
|
18240
|
-
"actionId": 604,
|
|
18241
|
-
"key": "Backflip_Sweep_Kick_inplace",
|
|
18242
|
-
"name": "Backflip Sweep Kick",
|
|
18243
|
-
"category": "BodyMovements",
|
|
18244
|
-
"subCategory": "PerformingStunt",
|
|
18245
|
-
"previewUrl": "https://cdn.meshy.ai/webapp-assets/feature-demo/animation/preview/biped/Backflip_Sweep_Kick.gif",
|
|
18246
|
-
"rigType": "style_02",
|
|
18247
|
-
"tag": "InPlace",
|
|
18248
|
-
"isDefault": false,
|
|
18249
|
-
"isFree": false,
|
|
18250
|
-
"createdAt": 1750829520975
|
|
18251
|
-
},
|
|
18252
|
-
{
|
|
18253
|
-
"actionId": 605,
|
|
18254
|
-
"key": "Back_Jump_inplace",
|
|
18255
|
-
"name": "Back Jump",
|
|
18256
|
-
"category": "BodyMovements",
|
|
18257
|
-
"subCategory": "Jumping",
|
|
18258
|
-
"previewUrl": "https://cdn.meshy.ai/webapp-assets/feature-demo/animation/preview/biped/Back_Jump.gif",
|
|
18259
|
-
"rigType": "style_02",
|
|
18260
|
-
"tag": "InPlace",
|
|
18261
|
-
"isDefault": false,
|
|
18262
|
-
"isFree": false,
|
|
18263
|
-
"createdAt": 1750829521003
|
|
18264
|
-
},
|
|
18265
|
-
{
|
|
18266
|
-
"actionId": 606,
|
|
18267
|
-
"key": "BackLeft_run_inplace",
|
|
18268
|
-
"name": "BackLeft Run",
|
|
18269
|
-
"category": "WalkAndRun",
|
|
18270
|
-
"subCategory": "Running",
|
|
18271
|
-
"previewUrl": "https://cdn.meshy.ai/webapp-assets/feature-demo/animation/preview/biped/BackLeft_run.gif",
|
|
18272
|
-
"rigType": "style_02",
|
|
18273
|
-
"tag": "InPlace",
|
|
18274
|
-
"isDefault": false,
|
|
18275
|
-
"isFree": false,
|
|
18276
|
-
"createdAt": 1750829521059
|
|
18277
|
-
},
|
|
18278
18214
|
{
|
|
18279
18215
|
"actionId": 607,
|
|
18280
18216
|
"key": "BackRight_Run_inplace",
|
|
@@ -18288,32 +18224,6 @@ var MESHY_ANIMATION_CATALOG = [
|
|
|
18288
18224
|
"isFree": false,
|
|
18289
18225
|
"createdAt": 1750829521105
|
|
18290
18226
|
},
|
|
18291
|
-
{
|
|
18292
|
-
"actionId": 608,
|
|
18293
|
-
"key": "BeHit_FlyUp_inplace",
|
|
18294
|
-
"name": "BeHit FlyUp",
|
|
18295
|
-
"category": "Fighting",
|
|
18296
|
-
"subCategory": "GettingHit",
|
|
18297
|
-
"previewUrl": "https://cdn.meshy.ai/webapp-assets/feature-demo/animation/preview/biped/BeHit_FlyUp.gif",
|
|
18298
|
-
"rigType": "style_02",
|
|
18299
|
-
"tag": "InPlace",
|
|
18300
|
-
"isDefault": false,
|
|
18301
|
-
"isFree": false,
|
|
18302
|
-
"createdAt": 1750829521290
|
|
18303
|
-
},
|
|
18304
|
-
{
|
|
18305
|
-
"actionId": 609,
|
|
18306
|
-
"key": "Boxing_Practice_inplace",
|
|
18307
|
-
"name": "Boxing Practice",
|
|
18308
|
-
"category": "Fighting",
|
|
18309
|
-
"subCategory": "Punching",
|
|
18310
|
-
"previewUrl": "https://cdn.meshy.ai/webapp-assets/feature-demo/animation/preview/biped/Boxing_Practice.gif",
|
|
18311
|
-
"rigType": "style_02",
|
|
18312
|
-
"tag": "InPlace",
|
|
18313
|
-
"isDefault": false,
|
|
18314
|
-
"isFree": false,
|
|
18315
|
-
"createdAt": 1750829521295
|
|
18316
|
-
},
|
|
18317
18227
|
{
|
|
18318
18228
|
"actionId": 610,
|
|
18319
18229
|
"key": "Carry_Heavy_Cannon_Forward_inplace",
|
|
@@ -18340,19 +18250,6 @@ var MESHY_ANIMATION_CATALOG = [
|
|
|
18340
18250
|
"isFree": false,
|
|
18341
18251
|
"createdAt": 1750829521423
|
|
18342
18252
|
},
|
|
18343
|
-
{
|
|
18344
|
-
"actionId": 612,
|
|
18345
|
-
"key": "Carry_Water_Bucket_Walk_inplace",
|
|
18346
|
-
"name": "Carry Water Bucket Walk",
|
|
18347
|
-
"category": "WalkAndRun",
|
|
18348
|
-
"subCategory": "Walking",
|
|
18349
|
-
"previewUrl": "https://cdn.meshy.ai/webapp-assets/feature-demo/animation/preview/biped/Carry_Water_Bucket_Walk.gif",
|
|
18350
|
-
"rigType": "style_02",
|
|
18351
|
-
"tag": "InPlace",
|
|
18352
|
-
"isDefault": false,
|
|
18353
|
-
"isFree": false,
|
|
18354
|
-
"createdAt": 1750829521448
|
|
18355
|
-
},
|
|
18356
18253
|
{
|
|
18357
18254
|
"actionId": 613,
|
|
18358
18255
|
"key": "Casual_Walk_inplace",
|
|
@@ -18665,58 +18562,6 @@ var MESHY_ANIMATION_CATALOG = [
|
|
|
18665
18562
|
"isFree": false,
|
|
18666
18563
|
"createdAt": 1750829523006
|
|
18667
18564
|
},
|
|
18668
|
-
{
|
|
18669
|
-
"actionId": 640,
|
|
18670
|
-
"key": "Jump_Over_Obstacle_inplace",
|
|
18671
|
-
"name": "Jump Over Obstacle",
|
|
18672
|
-
"category": "BodyMovements",
|
|
18673
|
-
"subCategory": "Jumping",
|
|
18674
|
-
"previewUrl": "https://cdn.meshy.ai/webapp-assets/feature-demo/animation/preview/biped/Jump_Over_Obstacle.gif",
|
|
18675
|
-
"rigType": "style_02",
|
|
18676
|
-
"tag": "InPlace",
|
|
18677
|
-
"isDefault": false,
|
|
18678
|
-
"isFree": false,
|
|
18679
|
-
"createdAt": 1750829523075
|
|
18680
|
-
},
|
|
18681
|
-
{
|
|
18682
|
-
"actionId": 641,
|
|
18683
|
-
"key": "Jump_Over_Obstacle_1_inplace",
|
|
18684
|
-
"name": "Jump Over Obstacle 1",
|
|
18685
|
-
"category": "BodyMovements",
|
|
18686
|
-
"subCategory": "Jumping",
|
|
18687
|
-
"previewUrl": "https://cdn.meshy.ai/webapp-assets/feature-demo/animation/preview/biped/Jump_Over_Obstacle_1.gif",
|
|
18688
|
-
"rigType": "style_02",
|
|
18689
|
-
"tag": "InPlace",
|
|
18690
|
-
"isDefault": false,
|
|
18691
|
-
"isFree": false,
|
|
18692
|
-
"createdAt": 1750829523089
|
|
18693
|
-
},
|
|
18694
|
-
{
|
|
18695
|
-
"actionId": 642,
|
|
18696
|
-
"key": "Jump_Over_Obstacle_2_inplace",
|
|
18697
|
-
"name": "Jump Over Obstacle 2",
|
|
18698
|
-
"category": "BodyMovements",
|
|
18699
|
-
"subCategory": "Jumping",
|
|
18700
|
-
"previewUrl": "https://cdn.meshy.ai/webapp-assets/feature-demo/animation/preview/biped/Jump_Over_Obstacle_2.gif",
|
|
18701
|
-
"rigType": "style_02",
|
|
18702
|
-
"tag": "InPlace",
|
|
18703
|
-
"isDefault": false,
|
|
18704
|
-
"isFree": false,
|
|
18705
|
-
"createdAt": 1750829523123
|
|
18706
|
-
},
|
|
18707
|
-
{
|
|
18708
|
-
"actionId": 643,
|
|
18709
|
-
"key": "Jump_Run_inplace",
|
|
18710
|
-
"name": "Jump Run",
|
|
18711
|
-
"category": "WalkAndRun",
|
|
18712
|
-
"subCategory": "Running",
|
|
18713
|
-
"previewUrl": "https://cdn.meshy.ai/webapp-assets/feature-demo/animation/preview/biped/Jump_Run.gif",
|
|
18714
|
-
"rigType": "style_02",
|
|
18715
|
-
"tag": "InPlace",
|
|
18716
|
-
"isDefault": false,
|
|
18717
|
-
"isFree": false,
|
|
18718
|
-
"createdAt": 1750829523234
|
|
18719
|
-
},
|
|
18720
18565
|
{
|
|
18721
18566
|
"actionId": 644,
|
|
18722
18567
|
"key": "Lean_Forward_Sprint_inplace",
|
|
@@ -18743,19 +18588,6 @@ var MESHY_ANIMATION_CATALOG = [
|
|
|
18743
18588
|
"isFree": false,
|
|
18744
18589
|
"createdAt": 1750829523289
|
|
18745
18590
|
},
|
|
18746
|
-
{
|
|
18747
|
-
"actionId": 646,
|
|
18748
|
-
"key": "Limping_Walk_1_inplace",
|
|
18749
|
-
"name": "Limping Walk 1",
|
|
18750
|
-
"category": "WalkAndRun",
|
|
18751
|
-
"subCategory": "Walking",
|
|
18752
|
-
"previewUrl": "https://cdn.meshy.ai/webapp-assets/feature-demo/animation/preview/biped/Limping_Walk_1.gif",
|
|
18753
|
-
"rigType": "style_02",
|
|
18754
|
-
"tag": "InPlace",
|
|
18755
|
-
"isDefault": false,
|
|
18756
|
-
"isFree": false,
|
|
18757
|
-
"createdAt": 1750829523312
|
|
18758
|
-
},
|
|
18759
18591
|
{
|
|
18760
18592
|
"actionId": 647,
|
|
18761
18593
|
"key": "Limping_Walk_2_inplace",
|
|
@@ -18782,19 +18614,6 @@ var MESHY_ANIMATION_CATALOG = [
|
|
|
18782
18614
|
"isFree": false,
|
|
18783
18615
|
"createdAt": 1750829523530
|
|
18784
18616
|
},
|
|
18785
|
-
{
|
|
18786
|
-
"actionId": 649,
|
|
18787
|
-
"key": "Lunge_Roundhouse_Kick_inplace",
|
|
18788
|
-
"name": "Lunge Roundhouse Kick",
|
|
18789
|
-
"category": "Fighting",
|
|
18790
|
-
"subCategory": "Punching",
|
|
18791
|
-
"previewUrl": "https://cdn.meshy.ai/webapp-assets/feature-demo/animation/preview/biped/Lunge_Roundhouse_Kick.gif",
|
|
18792
|
-
"rigType": "style_02",
|
|
18793
|
-
"tag": "InPlace",
|
|
18794
|
-
"isDefault": false,
|
|
18795
|
-
"isFree": false,
|
|
18796
|
-
"createdAt": 1750829523593
|
|
18797
|
-
},
|
|
18798
18617
|
{
|
|
18799
18618
|
"actionId": 650,
|
|
18800
18619
|
"key": "Mummy_Stagger_inplace",
|
|
@@ -18808,19 +18627,6 @@ var MESHY_ANIMATION_CATALOG = [
|
|
|
18808
18627
|
"isFree": false,
|
|
18809
18628
|
"createdAt": 1750829523629
|
|
18810
18629
|
},
|
|
18811
|
-
{
|
|
18812
|
-
"actionId": 651,
|
|
18813
|
-
"key": "Parkour_Vault_with_Roll_inplace",
|
|
18814
|
-
"name": "Parkour Vault with Roll",
|
|
18815
|
-
"category": "BodyMovements",
|
|
18816
|
-
"subCategory": "VaultingOverObstacle",
|
|
18817
|
-
"previewUrl": "https://cdn.meshy.ai/webapp-assets/feature-demo/animation/preview/biped/Parkour_Vault_with_Roll.gif",
|
|
18818
|
-
"rigType": "style_02",
|
|
18819
|
-
"tag": "InPlace",
|
|
18820
|
-
"isDefault": false,
|
|
18821
|
-
"isFree": false,
|
|
18822
|
-
"createdAt": 1750829523676
|
|
18823
|
-
},
|
|
18824
18630
|
{
|
|
18825
18631
|
"actionId": 652,
|
|
18826
18632
|
"key": "Proud_Strut_inplace",
|
|
@@ -18860,19 +18666,6 @@ var MESHY_ANIMATION_CATALOG = [
|
|
|
18860
18666
|
"isFree": false,
|
|
18861
18667
|
"createdAt": 1750829523813
|
|
18862
18668
|
},
|
|
18863
|
-
{
|
|
18864
|
-
"actionId": 656,
|
|
18865
|
-
"key": "Run_and_Leap_inplace",
|
|
18866
|
-
"name": "Run and Leap",
|
|
18867
|
-
"category": "BodyMovements",
|
|
18868
|
-
"subCategory": "Acting",
|
|
18869
|
-
"previewUrl": "https://cdn.meshy.ai/webapp-assets/feature-demo/animation/preview/biped/Run_and_Leap.gif",
|
|
18870
|
-
"rigType": "style_02",
|
|
18871
|
-
"tag": "InPlace",
|
|
18872
|
-
"isDefault": false,
|
|
18873
|
-
"isFree": false,
|
|
18874
|
-
"createdAt": 1750829523875
|
|
18875
|
-
},
|
|
18876
18669
|
{
|
|
18877
18670
|
"actionId": 657,
|
|
18878
18671
|
"key": "run_fast_10_inplace",
|
|
@@ -18886,19 +18679,6 @@ var MESHY_ANIMATION_CATALOG = [
|
|
|
18886
18679
|
"isFree": false,
|
|
18887
18680
|
"createdAt": 1750829523937
|
|
18888
18681
|
},
|
|
18889
|
-
{
|
|
18890
|
-
"actionId": 658,
|
|
18891
|
-
"key": "run_fast_2_inplace",
|
|
18892
|
-
"name": "Run Fast 2",
|
|
18893
|
-
"category": "WalkAndRun",
|
|
18894
|
-
"subCategory": "Running",
|
|
18895
|
-
"previewUrl": "https://cdn.meshy.ai/webapp-assets/feature-demo/animation/preview/biped/run_fast_2.gif",
|
|
18896
|
-
"rigType": "style_02",
|
|
18897
|
-
"tag": "InPlace",
|
|
18898
|
-
"isDefault": false,
|
|
18899
|
-
"isFree": false,
|
|
18900
|
-
"createdAt": 1750829524093
|
|
18901
|
-
},
|
|
18902
18682
|
{
|
|
18903
18683
|
"actionId": 659,
|
|
18904
18684
|
"key": "run_fast_3_inplace",
|
|
@@ -18912,32 +18692,6 @@ var MESHY_ANIMATION_CATALOG = [
|
|
|
18912
18692
|
"isFree": false,
|
|
18913
18693
|
"createdAt": 1750829524145
|
|
18914
18694
|
},
|
|
18915
|
-
{
|
|
18916
|
-
"actionId": 660,
|
|
18917
|
-
"key": "run_fast_4_inplace",
|
|
18918
|
-
"name": "Run Fast 4",
|
|
18919
|
-
"category": "WalkAndRun",
|
|
18920
|
-
"subCategory": "Running",
|
|
18921
|
-
"previewUrl": "https://cdn.meshy.ai/webapp-assets/feature-demo/animation/preview/biped/run_fast_4.gif",
|
|
18922
|
-
"rigType": "style_02",
|
|
18923
|
-
"tag": "InPlace",
|
|
18924
|
-
"isDefault": false,
|
|
18925
|
-
"isFree": false,
|
|
18926
|
-
"createdAt": 1750829524190
|
|
18927
|
-
},
|
|
18928
|
-
{
|
|
18929
|
-
"actionId": 661,
|
|
18930
|
-
"key": "run_fast_5_inplace",
|
|
18931
|
-
"name": "Run Fast 5",
|
|
18932
|
-
"category": "WalkAndRun",
|
|
18933
|
-
"subCategory": "Running",
|
|
18934
|
-
"previewUrl": "https://cdn.meshy.ai/webapp-assets/feature-demo/animation/preview/biped/run_fast_5.gif",
|
|
18935
|
-
"rigType": "style_02",
|
|
18936
|
-
"tag": "InPlace",
|
|
18937
|
-
"isDefault": false,
|
|
18938
|
-
"isFree": false,
|
|
18939
|
-
"createdAt": 1750829524239
|
|
18940
|
-
},
|
|
18941
18695
|
{
|
|
18942
18696
|
"actionId": 662,
|
|
18943
18697
|
"key": "run_fast_6_inplace",
|
|
@@ -18977,19 +18731,6 @@ var MESHY_ANIMATION_CATALOG = [
|
|
|
18977
18731
|
"isFree": false,
|
|
18978
18732
|
"createdAt": 1750829524385
|
|
18979
18733
|
},
|
|
18980
|
-
{
|
|
18981
|
-
"actionId": 665,
|
|
18982
|
-
"key": "run_fast_9_inplace",
|
|
18983
|
-
"name": "Run Fast 9",
|
|
18984
|
-
"category": "WalkAndRun",
|
|
18985
|
-
"subCategory": "Running",
|
|
18986
|
-
"previewUrl": "https://cdn.meshy.ai/webapp-assets/feature-demo/animation/preview/biped/run_fast_9.gif",
|
|
18987
|
-
"rigType": "style_02",
|
|
18988
|
-
"tag": "InPlace",
|
|
18989
|
-
"isDefault": false,
|
|
18990
|
-
"isFree": false,
|
|
18991
|
-
"createdAt": 1750829524420
|
|
18992
|
-
},
|
|
18993
18734
|
{
|
|
18994
18735
|
"actionId": 666,
|
|
18995
18736
|
"key": "Running_Reload_inplace",
|
|
@@ -19003,19 +18744,6 @@ var MESHY_ANIMATION_CATALOG = [
|
|
|
19003
18744
|
"isFree": false,
|
|
19004
18745
|
"createdAt": 1750829524436
|
|
19005
18746
|
},
|
|
19006
|
-
{
|
|
19007
|
-
"actionId": 667,
|
|
19008
|
-
"key": "Run_to_Walk_Transition_inplace",
|
|
19009
|
-
"name": "Run to Walk Transition",
|
|
19010
|
-
"category": "WalkAndRun",
|
|
19011
|
-
"subCategory": "Walking",
|
|
19012
|
-
"previewUrl": "https://cdn.meshy.ai/webapp-assets/feature-demo/animation/preview/biped/Run_to_Walk_Transition.gif",
|
|
19013
|
-
"rigType": "style_02",
|
|
19014
|
-
"tag": "InPlace",
|
|
19015
|
-
"isDefault": false,
|
|
19016
|
-
"isFree": false,
|
|
19017
|
-
"createdAt": 1750829524486
|
|
19018
|
-
},
|
|
19019
18747
|
{
|
|
19020
18748
|
"actionId": 668,
|
|
19021
18749
|
"key": "Skip_Forward_inplace",
|
|
@@ -19094,19 +18822,6 @@ var MESHY_ANIMATION_CATALOG = [
|
|
|
19094
18822
|
"isFree": false,
|
|
19095
18823
|
"createdAt": 1750829524902
|
|
19096
18824
|
},
|
|
19097
|
-
{
|
|
19098
|
-
"actionId": 674,
|
|
19099
|
-
"key": "Stumble_Walk_inplace",
|
|
19100
|
-
"name": "Stumble Walk",
|
|
19101
|
-
"category": "WalkAndRun",
|
|
19102
|
-
"subCategory": "Walking",
|
|
19103
|
-
"previewUrl": "https://cdn.meshy.ai/webapp-assets/feature-demo/animation/preview/biped/Stumble_Walk.gif",
|
|
19104
|
-
"rigType": "style_02",
|
|
19105
|
-
"tag": "InPlace",
|
|
19106
|
-
"isDefault": false,
|
|
19107
|
-
"isFree": false,
|
|
19108
|
-
"createdAt": 1750829524941
|
|
19109
|
-
},
|
|
19110
18825
|
{
|
|
19111
18826
|
"actionId": 675,
|
|
19112
18827
|
"key": "Stylish_Walk_inplace",
|
|
@@ -20513,6 +20228,7 @@ async function quote(url, token, body) {
|
|
|
20513
20228
|
return (await response.json()).quote;
|
|
20514
20229
|
}
|
|
20515
20230
|
function showAmbiguity(selector, candidates, log, json = false) {
|
|
20231
|
+
const none = candidates.length === 0 ? `\u201C${selector}\u201D matches no Meshy action (Meshy retires actions from time to time). Run \`genex animations search "<what the character should do>"\` and pass one of the ids it lists.` : null;
|
|
20516
20232
|
if (json) {
|
|
20517
20233
|
writeJson({
|
|
20518
20234
|
kind: "character",
|
|
@@ -20522,8 +20238,11 @@ function showAmbiguity(selector, candidates, log, json = false) {
|
|
|
20522
20238
|
actionId: candidate.actionId,
|
|
20523
20239
|
key: candidate.key,
|
|
20524
20240
|
name: candidate.name
|
|
20525
|
-
}))
|
|
20241
|
+
})),
|
|
20242
|
+
...none ? { message: none } : {}
|
|
20526
20243
|
});
|
|
20244
|
+
} else if (none) {
|
|
20245
|
+
log.error(none);
|
|
20527
20246
|
} else {
|
|
20528
20247
|
log.error(`\u201C${selector}\u201D is ambiguous. Choose an action id:`);
|
|
20529
20248
|
for (const candidate of candidates) {
|
|
@@ -23140,7 +22859,7 @@ async function runBlender(opts) {
|
|
|
23140
22859
|
return 1;
|
|
23141
22860
|
}
|
|
23142
22861
|
if (sub === "serve") {
|
|
23143
|
-
const { serveLocalBlender } = await import("./blender-serve-
|
|
22862
|
+
const { serveLocalBlender } = await import("./blender-serve-EH2MOW3U.js");
|
|
23144
22863
|
const port = Number(process.env.GENEX_BLENDER_PORT ?? 8088);
|
|
23145
22864
|
log.step(`Starting a local Blender service on port ${port}`);
|
|
23146
22865
|
log.plain(
|
|
@@ -23149,7 +22868,7 @@ async function runBlender(opts) {
|
|
|
23149
22868
|
return serveLocalBlender({ port, log });
|
|
23150
22869
|
}
|
|
23151
22870
|
if (sub === "mcp") {
|
|
23152
|
-
const { runBlenderMcp } = await import("./blender-mcp-
|
|
22871
|
+
const { runBlenderMcp } = await import("./blender-mcp-ZRCMYTXI.js");
|
|
23153
22872
|
return runBlenderMcp();
|
|
23154
22873
|
}
|
|
23155
22874
|
if (sub === "seat") {
|
|
@@ -23657,7 +23376,12 @@ import fs35 from "fs/promises";
|
|
|
23657
23376
|
import path36 from "path";
|
|
23658
23377
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
23659
23378
|
var SUBS3 = ["models", "bench", "price", "status", "cancel"];
|
|
23660
|
-
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN
|
|
23379
|
+
var LLM_BENCH_APPROVAL_REQUIRED = "STOP: `genex llm bench` runs the real model and spends YOUR OWN credits \u2014 your balance, not a player's, and a spend this build's asset allowance does not count. Re-run with --max-credits <n> --user-approved only after the person at the keyboard agreed to the number.";
|
|
23380
|
+
var LLM_MAX_COINS_DEPRECATED = "--max-coins is deprecated: it is read as --max-credits, the same number \u2014 a bench spends credits now.";
|
|
23381
|
+
function benchBalanceShortLine(spendable, maxCredits) {
|
|
23382
|
+
return `Your spendable credits (${spendable}) cannot cover one attempt at --max-credits ${maxCredits} \u2014 nothing was sent, nothing was spent. Add credits, or re-run with a lower --max-credits once the person at the keyboard agrees to it.`;
|
|
23383
|
+
}
|
|
23384
|
+
var LLM_BENCH_UNVERIFIED_LINE = "Credits pay only for a verified email, and this account's is not verified yet (credits_unverified) \u2014 nothing was sent, nothing was spent. Verify it, then re-run.";
|
|
23661
23385
|
var LLM_LANE_OFF_LINE = "The runtime LLM lane is off on this stand \u2014 in-game generate() answers 404 here, and nothing can be benchmarked.";
|
|
23662
23386
|
var LLM_TOOLS_CONVERT_HINT = [
|
|
23663
23387
|
"In-game model calls need a hosted game. This folder is a Genex Tools workspace, so nothing here",
|
|
@@ -23670,7 +23394,7 @@ function serverSlotCounts(body) {
|
|
|
23670
23394
|
return { held: body.slotsHeld, limit: body.slotLimit };
|
|
23671
23395
|
}
|
|
23672
23396
|
var PENDING_SLOT_RELEASE_MINUTES = 10;
|
|
23673
|
-
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so
|
|
23397
|
+
var LLM_STATUS_LEGEND = `"awaiting bill" = the call has stopped but the provider's bill is not final, so what it reserved stays held until the bill resolves; such a row stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`;
|
|
23674
23398
|
var LLM_STATUS_NO_COUNT_LINE = "This stand does not report how many slots are held across your projects, so no count is shown.";
|
|
23675
23399
|
var LLM_CANCEL_AWAITING_BILL = "this call already stopped; its bill is awaiting the provider";
|
|
23676
23400
|
var LLM_CANCEL_ALREADY_FINAL = "this call is already final";
|
|
@@ -23684,11 +23408,33 @@ var DEFAULT_SAMPLE_TIMEOUT_SEC = 180;
|
|
|
23684
23408
|
function parseProviderKey(value) {
|
|
23685
23409
|
return value === "ok" || value === "rejected" || value === "unknown" ? value : null;
|
|
23686
23410
|
}
|
|
23411
|
+
function bpsOrNull(value) {
|
|
23412
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
23413
|
+
}
|
|
23414
|
+
function standingCreditGrantOrNull(value) {
|
|
23415
|
+
if (!value || typeof value !== "object") return null;
|
|
23416
|
+
const v = value;
|
|
23417
|
+
const whole = (x) => typeof x === "number" && Number.isSafeInteger(x) && x >= 0;
|
|
23418
|
+
if (!whole(v.minCredits) || !whole(v.maxCredits) || !whole(v.stepCredits) || !whole(v.defaultCredits)) return null;
|
|
23419
|
+
return {
|
|
23420
|
+
minCredits: v.minCredits,
|
|
23421
|
+
maxCredits: v.maxCredits,
|
|
23422
|
+
stepCredits: v.stepCredits,
|
|
23423
|
+
defaultCredits: v.defaultCredits,
|
|
23424
|
+
...whole(v.minUsdCents) ? { minUsdCents: v.minUsdCents } : {},
|
|
23425
|
+
...whole(v.maxUsdCents) ? { maxUsdCents: v.maxUsdCents } : {},
|
|
23426
|
+
...whole(v.stepUsdCents) ? { stepUsdCents: v.stepUsdCents } : {}
|
|
23427
|
+
};
|
|
23428
|
+
}
|
|
23687
23429
|
async function readRuntimeLane(apiUrl, token) {
|
|
23688
23430
|
const empty = (state) => ({
|
|
23689
23431
|
state,
|
|
23690
23432
|
models: null,
|
|
23691
23433
|
recommendedDeclaredHeadroomBps: null,
|
|
23434
|
+
currency: null,
|
|
23435
|
+
feeBps: null,
|
|
23436
|
+
usdCentsPerCredit: null,
|
|
23437
|
+
standingCreditGrant: null,
|
|
23692
23438
|
externalProviders: null,
|
|
23693
23439
|
total: null,
|
|
23694
23440
|
featuredCount: null,
|
|
@@ -23707,7 +23453,11 @@ async function readRuntimeLane(apiUrl, token) {
|
|
|
23707
23453
|
return {
|
|
23708
23454
|
state: "live",
|
|
23709
23455
|
models: body.models,
|
|
23710
|
-
recommendedDeclaredHeadroomBps:
|
|
23456
|
+
recommendedDeclaredHeadroomBps: bpsOrNull(body.recommendedDeclaredHeadroomBps),
|
|
23457
|
+
currency: body.currency === "credits" || body.currency === "coins" ? body.currency : null,
|
|
23458
|
+
feeBps: bpsOrNull(body.feeBps),
|
|
23459
|
+
usdCentsPerCredit: typeof body.usdCentsPerCredit === "number" && Number.isSafeInteger(body.usdCentsPerCredit) && body.usdCentsPerCredit >= 1 ? body.usdCentsPerCredit : null,
|
|
23460
|
+
standingCreditGrant: standingCreditGrantOrNull(body.standingCreditGrant),
|
|
23711
23461
|
externalProviders: Array.isArray(body.externalProviders) ? body.externalProviders : null,
|
|
23712
23462
|
total: typeof body.total === "number" && Number.isFinite(body.total) ? body.total : null,
|
|
23713
23463
|
featuredCount: typeof body.featuredCount === "number" && Number.isFinite(body.featuredCount) ? body.featuredCount : null,
|
|
@@ -23723,18 +23473,45 @@ function percentile2(values, p) {
|
|
|
23723
23473
|
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
23724
23474
|
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
23725
23475
|
}
|
|
23726
|
-
function
|
|
23727
|
-
|
|
23476
|
+
function percentilePicos(values, p) {
|
|
23477
|
+
if (values.length === 0) return null;
|
|
23478
|
+
const sorted = [...values].sort((a, b) => a < b ? -1 : a > b ? 1 : 0);
|
|
23479
|
+
const rank2 = Math.ceil(p / 100 * sorted.length);
|
|
23480
|
+
return sorted[Math.min(sorted.length - 1, Math.max(0, rank2 - 1))];
|
|
23728
23481
|
}
|
|
23729
|
-
|
|
23730
|
-
|
|
23482
|
+
var PICOS_PER_CENT = 10000000000n;
|
|
23483
|
+
function ceilingCreditsFor(costPicos, terms) {
|
|
23484
|
+
if (costPicos <= 0n) return 1;
|
|
23485
|
+
const num = costPicos * BigInt(1e4 + terms.feeBps) * BigInt(1e4 + terms.headroomBps);
|
|
23486
|
+
const den = 100000000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
23487
|
+
return Math.max(1, Number((num + den - 1n) / den));
|
|
23488
|
+
}
|
|
23489
|
+
function averageCreditsPerCall(costsPicos, terms) {
|
|
23490
|
+
if (costsPicos.length === 0) return null;
|
|
23491
|
+
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
23492
|
+
return Number(sum * BigInt(1e4 + terms.feeBps)) / Number(BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT);
|
|
23493
|
+
}
|
|
23494
|
+
function perCallEstimateCreditsFor(costsPicos, terms) {
|
|
23495
|
+
if (costsPicos.length === 0) return null;
|
|
23496
|
+
const sum = costsPicos.reduce((a, b) => a + b, 0n);
|
|
23497
|
+
const num = sum * BigInt(1e4 + terms.feeBps);
|
|
23498
|
+
const den = BigInt(costsPicos.length) * 10000n * BigInt(terms.usdCentsPerCredit) * PICOS_PER_CENT;
|
|
23499
|
+
return Math.max(1, Number((num + den - 1n) / den));
|
|
23500
|
+
}
|
|
23501
|
+
function formatCredits(value) {
|
|
23502
|
+
if (!Number.isFinite(value)) return "\u2014";
|
|
23503
|
+
if (value >= 100) return String(Math.round(value));
|
|
23504
|
+
return String(Number(value.toPrecision(value < 1 ? 2 : 3)));
|
|
23505
|
+
}
|
|
23506
|
+
function usdFromPicos(picos) {
|
|
23507
|
+
if (picos === null || picos === void 0) return "unknown";
|
|
23508
|
+
const value = typeof picos === "bigint" ? picos : /^\d+$/.test(picos) ? BigInt(picos) : null;
|
|
23509
|
+
if (value === null) return "unknown";
|
|
23510
|
+
return `$${(Number(value) / 1e12).toFixed(6)}`;
|
|
23731
23511
|
}
|
|
23732
23512
|
function usdPerMillion(value) {
|
|
23733
23513
|
return `$${value.toFixed(value < 1 ? 4 : 2)}`;
|
|
23734
23514
|
}
|
|
23735
|
-
function usd(value) {
|
|
23736
|
-
return value === null ? "unknown" : `$${value.toFixed(6)}`;
|
|
23737
|
-
}
|
|
23738
23515
|
async function runLlm(opts = {}) {
|
|
23739
23516
|
const log = createLogger({ quiet: opts.quiet || opts.json });
|
|
23740
23517
|
const cwd = opts.cwd ?? process.cwd();
|
|
@@ -23760,16 +23537,23 @@ async function runLlm(opts = {}) {
|
|
|
23760
23537
|
return;
|
|
23761
23538
|
}
|
|
23762
23539
|
if (sub === "bench") {
|
|
23763
|
-
if (opts.maxCoins
|
|
23540
|
+
if (opts.maxCoins !== void 0) process.stderr.write(`${c.yellow("!")} ${LLM_MAX_COINS_DEPRECATED}
|
|
23541
|
+
`);
|
|
23542
|
+
if (opts.maxCoins !== void 0 && opts.maxCredits !== void 0 && opts.maxCoins !== opts.maxCredits) {
|
|
23543
|
+
fail5("llm bench", `Pass --max-credits alone \u2014 --max-coins is its deprecated name, and the two disagree (${opts.maxCredits} vs ${opts.maxCoins}).`);
|
|
23544
|
+
return;
|
|
23545
|
+
}
|
|
23546
|
+
const approvedCredits = opts.maxCredits ?? opts.maxCoins;
|
|
23547
|
+
if (approvedCredits === void 0 || opts.userApproved !== true) {
|
|
23764
23548
|
fail5("llm bench", LLM_BENCH_APPROVAL_REQUIRED);
|
|
23765
23549
|
return;
|
|
23766
23550
|
}
|
|
23767
|
-
if (!Number.isInteger(
|
|
23768
|
-
fail5("llm bench", `--max-
|
|
23551
|
+
if (!Number.isInteger(approvedCredits) || approvedCredits < 1) {
|
|
23552
|
+
fail5("llm bench", `--max-credits takes a whole number of credits, 1 or more (got ${String(approvedCredits)}).`);
|
|
23769
23553
|
return;
|
|
23770
23554
|
}
|
|
23771
23555
|
if (!opts.benchPrompt?.trim()) {
|
|
23772
|
-
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-
|
|
23556
|
+
fail5("llm bench", '`genex llm bench` needs the prompt your game would send, e.g. genex llm bench "<prompt>" --max-credits <n> --user-approved.');
|
|
23773
23557
|
return;
|
|
23774
23558
|
}
|
|
23775
23559
|
if (opts.jsonOutput && opts.textOutput) {
|
|
@@ -23861,6 +23645,10 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
23861
23645
|
models: null,
|
|
23862
23646
|
externalProviders: null,
|
|
23863
23647
|
recommendedDeclaredHeadroomBps: null,
|
|
23648
|
+
currency: null,
|
|
23649
|
+
feeBps: null,
|
|
23650
|
+
usdCentsPerCredit: null,
|
|
23651
|
+
standingCreditGrant: null,
|
|
23864
23652
|
total: null,
|
|
23865
23653
|
featuredCount: null,
|
|
23866
23654
|
// The stand was never asked, so the verdict is unknown — but the key is
|
|
@@ -23886,6 +23674,12 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
23886
23674
|
models: lane.models,
|
|
23887
23675
|
externalProviders: lane.externalProviders,
|
|
23888
23676
|
recommendedDeclaredHeadroomBps: lane.recommendedDeclaredHeadroomBps,
|
|
23677
|
+
// The credit terms a call is billed on, the server's — null on a stand
|
|
23678
|
+
// that predates credit billing, never a number the CLI filled in.
|
|
23679
|
+
currency: lane.currency,
|
|
23680
|
+
feeBps: lane.feeBps,
|
|
23681
|
+
usdCentsPerCredit: lane.usdCentsPerCredit,
|
|
23682
|
+
standingCreditGrant: lane.standingCreditGrant,
|
|
23889
23683
|
total: lane.total ?? lane.models?.length ?? null,
|
|
23890
23684
|
featuredCount: lane.featuredCount ?? (lane.models ? lane.models.filter((m) => m.featured).length : null),
|
|
23891
23685
|
// `null` when the lane is not live or the server predates the probe.
|
|
@@ -23932,8 +23726,9 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
23932
23726
|
);
|
|
23933
23727
|
for (const m of shown) {
|
|
23934
23728
|
const plan = m.personalPlan ? ` \xB7 personal plan: ${m.personalPlan}` : "";
|
|
23729
|
+
const kind = m.kind === "typesafe" ? " \xB7 judge (classifier): judge() under a budget, not generate() or bench" : "";
|
|
23935
23730
|
log.plain(
|
|
23936
|
-
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}`
|
|
23731
|
+
` ${c.cyan(m.id)} ${m.label} \u2014 ${usdPerMillion(m.inputUsdPerMillion)} in / ${usdPerMillion(m.outputUsdPerMillion)} out per million tokens${plan}${kind}`
|
|
23937
23732
|
);
|
|
23938
23733
|
}
|
|
23939
23734
|
if (hiddenCount > 0) {
|
|
@@ -23949,74 +23744,104 @@ async function reportModels(apiUrl, token, opts, log, toolsOnly = false) {
|
|
|
23949
23744
|
}
|
|
23950
23745
|
log.plain("");
|
|
23951
23746
|
if (lane.recommendedDeclaredHeadroomBps === null) {
|
|
23952
|
-
log.dim(" This stand serves no recommended headroom, so no
|
|
23747
|
+
log.dim(" This stand serves no recommended headroom, so no ceiling can be recommended from a bench.");
|
|
23748
|
+
} else {
|
|
23749
|
+
log.dim(` Recommended headroom over a benchmarked call's cost on this stand: ${lane.recommendedDeclaredHeadroomBps} bps.`);
|
|
23750
|
+
}
|
|
23751
|
+
if (lane.feeBps === null) {
|
|
23752
|
+
log.dim(" This stand serves no platform fee \u2014 it predates credit billing, so no ceiling can be recommended from a bench.");
|
|
23953
23753
|
} else {
|
|
23954
|
-
log.dim(`
|
|
23754
|
+
log.dim(` In-game calls are billed in the player's credits, as used: the provider's cost plus this stand's platform fee (${lane.feeBps} bps).`);
|
|
23955
23755
|
}
|
|
23956
23756
|
if (toolsOnly) {
|
|
23957
23757
|
printConvertHint();
|
|
23958
23758
|
return;
|
|
23959
23759
|
}
|
|
23960
|
-
log.dim(" Those per-million rates are the PROVIDER's; what
|
|
23961
|
-
log.dim(`
|
|
23760
|
+
log.dim(" Those per-million rates are the PROVIDER's list; what one call of YOUR prompt really costs,");
|
|
23761
|
+
log.dim(` and the per-call ceiling to declare, only a real call measures: ${c.cyan('genex llm bench "<prompt>" --max-credits <n> --user-approved')}`);
|
|
23962
23762
|
}
|
|
23963
23763
|
function wholeOrNull(value) {
|
|
23964
23764
|
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
23965
23765
|
}
|
|
23766
|
+
function costPicosOf(usage) {
|
|
23767
|
+
if (typeof usage?.costUsdPicos === "string" && /^\d+$/.test(usage.costUsdPicos)) return usage.costUsdPicos;
|
|
23768
|
+
if (typeof usage?.costUsd === "number" && Number.isFinite(usage.costUsd) && usage.costUsd >= 0) {
|
|
23769
|
+
return String(BigInt(Math.round(usage.costUsd * 1e12)));
|
|
23770
|
+
}
|
|
23771
|
+
return null;
|
|
23772
|
+
}
|
|
23966
23773
|
function sampleFrom(id, settled) {
|
|
23967
23774
|
const usage = settled?.usage;
|
|
23775
|
+
const picos = costPicosOf(usage);
|
|
23968
23776
|
return {
|
|
23969
23777
|
id,
|
|
23970
23778
|
status: settled?.status ?? null,
|
|
23971
23779
|
billingStatus: settled?.billingStatus ?? null,
|
|
23972
|
-
|
|
23973
|
-
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd : null,
|
|
23780
|
+
chargedCredits: wholeOrNull(settled?.chargedCredits),
|
|
23781
|
+
costUsd: typeof usage?.costUsd === "number" ? usage.costUsd : picos !== null ? Number(picos) / 1e12 : null,
|
|
23782
|
+
costUsdPicos: picos,
|
|
23974
23783
|
error: settled?.error ?? (settled ? null : "timed_out"),
|
|
23975
23784
|
providerMessage: typeof settled?.providerMessage === "string" && settled.providerMessage.trim() ? settled.providerMessage.trim() : null,
|
|
23976
23785
|
inputTokens: wholeOrNull(usage?.inputTokens),
|
|
23977
23786
|
outputTokens: wholeOrNull(usage?.outputTokens),
|
|
23978
23787
|
maxOutputTokens: wholeOrNull(usage?.maxOutputTokens),
|
|
23979
|
-
|
|
23788
|
+
fitCredits: typeof usage?.fitCredits === "number" && Number.isSafeInteger(usage.fitCredits) && usage.fitCredits >= 1 ? usage.fitCredits : null,
|
|
23980
23789
|
fitExceedsCap: typeof usage?.fitExceedsCap === "boolean" ? usage.fitExceedsCap : null,
|
|
23981
23790
|
truncated: typeof usage?.truncated === "boolean" ? usage.truncated : settled?.error === "provider_token_limit" ? true : null
|
|
23982
23791
|
};
|
|
23983
23792
|
}
|
|
23984
23793
|
var PROVIDER_TOKEN_LIMIT = "provider_token_limit";
|
|
23985
|
-
function benchRecommendation(results,
|
|
23986
|
-
const settled = results.filter(
|
|
23987
|
-
|
|
23988
|
-
|
|
23989
|
-
const
|
|
23990
|
-
const
|
|
23794
|
+
function benchRecommendation(results, terms) {
|
|
23795
|
+
const settled = results.filter(
|
|
23796
|
+
(r) => r.status === "succeeded" && r.billingStatus === "final" && r.chargedCredits !== null && r.costUsdPicos !== null
|
|
23797
|
+
);
|
|
23798
|
+
const costs = settled.map((r) => BigInt(r.costUsdPicos));
|
|
23799
|
+
const charged = settled.map((r) => r.chargedCredits);
|
|
23800
|
+
const fits = settled.map((r) => r.fitCredits).filter((v) => typeof v === "number");
|
|
23801
|
+
const costP95 = percentilePicos(costs, 95);
|
|
23802
|
+
const costMax = costs.length ? costs.reduce((a, b) => b > a ? b : a) : null;
|
|
23991
23803
|
const fitP95 = percentile2(fits, 95);
|
|
23992
23804
|
const fitMax = fits.length ? Math.max(...fits) : null;
|
|
23993
23805
|
const exceedsStandCap = results.some((r) => r.fitExceedsCap === true);
|
|
23994
23806
|
const cutOffSamples = results.filter((r) => r.error === PROVIDER_TOKEN_LIMIT && r.fitExceedsCap !== true).length;
|
|
23995
23807
|
const noRecommendationReason = exceedsStandCap ? "answer_exceeds_stand_cap" : cutOffSamples > 0 ? "answer_cut_off" : null;
|
|
23996
|
-
const
|
|
23997
|
-
const
|
|
23998
|
-
const
|
|
23808
|
+
const { headroomBps, feeBps, usdCentsPerCredit } = terms;
|
|
23809
|
+
const missingTerms = headroomBps === null ? "headroom" : feeBps === null || usdCentsPerCredit === null ? "credit_terms" : null;
|
|
23810
|
+
const full = headroomBps !== null && feeBps !== null && usdCentsPerCredit !== null ? { headroomBps, feeBps, usdCentsPerCredit } : null;
|
|
23811
|
+
const billing = feeBps !== null && usdCentsPerCredit !== null ? { feeBps, usdCentsPerCredit } : null;
|
|
23812
|
+
const costBased = full !== null && costP95 !== null ? ceilingCreditsFor(costP95, full) : null;
|
|
23813
|
+
const recommended = costBased === null || noRecommendationReason !== null ? null : Math.max(costBased, fitP95 ?? 0);
|
|
23814
|
+
const ceiling = recommended === null || costMax === null || full === null ? null : Math.max(ceilingCreditsFor(costMax, full), recommended, fitMax ?? 0);
|
|
23815
|
+
const average = billing !== null ? averageCreditsPerCall(costs, billing) : null;
|
|
23816
|
+
const estimate = ceiling === null || billing === null ? null : Math.min(ceiling, perCallEstimateCreditsFor(costs, billing) ?? 1);
|
|
23999
23817
|
return {
|
|
23818
|
+
costs,
|
|
24000
23819
|
charged,
|
|
24001
|
-
|
|
24002
|
-
|
|
24003
|
-
|
|
23820
|
+
costP50: percentilePicos(costs, 50),
|
|
23821
|
+
costP95,
|
|
23822
|
+
costMax,
|
|
23823
|
+
chargedP50: percentile2(charged, 50),
|
|
23824
|
+
chargedP95: percentile2(charged, 95),
|
|
23825
|
+
chargedMax: charged.length ? Math.max(...charged) : null,
|
|
24004
23826
|
fits,
|
|
24005
23827
|
fitP95,
|
|
24006
23828
|
fitMax,
|
|
24007
|
-
|
|
24008
|
-
|
|
24009
|
-
|
|
24010
|
-
|
|
23829
|
+
costBasedMaxCredits: costBased,
|
|
23830
|
+
recommendedMaxCredits: recommended,
|
|
23831
|
+
recommendedPerCallMaxCredits: ceiling,
|
|
23832
|
+
averageCreditsPerCall: average,
|
|
23833
|
+
recommendedPerCallEstimateCredits: estimate,
|
|
23834
|
+
lengthRaisedCeiling: recommended !== null && costBased !== null && recommended > costBased,
|
|
24011
23835
|
cutOffSamples,
|
|
24012
23836
|
exceedsStandCap,
|
|
24013
|
-
noRecommendationReason
|
|
23837
|
+
noRecommendationReason,
|
|
23838
|
+
missingTerms
|
|
24014
23839
|
};
|
|
24015
23840
|
}
|
|
24016
|
-
var LLM_LENGTH_RAISED_LINE = "The
|
|
24017
|
-
var LLM_EXCEEDS_STAND_CAP_LINE = "No
|
|
24018
|
-
function benchCutOffLine(cutOff,
|
|
24019
|
-
return `No
|
|
23841
|
+
var LLM_LENGTH_RAISED_LINE = "The ceiling also decides how long the answer may be: at the cost-based ceiling the answer would be cut off, so declare this one.";
|
|
23842
|
+
var LLM_EXCEEDS_STAND_CAP_LINE = "No ceiling recommended: this answer is longer than one call on this stand may produce. Ask for a shorter answer \u2014 fewer fields, shorter strings, a length the prompt states \u2014 and benchmark again.";
|
|
23843
|
+
function benchCutOffLine(cutOff, maxCredits) {
|
|
23844
|
+
return `No ceiling recommended: ${cutOff} sample${cutOff === 1 ? " was" : "s were"} cut off at this ceiling (--max-credits ${maxCredits}) before the answer was finished, so its real length is unknown. Re-run with a higher --max-credits.`;
|
|
24020
23845
|
}
|
|
24021
23846
|
var ACTIVE_STATUSES = /* @__PURE__ */ new Set(["requires_confirmation", "queued", "dispatching", "awaiting_external"]);
|
|
24022
23847
|
function rowState(row) {
|
|
@@ -24027,7 +23852,22 @@ function rowState(row) {
|
|
|
24027
23852
|
function rowHoldsSlot(row) {
|
|
24028
23853
|
if (typeof row.slotHeld === "boolean") return row.slotHeld;
|
|
24029
23854
|
const state = rowState(row);
|
|
24030
|
-
return state === "active" || state === "awaiting bill" && (row.reservedCoins ?? 0) > 0;
|
|
23855
|
+
return state === "active" || state === "awaiting bill" && ((row.reservedCoins ?? 0) > 0 || (row.reservedCredits ?? 0) > 0);
|
|
23856
|
+
}
|
|
23857
|
+
function rowMoney(row) {
|
|
23858
|
+
const credits = row.currency === "credits" || row.currency === void 0 && typeof row.reservedCredits === "number" && typeof row.reservedCoins !== "number";
|
|
23859
|
+
if (credits) {
|
|
23860
|
+
return {
|
|
23861
|
+
unit: "credits",
|
|
23862
|
+
reserved: typeof row.reservedCredits === "number" ? row.reservedCredits : null,
|
|
23863
|
+
charged: typeof row.chargedCredits === "number" ? row.chargedCredits : null
|
|
23864
|
+
};
|
|
23865
|
+
}
|
|
23866
|
+
return {
|
|
23867
|
+
unit: "coin",
|
|
23868
|
+
reserved: typeof row.reservedCoins === "number" ? row.reservedCoins : null,
|
|
23869
|
+
charged: typeof row.chargedCoins === "number" ? row.chargedCoins : null
|
|
23870
|
+
};
|
|
24031
23871
|
}
|
|
24032
23872
|
function providerRefusalStatus(error) {
|
|
24033
23873
|
const m = /^provider_http_(\d{3})$/.exec(error ?? "");
|
|
@@ -24046,7 +23886,7 @@ function formatAge(iso, now = Date.now()) {
|
|
|
24046
23886
|
}
|
|
24047
23887
|
async function runBench(args) {
|
|
24048
23888
|
const { apiUrl, token, projectId, cwd, opts, log } = args;
|
|
24049
|
-
const
|
|
23889
|
+
const maxCredits = opts.maxCredits ?? opts.maxCoins;
|
|
24050
23890
|
const samples = opts.samples ?? DEFAULT_BENCH_SAMPLES;
|
|
24051
23891
|
const prompt = opts.benchPrompt.trim();
|
|
24052
23892
|
const auth = { Authorization: `Bearer ${token}`, "Content-Type": "application/json" };
|
|
@@ -24068,7 +23908,7 @@ async function runBench(args) {
|
|
|
24068
23908
|
const lane = await readRuntimeLane(apiUrl, token);
|
|
24069
23909
|
if (lane.state !== "live" || !lane.models || lane.models.length === 0) {
|
|
24070
23910
|
if (lane.state === "off") {
|
|
24071
|
-
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
23911
|
+
if (opts.json) writeJsonLine({ ...emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCredits, opts), status: "off", error: null });
|
|
24072
23912
|
else {
|
|
24073
23913
|
log.plain(LLM_LANE_OFF_LINE);
|
|
24074
23914
|
log.dim(" Nothing was spent.");
|
|
@@ -24095,17 +23935,27 @@ async function runBench(args) {
|
|
|
24095
23935
|
);
|
|
24096
23936
|
return;
|
|
24097
23937
|
}
|
|
24098
|
-
const
|
|
23938
|
+
const walletBefore = await fetchCreditsSnapshot(apiUrl, token);
|
|
23939
|
+
if (walletBefore?.emailVerified === false) {
|
|
23940
|
+
benchFailed(opts, log, LLM_BENCH_UNVERIFIED_LINE);
|
|
23941
|
+
return;
|
|
23942
|
+
}
|
|
23943
|
+
if (walletBefore !== null && walletBefore.spendable < maxCredits) {
|
|
23944
|
+
benchFailed(opts, log, benchBalanceShortLine(walletBefore.spendable, maxCredits));
|
|
23945
|
+
return;
|
|
23946
|
+
}
|
|
23947
|
+
const balanceBefore = walletBefore?.spendable ?? null;
|
|
24099
23948
|
if (!opts.json) {
|
|
24100
23949
|
log.plain(c.bold("genex llm bench"));
|
|
24101
23950
|
log.dim(` ${apiUrl}`);
|
|
24102
23951
|
log.plain("");
|
|
24103
23952
|
log.plain(` Model ${c.cyan(modelId)}`);
|
|
24104
23953
|
log.plain(` Samples ${samples} real attempt${samples === 1 ? "" : "s"}, ${outputFormat} output`);
|
|
24105
|
-
log.plain(` Approved ${
|
|
23954
|
+
log.plain(` Approved ${maxCredits} credit${maxCredits === 1 ? "" : "s"} per attempt \u2014 at worst ${maxCredits * samples} credits for this run, from your own balance`);
|
|
24106
23955
|
log.plain(
|
|
24107
|
-
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore}
|
|
23956
|
+
` Balance ${balanceBefore === null ? "couldn't be read" : `${balanceBefore} credits spendable`}`
|
|
24108
23957
|
);
|
|
23958
|
+
log.dim(" Billed as used: each attempt is charged its real cost plus the platform fee, rounded up to a whole credit.");
|
|
24109
23959
|
log.plain("");
|
|
24110
23960
|
}
|
|
24111
23961
|
const base = `${apiUrl}/api/runtime/development/projects/${encodeURIComponent(projectId)}/generations`;
|
|
@@ -24131,7 +23981,7 @@ async function runBench(args) {
|
|
|
24131
23981
|
const res = await call(base, {
|
|
24132
23982
|
method: "POST",
|
|
24133
23983
|
body: JSON.stringify({
|
|
24134
|
-
|
|
23984
|
+
maxCredits,
|
|
24135
23985
|
request: {
|
|
24136
23986
|
idempotencyKey: `bench-${randomUUID2()}`,
|
|
24137
23987
|
modelId,
|
|
@@ -24161,13 +24011,14 @@ async function runBench(args) {
|
|
|
24161
24011
|
if (refusedAt !== null) providerRefused++;
|
|
24162
24012
|
if (!opts.json) {
|
|
24163
24013
|
const length = row.outputTokens !== null ? ` \xB7 ${row.outputTokens}${row.maxOutputTokens !== null ? ` of ${row.maxOutputTokens}` : ""} tokens out` : "";
|
|
24014
|
+
const charged = row.chargedCredits === null ? "\u2014" : String(row.chargedCredits);
|
|
24164
24015
|
log.plain(
|
|
24165
|
-
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${
|
|
24016
|
+
` ${row.status === "succeeded" ? c.green("\u2713") : c.yellow("!")} sample ${i + 1} ${charged.padStart(4)} credit${row.chargedCredits === 1 ? "" : "s"} charged \xB7 cost ${usdFromPicos(row.costUsdPicos)}${length}${row.error ? ` \xB7 ${row.error}` : ""}`
|
|
24166
24017
|
);
|
|
24167
24018
|
if (refusedAt !== null) printProviderRefusal(log, refusedAt, row.providerMessage, row.status === "unknown");
|
|
24168
24019
|
if (row.error === PROVIDER_TOKEN_LIMIT) {
|
|
24169
24020
|
log.dim(
|
|
24170
|
-
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no
|
|
24021
|
+
row.fitExceedsCap === true ? " The answer was cut off at this stand's own output limit \u2014 no ceiling makes room for it; it is not a sample." : ` The answer was cut off at this ceiling (--max-credits ${maxCredits}) before it was finished \u2014 it was charged, and it is not a sample.`
|
|
24171
24022
|
);
|
|
24172
24023
|
}
|
|
24173
24024
|
}
|
|
@@ -24175,16 +24026,19 @@ async function runBench(args) {
|
|
|
24175
24026
|
} finally {
|
|
24176
24027
|
process.removeListener("SIGINT", onSigint);
|
|
24177
24028
|
}
|
|
24178
|
-
const
|
|
24179
|
-
|
|
24180
|
-
|
|
24181
|
-
|
|
24182
|
-
|
|
24183
|
-
const
|
|
24029
|
+
const terms = {
|
|
24030
|
+
headroomBps: lane.recommendedDeclaredHeadroomBps,
|
|
24031
|
+
feeBps: lane.feeBps,
|
|
24032
|
+
usdCentsPerCredit: lane.usdCentsPerCredit
|
|
24033
|
+
};
|
|
24034
|
+
const rec = benchRecommendation(results, terms);
|
|
24035
|
+
const settledCount = rec.charged.length;
|
|
24184
24036
|
const lengthBlocked = rec.noRecommendationReason !== null;
|
|
24185
|
-
const balanceAfter = await
|
|
24037
|
+
const balanceAfter = (await fetchCreditsSnapshot(apiUrl, token))?.spendable ?? null;
|
|
24038
|
+
const picosOrNull = (v) => v === null ? null : String(v);
|
|
24186
24039
|
const record = {
|
|
24187
|
-
v:
|
|
24040
|
+
v: 2,
|
|
24041
|
+
currency: "credits",
|
|
24188
24042
|
ranAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
24189
24043
|
apiUrl,
|
|
24190
24044
|
projectId,
|
|
@@ -24193,28 +24047,34 @@ async function runBench(args) {
|
|
|
24193
24047
|
outputFormat,
|
|
24194
24048
|
samples,
|
|
24195
24049
|
completed: settledCount,
|
|
24196
|
-
|
|
24197
|
-
|
|
24198
|
-
|
|
24199
|
-
max,
|
|
24200
|
-
|
|
24201
|
-
|
|
24202
|
-
recommendedPerCallMaxCoins: ceiling,
|
|
24203
|
-
maxCoinsPerSample: maxCoins,
|
|
24204
|
-
fitCoins: rec.fits,
|
|
24050
|
+
maxCreditsPerSample: maxCredits,
|
|
24051
|
+
costUsdPicos: rec.costs.map(String),
|
|
24052
|
+
chargedCredits: rec.charged,
|
|
24053
|
+
cost: { p50: picosOrNull(rec.costP50), p95: picosOrNull(rec.costP95), max: picosOrNull(rec.costMax) },
|
|
24054
|
+
charged: { p50: rec.chargedP50, p95: rec.chargedP95, max: rec.chargedMax },
|
|
24055
|
+
fitCredits: rec.fits,
|
|
24205
24056
|
fitP95: rec.fitP95,
|
|
24206
|
-
|
|
24207
|
-
|
|
24057
|
+
fitMax: rec.fitMax,
|
|
24058
|
+
recommendedDeclaredHeadroomBps: terms.headroomBps,
|
|
24059
|
+
feeBps: terms.feeBps,
|
|
24060
|
+
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
24061
|
+
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
24062
|
+
recommendedMaxCredits: rec.recommendedMaxCredits,
|
|
24063
|
+
recommendedPerCallMaxCredits: rec.recommendedPerCallMaxCredits,
|
|
24064
|
+
averageCreditsPerCall: rec.averageCreditsPerCall,
|
|
24065
|
+
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
24066
|
+
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
24208
24067
|
cutOffSamples: rec.cutOffSamples,
|
|
24209
24068
|
noRecommendationReason: rec.noRecommendationReason
|
|
24210
24069
|
};
|
|
24211
24070
|
const savedTo = await saveBench(cwd, record);
|
|
24212
24071
|
if (opts.json) {
|
|
24213
|
-
const ok =
|
|
24072
|
+
const ok = settledCount > 0 && !lengthBlocked;
|
|
24214
24073
|
writeJsonLine({
|
|
24215
24074
|
command: "llm bench",
|
|
24216
24075
|
status: ok ? "ok" : "failed",
|
|
24217
24076
|
error: ok ? null : rec.noRecommendationReason ?? "no_sample_settled",
|
|
24077
|
+
currency: "credits",
|
|
24218
24078
|
apiUrl,
|
|
24219
24079
|
projectId,
|
|
24220
24080
|
modelId,
|
|
@@ -24223,35 +24083,54 @@ async function runBench(args) {
|
|
|
24223
24083
|
schemaPath: opts.schemaPath ?? null,
|
|
24224
24084
|
samples,
|
|
24225
24085
|
completed: settledCount,
|
|
24226
|
-
|
|
24086
|
+
maxCreditsPerSample: maxCredits,
|
|
24087
|
+
// Spendable credits before and after the run; null when unreadable.
|
|
24227
24088
|
balanceBefore,
|
|
24228
24089
|
balanceAfter,
|
|
24229
24090
|
results,
|
|
24230
24091
|
// The create-time refusal that ended the run, or null. Beside the
|
|
24231
24092
|
// samples, never among them: no attempt existed and nothing was spent.
|
|
24232
24093
|
refusal: refusal2,
|
|
24233
|
-
|
|
24234
|
-
//
|
|
24094
|
+
// Over the settled samples: the real provider cost (pico-dollar strings
|
|
24095
|
+
// and USD) and the whole credits charged.
|
|
24096
|
+
costUsdPicos: record.cost,
|
|
24097
|
+
costUsd: {
|
|
24098
|
+
p50: rec.costP50 === null ? null : Number(rec.costP50) / 1e12,
|
|
24099
|
+
p95: rec.costP95 === null ? null : Number(rec.costP95) / 1e12,
|
|
24100
|
+
max: rec.costMax === null ? null : Number(rec.costMax) / 1e12
|
|
24101
|
+
},
|
|
24102
|
+
chargedCredits: record.charged,
|
|
24103
|
+
// The answer's LENGTH, priced by the server per sample (`fitCredits`,
|
|
24235
24104
|
// headroom already on the tokens), over the settled samples.
|
|
24236
|
-
|
|
24237
|
-
|
|
24238
|
-
|
|
24105
|
+
fitCredits: { p95: rec.fitP95, max: rec.fitMax },
|
|
24106
|
+
recommendedDeclaredHeadroomBps: terms.headroomBps,
|
|
24107
|
+
feeBps: terms.feeBps,
|
|
24108
|
+
usdCentsPerCredit: terms.usdCentsPerCredit,
|
|
24109
|
+
missingTerms: rec.missingTerms,
|
|
24110
|
+
costBasedMaxCredits: rec.costBasedMaxCredits,
|
|
24111
|
+
lengthRaisedCeiling: rec.lengthRaisedCeiling,
|
|
24239
24112
|
cutOffSamples: rec.cutOffSamples,
|
|
24240
24113
|
exceedsStandCap: rec.exceedsStandCap,
|
|
24241
24114
|
noRecommendationReason: rec.noRecommendationReason,
|
|
24242
|
-
|
|
24243
|
-
|
|
24244
|
-
|
|
24115
|
+
recommendedMaxCredits: rec.recommendedMaxCredits,
|
|
24116
|
+
recommendedPerCallMaxCredits: rec.recommendedPerCallMaxCredits,
|
|
24117
|
+
averageCreditsPerCall: rec.averageCreditsPerCall,
|
|
24118
|
+
recommendedPerCallEstimateCredits: rec.recommendedPerCallEstimateCredits,
|
|
24245
24119
|
savedTo
|
|
24246
24120
|
});
|
|
24247
24121
|
if (!ok) process.exitCode = 1;
|
|
24248
24122
|
return;
|
|
24249
24123
|
}
|
|
24250
24124
|
log.plain("");
|
|
24125
|
+
const printMeasured = () => {
|
|
24126
|
+
log.plain(c.bold(" Real cost per call"));
|
|
24127
|
+
log.plain(` p50 ${usdFromPicos(rec.costP50)} p95 ${usdFromPicos(rec.costP95)} max ${usdFromPicos(rec.costMax)} (${settledCount} of ${samples} settled)`);
|
|
24128
|
+
log.plain(c.bold(" Charged credits"));
|
|
24129
|
+
log.plain(` p50 ${rec.chargedP50} p95 ${rec.chargedP95} max ${rec.chargedMax} (whole credits, rounded up per call)`);
|
|
24130
|
+
};
|
|
24251
24131
|
if (lengthBlocked) {
|
|
24252
|
-
if (
|
|
24253
|
-
|
|
24254
|
-
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
24132
|
+
if (settledCount > 0) {
|
|
24133
|
+
printMeasured();
|
|
24255
24134
|
log.plain("");
|
|
24256
24135
|
}
|
|
24257
24136
|
printRecommendation(log, record);
|
|
@@ -24259,17 +24138,16 @@ async function runBench(args) {
|
|
|
24259
24138
|
process.exitCode = 1;
|
|
24260
24139
|
return;
|
|
24261
24140
|
}
|
|
24262
|
-
if (
|
|
24141
|
+
if (settledCount === 0) {
|
|
24263
24142
|
log.error(
|
|
24264
|
-
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to
|
|
24143
|
+
providerRefused > 0 && providerRefused === results.length ? " No sample ran \u2014 the provider refused every attempt at its door \u2014 so there is nothing to measure from." : " No sample settled, so there is nothing to measure from."
|
|
24265
24144
|
);
|
|
24266
24145
|
process.exitCode = 1;
|
|
24267
24146
|
return;
|
|
24268
24147
|
}
|
|
24269
|
-
|
|
24270
|
-
log.plain(` p50 ${p50} p95 ${p95} max ${max} (${charged.length} of ${samples} settled)`);
|
|
24148
|
+
printMeasured();
|
|
24271
24149
|
if (balanceBefore !== null && balanceAfter !== null) {
|
|
24272
|
-
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter}
|
|
24150
|
+
log.plain(` Balance ${balanceBefore} \u2192 ${balanceAfter} credits spendable`);
|
|
24273
24151
|
}
|
|
24274
24152
|
log.plain("");
|
|
24275
24153
|
printRecommendation(log, record);
|
|
@@ -24281,43 +24159,58 @@ function printRecommendation(log, record) {
|
|
|
24281
24159
|
return;
|
|
24282
24160
|
}
|
|
24283
24161
|
if (record.noRecommendationReason === "answer_cut_off") {
|
|
24284
|
-
log.warn(` ${benchCutOffLine(record.cutOffSamples
|
|
24162
|
+
log.warn(` ${benchCutOffLine(record.cutOffSamples || 1, record.maxCreditsPerSample)}`);
|
|
24285
24163
|
return;
|
|
24286
24164
|
}
|
|
24287
|
-
|
|
24288
|
-
|
|
24165
|
+
const attempts = `${record.completed} settled attempt${record.completed === 1 ? "" : "s"} on ${record.modelId}`;
|
|
24166
|
+
if (record.recommendedMaxCredits === null || record.cost.p95 === null) {
|
|
24167
|
+
log.warn(" No ceiling recommended.");
|
|
24289
24168
|
log.dim(
|
|
24290
|
-
record.recommendedDeclaredHeadroomBps === null ? " This stand served no recommended headroom, and the multiplier is the server's to set \u2014" : "
|
|
24169
|
+
record.cost.p95 === null ? " No sample settled, so there is no p95 to build a ceiling on \u2014" : record.recommendedDeclaredHeadroomBps === null ? " This stand served no recommended headroom, and the multiplier is the server's to set \u2014" : " This stand served no platform fee or credit value, and those are the server's to set \u2014"
|
|
24291
24170
|
);
|
|
24292
24171
|
log.dim(" declaring a number from this run would be a guess dressed as a measurement.");
|
|
24172
|
+
printAverage(log, record, attempts);
|
|
24293
24173
|
return;
|
|
24294
24174
|
}
|
|
24295
|
-
log.plain(c.bold(` Declare
|
|
24296
|
-
if (record.
|
|
24175
|
+
log.plain(c.bold(` Declare maxCredits: ${record.recommendedMaxCredits}`));
|
|
24176
|
+
if (record.lengthRaisedCeiling) {
|
|
24297
24177
|
log.plain(` ${LLM_LENGTH_RAISED_LINE}`);
|
|
24298
24178
|
log.dim(
|
|
24299
|
-
` = the smallest
|
|
24179
|
+
` = the smallest ceiling whose answer allowance holds the answer plus this stand's headroom (p95 ${record.fitP95 ?? "\u2014"}), above the`
|
|
24300
24180
|
);
|
|
24301
24181
|
log.dim(
|
|
24302
|
-
`
|
|
24182
|
+
` cost-based ${record.costBasedMaxCredits ?? "\u2014"} (p95 real cost ${usdFromPicos(record.cost.p95)} with the platform fee, ${record.feeBps} bps, and ${record.recommendedDeclaredHeadroomBps} bps of headroom), over ${attempts}.`
|
|
24303
24183
|
);
|
|
24304
24184
|
} else {
|
|
24305
24185
|
log.dim(
|
|
24306
|
-
` = p95
|
|
24186
|
+
` = p95 real cost (${usdFromPicos(record.cost.p95)}) with this stand's platform fee (${record.feeBps} bps) and recommended headroom (${record.recommendedDeclaredHeadroomBps} bps), in whole credits,`
|
|
24307
24187
|
);
|
|
24308
|
-
log.dim(` measured over ${
|
|
24188
|
+
log.dim(` measured over ${attempts}.`);
|
|
24309
24189
|
}
|
|
24310
|
-
log.dim(" That number is a
|
|
24311
|
-
|
|
24312
|
-
|
|
24190
|
+
log.dim(" That number is a CEILING, not a price: a call is billed its real cost as used, never more than it \u2014");
|
|
24191
|
+
log.dim(" and it also sizes how long the answer may be.");
|
|
24192
|
+
if (record.recommendedPerCallMaxCredits !== null) {
|
|
24193
|
+
log.plain(c.bold(` Grant perCallMaxCredits: ${record.recommendedPerCallMaxCredits}`));
|
|
24313
24194
|
log.dim(
|
|
24314
|
-
record.
|
|
24195
|
+
record.lengthRaisedCeiling ? ` = the same over the worst sample (${usdFromPicos(record.cost.max)}) or the longest answer's ceiling, whichever is larger, and never below the ceiling above it.` : ` = the same over the worst sample (${usdFromPicos(record.cost.max)}), and never below the ceiling above it.`
|
|
24315
24196
|
);
|
|
24316
|
-
|
|
24197
|
+
}
|
|
24198
|
+
printAverage(log, record, attempts);
|
|
24199
|
+
}
|
|
24200
|
+
function printAverage(log, record, attempts) {
|
|
24201
|
+
if (record.averageCreditsPerCall === null) return;
|
|
24202
|
+
log.plain(` Per-call estimate: about ${formatCredits(record.averageCreditsPerCall)} credits per call`);
|
|
24203
|
+
log.dim(` = the average real cost with the platform fee, no headroom, over ${attempts}.`);
|
|
24204
|
+
if (record.recommendedPerCallEstimateCredits !== null) {
|
|
24205
|
+
log.plain(c.bold(` Grant perCallEstimateCredits: ${record.recommendedPerCallEstimateCredits}`));
|
|
24206
|
+
log.dim(` = that average rounded up to a whole credit. disclosure.estimatedCreditsPerPeriod = your calls per period \xD7 ${formatCredits(record.averageCreditsPerCall)}, rounded up.`);
|
|
24317
24207
|
}
|
|
24318
24208
|
}
|
|
24319
24209
|
async function explainBenchRefusal(res, call, base, log, opts, index, settledSoFar) {
|
|
24320
|
-
if (printedStructuredError(res))
|
|
24210
|
+
if (printedStructuredError(res)) {
|
|
24211
|
+
const printed2 = await res.json().catch(() => ({}));
|
|
24212
|
+
return { stop: true, refusal: { status: res.status, error: typeof printed2.error === "string" ? printed2.error : null, slotsHeld: null, slotLimit: null } };
|
|
24213
|
+
}
|
|
24321
24214
|
const body = await res.json().catch(() => ({}));
|
|
24322
24215
|
const refusal2 = { status: res.status, error: body.error ?? null, slotsHeld: null, slotLimit: null };
|
|
24323
24216
|
if (res.status === 429 && body.error === "generation_limit") {
|
|
@@ -24327,6 +24220,10 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24327
24220
|
if (!opts.json) log.error(` Sample ${index + 1} ${benchSlotsHeldSentence(counts)}`);
|
|
24328
24221
|
return { stop: true, refusal: refusal2 };
|
|
24329
24222
|
}
|
|
24223
|
+
if (res.status === 403 && body.error === "credits_unverified") {
|
|
24224
|
+
if (!opts.json) log.error(` ${LLM_BENCH_UNVERIFIED_LINE}`);
|
|
24225
|
+
return { stop: true, refusal: refusal2 };
|
|
24226
|
+
}
|
|
24330
24227
|
if (opts.json) return { stop: res.status !== 429, refusal: refusal2 };
|
|
24331
24228
|
if (res.status === 404 && body.error === "not_found") {
|
|
24332
24229
|
log.error(` ${LLM_LANE_OFF_LINE}`);
|
|
@@ -24342,7 +24239,7 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24342
24239
|
return { stop: true, refusal: refusal2 };
|
|
24343
24240
|
}
|
|
24344
24241
|
if (res.status === 402) {
|
|
24345
|
-
log.error(" Not enough
|
|
24242
|
+
log.error(" Not enough credits to start the attempt \u2014 a bench is paid from your own balance.");
|
|
24346
24243
|
return { stop: true, refusal: refusal2 };
|
|
24347
24244
|
}
|
|
24348
24245
|
log.error(
|
|
@@ -24351,7 +24248,7 @@ async function explainBenchRefusal(res, call, base, log, opts, index, settledSoF
|
|
|
24351
24248
|
return { stop: res.status >= 500 || res.status === 401 || res.status === 403, refusal: refusal2 };
|
|
24352
24249
|
}
|
|
24353
24250
|
function printProviderRefusal(log, status2, message, awaitingBill = false) {
|
|
24354
|
-
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so this call
|
|
24251
|
+
if (awaitingBill) log.dim(` The provider answered ${status2} after routing \u2014 its bill is not final, so what this call reserved stays held until it resolves (see \`genex llm status\`); it is not a sample.`);
|
|
24355
24252
|
else log.dim(" The provider refused this call at its door \u2014 no inference ran, it cost nothing, and it is not a sample.");
|
|
24356
24253
|
if (message) log.dim(` Provider said: ${message}`);
|
|
24357
24254
|
if (status2 === 401 || status2 === 403) {
|
|
@@ -24377,19 +24274,6 @@ async function pollSettled(call, url, timeoutSec) {
|
|
|
24377
24274
|
}
|
|
24378
24275
|
return last;
|
|
24379
24276
|
}
|
|
24380
|
-
async function readCoinBalance(apiUrl, token) {
|
|
24381
|
-
try {
|
|
24382
|
-
const res = await apiFetch(`${apiUrl}/api/coin/balance`, {
|
|
24383
|
-
headers: { Authorization: `Bearer ${token}` },
|
|
24384
|
-
signal: AbortSignal.timeout(6e3)
|
|
24385
|
-
});
|
|
24386
|
-
if (!res.ok) return null;
|
|
24387
|
-
const body = await res.json().catch(() => null);
|
|
24388
|
-
return typeof body?.spendable === "number" ? body.spendable : null;
|
|
24389
|
-
} catch {
|
|
24390
|
-
return null;
|
|
24391
|
-
}
|
|
24392
|
-
}
|
|
24393
24277
|
async function saveBench(cwd, record) {
|
|
24394
24278
|
try {
|
|
24395
24279
|
const file = path36.join(cwd, LLM_BENCH_FILE);
|
|
@@ -24405,9 +24289,10 @@ function benchFailed(opts, log, message) {
|
|
|
24405
24289
|
else log.error(message);
|
|
24406
24290
|
process.exitCode = 1;
|
|
24407
24291
|
}
|
|
24408
|
-
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples,
|
|
24292
|
+
function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCredits, opts) {
|
|
24409
24293
|
return {
|
|
24410
24294
|
command: "llm bench",
|
|
24295
|
+
currency: "credits",
|
|
24411
24296
|
apiUrl,
|
|
24412
24297
|
projectId,
|
|
24413
24298
|
modelId: opts.modelId ?? null,
|
|
@@ -24416,21 +24301,28 @@ function emptyBenchJson(apiUrl, projectId, prompt, outputFormat, samples, maxCoi
|
|
|
24416
24301
|
schemaPath: opts.schemaPath ?? null,
|
|
24417
24302
|
samples,
|
|
24418
24303
|
completed: 0,
|
|
24419
|
-
|
|
24304
|
+
maxCreditsPerSample: maxCredits,
|
|
24420
24305
|
balanceBefore: null,
|
|
24421
24306
|
balanceAfter: null,
|
|
24422
24307
|
results: [],
|
|
24423
24308
|
refusal: null,
|
|
24424
|
-
|
|
24425
|
-
|
|
24426
|
-
|
|
24427
|
-
|
|
24309
|
+
costUsdPicos: { p50: null, p95: null, max: null },
|
|
24310
|
+
costUsd: { p50: null, p95: null, max: null },
|
|
24311
|
+
chargedCredits: { p50: null, p95: null, max: null },
|
|
24312
|
+
fitCredits: { p95: null, max: null },
|
|
24313
|
+
recommendedDeclaredHeadroomBps: null,
|
|
24314
|
+
feeBps: null,
|
|
24315
|
+
usdCentsPerCredit: null,
|
|
24316
|
+
missingTerms: null,
|
|
24317
|
+
costBasedMaxCredits: null,
|
|
24318
|
+
lengthRaisedCeiling: false,
|
|
24428
24319
|
cutOffSamples: 0,
|
|
24429
24320
|
exceedsStandCap: false,
|
|
24430
24321
|
noRecommendationReason: null,
|
|
24431
|
-
|
|
24432
|
-
|
|
24433
|
-
|
|
24322
|
+
recommendedMaxCredits: null,
|
|
24323
|
+
recommendedPerCallMaxCredits: null,
|
|
24324
|
+
averageCreditsPerCall: null,
|
|
24325
|
+
recommendedPerCallEstimateCredits: null,
|
|
24434
24326
|
savedTo: null
|
|
24435
24327
|
};
|
|
24436
24328
|
}
|
|
@@ -24493,7 +24385,8 @@ async function reportStatus(args) {
|
|
|
24493
24385
|
for (const row of rows) {
|
|
24494
24386
|
const state = rowState(row);
|
|
24495
24387
|
const mark = state === "active" ? c.cyan("\u25CF") : state === "awaiting bill" ? c.yellow("\u25D0") : c.green("\u25CB");
|
|
24496
|
-
const
|
|
24388
|
+
const money2 = rowMoney(row);
|
|
24389
|
+
const reserved = money2.reserved === null ? "reserved \u2014" : `${money2.reserved} ${money2.unit} reserved`;
|
|
24497
24390
|
log.plain(
|
|
24498
24391
|
` ${mark} ${row.id ?? "?"} ${row.modelId ?? "?"} ${state.padEnd(13)} ${formatAge(row.createdAt, now).padStart(4)} old ${reserved}${row.error ? ` ${row.error}` : ""}${rowHoldsSlot(row) ? "" : c.dim(" (no slot)")}`
|
|
24499
24392
|
);
|
|
@@ -24565,10 +24458,11 @@ async function cancelCall(args) {
|
|
|
24565
24458
|
if (!opts.json) {
|
|
24566
24459
|
log.warn(` ${row.id} (${row.status ?? "?"}) \u2014 ${sentence}.`);
|
|
24567
24460
|
if (state === "awaiting bill") {
|
|
24568
|
-
log.dim(` Nothing to cancel:
|
|
24461
|
+
log.dim(` Nothing to cancel: what it reserved releases when the bill resolves, and it stops holding a slot ${PENDING_SLOT_RELEASE_MINUTES} minutes after dispatch.`);
|
|
24569
24462
|
if (row.providerMessage?.trim()) log.dim(` Provider said: ${row.providerMessage.trim()}`);
|
|
24570
24463
|
} else {
|
|
24571
|
-
|
|
24464
|
+
const money2 = rowMoney(row);
|
|
24465
|
+
log.dim(` ${money2.charged === null ? "charge unknown" : `${money2.charged} ${money2.unit} charged`}${row.error ? ` \xB7 ${row.error}` : ""}.`);
|
|
24572
24466
|
}
|
|
24573
24467
|
}
|
|
24574
24468
|
return done("ok", null, state === "awaiting bill" ? "awaiting_bill" : "already_final", row);
|
|
@@ -24590,78 +24484,126 @@ async function cancelCall(args) {
|
|
|
24590
24484
|
const after = await res.json().catch(() => null) ?? row;
|
|
24591
24485
|
if (!opts.json) {
|
|
24592
24486
|
log.success(` Cancelled ${row.id} (was ${row.status ?? "active"}).`);
|
|
24593
|
-
log.dim(`
|
|
24487
|
+
log.dim(` What it reserved releases once the bill settles; ${c.cyan("genex llm status")} shows it until then.`);
|
|
24594
24488
|
}
|
|
24595
24489
|
return done("ok", null, "cancelled", after);
|
|
24596
24490
|
}
|
|
24597
|
-
async function
|
|
24598
|
-
let
|
|
24491
|
+
async function readSavedBench(cwd) {
|
|
24492
|
+
let parsed;
|
|
24599
24493
|
try {
|
|
24600
|
-
|
|
24494
|
+
parsed = JSON.parse(await fs35.readFile(path36.join(cwd, LLM_BENCH_FILE), "utf8"));
|
|
24601
24495
|
} catch {
|
|
24602
|
-
|
|
24496
|
+
return null;
|
|
24603
24497
|
}
|
|
24604
|
-
if (!
|
|
24498
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null;
|
|
24499
|
+
const v = parsed;
|
|
24500
|
+
if (v.v === 2 && v.currency === "credits" && v.cost && v.charged) return { kind: "credits", record: parsed };
|
|
24501
|
+
return { kind: "coins", record: parsed };
|
|
24502
|
+
}
|
|
24503
|
+
var LLM_PRICE_COIN_TERMS_LINE = "This bench was measured under the old COIN terms, when a game declared a fixed price per call. In-game calls are now billed in the player's credits, as used, and a game declares a per-call CEILING (maxCredits) instead \u2014 a coin price is not that number. Re-run the bench for the figures to declare.";
|
|
24504
|
+
async function reportSavedPrice(cwd, opts, log) {
|
|
24505
|
+
const saved = await readSavedBench(cwd);
|
|
24506
|
+
const rerun = 'genex llm bench "<prompt>" --max-credits <n> --user-approved';
|
|
24507
|
+
const creditKeys = (record2) => ({
|
|
24508
|
+
costUsdPicos: record2?.cost ?? { p50: null, p95: null, max: null },
|
|
24509
|
+
chargedCredits: record2?.charged ?? { p50: null, p95: null, max: null },
|
|
24510
|
+
fitP95: record2?.fitP95 ?? null,
|
|
24511
|
+
fitMax: record2?.fitMax ?? null,
|
|
24512
|
+
recommendedDeclaredHeadroomBps: record2?.recommendedDeclaredHeadroomBps ?? null,
|
|
24513
|
+
feeBps: record2?.feeBps ?? null,
|
|
24514
|
+
usdCentsPerCredit: record2?.usdCentsPerCredit ?? null,
|
|
24515
|
+
costBasedMaxCredits: record2?.costBasedMaxCredits ?? null,
|
|
24516
|
+
lengthRaisedCeiling: record2?.lengthRaisedCeiling ?? false,
|
|
24517
|
+
cutOffSamples: record2?.cutOffSamples ?? 0,
|
|
24518
|
+
noRecommendationReason: record2?.noRecommendationReason ?? null,
|
|
24519
|
+
recommendedMaxCredits: record2?.recommendedMaxCredits ?? null,
|
|
24520
|
+
recommendedPerCallMaxCredits: record2?.recommendedPerCallMaxCredits ?? null,
|
|
24521
|
+
averageCreditsPerCall: record2?.averageCreditsPerCall ?? null,
|
|
24522
|
+
recommendedPerCallEstimateCredits: record2?.recommendedPerCallEstimateCredits ?? null
|
|
24523
|
+
});
|
|
24524
|
+
if (!saved) {
|
|
24605
24525
|
if (opts.json) {
|
|
24606
24526
|
writeJsonLine({
|
|
24607
24527
|
command: "llm price",
|
|
24608
24528
|
status: "failed",
|
|
24609
24529
|
error: "no_bench",
|
|
24530
|
+
currency: null,
|
|
24610
24531
|
ranAt: null,
|
|
24611
24532
|
modelId: null,
|
|
24612
24533
|
samples: null,
|
|
24613
24534
|
completed: null,
|
|
24614
|
-
|
|
24615
|
-
|
|
24616
|
-
max: null,
|
|
24617
|
-
fitP95: null,
|
|
24618
|
-
chargedBasedEstimateCoins: null,
|
|
24619
|
-
lengthRaisedPrice: false,
|
|
24620
|
-
cutOffSamples: 0,
|
|
24621
|
-
noRecommendationReason: null,
|
|
24622
|
-
recommendedDeclaredHeadroomBps: null,
|
|
24623
|
-
recommendedEstimateCoins: null,
|
|
24624
|
-
recommendedPerCallMaxCoins: null,
|
|
24535
|
+
...creditKeys(null),
|
|
24536
|
+
legacyCoinTerms: null,
|
|
24625
24537
|
savedTo: null
|
|
24626
24538
|
});
|
|
24627
24539
|
} else {
|
|
24628
24540
|
log.error("Nothing has been benchmarked in this folder yet.");
|
|
24629
|
-
log.dim(` Run ${c.cyan(
|
|
24541
|
+
log.dim(` Run ${c.cyan(rerun)} first.`);
|
|
24542
|
+
}
|
|
24543
|
+
process.exitCode = 1;
|
|
24544
|
+
return;
|
|
24545
|
+
}
|
|
24546
|
+
if (saved.kind === "coins") {
|
|
24547
|
+
const old = saved.record;
|
|
24548
|
+
const legacy = {
|
|
24549
|
+
p50: old.p50 ?? null,
|
|
24550
|
+
p95: old.p95 ?? null,
|
|
24551
|
+
max: old.max ?? null,
|
|
24552
|
+
recommendedEstimateCoins: old.recommendedEstimateCoins ?? null,
|
|
24553
|
+
recommendedPerCallMaxCoins: old.recommendedPerCallMaxCoins ?? null,
|
|
24554
|
+
maxCoinsPerSample: old.maxCoinsPerSample ?? null,
|
|
24555
|
+
noRecommendationReason: old.noRecommendationReason ?? null
|
|
24556
|
+
};
|
|
24557
|
+
if (opts.json) {
|
|
24558
|
+
writeJsonLine({
|
|
24559
|
+
command: "llm price",
|
|
24560
|
+
status: "failed",
|
|
24561
|
+
error: "coin_terms",
|
|
24562
|
+
currency: "coins",
|
|
24563
|
+
ranAt: old.ranAt ?? null,
|
|
24564
|
+
modelId: old.modelId ?? null,
|
|
24565
|
+
samples: old.samples ?? null,
|
|
24566
|
+
completed: old.completed ?? null,
|
|
24567
|
+
...creditKeys(null),
|
|
24568
|
+
legacyCoinTerms: legacy,
|
|
24569
|
+
savedTo: LLM_BENCH_FILE
|
|
24570
|
+
});
|
|
24571
|
+
} else {
|
|
24572
|
+
log.plain(c.bold("genex llm price"));
|
|
24573
|
+
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${old.ranAt ?? "at an unknown time"}${old.modelId ? ` on ${old.modelId}` : ""}`);
|
|
24574
|
+
log.plain("");
|
|
24575
|
+
log.warn(` ${LLM_PRICE_COIN_TERMS_LINE}`);
|
|
24576
|
+
const measured = legacy.p95 === null ? "no settled sample" : `charged coins p50 ${legacy.p50} \xB7 p95 ${legacy.p95} \xB7 max ${legacy.max}`;
|
|
24577
|
+
const recommended = legacy.recommendedEstimateCoins === null ? "no price recommended" : `it recommended estimateCoins ${legacy.recommendedEstimateCoins}${legacy.recommendedPerCallMaxCoins != null ? ` and perCallMaxCoins ${legacy.recommendedPerCallMaxCoins}` : ""}`;
|
|
24578
|
+
log.dim(` What it measured then: ${measured}; ${recommended}.`);
|
|
24579
|
+
log.dim(` Re-run: ${c.cyan(rerun)}`);
|
|
24630
24580
|
}
|
|
24631
24581
|
process.exitCode = 1;
|
|
24632
24582
|
return;
|
|
24633
24583
|
}
|
|
24584
|
+
const record = saved.record;
|
|
24634
24585
|
if (opts.json) {
|
|
24635
24586
|
writeJsonLine({
|
|
24636
24587
|
command: "llm price",
|
|
24637
|
-
status: record.
|
|
24638
|
-
error: record.
|
|
24588
|
+
status: record.recommendedMaxCredits === null ? "failed" : "ok",
|
|
24589
|
+
error: record.recommendedMaxCredits === null ? record.noRecommendationReason ?? "no_recommendation" : null,
|
|
24590
|
+
currency: "credits",
|
|
24639
24591
|
ranAt: record.ranAt ?? null,
|
|
24640
24592
|
modelId: record.modelId ?? null,
|
|
24641
24593
|
samples: record.samples ?? null,
|
|
24642
24594
|
completed: record.completed ?? null,
|
|
24643
|
-
|
|
24644
|
-
|
|
24645
|
-
max: record.max ?? null,
|
|
24646
|
-
// Absent from a file an older CLI wrote; null / false / 0 then, never missing.
|
|
24647
|
-
fitP95: record.fitP95 ?? null,
|
|
24648
|
-
chargedBasedEstimateCoins: record.chargedBasedEstimateCoins ?? null,
|
|
24649
|
-
lengthRaisedPrice: record.lengthRaisedPrice ?? false,
|
|
24650
|
-
cutOffSamples: record.cutOffSamples ?? 0,
|
|
24651
|
-
noRecommendationReason: record.noRecommendationReason ?? null,
|
|
24652
|
-
recommendedDeclaredHeadroomBps: record.recommendedDeclaredHeadroomBps ?? null,
|
|
24653
|
-
recommendedEstimateCoins: record.recommendedEstimateCoins ?? null,
|
|
24654
|
-
recommendedPerCallMaxCoins: record.recommendedPerCallMaxCoins ?? null,
|
|
24595
|
+
...creditKeys(record),
|
|
24596
|
+
legacyCoinTerms: null,
|
|
24655
24597
|
savedTo: LLM_BENCH_FILE
|
|
24656
24598
|
});
|
|
24657
|
-
if (record.
|
|
24599
|
+
if (record.recommendedMaxCredits === null) process.exitCode = 1;
|
|
24658
24600
|
return;
|
|
24659
24601
|
}
|
|
24660
24602
|
log.plain(c.bold("genex llm price"));
|
|
24661
24603
|
log.dim(` from ${LLM_BENCH_FILE}, benchmarked ${record.ranAt}`);
|
|
24662
24604
|
log.plain("");
|
|
24663
24605
|
printRecommendation(log, record);
|
|
24664
|
-
if (record.
|
|
24606
|
+
if (record.recommendedMaxCredits === null) process.exitCode = 1;
|
|
24665
24607
|
}
|
|
24666
24608
|
function writeJsonLine(value) {
|
|
24667
24609
|
process.stdout.write(`${JSON.stringify(value)}
|
|
@@ -24921,7 +24863,7 @@ function runtimeRow(state, lane, toolsOnly = false) {
|
|
|
24921
24863
|
return {
|
|
24922
24864
|
label: "Runtime",
|
|
24923
24865
|
value: `in-game LLM lane live \xB7 ${n} model${n === 1 ? "" : "s"}${keyRefused ? ` \xB7 ${c.yellow("provider key refused")}` : ""}`,
|
|
24924
|
-
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : '
|
|
24866
|
+
fix: keyRefused ? LLM_PROVIDER_KEY_REJECTED_LINE : toolsOnly ? convertFix : 'Measure a call before a game declares its per-call ceiling: `npx genex llm bench "<prompt>" --max-credits <n> --user-approved`.'
|
|
24925
24867
|
};
|
|
24926
24868
|
}
|
|
24927
24869
|
case "off":
|
|
@@ -26859,16 +26801,16 @@ ${c.bold("Usage")}
|
|
|
26859
26801
|
status | cancel. "models" says whether this
|
|
26860
26802
|
stand serves the runtime LLM lane at all, and
|
|
26861
26803
|
at what provider rates. "bench" runs the real
|
|
26862
|
-
model on YOUR OWN
|
|
26863
|
-
attempt
|
|
26804
|
+
model on YOUR OWN credits and reports what each
|
|
26805
|
+
attempt really cost; it needs --max-credits <n>
|
|
26864
26806
|
--user-approved, like any spend the player has
|
|
26865
26807
|
to agree to. "price" reprints the last run's
|
|
26866
|
-
recommended
|
|
26808
|
+
recommended maxCredits. "status" lists this
|
|
26867
26809
|
project's open calls and the slots they hold;
|
|
26868
26810
|
"cancel <id>" stops an active one. Declare a
|
|
26869
|
-
game's
|
|
26870
|
-
a
|
|
26871
|
-
|
|
26811
|
+
game's per-call ceiling from a bench, never
|
|
26812
|
+
from a guess: the player is billed as used, and
|
|
26813
|
+
the ceiling also sizes how long an answer may be.
|
|
26872
26814
|
genex player <sub> [options] Answer the in-game model requests you approve, on your
|
|
26873
26815
|
OWN Claude or ChatGPT subscription: install | run |
|
|
26874
26816
|
status | stop | uninstall. "install" signs this machine
|
|
@@ -27323,7 +27265,7 @@ ${c.bold("Examples")}
|
|
|
27323
27265
|
genex shop remove sku_123
|
|
27324
27266
|
genex llm models
|
|
27325
27267
|
genex llm models --all
|
|
27326
|
-
genex llm bench "Reply with one short taunt." --samples 5 --max-
|
|
27268
|
+
genex llm bench "Reply with one short taunt." --samples 5 --max-credits 20 --user-approved
|
|
27327
27269
|
genex llm price
|
|
27328
27270
|
genex llm status
|
|
27329
27271
|
genex llm cancel <id>
|
|
@@ -27560,22 +27502,24 @@ ${c.bold("How the allowance works")}
|
|
|
27560
27502
|
|
|
27561
27503
|
Every number printed is live from your account; prices can change without a deploy.
|
|
27562
27504
|
`,
|
|
27563
|
-
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what should the game declare?
|
|
27505
|
+
llm: `${c.bold("genex llm")} \u2014 in-game model calls: is the lane live, what does one call cost, what ceiling should the game declare?
|
|
27564
27506
|
|
|
27565
27507
|
${c.bold("Usage")}
|
|
27566
27508
|
genex llm models [--all] [--json] Which models this stand serves, their provider rates per
|
|
27567
|
-
million tokens,
|
|
27568
|
-
|
|
27569
|
-
|
|
27570
|
-
|
|
27571
|
-
on this stand: it says so
|
|
27572
|
-
|
|
27573
|
-
|
|
27574
|
-
|
|
27575
|
-
|
|
27576
|
-
|
|
27577
|
-
|
|
27578
|
-
|
|
27509
|
+
million tokens, the platform fee a call is billed at, and
|
|
27510
|
+
the headroom it recommends over a benchmark. The catalog
|
|
27511
|
+
is synced from OpenRouter, so the FEATURED short list is
|
|
27512
|
+
printed by default and --all prints every row (--json
|
|
27513
|
+
always carries every row). Off on this stand: it says so
|
|
27514
|
+
and exits 0 \u2014 build the feature without a model rather
|
|
27515
|
+
than promising a 404.
|
|
27516
|
+
genex llm bench "<prompt>" --max-credits <n> --user-approved [options]
|
|
27517
|
+
Run the real model through the development lane and
|
|
27518
|
+
report what each attempt really COST and was charged.
|
|
27519
|
+
Refused without both flags, before any network call:
|
|
27520
|
+
this spends YOUR OWN CREDITS, a spend the build's asset
|
|
27521
|
+
allowance does not count. Needs a folder linked to a
|
|
27522
|
+
game you own, and a balance that covers one attempt.
|
|
27579
27523
|
genex llm price [--json] The recommendation from the last bench in this folder.
|
|
27580
27524
|
Reads a file; spends nothing.
|
|
27581
27525
|
genex llm status [--json] This project's open development calls: active, or
|
|
@@ -27592,29 +27536,32 @@ ${c.bold("Bench options")}
|
|
|
27592
27536
|
--samples <n> How many real attempts (default ${DEFAULT_BENCH_SAMPLES}). More samples, better p95.
|
|
27593
27537
|
--schema <file> A JSON file holding the output schema; implies JSON output.
|
|
27594
27538
|
--json-output Ask for JSON without a schema. --text Ask for text (the default).
|
|
27595
|
-
--max-
|
|
27539
|
+
--max-credits <n> The per-attempt ceiling YOU approved \u2014 a bench runs on your own credits,
|
|
27596
27540
|
never a player's. The run's worst case is that number times --samples,
|
|
27597
|
-
and it is printed before anything starts.
|
|
27541
|
+
and it is printed before anything starts. (--max-coins is its deprecated
|
|
27542
|
+
name, read as the same number.)
|
|
27598
27543
|
--timeout <seconds> Bounds the wait for each attempt.
|
|
27599
27544
|
--json One machine-readable object; every key is always present, null when
|
|
27600
27545
|
the CLI could not learn it.
|
|
27601
27546
|
|
|
27602
27547
|
${c.bold("Why a benchmark and not an estimate")}
|
|
27603
|
-
|
|
27604
|
-
|
|
27605
|
-
|
|
27606
|
-
|
|
27607
|
-
|
|
27608
|
-
|
|
27609
|
-
|
|
27610
|
-
|
|
27611
|
-
|
|
27548
|
+
An in-game call is paid from the PLAYER's credits and billed as used: its real provider cost
|
|
27549
|
+
plus the platform fee, never more than the per-call CEILING the game declares (\`maxCredits\`,
|
|
27550
|
+
or \`perCallMaxCredits\` on a standing budget). The ceiling is not a price, but it is still a
|
|
27551
|
+
number with two jobs: it bounds what one call may cost, and it funds how LONG the answer may
|
|
27552
|
+
be. Three layers sit between the provider's rate card and it \u2014 the provider's cost for YOUR
|
|
27553
|
+
prompt, the platform fee, and your headroom \u2014 and only a real call measures the first. The
|
|
27554
|
+
bench is billed to YOU, on the development lane; the player-funded lane is what the
|
|
27555
|
+
published game uses. The recommendation is the p95 real cost with the server's fee and
|
|
27556
|
+
headroom applied, in whole credits; a stand that serves no headroom or no fee gets no
|
|
27557
|
+
recommendation, because those multipliers are not the CLI's to invent.
|
|
27612
27558
|
|
|
27613
|
-
|
|
27614
|
-
|
|
27615
|
-
|
|
27616
|
-
|
|
27617
|
-
|
|
27559
|
+
Because the ceiling sizes each call's output allowance, the recommendation is never below
|
|
27560
|
+
the smallest ceiling that leaves room for the benchmarked answer, as the server computes it.
|
|
27561
|
+
A sample cut off at your --max-credits (provider_token_limit) is not a sample \u2014 re-run with a
|
|
27562
|
+
higher --max-credits; an answer longer than one call on this stand may produce gets no
|
|
27563
|
+
ceiling at all \u2014 ask for a shorter one. The run also prints the measured average per call,
|
|
27564
|
+
fee included, which is the honest estimate a budget's disclosure is built on.
|
|
27618
27565
|
|
|
27619
27566
|
Results land in ${LLM_BENCH_FILE} \u2014 gitignored, machine-local, re-read by \`genex llm price\`.
|
|
27620
27567
|
`,
|
|
@@ -27829,11 +27776,13 @@ function parseArgs(argv) {
|
|
|
27829
27776
|
"--limit",
|
|
27830
27777
|
// `genex budget --assets <credits>` — the allowance the player approved.
|
|
27831
27778
|
"--assets",
|
|
27832
|
-
// `genex llm bench` value flags. `--max-
|
|
27833
|
-
//
|
|
27779
|
+
// `genex llm bench` value flags. `--max-credits` is the same approval
|
|
27780
|
+
// shape as `--assets`, but its own gate: the asset allowance does not
|
|
27781
|
+
// count a bench. `--max-coins` is its deprecated name (same number).
|
|
27834
27782
|
"--model",
|
|
27835
27783
|
"--samples",
|
|
27836
27784
|
"--schema",
|
|
27785
|
+
"--max-credits",
|
|
27837
27786
|
"--max-coins",
|
|
27838
27787
|
// `genex player install --label <name>` — what this machine is called in
|
|
27839
27788
|
// the dashboard. Its own key rather than the generic 2nd positional,
|
|
@@ -28348,12 +28297,14 @@ function applyValueFlag(options, flag, value) {
|
|
|
28348
28297
|
options.samples = n;
|
|
28349
28298
|
break;
|
|
28350
28299
|
}
|
|
28300
|
+
case "--max-credits":
|
|
28351
28301
|
case "--max-coins": {
|
|
28352
28302
|
const n = Number(value);
|
|
28353
28303
|
if (!Number.isInteger(n) || n < 1) {
|
|
28354
|
-
throw new Error(`Invalid
|
|
28304
|
+
throw new Error(`Invalid ${flag} value: ${value} (whole credits, 1 or more)`);
|
|
28355
28305
|
}
|
|
28356
|
-
options.
|
|
28306
|
+
if (flag === "--max-credits") options.maxCredits = n;
|
|
28307
|
+
else options.maxCoins = n;
|
|
28357
28308
|
break;
|
|
28358
28309
|
}
|
|
28359
28310
|
case "--timeout": {
|
|
@@ -28645,9 +28596,9 @@ async function main() {
|
|
|
28645
28596
|
await runBudget(parsed.options);
|
|
28646
28597
|
break;
|
|
28647
28598
|
// The in-game model lane: is it live here, what does one call really
|
|
28648
|
-
// cost, what should the game declare. `bench`
|
|
28649
|
-
//
|
|
28650
|
-
//
|
|
28599
|
+
// cost, what ceiling should the game declare. `bench` spends the
|
|
28600
|
+
// builder's own credits OUTSIDE the asset allowance, so it carries its
|
|
28601
|
+
// own --max-credits/--user-approved gate.
|
|
28651
28602
|
case "llm":
|
|
28652
28603
|
await runLlm(parsed.options);
|
|
28653
28604
|
break;
|